From 1a7a3c163fb51aeaf4530d8f8f15239b2ea5f9a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Kokosi=C5=84ski?= Date: Wed, 31 Jul 2019 18:56:16 -0700 Subject: [PATCH 1/2] Cardinality of fiter compared to zero is rewritten to any_match/none_match. --- .mailmap | 25 + .travis.yml | 6 +- CONTRIBUTING.md | 4 +- README.md | 25 +- pom.xml | 246 ++- presto-accumulo/pom.xml | 24 +- .../presto/accumulo/AccumuloClient.java | 2 +- .../presto/accumulo/AccumuloConnector.java | 4 +- .../accumulo/AccumuloConnectorFactory.java | 4 +- .../presto/accumulo/AccumuloModule.java | 10 +- .../presto/accumulo/AccumuloTableManager.java | 2 +- .../presto/accumulo/conf/AccumuloConfig.java | 6 +- .../index/ColumnCardinalityCache.java | 6 +- .../presto/accumulo/index/IndexLookup.java | 6 +- .../presto/accumulo/io/AccumuloPageSink.java | 2 +- .../presto/accumulo/io/AccumuloRecordSet.java | 2 +- .../metadata/ZooKeeperMetadataManager.java | 2 +- .../facebook/presto/accumulo/model/Field.java | 2 +- .../facebook/presto/accumulo/model/Row.java | 19 +- .../serializers/AccumuloRowSerializer.java | 4 +- .../presto/accumulo/AccumuloQueryRunner.java | 2 +- .../accumulo/model/TestAccumuloSplit.java | 2 +- .../presto/accumulo/model/TestRow.java | 2 +- .../AbstractTestAccumuloRowSerializer.java | 6 +- presto-array/pom.xml | 2 +- .../com/facebook/presto/array/Arrays.java | 139 ++ presto-atop/pom.xml | 16 +- .../facebook/presto/atop/AtopConnector.java | 4 +- .../presto/atop/AtopConnectorConfig.java | 4 +- .../presto/atop/AtopConnectorFactory.java | 6 +- .../com/facebook/presto/atop/AtopModule.java | 2 +- .../facebook/presto/atop/AtopPageSource.java | 6 + .../presto/atop/AtopProcessFactory.java | 2 +- .../presto/atop/LocalAtopQueryRunner.java | 2 +- .../presto/atop/TestAtopConnectorConfig.java | 6 +- .../facebook/presto/atop/TestAtopPlugin.java | 2 +- .../facebook/presto/atop/TestAtopSplit.java | 2 +- presto-base-jdbc/pom.xml | 38 +- .../presto/plugin/jdbc/BaseJdbcClient.java | 296 +++- .../presto/plugin/jdbc/BaseJdbcConfig.java | 36 +- .../presto/plugin/jdbc/ConnectionFactory.java | 2 +- .../plugin/jdbc/DriverConnectionFactory.java | 2 +- .../presto/plugin/jdbc/JdbcClient.java | 38 +- .../presto/plugin/jdbc/JdbcConnector.java | 34 +- .../plugin/jdbc/JdbcConnectorFactory.java | 14 +- .../presto/plugin/jdbc/JdbcIdentity.java | 79 + .../presto/plugin/jdbc/JdbcMetadata.java | 40 +- .../plugin/jdbc/JdbcMetadataConfig.java | 4 +- .../presto/plugin/jdbc/JdbcModule.java | 2 +- .../presto/plugin/jdbc/JdbcPageSink.java | 7 +- .../plugin/jdbc/JdbcPageSinkProvider.java | 4 +- .../presto/plugin/jdbc/JdbcRecordCursor.java | 4 +- .../presto/plugin/jdbc/JdbcSplit.java | 7 +- .../presto/plugin/jdbc/JdbcSplitManager.java | 2 +- .../presto/plugin/jdbc/JdbcTableHandle.java | 2 + .../plugin/jdbc/JdbcTableLayoutHandle.java | 18 +- .../presto/plugin/jdbc/QueryBuilder.java | 8 +- .../plugin/jdbc/RemoteTableNameCacheKey.java | 70 + .../optimization/JdbcComputePushdown.java | 163 ++ .../jdbc/optimization/JdbcExpression.java | 83 ++ .../JdbcFilterToSqlTranslator.java | 181 +++ .../JdbcPlanOptimizerProvider.java | 77 + .../function/JdbcTranslationUtil.java | 41 + .../function/OperatorTranslators.java | 69 + .../presto/plugin/jdbc/JdbcQueryRunner.java | 2 +- .../presto/plugin/jdbc/MetadataUtil.java | 6 +- .../plugin/jdbc/TestBaseJdbcConfig.java | 16 +- .../presto/plugin/jdbc/TestJdbcClient.java | 30 +- .../plugin/jdbc/TestJdbcColumnHandle.java | 2 +- .../jdbc/TestJdbcIntegrationSmokeTest.java | 44 + .../plugin/jdbc/TestJdbcMetadataConfig.java | 6 +- .../plugin/jdbc/TestJdbcQueryBuilder.java | 2 +- .../jdbc/TestJdbcRecordSetProvider.java | 11 +- .../presto/plugin/jdbc/TestJdbcSplit.java | 4 +- .../plugin/jdbc/TestJdbcTableHandle.java | 2 +- .../presto/plugin/jdbc/TestingDatabase.java | 12 +- .../plugin/jdbc/TestingH2JdbcModule.java | 2 +- .../optimization/TestJdbcComputePushdown.java | 334 +++++ presto-benchmark-driver/pom.xml | 10 +- .../driver/BenchmarkDriverOptions.java | 1 - .../driver/BenchmarkQueryRunner.java | 24 +- .../driver/PrestoBenchmarkDriver.java | 10 +- .../presto/benchmark/driver/Suite.java | 2 +- presto-benchmark-runner/pom.xml | 96 ++ .../benchmark/framework/BenchmarkQuery.java | 104 ++ .../framework/BenchmarkRunnerConfig.java | 52 + .../benchmark/framework/BenchmarkSuite.java | 67 + .../framework/BenchmarkSuiteInfo.java | 87 ++ .../framework/ConcurrentExecutionPhase.java | 72 + .../framework/MySqlBenchmarkSuiteConfig.java | 36 + .../framework/PhaseSpecification.java | 50 + .../framework/StreamExecutionPhase.java | 75 + .../source/BenchmarkSuiteConfig.java | 64 + .../benchmark/source/BenchmarkSuiteDao.java | 87 ++ .../source/BenchmarkSuiteModule.java | 58 + .../source/BenchmarkSuiteSupplier.java | 23 + .../source/DbBenchmarkSuiteSupplier.java | 54 + .../PhaseSpecificationsColumnMapper.java | 39 + .../source/StringToStringMapColumnMapper.java | 38 + .../presto/benchmark/BenchmarkTestUtil.java | 122 ++ .../framework/TestBenchmarkRunnerConfig.java | 48 + .../source/TestBenchmarkSuiteConfig.java | 51 + .../source/TestBenchmarkSuiteModule.java | 103 ++ .../source/TestDbBenchmarkSuiteSupplier.java | 78 + presto-benchmark/pom.xml | 12 +- .../benchmark/AbstractOperatorBenchmark.java | 14 +- .../presto/benchmark/BenchmarkSuite.java | 2 +- .../benchmark/CountAggregationBenchmark.java | 2 +- .../DoubleSumAggregationBenchmark.java | 2 +- .../presto/benchmark/HandTpchQuery1.java | 2 +- .../presto/benchmark/HandTpchQuery6.java | 4 +- .../benchmark/HashAggregationBenchmark.java | 2 +- .../presto/benchmark/HashJoinBenchmark.java | 2 +- .../JsonAvgBenchmarkResultWriter.java | 2 +- .../benchmark/PredicateFilterBenchmark.java | 2 +- .../benchmark/MemoryLocalQueryRunner.java | 4 +- presto-benchto-benchmarks/pom.xml | 4 +- .../planner/AbstractCostBasedPlanTest.java | 2 +- .../resources/sql/presto/tpcds/q01.plan.txt | 10 +- .../resources/sql/presto/tpcds/q02.plan.txt | 10 +- .../resources/sql/presto/tpcds/q03.plan.txt | 2 +- .../resources/sql/presto/tpcds/q04.plan.txt | 58 +- .../resources/sql/presto/tpcds/q05.plan.txt | 14 +- .../resources/sql/presto/tpcds/q06.plan.txt | 14 +- .../resources/sql/presto/tpcds/q07.plan.txt | 2 +- .../resources/sql/presto/tpcds/q08.plan.txt | 18 +- .../resources/sql/presto/tpcds/q10.plan.txt | 12 +- .../resources/sql/presto/tpcds/q11.plan.txt | 32 +- .../resources/sql/presto/tpcds/q12.plan.txt | 4 +- .../resources/sql/presto/tpcds/q14_1.plan.txt | 46 +- .../resources/sql/presto/tpcds/q14_2.plan.txt | 28 +- .../resources/sql/presto/tpcds/q15.plan.txt | 10 +- .../resources/sql/presto/tpcds/q16.plan.txt | 6 +- .../resources/sql/presto/tpcds/q17.plan.txt | 10 +- .../resources/sql/presto/tpcds/q18.plan.txt | 12 +- .../resources/sql/presto/tpcds/q19.plan.txt | 10 +- .../resources/sql/presto/tpcds/q20.plan.txt | 4 +- .../resources/sql/presto/tpcds/q21.plan.txt | 2 +- .../resources/sql/presto/tpcds/q22.plan.txt | 4 +- .../resources/sql/presto/tpcds/q23_1.plan.txt | 28 +- .../resources/sql/presto/tpcds/q23_2.plan.txt | 40 +- .../resources/sql/presto/tpcds/q24_1.plan.txt | 18 +- .../resources/sql/presto/tpcds/q24_2.plan.txt | 18 +- .../resources/sql/presto/tpcds/q25.plan.txt | 10 +- .../resources/sql/presto/tpcds/q26.plan.txt | 2 +- .../resources/sql/presto/tpcds/q27.plan.txt | 8 +- .../resources/sql/presto/tpcds/q28.plan.txt | 12 +- .../resources/sql/presto/tpcds/q29.plan.txt | 10 +- .../resources/sql/presto/tpcds/q30.plan.txt | 14 +- .../resources/sql/presto/tpcds/q31.plan.txt | 18 +- .../resources/sql/presto/tpcds/q32.plan.txt | 4 +- .../resources/sql/presto/tpcds/q33.plan.txt | 14 +- .../resources/sql/presto/tpcds/q34.plan.txt | 6 +- .../resources/sql/presto/tpcds/q35.plan.txt | 20 +- .../resources/sql/presto/tpcds/q36.plan.txt | 6 +- .../resources/sql/presto/tpcds/q37.plan.txt | 2 +- .../resources/sql/presto/tpcds/q38.plan.txt | 16 +- .../resources/sql/presto/tpcds/q39_1.plan.txt | 8 +- .../resources/sql/presto/tpcds/q39_2.plan.txt | 8 +- .../resources/sql/presto/tpcds/q40.plan.txt | 2 +- .../resources/sql/presto/tpcds/q41.plan.txt | 4 +- .../resources/sql/presto/tpcds/q42.plan.txt | 2 +- .../resources/sql/presto/tpcds/q43.plan.txt | 2 +- .../resources/sql/presto/tpcds/q44.plan.txt | 20 +- .../resources/sql/presto/tpcds/q45.plan.txt | 10 +- .../resources/sql/presto/tpcds/q46.plan.txt | 12 +- .../resources/sql/presto/tpcds/q47.plan.txt | 12 +- .../resources/sql/presto/tpcds/q48.plan.txt | 4 +- .../resources/sql/presto/tpcds/q49.plan.txt | 26 +- .../resources/sql/presto/tpcds/q50.plan.txt | 2 +- .../resources/sql/presto/tpcds/q51.plan.txt | 10 +- .../resources/sql/presto/tpcds/q52.plan.txt | 2 +- .../resources/sql/presto/tpcds/q53.plan.txt | 4 +- .../resources/sql/presto/tpcds/q54.plan.txt | 10 +- .../resources/sql/presto/tpcds/q55.plan.txt | 2 +- .../resources/sql/presto/tpcds/q56.plan.txt | 14 +- .../resources/sql/presto/tpcds/q57.plan.txt | 12 +- .../resources/sql/presto/tpcds/q58.plan.txt | 6 +- .../resources/sql/presto/tpcds/q59.plan.txt | 8 +- .../resources/sql/presto/tpcds/q60.plan.txt | 14 +- .../resources/sql/presto/tpcds/q61.plan.txt | 8 +- .../resources/sql/presto/tpcds/q62.plan.txt | 2 +- .../resources/sql/presto/tpcds/q63.plan.txt | 4 +- .../resources/sql/presto/tpcds/q64.plan.txt | 136 +- .../resources/sql/presto/tpcds/q65.plan.txt | 8 +- .../resources/sql/presto/tpcds/q66.plan.txt | 6 +- .../resources/sql/presto/tpcds/q67.plan.txt | 6 +- .../resources/sql/presto/tpcds/q68.plan.txt | 12 +- .../resources/sql/presto/tpcds/q69.plan.txt | 16 +- .../resources/sql/presto/tpcds/q70.plan.txt | 12 +- .../resources/sql/presto/tpcds/q71.plan.txt | 2 +- .../resources/sql/presto/tpcds/q72.plan.txt | 6 +- .../resources/sql/presto/tpcds/q73.plan.txt | 6 +- .../resources/sql/presto/tpcds/q74.plan.txt | 32 +- .../resources/sql/presto/tpcds/q75.plan.txt | 38 +- .../resources/sql/presto/tpcds/q76.plan.txt | 16 +- .../resources/sql/presto/tpcds/q77.plan.txt | 16 +- .../resources/sql/presto/tpcds/q78.plan.txt | 10 +- .../resources/sql/presto/tpcds/q79.plan.txt | 2 +- .../resources/sql/presto/tpcds/q80.plan.txt | 10 +- .../resources/sql/presto/tpcds/q81.plan.txt | 6 +- .../resources/sql/presto/tpcds/q82.plan.txt | 2 +- .../resources/sql/presto/tpcds/q83.plan.txt | 6 +- .../resources/sql/presto/tpcds/q85.plan.txt | 10 +- .../resources/sql/presto/tpcds/q86.plan.txt | 6 +- .../resources/sql/presto/tpcds/q87.plan.txt | 16 +- .../resources/sql/presto/tpcds/q89.plan.txt | 4 +- .../resources/sql/presto/tpcds/q91.plan.txt | 2 +- .../resources/sql/presto/tpcds/q92.plan.txt | 4 +- .../resources/sql/presto/tpcds/q93.plan.txt | 6 +- .../resources/sql/presto/tpcds/q94.plan.txt | 6 +- .../resources/sql/presto/tpcds/q95.plan.txt | 14 +- .../resources/sql/presto/tpcds/q97.plan.txt | 4 +- .../resources/sql/presto/tpcds/q98.plan.txt | 4 +- .../resources/sql/presto/tpcds/q99.plan.txt | 2 +- .../resources/sql/presto/tpch/q01.plan.txt | 2 +- .../resources/sql/presto/tpch/q02.plan.txt | 8 +- .../resources/sql/presto/tpch/q03.plan.txt | 2 +- .../resources/sql/presto/tpch/q04.plan.txt | 6 +- .../resources/sql/presto/tpch/q05.plan.txt | 6 +- .../resources/sql/presto/tpch/q07.plan.txt | 6 +- .../resources/sql/presto/tpch/q08.plan.txt | 10 +- .../resources/sql/presto/tpch/q09.plan.txt | 20 +- .../resources/sql/presto/tpch/q10.plan.txt | 4 +- .../resources/sql/presto/tpch/q11.plan.txt | 2 +- .../resources/sql/presto/tpch/q12.plan.txt | 6 +- .../resources/sql/presto/tpch/q13.plan.txt | 6 +- .../resources/sql/presto/tpch/q14.plan.txt | 4 +- .../resources/sql/presto/tpch/q15.plan.txt | 6 +- .../resources/sql/presto/tpch/q16.plan.txt | 10 +- .../resources/sql/presto/tpch/q17.plan.txt | 4 +- .../resources/sql/presto/tpch/q18.plan.txt | 12 +- .../resources/sql/presto/tpch/q19.plan.txt | 4 +- .../resources/sql/presto/tpch/q20.plan.txt | 12 +- .../resources/sql/presto/tpch/q21.plan.txt | 20 +- .../resources/sql/presto/tpch/q22.plan.txt | 6 +- presto-blackhole/pom.xml | 10 +- .../blackhole/BlackHoleConnectorFactory.java | 2 +- .../plugin/blackhole/BlackHolePageSink.java | 2 +- .../plugin/blackhole/BlackHolePageSource.java | 12 +- .../blackhole/BlackHoleQueryRunner.java | 6 +- .../plugin/blackhole/TestBlackHoleSmoke.java | 2 +- presto-bytecode/pom.xml | 2 +- presto-cache/pom.xml | 105 ++ .../facebook/presto/cache/CacheConfig.java | 103 ++ .../facebook/presto/cache/CacheManager.java | 34 + .../com/facebook/presto/cache/CacheStats.java | 61 + .../presto/cache/CachingFileSystem.java | 139 ++ .../presto/cache/CachingInputStream.java | 66 + .../presto/cache/FileReadRequest.java | 70 + .../presto/cache/ForCachingFileSystem.java | 31 + .../presto/cache/LocalRangeCacheManager.java | 498 +++++++ .../presto/cache/NoOpCacheManager.java | 18 +- .../presto/cache/TestCacheConfig.java | 65 + .../cache/TestLocalRangeCacheManager.java | 255 ++++ presto-cassandra/pom.xml | 14 +- .../cassandra/CassandraClientConfig.java | 8 +- .../cassandra/CassandraClientModule.java | 6 +- .../presto/cassandra/CassandraConnector.java | 4 +- .../cassandra/CassandraConnectorFactory.java | 4 +- .../presto/cassandra/CassandraMetadata.java | 2 +- .../cassandra/CassandraPartitionManager.java | 2 +- .../cassandra/CassandraRecordSetProvider.java | 2 +- .../cassandra/NativeCassandraSession.java | 4 +- .../cassandra/RebindSafeMBeanServer.java | 2 +- .../presto/cassandra/ReopeningCluster.java | 2 +- .../presto/cassandra/EmbeddedCassandra.java | 4 +- .../cassandra/TestCassandraClientConfig.java | 2 +- .../cassandra/TestCassandraColumnHandle.java | 4 +- .../cassandra/TestCassandraConnector.java | 7 +- .../presto/cassandra/TestCassandraSplit.java | 2 +- .../cassandra/TestCassandraTableHandle.java | 2 +- .../cassandra/TestJsonCassandraHandles.java | 4 +- presto-cli/pom.xml | 10 +- .../facebook/presto/cli/ClientOptions.java | 1 - .../java/com/facebook/presto/cli/Console.java | 13 +- .../java/com/facebook/presto/cli/Pager.java | 2 +- .../java/com/facebook/presto/cli/Query.java | 5 - .../presto/cli/QueryPreprocessor.java | 2 +- .../facebook/presto/cli/StatusPrinter.java | 2 +- .../presto/cli/TableNameCompleter.java | 2 +- .../facebook/presto/cli/TestQueryRunner.java | 5 +- presto-client/pom.xml | 6 +- .../facebook/presto/client/ClientSession.java | 18 - .../client/ClientTypeSignatureParameter.java | 2 +- .../presto/client/FixJsonDataUtils.java | 2 + .../facebook/presto/client/JsonResponse.java | 2 +- .../facebook/presto/client/OkHttpUtil.java | 2 +- .../facebook/presto/client/PrestoHeaders.java | 3 - .../presto/client/StatementClient.java | 2 - .../presto/client/StatementClientV1.java | 22 +- .../client/TestClientTypeSignature.java | 6 +- .../presto/client/TestFixJsonDataUtils.java | 1 + .../presto/client/TestQueryResults.java | 4 +- .../presto/client/TestServerInfo.java | 4 +- presto-docs/pom.xml | 2 +- .../src/main/sphinx/admin/properties.rst | 13 + presto-docs/src/main/sphinx/admin/spill.rst | 18 + presto-docs/src/main/sphinx/conf.py | 4 +- presto-docs/src/main/sphinx/connector.rst | 1 + .../main/sphinx/connector/elasticsearch.rst | 8 +- .../src/main/sphinx/connector/hive.rst | 42 + .../main/sphinx/connector/kafka-tutorial.rst | 2 +- .../src/main/sphinx/connector/pinot.rst | 100 ++ presto-docs/src/main/sphinx/functions.rst | 1 + .../src/main/sphinx/functions/aggregate.rst | 376 ++++- .../src/main/sphinx/functions/array.rst | 46 +- .../src/main/sphinx/functions/geospatial.rst | 95 +- presto-docs/src/main/sphinx/functions/ip.rst | 12 + .../src/main/sphinx/functions/string.rst | 4 +- .../src/main/sphinx/functions/window.rst | 12 +- .../src/main/sphinx/installation/verifier.rst | 71 +- .../src/main/sphinx/language/reserved.rst | 1 - .../src/main/sphinx/language/types.rst | 32 +- presto-docs/src/main/sphinx/release.rst | 6 + .../src/main/sphinx/release/release-0.224.rst | 45 + .../src/main/sphinx/release/release-0.225.rst | 50 + .../src/main/sphinx/release/release-0.226.rst | 20 + .../src/main/sphinx/release/release-0.227.rst | 57 + .../src/main/sphinx/release/release-0.228.rst | 21 + .../src/main/sphinx/release/release-0.229.rst | 36 + presto-elasticsearch/pom.xml | 20 +- .../elasticsearch/ElasticsearchClient.java | 8 +- .../elasticsearch/ElasticsearchConnector.java | 4 +- .../ElasticsearchConnectorConfig.java | 6 +- .../ElasticsearchConnectorFactory.java | 4 +- .../ElasticsearchConnectorModule.java | 6 +- .../ElasticsearchQueryBuilder.java | 4 +- .../ElasticsearchRecordCursor.java | 91 +- ...ElasticsearchTableDescriptionProvider.java | 2 +- .../elasticsearch/ElasticsearchUtils.java | 24 +- .../presto/elasticsearch/RetryDriver.java | 2 +- .../presto/elasticsearch/CodecSupplier.java | 6 +- .../ElasticsearchQueryRunner.java | 8 +- .../TestElasticsearchConnectorConfig.java | 6 +- ...TestElasticsearchIntegrationSmokeTest.java | 26 + .../TestingElasticsearchConnectorFactory.java | 4 +- .../resources/queryrunner/nested.data.json | 25 + presto-example-http/pom.xml | 16 +- .../presto/example/ExampleClient.java | 2 +- .../presto/example/ExampleConfig.java | 2 +- .../presto/example/ExampleConnector.java | 4 +- .../example/ExampleConnectorFactory.java | 4 +- .../presto/example/ExampleModule.java | 8 +- .../presto/example/ExampleHttpServer.java | 12 +- .../facebook/presto/example/MetadataUtil.java | 8 +- .../example/TestExampleColumnHandle.java | 2 +- .../presto/example/TestExampleConfig.java | 2 +- .../presto/example/TestExampleSplit.java | 4 +- .../example/TestExampleTableHandle.java | 6 +- presto-expressions/pom.xml | 48 + .../DefaultRowExpressionTraversalVisitor.java | 10 +- .../expressions/LogicalRowExpressions.java | 812 ++++++++++ .../RowExpressionNodeInliner.java | 4 +- .../expressions}/RowExpressionRewriter.java | 10 +- .../RowExpressionTreeRewriter.java | 11 +- .../translator/FunctionTranslator.java | 78 + .../translator/RowExpressionTranslator.java | 50 + .../RowExpressionTreeTranslator.java | 91 ++ .../translator/TranslatedExpression.java | 61 + .../TranslatorAnnotationParser.java | 292 ++++ presto-geospatial-toolkit/pom.xml | 19 +- .../presto/geospatial/GeometryUtils.java | 116 ++ .../presto/geospatial/KdbTreeUtils.java | 4 +- .../facebook/presto/geospatial/Rectangle.java | 17 +- .../presto/geospatial/rtree/Flatbush.java | 306 ++++ .../presto/geospatial/rtree/HasExtent.java | 22 + .../presto/geospatial/rtree/HilbertIndex.java | 148 ++ ...metrySerde.java => EsriGeometrySerde.java} | 50 +- .../geospatial/serde/JtsGeometrySerde.java | 7 + .../presto/geospatial/TestGeometryUtils.java | 72 + .../rtree/BenchmarkFlatbushBuild.java | 97 ++ .../rtree/BenchmarkFlatbushQuery.java | 105 ++ .../rtree/BenchmarkJtsStrTreeBuild.java | 96 ++ .../rtree/BenchmarkJtsStrTreeQuery.java | 115 ++ .../geospatial/rtree/RtreeTestUtils.java | 50 + .../presto/geospatial/rtree/TestFlatbush.java | 279 ++++ .../geospatial/rtree/TestHilbertIndex.java | 73 + .../serde/BenchmarkGeometrySerde.java | 6 +- .../serde/TestGeometrySerialization.java | 40 +- presto-geospatial/pom.xml | 14 +- .../presto/plugin/geospatial/BingTile.java | 4 +- .../plugin/geospatial/BingTileFunctions.java | 58 +- .../plugin/geospatial/GeoFunctions.java | 637 ++++---- .../plugin/geospatial/GeometryType.java | 2 +- ...PartitioningInternalAggregateFunction.java | 9 +- .../SpatialPartitioningStateFactory.java | 2 +- .../geospatial/SphericalGeographyType.java | 2 +- .../aggregation/ConvexHullAggregation.java | 6 +- .../aggregation/GeometryStateSerializer.java | 6 +- .../aggregation/GeometryUnionAgg.java | 6 +- .../BenchmarkEnvelopeIntersection.java | 2 +- .../plugin/geospatial/BenchmarkSTArea.java | 10 +- .../geospatial/BenchmarkSTContains.java | 4 +- .../geospatial/BenchmarkSTIntersects.java | 2 +- .../plugin/geospatial/GeoQueryRunner.java | 4 +- .../geospatial/TestBingTileFunctions.java | 32 +- .../plugin/geospatial/TestGeoFunctions.java | 517 +++---- .../plugin/geospatial/TestGeoRelations.java | 286 ++++ ...RewriteSpatialPartitioningAggregation.java | 2 +- .../geospatial/TestSpatialJoinOperator.java | 12 +- .../geospatial/TestSpatialJoinPlanning.java | 28 +- .../plugin/geospatial/TestSpatialJoins.java | 67 +- ...patialPartitioningInternalAggregation.java | 27 +- .../geospatial/TestSphericalGeoFunctions.java | 35 +- .../AbstractTestGeoAggregationFunctions.java | 6 +- .../aggregation/TestGeometryStateFactory.java | 9 +- presto-hive-hadoop2/pom.xml | 23 +- .../hive/AbstractTestHiveFileSystemS3.java | 5 +- presto-hive-metastore/pom.xml | 226 +++ .../presto/hive/ForCachingHiveMetastore.java | 0 .../hive/ForRecordingHiveMetastore.java | 0 .../presto/hive/HdfsConfiguration.java | 0 .../facebook/presto/hive/HdfsEnvironment.java | 12 +- .../presto/hive/HiveBasicStatistics.java | 0 .../presto/hive/HiveBucketProperty.java | 2 +- .../presto/hive/HiveStorageFormat.java | 51 - .../com/facebook/presto/hive/HiveType.java | 11 +- .../facebook/presto/hive/HiveTypeName.java | 3 +- .../hive/HiveViewNotSupportedException.java | 0 .../facebook/presto/hive/LocationHandle.java | 19 + .../presto/hive/MetastoreClientConfig.java | 184 +++ .../presto/hive/MetastoreClientModule.java | 33 + .../presto/hive/MetastoreErrorCode.java | 55 + .../hive/PartitionNotFoundException.java | 0 .../hive/PartitionOfflineException.java | 2 +- .../com/facebook/presto/hive/RetryDriver.java | 2 +- .../hive/SchemaAlreadyExistsException.java | 0 .../hive/TableAlreadyExistsException.java | 0 .../presto/hive/TableOfflineException.java | 2 +- .../facebook/presto/hive/TypeTranslator.java | 0 .../GenericExceptionAction.java | 0 .../authentication/HdfsAuthentication.java | 0 .../HiveMetastoreAuthentication.java | 0 .../NoHiveMetastoreAuthentication.java | 0 .../hive/metastore/BooleanStatistics.java | 0 .../hive/metastore/CachingHiveMetastore.java | 13 +- .../presto/hive/metastore/Column.java | 12 + .../presto/hive/metastore/Database.java | 0 .../presto/hive/metastore/DateStatistics.java | 0 .../hive/metastore/DecimalStatistics.java | 0 .../hive/metastore/DoubleStatistics.java | 0 .../hive/metastore/ExtendedHiveMetastore.java | 1 - .../hive/metastore/HiveColumnStatistics.java | 0 .../hive/metastore/HiveMetastoreModule.java | 4 +- .../hive/metastore/HivePageSinkMetadata.java | 0 .../HivePageSinkMetadataProvider.java | 0 .../hive/metastore/HivePartitionName.java | 2 +- .../hive/metastore/HivePrivilegeInfo.java | 0 .../presto/hive/metastore/HiveTableName.java | 0 .../hive/metastore/IntegerStatistics.java | 0 .../hive/metastore/MetastoreConfig.java | 2 +- .../presto/hive/metastore/MetastoreUtil.java | 621 ++++++++ .../presto/hive/metastore/Partition.java | 0 .../hive/metastore/PartitionFilter.java | 0 .../hive/metastore}/PartitionStatistics.java | 4 +- .../metastore/PartitionWithStatistics.java | 4 +- .../hive/metastore/PrestoTableType.java | 0 .../hive/metastore/PrincipalPrivileges.java | 0 .../metastore/RecordingHiveMetastore.java | 47 +- .../SemiTransactionalHiveMetastore.java | 90 +- .../presto/hive/metastore/SortingColumn.java | 2 +- .../presto/hive/metastore}/Statistics.java | 19 +- .../presto/hive/metastore/Storage.java | 0 .../presto/hive/metastore/StorageFormat.java | 4 +- .../facebook/presto/hive/metastore/Table.java | 0 .../hive/metastore/UserDatabaseKey.java | 0 .../presto/hive/metastore/UserTableKey.java | 0 .../hive/metastore/file/DatabaseMetadata.java | 0 .../metastore/file/FileHiveMetastore.java | 26 +- .../file/FileHiveMetastoreConfig.java | 4 +- .../metastore/file/FileMetastoreModule.java | 2 +- .../metastore/file/PartitionMetadata.java | 2 +- .../metastore/file/PermissionMetadata.java | 0 .../hive/metastore/file/TableMetadata.java | 0 .../metastore/glue/ForGlueHiveMetastore.java | 29 + .../metastore/glue/GlueExpressionUtil.java | 2 +- .../metastore/glue/GlueHiveMetastore.java | 76 +- .../glue/GlueHiveMetastoreConfig.java | 36 +- .../metastore/glue/GlueMetastoreModule.java | 23 +- .../glue/converter/GlueInputConverter.java | 2 +- .../glue/converter/GlueToPrestoConverter.java | 2 +- .../thrift/BridgingHiveMetastore.java | 6 +- .../hive/metastore/thrift/HiveCluster.java | 0 .../hive/metastore/thrift/HiveMetastore.java | 4 +- .../thrift/HiveMetastoreApiStats.java | 4 +- .../metastore/thrift/HiveMetastoreClient.java | 0 .../thrift/HiveMetastoreClientFactory.java | 6 +- .../metastore/thrift/StaticHiveCluster.java | 0 .../thrift/StaticMetastoreConfig.java | 4 +- .../metastore/thrift/ThriftHiveMetastore.java | 6 +- .../thrift/ThriftHiveMetastoreClient.java | 0 .../thrift/ThriftHiveMetastoreStats.java | 0 .../thrift/ThriftMetastoreModule.java | 8 +- .../metastore/thrift/ThriftMetastoreUtil.java | 8 +- .../hive/metastore/thrift/Transport.java | 0 .../hive/TestPartitionOfflineException.java | 0 .../hive/TestTableOfflineException.java | 0 .../metastore/TestCachingHiveMetastore.java | 2 +- .../hive/metastore/TestHiveMetastoreUtil.java | 37 +- .../metastore/TestMetastoreClientConfig.java | 77 + .../hive/metastore/TestMetastoreConfig.java | 6 +- .../metastore/TestRecordingHiveMetastore.java | 7 +- .../presto/hive/metastore/TestStorage.java | 4 +- .../metastore/UnimplementedHiveMetastore.java | 1 - .../glue/TestGlueExpressionUtil.java | 0 .../glue/TestGlueHiveMetastoreConfig.java | 16 +- .../thrift/InMemoryHiveMetastore.java | 4 +- .../thrift/MockHiveMetastoreClient.java | 0 .../MockHiveMetastoreClientFactory.java | 0 .../thrift/TestStaticHiveCluster.java | 2 +- .../thrift/TestStaticMetastoreConfig.java | 6 +- .../thrift/TestThriftHiveMetastoreUtil.java | 2 +- .../metastore/thrift/TestingHiveCluster.java | 10 +- presto-hive/pom.xml | 139 +- .../hive/BackgroundHiveSplitLoader.java | 58 +- .../presto/hive/BucketAdaptation.java | 59 + .../hive/CreateEmptyPartitionProcedure.java | 2 +- .../facebook/presto/hive/DirectoryLister.java | 9 +- .../hive/DynamicConfigurationProvider.java | 25 + .../hive/FileFormatDataSourceStats.java | 4 +- .../com/facebook/presto/hive/FileOpener.java | 27 + .../presto/hive/FilteringPageSource.java | 288 ++++ .../hive/GenericHiveRecordCursorProvider.java | 2 +- .../presto/hive/HadoopDirectoryLister.java | 37 +- .../presto/hive/HadoopFileOpener.java | 32 + ...java => HdfsConfigurationInitializer.java} | 14 +- .../presto/hive/HiveAnalyzeProperties.java | 2 +- .../hive/HiveBatchPageSourceFactory.java | 9 +- .../facebook/presto/hive/HiveBucketing.java | 88 +- .../presto/hive/HiveClientConfig.java | 233 +-- .../presto/hive/HiveClientModule.java | 88 +- .../com/facebook/presto/hive/HiveCoercer.java | 555 +++++++ .../presto/hive/HiveCoercionRecordCursor.java | 6 +- .../presto/hive/HiveColumnHandle.java | 3 + .../facebook/presto/hive/HiveConnector.java | 4 +- .../presto/hive/HiveConnectorFactory.java | 14 +- .../facebook/presto/hive/HiveErrorCode.java | 28 +- .../facebook/presto/hive/HiveEventClient.java | 4 +- .../facebook/presto/hive/HiveFileInfo.java | 83 ++ .../presto/hive/HiveHdfsConfiguration.java | 25 +- .../presto/hive/HiveLocationService.java | 40 +- .../facebook/presto/hive/HiveMetadata.java | 493 ++++-- .../presto/hive/HiveMetadataFactory.java | 35 +- .../facebook/presto/hive/HivePageSink.java | 6 +- .../presto/hive/HivePageSinkProvider.java | 21 +- .../facebook/presto/hive/HivePageSource.java | 330 +--- .../presto/hive/HivePageSourceProvider.java | 253 ++-- .../presto/hive/HivePartitionKey.java | 2 +- .../presto/hive/HivePartitionManager.java | 109 +- .../presto/hive/HivePartitionMetadata.java | 11 +- .../hive/HivePartitionObjectBuilder.java | 51 + .../presto/hive/HivePartitionResult.java | 20 + .../hive/HiveSelectivePageSourceFactory.java | 15 +- .../presto/hive/HiveSessionProperties.java | 70 +- .../com/facebook/presto/hive/HiveSplit.java | 79 +- .../presto/hive/HiveSplitManager.java | 98 +- .../presto/hive/HiveSplitPartitionInfo.java | 41 +- .../facebook/presto/hive/HiveSplitSource.java | 50 +- .../presto/hive/HiveStagingFileCommitter.java | 30 +- .../presto/hive/HiveTableLayoutHandle.java | 72 +- .../presto/hive/HiveTableProperties.java | 3 + .../presto/hive/HiveTypeTranslator.java | 6 +- .../com/facebook/presto/hive/HiveUtil.java | 168 ++- .../facebook/presto/hive/HiveWriteUtils.java | 195 +-- .../presto/hive/HiveWriterFactory.java | 115 +- .../facebook/presto/hive/HiveWriterStats.java | 2 +- .../presto/hive/HiveZeroRowFileCreator.java | 67 +- .../presto/hive/InternalHiveSplit.java | 25 +- .../facebook/presto/hive/LocationService.java | 18 +- .../facebook/presto/hive/NamenodeStats.java | 4 +- .../presto/hive/OrcFileWriterConfig.java | 2 +- .../presto/hive/OrcFileWriterFactory.java | 5 +- .../presto/hive/ParquetFileWriterConfig.java | 2 +- .../presto/hive/PartitionObjectBuilder.java | 27 + .../presto/hive/RcFileFileWriterFactory.java | 3 +- .../presto/hive/RebindSafeMBeanServer.java | 2 +- .../presto/hive/S3SelectLineRecordReader.java | 12 +- .../presto/hive/S3SelectPushdown.java | 2 +- .../hive/S3SelectRecordCursorProvider.java | 2 +- .../presto/hive/SortingFileWriter.java | 34 +- .../presto/hive/SortingFileWriterFactory.java | 108 ++ .../presto/hive/StagingFileCommitter.java | 3 +- .../presto/hive/SubfieldExtractor.java | 136 +- .../presto/hive/TransactionalMetadata.java | 3 + .../presto/hive/WriteCompletedEvent.java | 6 +- .../authentication/AuthenticationModules.java | 20 +- .../authentication/HdfsKerberosConfig.java | 4 +- .../HiveAuthenticationModule.java | 4 +- .../KerberosAuthentication.java | 2 +- .../KerberosHadoopAuthentication.java | 6 +- .../MetastoreKerberosConfig.java | 4 +- .../hive/gcs/GcsAccessTokenProvider.java | 48 + .../hive/gcs/GcsConfigurationInitializer.java | 21 + .../hive/gcs/GcsConfigurationProvider.java | 42 + .../presto/hive/gcs/HiveGcsConfig.java | 51 + .../gcs/HiveGcsConfigurationInitializer.java | 54 + .../presto/hive/gcs/HiveGcsModule.java | 37 + .../presto/hive/metastore/MetastoreUtil.java | 297 ---- .../hive/orc/DwrfBatchPageSourceFactory.java | 46 +- .../orc/DwrfSelectivePageSourceFactory.java | 46 +- .../presto/hive/orc/HdfsOrcDataSource.java | 2 +- .../presto/hive/orc/OrcBatchPageSource.java | 27 +- .../hive/orc/OrcBatchPageSourceFactory.java | 82 +- .../hive/orc/OrcSelectivePageSource.java | 6 + .../orc/OrcSelectivePageSourceFactory.java | 376 ++++- .../hive/parquet/HdfsParquetDataSource.java | 2 +- .../hive/parquet/ParquetPageSource.java | 66 +- .../parquet/ParquetPageSourceFactory.java | 202 ++- .../presto/hive/rcfile/RcFilePageSource.java | 13 +- .../hive/rcfile/RcFilePageSourceFactory.java | 24 +- .../hive/rule/HivePlanOptimizerProvider.java | 8 +- .../facebook/presto/hive/s3/HiveS3Config.java | 6 +- .../facebook/presto/hive/s3/HiveS3Module.java | 7 +- .../presto/hive/s3/PrestoS3FileSystem.java | 181 ++- .../hive/s3/PrestoS3FileSystemStats.java | 12 +- .../presto/hive/s3/S3FileSystemType.java | 1 + .../hive/security/HiveSecurityModule.java | 4 +- .../hive/security/LegacyAccessControl.java | 14 +- .../hive/security/LegacySecurityConfig.java | 4 +- .../hive/security/LegacySecurityModule.java | 2 +- .../presto/hive/security/SecurityConfig.java | 2 +- .../security/SqlStandardAccessControl.java | 28 +- .../MetastoreHiveStatisticsProvider.java | 6 +- .../facebook/presto/hive/util/AsyncQueue.java | 4 +- .../presto/hive/util/HiveFileIterator.java | 55 +- .../hive/util/InternalHiveSplitFactory.java | 34 +- .../presto/hive/util/ResumableTasks.java | 2 +- .../presto/hive/util/TempFileReader.java | 26 +- .../presto/hive/AbstractTestHiveClient.java | 623 ++++++-- .../hive/AbstractTestHiveClientLocal.java | 3 +- .../hive/AbstractTestHiveFileFormats.java | 2 +- .../hive/AbstractTestHiveFileSystem.java | 69 +- .../presto/hive/HiveBenchmarkQueryRunner.java | 2 +- .../facebook/presto/hive/HiveQueryRunner.java | 79 +- .../facebook/presto/hive/HiveTestUtils.java | 76 +- .../hive/TestBackgroundHiveSplitLoader.java | 75 +- .../presto/hive/TestCoercingFilters.java | 73 + .../presto/hive/TestDomainTranslator.java | 91 +- .../presto/hive/TestFileSystemCache.java | 5 +- .../presto/hive/TestHiveClientConfig.java | 55 +- .../hive/TestHiveClientFileMetastore.java | 8 +- ...ntInMemoryMetastoreWithFilterPushdown.java | 59 + .../presto/hive/TestHiveColumnHandle.java | 2 +- .../presto/hive/TestHiveConnectorFactory.java | 4 +- ...tedQueriesWithExchangeMaterialization.java | 18 - ...tedQueriesWithOptimizedRepartitioning.java | 50 + .../presto/hive/TestHiveFileFormats.java | 320 +++- .../hive/TestHiveIntegrationSmokeTest.java | 353 ++++- .../presto/hive/TestHiveLogicalPlanner.java | 502 ++++++- .../presto/hive/TestHivePageSink.java | 91 +- .../presto/hive/TestHivePartitionManager.java | 185 +++ .../TestHivePushdownDistributedQueries.java | 56 + .../hive/TestHivePushdownFilterQueries.java | 936 +++++++++++- .../TestHivePushdownIntegrationSmokeTest.java | 43 + .../TestHiveRecoverableGroupedExecution.java | 413 ++--- .../facebook/presto/hive/TestHiveRoles.java | 14 +- .../facebook/presto/hive/TestHiveSplit.java | 43 +- .../presto/hive/TestHiveSplitSource.java | 61 +- .../presto/hive/TestHiveTableHandle.java | 2 +- .../presto/hive/TestHiveTypeTranslator.java | 2 +- .../facebook/presto/hive/TestHiveUtil.java | 17 +- .../presto/hive/TestHiveWriteUtils.java | 4 +- .../presto/hive/TestJsonHiveHandles.java | 4 +- .../TestOrcBatchPageSourceMemoryTracking.java | 78 +- .../presto/hive/TestOrcFileWriterConfig.java | 6 +- .../hive/TestParquetDistributedQueries.java | 97 ++ .../hive/TestParquetFileWriterConfig.java | 6 +- .../presto/hive/TestPartitionUpdate.java | 4 +- .../presto/hive/TestSubfieldExtractor.java | 84 +- ...TestingSemiTransactionalHiveMetastore.java | 340 +++++ .../TestHdfsKerberosConfig.java | 2 +- .../TestMetastoreKerberosConfig.java | 2 +- .../presto/hive/benchmark/FileFormat.java | 36 +- .../benchmark/HiveFileFormatBenchmark.java | 3 +- .../presto/hive/gcs/TestHiveGcsConfig.java | 49 + .../glue/TestHiveClientGlueMetastore.java | 50 +- .../parquet/AbstractTestParquetReader.java | 79 + .../presto/hive/parquet/ParquetTester.java | 119 +- .../parquet/write/TestDataWritableWriter.java | 2 +- .../facebook/presto/hive/s3/MockAmazonS3.java | 10 +- .../presto/hive/s3/TestHiveS3Config.java | 6 +- .../hive/s3/TestPrestoS3FileSystem.java | 73 +- .../security/TestLegacySecurityConfig.java | 6 +- .../TestMetastoreHiveStatisticsProvider.java | 4 +- .../presto/hive/util/TestAsyncQueue.java | 6 +- .../presto/hive/util/TestStatistics.java | 8 +- presto-jdbc/pom.xml | 25 +- .../presto/jdbc/PrestoConnection.java | 3 - .../facebook/presto/jdbc/QueryExecutor.java | 4 +- .../presto/jdbc/TestJdbcConnection.java | 68 +- .../jdbc/TestJdbcPreparedStatement.java | 2 +- .../presto/jdbc/TestJdbcResultSet.java | 2 +- .../presto/jdbc/TestJdbcWarnings.java | 2 +- .../jdbc/TestPrestoDatabaseMetaData.java | 4 +- .../presto/jdbc/TestPrestoDriver.java | 10 +- .../presto/jdbc/TestPrestoDriverAuth.java | 4 +- .../presto/jdbc/TestProgressMonitor.java | 6 +- .../presto/jdbc/TestQueryExecutor.java | 4 +- presto-jmx/pom.xml | 14 +- .../presto/connector/jmx/JmxConnector.java | 2 +- .../connector/jmx/JmxConnectorConfig.java | 2 +- .../connector/jmx/JmxConnectorFactory.java | 4 +- .../connector/jmx/JmxPeriodicSampler.java | 4 +- .../jmx/util/RebindSafeMBeanServer.java | 2 +- .../presto/connector/jmx/JmxQueryRunner.java | 2 +- .../presto/connector/jmx/MetadataUtil.java | 6 +- .../connector/jmx/TestJmxColumnHandle.java | 2 +- .../connector/jmx/TestJmxConnectorConfig.java | 8 +- .../connector/jmx/TestJmxTableHandle.java | 2 +- presto-kafka/pom.xml | 14 +- .../facebook/presto/kafka/KafkaConnector.java | 4 +- .../presto/kafka/KafkaConnectorConfig.java | 2 +- .../presto/kafka/KafkaConnectorFactory.java | 4 +- .../presto/kafka/KafkaConnectorModule.java | 6 +- .../facebook/presto/kafka/KafkaRecordSet.java | 2 +- .../kafka/KafkaSimpleConsumerManager.java | 2 +- .../presto/kafka/KafkaSplitManager.java | 2 +- .../kafka/KafkaTableDescriptionSupplier.java | 4 +- .../presto/kafka/KafkaQueryRunner.java | 8 +- .../kafka/TestKafkaConnectorConfig.java | 2 +- .../presto/kafka/TestKafkaPlugin.java | 2 +- .../presto/kafka/util/CodecSupplier.java | 6 +- .../facebook/presto/kafka/util/TestUtils.java | 2 +- presto-kudu/pom.xml | 12 +- .../presto/kudu/KuduClientConfig.java | 2 +- .../presto/kudu/KuduClientSession.java | 2 +- .../facebook/presto/kudu/KuduConnector.java | 4 +- .../presto/kudu/KuduConnectorFactory.java | 4 +- .../facebook/presto/kudu/KuduMetadata.java | 2 +- .../com/facebook/presto/kudu/KuduModule.java | 2 +- .../presto/kudu/KuduRecordCursor.java | 2 +- .../presto/kudu/KuduUpdatablePageSource.java | 6 + .../presto/kudu/KuduQueryRunnerFactory.java | 2 +- presto-local-file/pom.xml | 12 +- .../presto/localfile/LocalFileConfig.java | 4 +- .../presto/localfile/LocalFileConnector.java | 4 +- .../localfile/LocalFileConnectorFactory.java | 2 +- .../presto/localfile/LocalFileModule.java | 2 +- .../presto/localfile/MetadataUtil.java | 6 +- .../presto/localfile/TestLocalFileConfig.java | 2 +- .../presto/localfile/TestLocalFileSplit.java | 4 +- presto-main/etc/catalog/raptor.properties | 2 +- presto-main/pom.xml | 63 +- .../facebook/presto/FullConnectorSession.java | 21 +- .../java/com/facebook/presto/Session.java | 58 +- .../presto/SessionRepresentation.java | 21 - .../presto/SystemSessionProperties.java | 89 +- .../connector/ConnectorContextInstance.java | 12 +- .../presto/connector/ConnectorManager.java | 68 +- .../system/GlobalSystemConnector.java | 10 +- .../system/SystemConnectorSessionUtil.java | 4 - .../connector/system/TaskSystemTable.java | 6 +- .../presto/cost/AggregationStatsRule.java | 6 +- .../presto/cost/CachingCostProvider.java | 13 +- .../presto/cost/CachingStatsProvider.java | 2 +- ...ConnectorFilterStatsCalculatorService.java | 123 ++ .../facebook/presto/cost/CostCalculator.java | 5 +- .../cost/CostCalculatorUsingExchanges.java | 22 +- .../CostCalculatorWithEstimatedExchanges.java | 23 +- .../presto/cost/FilterStatsCalculator.java | 38 +- .../facebook/presto/cost/FilterStatsRule.java | 2 +- .../facebook/presto/cost/JoinStatsRule.java | 6 +- .../facebook/presto/cost/LimitStatsRule.java | 2 +- .../presto/cost/ProjectStatsRule.java | 2 +- .../presto/cost/ScalarStatsCalculator.java | 43 +- .../SimpleFilterProjectSemiJoinStatsRule.java | 8 +- .../presto/cost/SpatialJoinStatsRule.java | 2 +- .../presto/cost/StatsCalculatorModule.java | 14 +- .../com/facebook/presto/cost/StatsUtil.java | 18 + .../presto/cost/TableScanStatsRule.java | 21 +- .../facebook/presto/cost/UnionStatsRule.java | 15 +- .../facebook/presto/event/QueryMonitor.java | 105 +- .../presto/event/QueryMonitorConfig.java | 2 +- .../facebook/presto/event/SplitMonitor.java | 5 +- .../eventlistener/EventListenerManager.java | 2 +- ...ats.java => BasicStageExecutionStats.java} | 12 +- .../facebook/presto/execution/CallTask.java | 2 +- .../presto/execution/ClusterSizeMonitor.java | 2 +- .../presto/execution/CreateFunctionTask.java | 97 ++ .../presto/execution/CreateViewTask.java | 2 +- .../execution/DataDefinitionExecution.java | 6 + .../presto/execution/DropFunctionTask.java | 71 + .../execution/FailedQueryExecution.java | 6 + .../execution/MemoryRevokingScheduler.java | 2 +- .../presto/execution/NodeTaskMap.java | 2 +- .../presto/execution/QueryExecutionMBean.java | 2 +- .../facebook/presto/execution/QueryInfo.java | 11 - .../presto/execution/QueryManagerConfig.java | 53 +- .../presto/execution/QueryStateMachine.java | 186 ++- .../facebook/presto/execution/QueryStats.java | 11 + .../presto/execution/QueryTracker.java | 68 +- .../presto/execution/RemoteTaskFactory.java | 4 +- .../presto/execution/SetPathTask.java | 80 - .../presto/execution/SqlQueryExecution.java | 17 +- .../presto/execution/SqlQueryManager.java | 42 +- .../execution/SqlQueryManagerStats.java | 19 +- .../presto/execution/SqlStageExecution.java | 95 +- .../facebook/presto/execution/SqlTask.java | 19 +- .../presto/execution/SqlTaskExecution.java | 2 +- .../execution/SqlTaskExecutionFactory.java | 9 +- .../presto/execution/SqlTaskIoStats.java | 2 +- .../presto/execution/SqlTaskManager.java | 24 +- .../presto/execution/StageExecutionId.java | 87 ++ .../presto/execution/StageExecutionInfo.java | 82 + ...ageState.java => StageExecutionState.java} | 6 +- ...e.java => StageExecutionStateMachine.java} | 165 +- ...ageStats.java => StageExecutionStats.java} | 14 +- .../facebook/presto/execution/StageInfo.java | 56 +- .../presto/execution/StateMachine.java | 2 +- .../com/facebook/presto/execution/TaskId.java | 63 +- .../execution/TaskManagementExecutor.java | 4 +- .../presto/execution/TaskManager.java | 10 +- .../presto/execution/TaskManagerConfig.java | 24 +- .../presto/execution/TaskStateMachine.java | 2 +- ...ry.java => TrackingRemoteTaskFactory.java} | 29 +- .../execution/buffer/LazyOutputBuffer.java | 2 +- .../presto/execution/buffer/PagesSerde.java | 77 +- .../executor/MultilevelSplitQueue.java | 2 +- .../executor/PrioritizedSplitRunner.java | 8 +- .../execution/executor/TaskExecutor.java | 16 +- .../resourceGroups/InternalResourceGroup.java | 2 +- .../InternalResourceGroupManager.java | 6 +- .../scheduler/AllAtOnceExecutionSchedule.java | 10 +- .../scheduler/ExecutionWriterTarget.java | 137 ++ .../FixedSourcePartitionedScheduler.java | 28 +- .../scheduler/NetworkLocationCache.java | 4 +- .../presto/execution/scheduler/NodeMap.java | 12 +- .../execution/scheduler/NodeScheduler.java | 81 +- .../scheduler/NodeSchedulerConfig.java | 6 +- .../scheduler/NodeSchedulerExporter.java | 2 +- .../scheduler/PhasedExecutionSchedule.java | 10 +- .../scheduler/SimpleNodeSelector.java | 34 +- .../scheduler/SourcePartitionedScheduler.java | 6 +- .../scheduler/SplitSchedulerStats.java | 6 +- .../scheduler/SqlQueryScheduler.java | 767 ++++++---- .../execution/scheduler/TableWriteInfo.java | 222 +++ .../scheduler/TopologyAwareNodeSelector.java | 4 +- .../warnings/WarningCollectorConfig.java | 2 +- .../warnings/WarningCollectorModule.java | 2 +- .../failureDetector/FailureDetector.java | 2 +- .../FailureDetectorConfig.java | 4 +- .../FailureDetectorModule.java | 4 +- .../HeartbeatFailureDetector.java | 28 +- .../failureDetector/NoOpFailureDetector.java | 2 +- .../memory/ClusterMemoryLeakDetector.java | 2 +- .../presto/memory/ClusterMemoryManager.java | 43 +- .../presto/memory/MemoryManagerConfig.java | 6 +- .../facebook/presto/memory/MemoryPool.java | 19 +- .../presto/memory/NodeMemoryConfig.java | 4 +- .../facebook/presto/memory/QueryContext.java | 19 +- .../presto/memory/RemoteNodeMemory.java | 18 +- .../memory/ReservedSystemMemoryConfig.java | 2 +- .../presto/metadata/BuiltInFunction.java | 26 + ...Handle.java => BuiltInFunctionHandle.java} | 19 +- ...iltInFunctionNamespaceHandleResolver.java} | 5 +- ...a => BuiltInFunctionNamespaceManager.java} | 527 ++----- .../facebook/presto/metadata/CastType.java | 21 +- .../presto/metadata/DiscoveryNodeManager.java | 16 +- .../presto/metadata/FunctionExtractor.java | 4 +- .../metadata/FunctionInvokerProvider.java | 22 +- .../presto/metadata/FunctionListBuilder.java | 10 +- .../presto/metadata/FunctionManager.java | 609 +++++++- .../presto/metadata/HandleJsonModule.java | 2 +- .../presto/metadata/HandleResolver.java | 5 +- .../facebook/presto/metadata/Metadata.java | 12 +- .../presto/metadata/MetadataManager.java | 48 +- .../metadata/OperatorSignatureUtils.java | 43 - .../metadata/PolymorphicScalarFunction.java | 18 +- .../PolymorphicScalarFunctionBuilder.java | 13 +- .../presto/metadata/RemoteNodeState.java | 18 +- .../metadata/SessionPropertyManager.java | 4 +- .../presto/metadata/SignatureBuilder.java | 13 +- .../metadata/SpecializedFunctionKey.java | 6 +- .../metadata/SqlAggregationFunction.java | 6 +- .../facebook/presto/metadata/SqlOperator.java | 4 +- .../presto/metadata/SqlScalarFunction.java | 6 +- .../presto/metadata/StaticCatalogStore.java | 2 +- .../metadata/StaticCatalogStoreConfig.java | 4 +- .../StaticFunctionNamespaceStore.java | 90 ++ ...> StaticFunctionNamespaceStoreConfig.java} | 28 +- .../presto/operator/AggregationOperator.java | 2 +- .../facebook/presto/operator/Aggregator.java | 2 +- .../presto/operator/ArrayOfRowsUnnester.java | 92 -- .../presto/operator/ArrayUnnester.java | 68 - .../operator/AssignUniqueIdOperator.java | 2 +- .../presto/operator/DeleteOperator.java | 9 +- .../com/facebook/presto/operator/Driver.java | 4 +- .../presto/operator/DriverContext.java | 2 +- .../presto/operator/ExchangeClient.java | 52 +- .../presto/operator/ExchangeClientConfig.java | 17 +- .../operator/ExchangeClientFactory.java | 13 +- .../presto/operator/ExchangeOperator.java | 2 +- .../operator/ExplainAnalyzeOperator.java | 2 +- .../presto/operator/GroupByIdBlock.java | 6 +- .../operator/HashAggregationOperator.java | 2 +- .../presto/operator/HashBuilderOperator.java | 4 +- .../presto/operator/HashGenerator.java | 12 +- .../presto/operator/HashSemiJoinOperator.java | 2 +- .../presto/operator/HttpPageBufferClient.java | 46 +- .../presto/operator/LookupJoinOperator.java | 6 +- .../presto/operator/LookupOuterOperator.java | 2 +- .../facebook/presto/operator/MapUnnester.java | 71 - .../presto/operator/MergeOperator.java | 2 +- .../presto/operator/MoreByteArrays.java | 100 ++ .../operator/NestedLoopJoinOperator.java | 2 +- .../presto/operator/OperatorContext.java | 25 +- .../presto/operator/OperatorInfo.java | 5 +- .../presto/operator/OperatorStats.java | 24 + .../presto/operator/PageSourceOperator.java | 4 +- .../facebook/presto/operator/PagesIndex.java | 8 +- .../presto/operator/PagesRTreeIndex.java | 58 +- .../operator/PagesSpatialIndexSupplier.java | 64 +- .../presto/operator/PipelineContext.java | 4 +- .../presto/operator/PipelineStats.java | 2 +- .../ScanFilterAndProjectOperator.java | 19 +- .../presto/operator/SimpleArrayAllocator.java | 2 +- .../operator/SimplePagesHashStrategy.java | 2 +- .../operator/SpatialIndexBuilderOperator.java | 3 +- .../presto/operator/SpatialJoinOperator.java | 2 +- .../operator/SpilledLookupSourceHandle.java | 2 +- .../StreamingAggregationOperator.java | 2 +- .../presto/operator/TableCommitContext.java | 30 +- .../presto/operator/TableFinishInfo.java | 26 +- .../presto/operator/TableFinishOperator.java | 155 +- .../presto/operator/TableScanOperator.java | 17 +- .../presto/operator/TableWriterMergeInfo.java | 74 + .../operator/TableWriterMergeOperator.java | 356 +++++ .../presto/operator/TableWriterOperator.java | 61 +- .../presto/operator/TableWriterUtils.java | 140 ++ .../facebook/presto/operator/TaskContext.java | 11 +- .../facebook/presto/operator/TaskStats.java | 12 + .../presto/operator/UncheckedByteArrays.java | 94 ++ .../presto/operator/UnnestOperator.java | 262 ---- .../operator/WindowFunctionDefinition.java | 18 +- .../AbstractMinMaxAggregationFunction.java | 6 +- .../AbstractMinMaxNAggregationFunction.java | 4 +- .../AggregationImplementation.java | 4 +- .../aggregation/AggregationUtils.java | 35 + .../ApproximateCountDistinctAggregation.java | 2 +- ...proximateDoublePercentileAggregations.java | 2 +- ...mateDoublePercentileArrayAggregations.java | 2 +- ...ApproximateLongPercentileAggregations.java | 2 +- ...ximateLongPercentileArrayAggregations.java | 2 +- ...ApproximateRealPercentileAggregations.java | 2 +- ...ximateRealPercentileArrayAggregations.java | 2 +- .../ApproximateSetAggregation.java | 2 +- .../aggregation/HyperLogLogUtils.java | 2 +- .../MergeHyperLogLogAggregation.java | 2 +- .../MergeQuantileDigestFunction.java | 2 +- .../aggregation/ParametricAggregation.java | 4 +- .../QuantileDigestAggregationFunction.java | 2 +- .../ReduceAggregationFunction.java | 4 +- .../presto/operator/aggregation/TypedSet.java | 19 +- .../InMemoryHashAggregationBuilder.java | 4 +- .../MergingHashAggregationBuilder.java | 2 +- .../SpillableHashAggregationBuilder.java | 4 +- .../DifferentialEntropyAggregation.java | 111 ++ .../DifferentialEntropyState.java | 28 + .../DifferentialEntropyStateFactory.java | 110 ++ .../DifferentialEntropyStateSerializer.java | 57 + .../DifferentialEntropyStateStrategy.java | 230 +++ .../EntropyCalculations.java | 52 + .../FixedHistogramJacknifeStateStrategy.java | 181 +++ .../FixedHistogramMleStateStrategy.java | 146 ++ .../FixedHistogramStateStrategyUtils.java | 96 ++ ...nweightedReservoirSampleStateStrategy.java | 141 ++ .../WeightedReservoirSampleStateStrategy.java | 134 ++ .../FixedDoubleBreakdownHistogram.java | 125 +- .../fixedhistogram/FixedDoubleHistogram.java | 45 +- .../fixedhistogram/FixedHistogramUtils.java | 8 +- .../minmaxby/AbstractMinMaxBy.java | 8 +- .../UnweightedDoubleReservoirSample.java | 173 +++ .../WeightedDoubleReservoirSample.java | 219 +++ .../state/DigestAndPercentileArrayState.java | 2 +- .../DigestAndPercentileArrayStateFactory.java | 2 +- ...gestAndPercentileArrayStateSerializer.java | 2 +- .../state/DigestAndPercentileState.java | 2 +- .../DigestAndPercentileStateFactory.java | 2 +- .../DigestAndPercentileStateSerializer.java | 2 +- .../aggregation/state/HyperLogLogState.java | 2 +- .../state/HyperLogLogStateFactory.java | 2 +- .../state/HyperLogLogStateSerializer.java | 2 +- .../state/PrecisionRecallStateSerializer.java | 4 +- .../state/QuantileDigestState.java | 2 +- .../state/QuantileDigestStateFactory.java | 2 +- .../state/QuantileDigestStateSerializer.java | 2 +- .../aggregation/state/StateCompiler.java | 3 +- .../ScalarImplementationDependency.java | 2 +- .../operator/exchange/LocalExchange.java | 115 +- .../exchange/PartitioningExchanger.java | 50 +- .../index/DynamicTupleFilterFactory.java | 4 +- .../index/FieldSetFilteringRecordSet.java | 2 +- .../operator/index/IndexJoinLookupStats.java | 2 +- .../presto/operator/index/IndexLoader.java | 36 +- .../index/IndexLookupSourceFactory.java | 8 +- .../presto/operator/index/UpdateRequest.java | 2 +- .../AbstractBlockEncodingBuffer.java | 465 ++++++ .../repartition/ArrayBlockEncodingBuffer.java | 242 +++ .../repartition/BlockEncodingBuffer.java | 54 + .../ByteArrayBlockEncodingBuffer.java | 142 ++ .../repartition/DecodedBlockNode.java | 82 + .../Int128ArrayBlockEncodingBuffer.java | 153 ++ .../IntArrayBlockEncodingBuffer.java | 144 ++ .../LongArrayBlockEncodingBuffer.java | 128 ++ .../repartition/MapBlockEncodingBuffer.java | 335 +++++ .../OptimizedPartitionedOutputOperator.java | 734 +++++++++ .../repartition/PartitionedOutputInfo.java | 83 ++ .../PartitionedOutputOperator.java | 93 +- .../repartition/RowBlockEncodingBuffer.java | 284 ++++ .../ShortArrayBlockEncodingBuffer.java | 144 ++ .../VariableWidthBlockEncodingBuffer.java | 197 +++ .../scalar/AbstractGreatestLeast.java | 23 +- .../presto/operator/scalar/ApplyFunction.java | 14 +- .../scalar/ArrayAllMatchFunction.java | 183 +++ .../scalar/ArrayAnyMatchFunction.java | 183 +++ .../scalar/ArrayCombinationsFunction.java | 137 ++ .../operator/scalar/ArrayConcatFunction.java | 13 +- .../operator/scalar/ArrayConstructor.java | 12 +- .../scalar/ArrayDistinctFunction.java | 166 ++- .../operator/scalar/ArrayExceptFunction.java | 3 +- .../operator/scalar/ArrayFilterFunction.java | 315 ++-- .../operator/scalar/ArrayFlattenFunction.java | 12 +- .../scalar/ArrayIntersectFunction.java | 22 +- .../presto/operator/scalar/ArrayJoin.java | 28 +- .../scalar/ArrayNoneMatchFunction.java | 112 ++ .../operator/scalar/ArrayReduceFunction.java | 16 +- .../operator/scalar/ArraySortFunction.java | 151 +- .../scalar/ArraySubscriptOperator.java | 8 +- .../operator/scalar/ArrayToArrayCast.java | 14 +- .../scalar/ArrayToElementConcatFunction.java | 14 +- .../operator/scalar/ArrayToJsonCast.java | 8 +- .../scalar/ArrayTransformFunction.java | 14 +- .../BlockToBooleanFunction.java} | 16 +- .../scalar/BooleanToBooleanFunction.java | 23 + ... BuiltInScalarFunctionImplementation.java} | 14 +- .../scalar/CastFromUnknownOperator.java | 8 +- .../operator/scalar/ConcatFunction.java | 12 +- .../operator/scalar/DateTimeFunctions.java | 2 +- .../scalar/DoubleToBooleanFunction.java | 23 + .../scalar/ElementToArrayConcatFunction.java | 13 +- .../operator/scalar/FailureFunction.java | 20 +- .../presto/operator/scalar/Greatest.java | 5 +- .../operator/scalar/HyperLogLogFunctions.java | 2 +- .../presto/operator/scalar/IdentityCast.java | 8 +- .../operator/scalar/InvokeFunction.java | 10 +- .../operator/scalar/IpPrefixFunctions.java | 74 + .../presto/operator/scalar/JsonFunctions.java | 2 +- .../scalar/JsonStringToArrayCast.java | 6 +- .../operator/scalar/JsonStringToMapCast.java | 6 +- .../operator/scalar/JsonStringToRowCast.java | 6 +- .../operator/scalar/JsonToArrayCast.java | 8 +- .../presto/operator/scalar/JsonToMapCast.java | 8 +- .../presto/operator/scalar/JsonToRowCast.java | 8 +- .../presto/operator/scalar/Least.java | 5 +- .../scalar/LongToBooleanFunction.java | 23 + .../operator/scalar/MapConcatFunction.java | 13 +- .../operator/scalar/MapConstructor.java | 18 +- .../operator/scalar/MapElementAtFunction.java | 14 +- .../operator/scalar/MapFilterFunction.java | 14 +- .../operator/scalar/MapHashCodeOperator.java | 12 +- .../operator/scalar/MapSubscriptOperator.java | 8 +- .../presto/operator/scalar/MapToJsonCast.java | 8 +- .../presto/operator/scalar/MapToMapCast.java | 10 +- .../scalar/MapTransformKeyFunction.java | 19 +- .../scalar/MapTransformValueFunction.java | 32 +- .../operator/scalar/MapZipWithFunction.java | 14 +- .../operator/scalar/ParametricScalar.java | 12 +- .../scalar/QuantileDigestFunctions.java | 2 +- .../scalar/Re2JCastToRegexpFunction.java | 8 +- .../scalar/RowComparisonOperator.java | 2 +- .../scalar/RowDistinctFromOperator.java | 14 +- .../operator/scalar/RowEqualOperator.java | 10 +- .../scalar/RowGreaterThanOperator.java | 8 +- .../scalar/RowGreaterThanOrEqualOperator.java | 8 +- .../operator/scalar/RowHashCodeOperator.java | 8 +- .../scalar/RowIndeterminateOperator.java | 12 +- .../operator/scalar/RowLessThanOperator.java | 8 +- .../scalar/RowLessThanOrEqualOperator.java | 8 +- .../operator/scalar/RowNotEqualOperator.java | 8 +- .../presto/operator/scalar/RowToJsonCast.java | 8 +- .../presto/operator/scalar/RowToRowCast.java | 10 +- .../operator/scalar/SessionFunctions.java | 10 - .../scalar/SliceToBooleanFunction.java | 24 + .../operator/scalar/TryCastFunction.java | 12 +- .../presto/operator/scalar/ZipFunction.java | 15 +- .../operator/scalar/ZipWithFunction.java | 14 +- .../ParametricScalarImplementation.java | 26 +- .../ScalarFromAnnotationsParser.java | 1 - .../ScalarImplementationHeader.java | 11 +- .../operator/unnest/ArrayOfRowsUnnester.java | 120 ++ .../presto/operator/unnest/ArrayUnnester.java | 86 ++ .../presto/operator/unnest/MapUnnester.java | 95 ++ .../unnest/ReplicatedBlockBuilder.java | 77 + .../operator/unnest/UnnestBlockBuilder.java | 265 ++++ .../operator/unnest/UnnestOperator.java | 306 ++++ .../unnest/UnnestOperatorBlockUtil.java | 44 + .../presto/operator/unnest/Unnester.java | 125 ++ .../AbstractWindowFunctionSupplier.java | 8 +- .../window/AggregateWindowFunction.java | 2 +- .../operator/window/FirstValueFunction.java | 19 +- .../presto/operator/window/LagFunction.java | 26 +- .../operator/window/LastValueFunction.java | 19 +- .../presto/operator/window/LeadFunction.java | 26 +- .../operator/window/NthValueFunction.java | 21 +- .../ReflectionWindowFunctionSupplier.java | 19 +- .../operator/window/SqlWindowFunction.java | 4 +- .../window/WindowAnnotationsParser.java | 8 +- .../window/WindowFunctionSupplier.java | 2 +- .../presto/security/AccessControlManager.java | 4 +- .../FileBasedSystemAccessControl.java | 2 +- .../server/AsyncHttpExecutionMBean.java | 2 +- .../presto/server/CoordinatorModule.java | 33 +- .../server/EmbeddedDiscoveryConfig.java | 2 +- .../presto/server/ExchangeExecutionMBean.java | 2 +- .../GenerateTraceTokenRequestFilter.java | 8 +- .../server/GracefulShutdownHandler.java | 6 +- .../presto/server/GracefulShutdownModule.java | 2 +- .../server/HttpRequestSessionContext.java | 273 ++-- .../server/InternalCommunicationConfig.java | 64 +- .../server/InternalCommunicationModule.java | 17 +- .../facebook/presto/server/PluginManager.java | 12 +- .../presto/server/PluginManagerConfig.java | 2 +- .../facebook/presto/server/PrestoServer.java | 40 +- .../presto/server/QuerySessionSupplier.java | 9 - .../presto/server/RequestHelpers.java | 2 +- .../facebook/presto/server/ServerConfig.java | 2 +- .../presto/server/ServerInfoResource.java | 2 +- .../presto/server/ServerMainModule.java | 46 +- .../presto/server/SessionContext.java | 5 - .../server/SessionPropertyDefaults.java | 4 +- .../server/StatementHttpExecutionMBean.java | 2 +- .../presto/server/StatusResource.java | 2 +- .../facebook/presto/server/TaskResource.java | 11 +- .../presto/server/TaskUpdateRequest.java | 13 +- .../presto/server/ThrowableMapper.java | 2 +- .../presto/server/WorkerResource.java | 12 +- .../server/protocol/PurgeQueriesRunnable.java | 2 +- .../presto/server/protocol/Query.java | 50 +- .../server/protocol/StatementResource.java | 21 +- .../ContinuousTaskStatusFetcher.java | 14 +- .../remotetask/HttpLocationFactory.java | 4 +- .../server/remotetask/HttpRemoteTask.java | 63 +- .../remotetask/HttpRemoteTaskFactory.java | 23 +- .../server/remotetask/RemoteTaskStats.java | 60 +- .../remotetask/RequestErrorTracker.java | 4 +- .../remotetask/SimpleHttpResponseHandler.java | 7 +- .../server/remotetask/TaskInfoFetcher.java | 14 +- .../security/JsonWebTokenAuthenticator.java | 2 +- .../server/security/JsonWebTokenConfig.java | 2 +- .../security/KerberosAuthenticator.java | 2 +- .../server/security/KerberosConfig.java | 2 +- .../PasswordAuthenticatorManager.java | 2 +- .../server/security/SecurityConfig.java | 6 +- .../server/security/ServerSecurityModule.java | 6 +- .../smile/AdaptingJsonResponseHandler.java | 12 +- .../presto/server/smile/BaseResponse.java | 2 +- .../smile/FullSmileResponseHandler.java | 10 +- .../presto/server/smile/JsonCodecWrapper.java | 2 +- .../server/smile/JsonResponseWrapper.java | 4 +- .../server/smile/SmileBodyGenerator.java | 2 +- .../smile/SmileObjectMapperProvider.java | 2 +- .../server/testing/TestingPrestoServer.java | 106 +- .../FileSingleStreamSpillerFactory.java | 4 +- .../spiller/GenericPartitioningSpiller.java | 2 +- .../presto/spiller/NodeSpillConfig.java | 2 +- .../split/CloseableSplitSourceProvider.java | 2 +- .../split/ConnectorAwareSplitSource.java | 2 +- .../presto/split/EmptySplitPageSource.java | 6 + .../presto/split/MappedPageSource.java | 6 + .../presto/split/PageSourceManager.java | 6 +- .../presto/split/PageSourceProvider.java | 3 +- .../sql/InterpretedFunctionInvoker.java | 16 +- .../presto/sql/SqlEnvironmentConfig.java | 18 +- .../java/com/facebook/presto/sql/SqlPath.java | 97 -- .../sql/analyzer/ExpressionAnalyzer.java | 107 +- .../presto/sql/analyzer/FeaturesConfig.java | 79 +- .../sql/analyzer/SemanticErrorCode.java | 2 + .../sql/analyzer/StatementAnalyzer.java | 69 +- .../presto/sql/gen/ArrayGeneratorUtils.java | 4 +- .../facebook/presto/sql/gen/BodyCompiler.java | 3 +- .../sql/gen/BytecodeGeneratorContext.java | 6 +- .../presto/sql/gen/BytecodeUtils.java | 16 +- .../presto/sql/gen/CompilerOperations.java | 3 +- .../sql/gen/CursorProcessorCompiler.java | 55 +- .../presto/sql/gen/ExpressionCompiler.java | 46 +- .../sql/gen/FunctionCallCodeGenerator.java | 10 +- .../presto/sql/gen/InCodeGenerator.java | 10 +- .../gen/InvokeFunctionBytecodeExpression.java | 8 +- .../facebook/presto/sql/gen/JoinCompiler.java | 4 +- .../sql/gen/JoinFilterFunctionCompiler.java | 67 +- .../sql/gen/LambdaBytecodeGenerator.java | 41 +- .../presto/sql/gen/NullIfCodeGenerator.java | 6 +- .../presto/sql/gen/OrderingCompiler.java | 2 +- .../presto/sql/gen/PageFunctionCompiler.java | 115 +- .../presto/sql/gen/RowExpressionCompiler.java | 93 +- .../gen/RowExpressionPredicateCompiler.java | 99 +- .../sql/gen/SqlTypeBytecodeExpression.java | 13 +- .../presto/sql/gen/SwitchCodeGenerator.java | 2 +- .../presto/sql/planner/CompilerConfig.java | 4 +- .../ConnectorPlanOptimizerManager.java | 18 +- .../planner/EffectivePredicateExtractor.java | 14 +- .../presto/sql/planner/EqualityInference.java | 1 + .../planner/ExpressionDomainTranslator.java | 4 +- .../sql/planner/ExpressionExtractor.java | 7 +- .../sql/planner/ExpressionInterpreter.java | 120 +- .../presto/sql/planner/InputExtractor.java | 29 +- .../presto/sql/planner/Interpreters.java | 13 +- .../presto/sql/planner/LiteralEncoder.java | 43 +- .../sql/planner/LiteralInterpreter.java | 5 + .../sql/planner/LocalExecutionPlanner.java | 434 ++++-- .../presto/sql/planner/LogicalPlanner.java | 76 +- .../sql/planner/LookupSymbolResolver.java | 54 - .../sql/planner/NoOpVariableResolver.java | 28 + .../sql/planner/NodePartitioningManager.java | 38 +- .../sql/planner/NullabilityAnalyzer.java | 92 +- .../presto/sql/planner/OrderingScheme.java | 112 -- .../presto/sql/planner/OutputExtractor.java | 26 +- .../presto/sql/planner/Partitioning.java | 317 ++-- .../planner/PartitioningProviderManager.java | 48 + .../sql/planner/PartitioningScheme.java | 2 +- .../presto/sql/planner/PlanBuilder.java | 4 +- .../presto/sql/planner/PlanFragmenter.java | 169 ++- .../presto/sql/planner/PlanOptimizers.java | 81 +- .../sql/planner/PlanVariableAllocator.java | 16 +- .../presto/sql/planner/PlannerUtils.java | 23 +- .../presto/sql/planner/QueryPlanner.java | 61 +- .../presto/sql/planner/RelationPlanner.java | 39 +- .../RowExpressionEqualityInference.java | 502 +++++++ .../sql/planner/RowExpressionInterpreter.java | 268 +++- .../RowExpressionPredicateExtractor.java | 425 ++++++ .../planner/RowExpressionVariableInliner.java | 18 +- .../sql/planner/SortExpressionExtractor.java | 2 +- .../sql/planner/SplitSourceFactory.java | 122 +- .../planner/StatisticsAggregationPlanner.java | 2 +- .../sql/planner/SubExpressionExtractor.java | 1 + .../presto/sql/planner/SubqueryPlanner.java | 6 +- .../presto/sql/planner/TypeProvider.java | 5 + ...bolResolver.java => VariableResolver.java} | 13 +- .../sql/planner/VariablesExtractor.java | 2 +- .../planner/iterative/IterativeOptimizer.java | 2 +- .../sql/planner/iterative/RuleStats.java | 2 +- .../rule/AddIntermediateAggregations.java | 12 +- .../iterative/rule/CreatePartialTopN.java | 8 +- .../iterative/rule/DesugarCurrentPath.java | 49 - .../rule/DetermineJoinDistributionType.java | 3 - .../DetermineSemiJoinDistributionType.java | 3 - .../iterative/rule/EliminateCrossJoins.java | 6 +- .../iterative/rule/EvaluateZeroLimit.java | 2 +- .../rule/ExpressionRewriteRuleSet.java | 8 +- .../iterative/rule/ExtractSpatialJoins.java | 8 +- .../iterative/rule/GatherAndMergeWindows.java | 14 +- .../rule/ImplementFilteredAggregations.java | 8 +- .../iterative/rule/InlineProjections.java | 110 +- .../rule/MergeLimitWithDistinct.java | 4 +- .../iterative/rule/MergeLimitWithSort.java | 4 +- .../iterative/rule/MergeLimitWithTopN.java | 4 +- .../planner/iterative/rule/MergeLimits.java | 4 +- ...ipleDistinctAggregationToMarkDistinct.java | 4 +- .../iterative/rule/PickTableLayout.java | 278 +++- .../rule/ProjectOffPushDownRule.java | 2 +- .../rule/PruneAggregationColumns.java | 2 +- .../rule/PruneAggregationSourceColumns.java | 2 +- .../rule/PruneCountAggregationOverScalar.java | 2 +- .../iterative/rule/PruneCrossJoinColumns.java | 4 +- .../iterative/rule/PruneLimitColumns.java | 2 +- .../rule/PruneOrderByInAggregation.java | 4 +- .../iterative/rule/PruneProjectColumns.java | 2 +- .../iterative/rule/PruneSemiJoinColumns.java | 2 +- .../PruneSemiJoinFilteringSourceColumns.java | 2 +- .../iterative/rule/PruneTopNColumns.java | 4 +- .../iterative/rule/PruneWindowColumns.java | 4 +- .../rule/PushAggregationThroughOuterJoin.java | 24 +- .../rule/PushLimitThroughMarkDistinct.java | 2 +- .../rule/PushLimitThroughOuterJoin.java | 7 +- .../rule/PushLimitThroughProject.java | 4 +- .../rule/PushLimitThroughSemiJoin.java | 2 +- ...PushPartialAggregationThroughExchange.java | 23 +- .../PushPartialAggregationThroughJoin.java | 8 +- .../rule/PushProjectionThroughExchange.java | 50 +- .../rule/PushProjectionThroughUnion.java | 33 +- ...shRemoteExchangeThroughAssignUniqueId.java | 1 + .../rule/PushTableWriteThroughUnion.java | 13 +- .../iterative/rule/PushTopNThroughUnion.java | 7 +- .../RemoveRedundantIdentityProjections.java | 2 +- .../planner/iterative/rule/ReorderJoins.java | 2 +- ...RewriteSpatialPartitioningAggregation.java | 18 +- .../rule/RowExpressionRewriteRuleSet.java} | 413 ++--- .../rule/SimplifyArrayOperations.java | 25 + .../rule/SimplifyArrayOperationsRewriter.java | 222 +++ .../rule/SimplifyCountOverConstant.java | 6 +- .../iterative/rule/SimplifyExpressions.java | 4 +- .../rule/SimplifyRowExpressions.java | 135 ++ .../SingleDistinctAggregationToGroupBy.java | 8 +- .../TransformCorrelatedInPredicateToJoin.java | 8 +- ...formCorrelatedScalarAggregationToJoin.java | 4 +- .../TransformCorrelatedScalarSubquery.java | 2 +- ...mCorrelatedSingleRowSubqueryToProject.java | 4 +- .../TransformExistsApplyToLateralNode.java | 15 +- .../iterative/rule/TranslateExpressions.java | 187 +++ .../sql/planner/iterative/rule/Util.java | 9 +- .../optimizations/ActualProperties.java | 80 +- .../planner/optimizations/AddExchanges.java | 118 +- .../optimizations/AddLocalExchanges.java | 117 +- .../optimizations/AggregationNodeUtils.java | 4 +- .../ApplyConnectorOptimization.java | 16 +- .../planner/optimizations/ApplyNodeUtil.java | 2 +- .../optimizations/BeginTableWrite.java | 255 ---- .../CheckSubqueryNodesAreRewritten.java | 11 +- .../DistinctOutputQueryUtil.java | 10 +- .../optimizations/ExpressionEquivalence.java | 20 +- .../HashGenerationOptimizer.java | 26 +- .../ImplementIntersectAndExceptAsUnion.java | 25 +- .../optimizations/IndexJoinOptimizer.java | 4 +- .../planner/optimizations/LimitPushDown.java | 38 +- .../MetadataDeleteOptimizer.java | 3 +- .../optimizations/MetadataQueryOptimizer.java | 19 +- .../OptimizeMixedDistinctAggregations.java | 20 +- .../planner/optimizations/OptimizerStats.java | 2 +- .../optimizations/PlanNodeDecorrelator.java | 10 +- .../optimizations/PredicatePushDown.java | 18 +- .../optimizations/PreferredProperties.java | 7 +- .../optimizations/PropertyDerivations.java | 116 +- .../PruneUnreferencedOutputs.java | 116 +- .../optimizations/PushdownSubfields.java | 55 +- .../optimizations/QueryCardinalityUtil.java | 6 +- .../ReplicateSemiJoinInDelete.java | 1 - .../RowExpressionPredicatePushDown.java | 1328 +++++++++++++++++ .../ScalarAggregationToJoinRewriter.java | 10 +- .../optimizations/SetFlatteningOptimizer.java | 33 +- .../optimizations/SetOperationNodeUtils.java | 59 + .../StatsRecordingPlanOptimizer.java | 7 + .../StreamPropertyDerivations.java | 28 +- .../planner/optimizations/SymbolMapper.java | 47 +- ...uantifiedComparisonApplyToLateralJoin.java | 10 +- .../UnaliasSymbolReferences.java | 158 +- .../optimizations/WindowFilterPushDown.java | 19 +- .../planner/optimizations/WindowNodeUtil.java | 2 +- .../optimizations/joins/JoinGraph.java | 2 +- .../presto/sql/planner/plan/ApplyNode.java | 26 +- .../sql/planner/plan/AssignmentUtils.java | 15 +- .../presto/sql/planner/plan/DeleteNode.java | 12 +- .../presto/sql/planner/plan/ExchangeNode.java | 33 +- .../sql/planner/plan/InternalPlanVisitor.java | 40 +- .../presto/sql/planner/plan/LimitNode.java | 97 -- .../sql/planner/plan/MetadataDeleteNode.java | 14 +- .../presto/sql/planner/plan/Patterns.java | 10 + .../sql/planner/plan/RemoteSourceNode.java | 14 +- .../sql/planner/plan/SetOperationNode.java | 125 -- .../presto/sql/planner/plan/SortNode.java | 15 +- .../planner/plan/StatisticAggregations.java | 74 +- .../planner/plan/StatisticsWriterNode.java | 105 +- .../sql/planner/plan/TableFinishNode.java | 9 +- .../planner/plan/TableWriterMergeNode.java | 119 ++ .../sql/planner/plan/TableWriterNode.java | 131 +- .../sql/planner/plan/TopNRowNumberNode.java | 2 +- .../presto/sql/planner/plan/WindowNode.java | 19 +- .../HashCollisionPlanNodeStats.java | 6 +- .../planner/planPrinter/IOPlanPrinter.java | 41 +- .../sql/planner/planPrinter/JsonRenderer.java | 2 +- .../planner/planPrinter/PlanNodeStats.java | 19 + .../planPrinter/PlanNodeStatsSummarizer.java | 14 +- .../sql/planner/planPrinter/PlanPrinter.java | 147 +- .../planPrinter/RowExpressionFormatter.java | 44 +- .../planPrinter/WindowPlanNodeStats.java | 6 +- .../sql/planner/sanity/TypeValidator.java | 14 +- ...ValidateAggregationsWithDefaultValues.java | 8 +- .../sanity/ValidateDependenciesChecker.java | 67 +- .../sanity/ValidateStreamingAggregations.java | 2 +- .../sanity/VerifyNoFilteredAggregations.java | 2 +- .../sanity/VerifyNoOriginalExpression.java | 14 +- .../ConnectorRowExpressionService.java | 13 +- .../presto/sql/relational/Expressions.java | 17 + .../sql/relational/FunctionResolution.java | 33 +- .../sql/relational/ProjectNodeUtils.java | 2 +- .../RowExpressionDomainTranslator.java | 64 +- .../relational/RowExpressionOptimizer.java | 20 +- .../sql/relational/SqlFunctionUtils.java | 249 ++++ .../SqlToRowExpressionTranslator.java | 88 +- .../optimizer/ExpressionOptimizer.java | 273 ---- .../sql/rewrite/ShowQueriesRewrite.java | 7 +- .../presto/sql/rewrite/ShowStatsRewrite.java | 4 +- .../InMemoryFunctionNamespaceManager.java | 118 ++ .../presto/testing/LocalQueryRunner.java | 68 +- .../presto/testing/MaterializedResult.java | 3 + .../testing/TestingConnectorContext.java | 21 + .../testing/TestingConnectorSession.java | 36 +- .../presto/testing/TestingMetadata.java | 19 + .../presto/testing/TestingSession.java | 2 - .../presto/testing/TestingTaskContext.java | 8 +- .../TestingWarningCollectorConfig.java | 4 +- .../TestingWarningCollectorModule.java | 2 +- .../DelegatingTransactionManager.java | 166 +++ .../InMemoryTransactionManager.java | 157 +- .../transaction/NoOpTransactionManager.java | 17 +- .../transaction/TransactionBuilder.java | 4 +- .../transaction/TransactionManager.java | 15 +- .../transaction/TransactionManagerConfig.java | 26 +- .../type/DecimalInequalityOperators.java | 6 +- .../presto/type/HyperLogLogOperators.java | 2 +- .../presto/type/IpPrefixOperators.java | 273 ++++ .../facebook/presto/type/IpPrefixType.java | 158 ++ .../com/facebook/presto/type/Re2JRegexp.java | 2 +- .../facebook/presto/type/TypeRegistry.java | 4 +- .../presto/type/setdigest/SetDigest.java | 2 +- .../type/setdigest/SetDigestFunctions.java | 2 +- .../facebook/presto/util/CompilerUtils.java | 2 +- .../presto/util/FastutilSetHelper.java | 12 +- .../presto/util/FinalizerService.java | 4 +- .../facebook/presto/util/GcStatusMonitor.java | 237 +++ .../facebook/presto/util/GraphvizPrinter.java | 45 +- .../presto/util/SpatialJoinUtils.java | 40 +- .../facebook/presto/util/StatementUtils.java | 6 +- .../presto/util/StringTableUtils.java | 65 + .../main/resources/webapp/assets/presto.css | 4 + .../resources/webapp/dist/embedded_plan.js | 2 +- .../src/main/resources/webapp/dist/plan.js | 2 +- .../src/main/resources/webapp/dist/query.js | 2 +- .../src/main/resources/webapp/dist/stage.js | 2 +- .../src/main/resources/webapp/dist/worker.js | 2 +- .../webapp/src/components/ClusterHUD.jsx | 8 +- .../webapp/src/components/LivePlan.jsx | 12 +- .../webapp/src/components/QueryDetail.jsx | 256 ++-- .../webapp/src/components/QueryHeader.jsx | 7 +- .../webapp/src/components/StageDetail.jsx | 9 +- .../webapp/src/components/WorkerStatus.jsx | 10 +- .../src/components/WorkerThreadList.jsx | 4 +- .../src/main/resources/webapp/timeline.html | 2 +- .../com/facebook/presto/SessionTestUtils.java | 7 - .../presto/block/AbstractTestBlock.java | 8 +- .../presto/block/BlockAssertions.java | 304 +++- .../presto/block/ColumnarTestUtils.java | 6 +- .../presto/block/TestColumnarArray.java | 1 + .../presto/block/TestColumnarMap.java | 14 +- .../facebook/presto/block/TestMapBlock.java | 110 +- .../facebook/presto/block/TestRowBlock.java | 2 +- .../connector/system/TestSystemSplit.java | 4 +- .../presto/cost/TestCostCalculator.java | 120 +- .../cost/TestFilterStatsCalculator.java | 4 +- .../cost/TestScalarStatsCalculator.java | 2 +- .../cost/TestSemiJoinStatsCalculator.java | 4 +- .../presto/cost/TestValuesNodeStats.java | 11 +- .../presto/event/TestQueryMonitorConfig.java | 6 +- .../execution/BenchmarkNodeScheduler.java | 4 +- .../presto/execution/MockQueryExecution.java | 8 +- .../execution/MockRemoteTaskFactory.java | 18 +- .../presto/execution/TaskTestUtils.java | 14 +- .../facebook/presto/execution/TestColumn.java | 2 +- .../presto/execution/TestCommitTask.java | 4 +- .../presto/execution/TestCreateTableTask.java | 2 +- .../presto/execution/TestDeallocateTask.java | 2 +- .../facebook/presto/execution/TestInput.java | 2 +- .../TestMemoryRevokingScheduler.java | 14 +- .../presto/execution/TestNodeScheduler.java | 114 +- .../execution/TestNodeSchedulerConfig.java | 2 +- .../facebook/presto/execution/TestOutput.java | 2 +- .../execution/TestPageSplitterUtil.java | 4 +- .../presto/execution/TestPlannerWarnings.java | 12 +- .../presto/execution/TestPrepareTask.java | 2 +- .../execution/TestQueryManagerConfig.java | 15 +- .../execution/TestQueryStateMachine.java | 91 +- .../presto/execution/TestQueryStats.java | 12 +- .../execution/TestResetSessionTask.java | 4 +- .../presto/execution/TestRollbackTask.java | 4 +- .../presto/execution/TestSetPathTask.java | 132 -- .../presto/execution/TestSetRoleTask.java | 2 +- .../presto/execution/TestSetSessionTask.java | 4 +- .../execution/TestSqlStageExecution.java | 21 +- .../presto/execution/TestSqlTask.java | 24 +- .../execution/TestSqlTaskExecution.java | 8 +- .../presto/execution/TestSqlTaskManager.java | 13 +- ...va => TestStageExecutionStateMachine.java} | 142 +- ...tats.java => TestStageExecutionStats.java} | 18 +- .../execution/TestStartTransactionTask.java | 4 +- .../presto/execution/TestStateMachine.java | 4 +- .../execution/TestTaskManagerConfig.java | 9 +- .../execution/buffer/BufferTestUtils.java | 2 +- .../buffer/TestArbitraryOutputBuffer.java | 2 +- .../buffer/TestBroadcastOutputBuffer.java | 2 +- .../buffer/TestPartitionedOutputBuffer.java | 2 +- .../executor/SimulationController.java | 4 +- .../executor/TaskExecutorSimulator.java | 2 +- .../execution/executor/TestTaskExecutor.java | 32 +- .../resourceGroups/TestResourceGroups.java | 6 +- .../TestStochasticPriorityQueue.java | 4 +- .../scheduler/TestFixedCountScheduler.java | 6 +- .../TestPhasedExecutionSchedule.java | 14 +- .../TestSourcePartitionedScheduler.java | 17 +- .../TestHeartbeatFailureDetector.java | 28 +- .../memory/TestMemoryManagerConfig.java | 6 +- .../presto/memory/TestMemoryPools.java | 13 +- .../presto/memory/TestMemoryTracking.java | 6 +- .../presto/memory/TestNodeMemoryConfig.java | 4 +- .../presto/memory/TestQueryContext.java | 12 +- .../TestReservedSystemMemoryConfig.java | 2 +- .../memory/TestSystemMemoryBlocking.java | 13 +- .../presto/metadata/AbstractMockMetadata.java | 10 +- .../metadata/TestDiscoveryNodeManager.java | 22 +- .../metadata/TestFunctionInvokerProvider.java | 12 +- ...amespace.java => TestFunctionManager.java} | 98 +- .../TestFunctionNamespaceManager.java | 163 ++ .../TestInformationSchemaMetadata.java | 8 +- .../TestInformationSchemaTableHandle.java | 4 +- .../TestPolymorphicScalarFunction.java | 20 +- .../metadata/TestQualifiedTablePrefix.java | 4 +- .../presto/metadata/TestSignature.java | 12 +- .../TestStaticCatalogStoreConfig.java | 6 +- ...estStaticFunctionNamespaceStoreConfig.java | 47 + .../metadata/TestSystemTableHandle.java | 4 +- .../BenchmarkCompressToByteBuffer.java | 100 ++ ...kHashAndStreamingAggregationOperators.java | 4 +- .../BenchmarkHashBuildAndJoinOperators.java | 4 +- .../operator/BenchmarkMoreByteArrays.java | 154 ++ .../BenchmarkPartitionedOutputOperator.java | 229 --- .../presto/operator/BenchmarkReadBlock.java | 297 ++++ .../BenchmarkUncheckedByteArrays.java | 287 ++++ .../operator/BenchmarkUnnestOperator.java | 424 ++++++ .../operator/BenchmarkWindowOperator.java | 2 +- .../presto/operator/GenericLongFunction.java | 14 +- .../operator/GroupByHashYieldAssertion.java | 10 +- .../MockExchangeRequestProcessor.java | 21 +- .../presto/operator/OperatorAssertion.java | 6 +- .../presto/operator/PageAssertions.java | 103 ++ .../operator/TestAggregationOperator.java | 4 +- .../TestAnnotationEngineForAggregates.java | 36 +- .../TestAnnotationEngineForScalars.java | 44 +- .../operator/TestColumnarPageProcessor.java | 2 +- .../operator/TestDistinctLimitOperator.java | 4 +- .../facebook/presto/operator/TestDriver.java | 27 +- .../presto/operator/TestDriverStats.java | 2 +- .../presto/operator/TestExchangeClient.java | 133 +- .../operator/TestExchangeClientConfig.java | 15 +- .../presto/operator/TestExchangeOperator.java | 13 +- .../TestFilterAndProjectOperator.java | 6 +- .../presto/operator/TestGroupIdOperator.java | 2 +- .../operator/TestGroupedTopNBuilder.java | 2 +- .../operator/TestHashAggregationOperator.java | 49 +- .../presto/operator/TestHashJoinOperator.java | 19 +- .../operator/TestHashSemiJoinOperator.java | 6 +- .../operator/TestHttpPageBufferClient.java | 53 +- .../presto/operator/TestLimitOperator.java | 2 +- .../operator/TestMarkDistinctOperator.java | 4 +- .../presto/operator/TestMergeOperator.java | 12 +- .../presto/operator/TestMoreByteArrays.java | 91 ++ .../operator/TestNestedLoopBuildOperator.java | 2 +- .../operator/TestNestedLoopJoinOperator.java | 2 +- .../presto/operator/TestOperatorStats.java | 11 +- .../presto/operator/TestOrderByOperator.java | 2 +- .../TestPartitionedOutputOperator.java | 218 +++ .../presto/operator/TestPipelineStats.java | 6 +- .../operator/TestRowNumberOperator.java | 6 +- .../TestScanFilterAndProjectOperator.java | 61 +- .../TestStreamingAggregationOperator.java | 4 +- .../operator/TestTableFinishOperator.java | 18 +- .../operator/TestTableWriterOperator.java | 15 +- .../presto/operator/TestTaskStats.java | 4 +- .../presto/operator/TestTopNOperator.java | 2 +- .../operator/TestTopNRowNumberOperator.java | 4 +- .../operator/TestUncheckedByteArrays.java | 122 ++ .../presto/operator/TestWindowOperator.java | 2 +- .../TestingExchangeHttpClientHandler.java | 10 +- .../AbstractTestAggregationFunction.java | 4 +- .../AbstractTestApproximateCountDistinct.java | 2 +- .../aggregation/AggregationTestUtils.java | 5 + .../aggregation/TestCountNullAggregation.java | 2 +- .../TestMergeHyperLogLogAggregation.java | 2 +- .../TestMergeQuantileDigestFunction.java | 2 +- ...TestQuantileDigestAggregationFunction.java | 2 +- ...AbstractTestFixedHistogramAggregation.java | 112 ++ .../AbstractTestReservoirAggregation.java | 43 + .../AbstractTestStateStrategy.java | 75 + .../TestEntropyCalculations.java | 48 + ...TestFixedHistogramJacknifeAggregation.java | 130 ++ ...stFixedHistogramJacknifeStateStrategy.java | 23 + .../TestFixedHistogramMleAggregation.java | 119 ++ .../TestFixedHistogramMleStateStrategy.java | 23 + .../TestIllegalMethodAggregation.java | 71 + .../TestUnweightedReservoirAggregation.java | 67 + ...nweightedReservoirSampleStateStrategy.java | 23 + .../TestWeightedReservoirAggregation.java | 83 ++ ...tWeightedReservoirSampleStateStrategy.java | 23 + .../TestFixedDoubleBreakdownHistogram.java | 35 +- .../TestFixedDoubleHistogram.java | 28 +- .../TestUnweightedDoubleReservoirSample.java | 77 + .../TestWeightedDoubleReservoirSample.java | 109 ++ .../operator/exchange/TestLocalExchange.java | 42 +- .../index/TestTupleFilterProcessor.java | 1 + ...enchmarkDictionaryBlockGetSizeInBytes.java | 2 +- .../TestDictionaryAwarePageProjection.java | 6 +- .../operator/project/TestPageProcessor.java | 6 +- .../BenchmarkPartitionedOutputOperator.java | 383 +++++ .../repartition/TestBlockEncodingBuffers.java | 696 +++++++++ ...estOptimizedPartitionedOutputOperator.java | 862 +++++++++++ .../scalar/AbstractTestFunctions.java | 19 +- ....java => AbstractTestRegexpFunctions.java} | 30 +- .../scalar/BenchmarkArrayDistinct.java | 4 +- .../operator/scalar/BenchmarkArrayFilter.java | 16 +- .../BenchmarkArrayHashCodeOperator.java | 9 +- .../scalar/BenchmarkArrayIntersect.java | 2 +- .../operator/scalar/BenchmarkArrayJoin.java | 2 +- .../operator/scalar/BenchmarkArraySort.java | 4 +- .../scalar/BenchmarkArraySubscript.java | 2 +- .../scalar/BenchmarkArrayTransform.java | 2 +- .../scalar/BenchmarkEqualsOperator.java | 2 +- .../scalar/BenchmarkJsonToArrayCast.java | 2 +- .../scalar/BenchmarkJsonToMapCast.java | 2 +- .../operator/scalar/BenchmarkMapConcat.java | 4 +- .../scalar/BenchmarkMapSubscript.java | 4 +- .../scalar/BenchmarkMapToMapCast.java | 4 +- .../scalar/BenchmarkRowToRowCast.java | 2 +- .../scalar/BenchmarkTransformKey.java | 2 +- .../scalar/BenchmarkTransformValue.java | 2 +- .../operator/scalar/FunctionAssertions.java | 109 +- .../scalar/TestArrayCombinationsFunction.java | 154 ++ .../scalar/TestArrayIntersectFunction.java | 68 + .../scalar/TestArrayMatchFunctions.java | 63 + .../scalar/TestArraySortFunction.java | 42 + .../scalar/TestDateTimeFunctionsBase.java | 2 +- .../operator/scalar/TestFailureFunction.java | 17 +- .../scalar/TestHyperLogLogFunctions.java | 2 +- .../scalar/TestIpPrefixFunctions.java | 60 + .../scalar/TestJoniRegexpFunctions.java | 25 + .../operator/scalar/TestLambdaExpression.java | 2 +- .../scalar/TestPageProcessorCompiler.java | 15 +- ...ametricScalarImplementationValidation.java | 12 +- ...idedBlockBuilderReturnPlaceConvention.java | 26 +- .../scalar/TestRe2jRegexpFunctions.java | 25 + .../unnest/TestArrayOfRowsUnnester.java | 232 +++ .../operator/unnest/TestArrayUnnester.java | 163 ++ .../operator/unnest/TestMapUnnester.java | 184 +++ .../unnest/TestReplicatedBlockBuilder.java | 113 ++ .../unnest/TestUnnestBlockBuilder.java | 158 ++ .../{ => unnest}/TestUnnestOperator.java | 6 +- .../operator/unnest/TestUnnesterUtil.java | 248 +++ .../window/AbstractTestWindowFunction.java | 2 +- .../window/TestFirstValueFunction.java | 268 ++++ .../operator/window/TestLagFunction.java | 104 ++ .../window/TestLastValueFunction.java | 102 ++ .../operator/window/TestLeadFunction.java | 104 ++ .../operator/window/TestNthValueFunction.java | 74 + .../operator/window/WindowAssertions.java | 2 +- .../presto/server/TestBasicQueryInfo.java | 3 +- .../server/TestEmbeddedDiscoveryConfig.java | 6 +- .../server/TestFailureDetectorConfig.java | 2 +- .../server/TestGenerateTokenFilter.java | 24 +- .../server/TestHttpRequestSessionContext.java | 53 +- .../TestInternalCommunicationConfig.java | 24 +- .../presto/server/TestNodeResource.java | 12 +- .../server/TestPluginManagerConfig.java | 2 +- .../presto/server/TestQueryProgressStats.java | 2 +- .../presto/server/TestQueryResource.java | 22 +- .../server/TestQuerySessionSupplier.java | 59 - .../presto/server/TestQueryStateInfo.java | 2 +- .../server/TestQueryStateInfoResource.java | 26 +- .../facebook/presto/server/TestServer.java | 33 +- .../presto/server/TestServerConfig.java | 6 +- .../server/TestSessionPropertyDefaults.java | 2 +- .../presto/server/remotetask/TestBackoff.java | 2 +- .../server/remotetask/TestHttpRemoteTask.java | 25 +- .../security/TestJsonWebTokenConfig.java | 4 +- .../server/security/TestKerberosConfig.java | 2 +- .../server/security/TestSecurityConfig.java | 2 +- .../presto/server/smile/ImmutablePerson.java | 2 +- .../facebook/presto/server/smile/Person.java | 2 +- .../smile/TestFullSmileResponseHandler.java | 12 +- .../presto/server/smile/TestSmileSupport.java | 2 +- .../TestGenericPartitioningSpiller.java | 2 +- .../presto/spiller/TestNodeSpillConfig.java | 6 +- .../split/TestBufferingSplitSource.java | 4 +- .../presto/sql/TestExpressionInterpreter.java | 141 +- .../presto/sql/TestRowExpressionSerde.java | 29 +- .../presto/sql/TestSqlEnvironmentConfig.java | 17 +- .../sql/TestingRowExpressionTranslator.java | 19 +- .../presto/sql/analyzer/TestAnalyzer.java | 4 +- .../sql/analyzer/TestFeaturesConfig.java | 29 +- .../sql/gen/BenchmarkPageProcessor.java | 3 +- .../sql/gen/InCodeGeneratorBenchmark.java | 2 +- .../sql/gen/PageProcessorBenchmark.java | 10 +- .../sql/gen/TestExpressionCompiler.java | 37 +- .../presto/sql/gen/TestInCodeGenerator.java | 16 +- .../sql/gen/TestPageFunctionCompiler.java | 36 +- .../TestRowExpressionPredicateCompiler.java | 20 +- .../TestVarArgsToArrayAdapterGenerator.java | 14 +- .../presto/sql/planner/TestCanonicalize.java | 2 +- .../sql/planner/TestCompilerConfig.java | 6 +- .../TestEffectivePredicateExtractor.java | 36 +- .../TestExpressionDomainTranslator.java | 2 +- .../planner/TestLocalExecutionPlanner.java | 2 +- .../sql/planner/TestLogicalPlanner.java | 62 +- .../sql/planner/TestNullabilityAnalyzer.java | 104 ++ .../sql/planner/TestPredicatePushdown.java | 25 +- .../sql/planner/TestQuantifiedComparison.java | 2 +- .../TestRowExpressionDomainTranslator.java | 23 +- .../TestRowExpressionEqualityInference.java | 524 +++++++ .../planner/TestRowExpressionFormatter.java | 18 +- .../TestRowExpressionPredicateExtractor.java | 798 ++++++++++ .../planner/TestRowExpressionRewriter.java | 6 +- .../TestRowExpressionVariableInliner.java | 14 +- .../presto/sql/planner/TestTypeValidator.java | 45 +- .../sql/planner/TestingWriterTarget.java | 17 + .../AggregationFunctionMatcher.java | 12 +- .../assertions/AggregationMatcher.java | 4 +- .../assertions/AggregationStepMatcher.java | 4 +- .../sql/planner/assertions/BasePlanTest.java | 25 +- .../planner/assertions/ExpressionMatcher.java | 2 +- .../sql/planner/assertions/LimitMatcher.java | 2 +- .../sql/planner/assertions/PlanAssert.java | 5 +- .../planner/assertions/PlanMatchPattern.java | 16 +- .../assertions/PlanMatchingVisitor.java | 4 +- .../assertions/RowExpressionVerifier.java | 8 +- .../assertions/SpecificationProvider.java | 20 +- .../StrictAssignedSymbolsMatcher.java | 2 +- .../sql/planner/assertions/SymbolAliases.java | 2 +- .../sql/planner/assertions/TopNMatcher.java | 2 +- .../presto/sql/planner/assertions/Util.java | 13 +- .../assertions/WindowFunctionMatcher.java | 2 +- .../iterative/TestIterativeOptimizer.java | 2 +- .../sql/planner/iterative/TestRuleIndex.java | 4 +- .../rule/TestAddIntermediateAggregations.java | 8 +- .../TestDetermineJoinDistributionType.java | 20 +- ...TestDetermineSemiJoinDistributionType.java | 4 +- .../rule/TestEliminateCrossJoins.java | 2 +- .../iterative/rule/TestEvaluateZeroLimit.java | 4 +- .../rule/TestEvaluateZeroSample.java | 4 +- .../iterative/rule/TestInlineProjections.java | 53 +- .../iterative/rule/TestJoinEnumerator.java | 5 +- .../iterative/rule/TestJoinNodeFlattener.java | 2 +- .../rule/TestMergeAdjacentWindows.java | 5 +- .../iterative/rule/TestPickTableLayout.java | 95 ++ .../rule/TestPruneAggregationColumns.java | 4 +- .../TestPruneAggregationSourceColumns.java | 4 +- .../TestPruneCountAggregationOverScalar.java | 6 +- .../rule/TestPruneFilterColumns.java | 2 +- .../iterative/rule/TestPruneJoinColumns.java | 2 +- .../iterative/rule/TestPruneLimitColumns.java | 2 +- .../rule/TestPruneOrderByInAggregation.java | 4 +- .../iterative/rule/TestPruneTopNColumns.java | 2 +- .../rule/TestPruneValuesColumns.java | 4 +- .../rule/TestPruneWindowColumns.java | 12 +- .../TestPushAggregationThroughOuterJoin.java | 18 +- .../TestPushLimitThroughMarkDistinct.java | 2 +- ...TestPushPartialAggregationThroughJoin.java | 2 +- .../TestPushProjectionThroughExchange.java | 39 +- .../iterative/rule/TestRemoveFullSample.java | 4 +- .../rule/TestRemoveTrivialFilters.java | 2 +- ...estRemoveUnreferencedScalarApplyNodes.java | 2 +- .../iterative/rule/TestReorderJoins.java | 2 +- .../rule/TestSimplifyExpressions.java | 32 +- ...estSingleDistinctAggregationToGroupBy.java | 2 +- ...stSwapAdjacentWindowsBySpecifications.java | 5 +- ...TestTransformCorrelatedScalarSubquery.java | 6 +- ...TestTransformExistsApplyToLateralJoin.java | 2 +- ...rrelatedInPredicateSubqueryToSemiJoin.java | 2 +- .../rule}/TestTranslateExpressions.java | 86 +- .../iterative/rule/test/BaseRuleTest.java | 2 +- .../iterative/rule/test/PlanBuilder.java | 82 +- .../iterative/rule/test/RuleAssert.java | 21 +- .../iterative/rule/test/TestRuleTester.java | 4 +- .../TestConnectorOptimization.java | 8 +- .../optimizations/TestEliminateSorts.java | 28 +- .../TestForceSingleNodeOutput.java | 2 +- .../TestFullOuterJoinWithCoalesce.java | 184 +++ .../optimizations/TestLocalProperties.java | 2 +- .../optimizations/TestMergeWindows.java | 2 +- ...TestOptimizeMixedDistinctAggregations.java | 4 +- .../optimizations/TestReorderWindows.java | 4 +- .../TestSetFlatteningOptimizer.java | 2 +- .../sql/planner/optimizations/TestUnion.java | 4 +- ...tAssingments.java => TestAssignments.java} | 12 +- .../plan/TestStatisticsWriterNode.java | 23 +- .../sql/planner/plan/TestWindowNode.java | 21 +- ...ValidateAggregationsWithDefaultValues.java | 6 +- .../TestValidateStreamingAggregations.java | 2 +- .../TestVerifyNoOriginalExpression.java | 49 +- .../sanity/TestVerifyOnlyOneOutputNode.java | 4 +- .../presto/sql/query/QueryAssertions.java | 7 +- .../sql/query/TestSessionFunctions.java | 21 - .../presto/sql/query/TestSubqueries.java | 21 +- .../relational/TestLogicalRowExpressions.java | 265 +++- .../TestRowExpressionOptimizer.java} | 36 +- .../TestRowExpressionTranslator.java | 295 ++++ .../sql/relational/TestSubExpressions.java | 65 + .../presto/tests/LogTestDurationListener.java | 190 +++ .../transaction/TestTransactionManager.java | 19 +- .../TestTransactionManagerConfig.java | 9 +- .../presto/type/AbstractTestType.java | 2 +- .../type/BenchmarkDecimalOperators.java | 8 +- .../presto/type/TestIpPrefixOperators.java | 200 +++ .../presto/type/TestIpPrefixType.java | 85 ++ .../presto/type/TestSmallintOperators.java | 6 +- .../presto/type/TestTinyintOperators.java | 6 +- .../presto/type/setdigest/TestSetDigest.java | 2 +- .../presto/util/TestPowerOfTwoValidator.java | 93 +- .../presto/util/TestStringTableUtils.java | 67 + .../services/org.testng.ITestNGListener | 1 + presto-matching/pom.xml | 2 +- presto-memory-context/pom.xml | 4 +- .../AbstractAggregatedMemoryContext.java | 4 +- presto-memory/pom.xml | 14 +- .../presto/plugin/memory/MemoryConfig.java | 2 +- .../plugin/memory/MemoryConnectorFactory.java | 4 +- .../plugin/memory/MemoryDataFragment.java | 4 +- .../presto/plugin/memory/MemoryModule.java | 2 +- .../plugin/memory/MemoryQueryRunner.java | 6 +- .../plugin/memory/TestMemoryMetadata.java | 2 +- .../plugin/memory/TestMemoryWorkerCrash.java | 2 +- presto-ml/pom.xml | 6 +- .../facebook/presto/ml/AbstractSvmModel.java | 2 +- ...eClassifierPredictionsStateSerializer.java | 2 +- .../presto/ml/MLFeaturesFunctions.java | 2 +- .../presto/ml/AbstractTestMLFunctions.java | 2 +- .../ml/TestEvaluateClassifierPredictions.java | 2 +- .../com/facebook/presto/ml/TestMLQueries.java | 2 +- presto-mongodb/pom.xml | 12 +- .../presto/mongodb/MongoClientConfig.java | 19 +- .../presto/mongodb/MongoClientModule.java | 2 +- .../presto/mongodb/MongoConnectorFactory.java | 4 +- .../presto/mongodb/MongoMetadata.java | 2 +- .../presto/mongodb/MongoPageSink.java | 6 +- .../presto/mongodb/MongoPageSource.java | 18 +- .../facebook/presto/mongodb/MongoSession.java | 2 +- .../presto/mongodb/MongoQueryRunner.java | 2 +- .../presto/mongodb/TestMongoClientConfig.java | 102 ++ .../presto/mongodb/TestMongoSplit.java | 2 +- .../presto/mongodb/TestMongoTableHandle.java | 2 +- presto-mysql/pom.xml | 30 +- .../presto/plugin/mysql/MySqlClient.java | 40 +- .../plugin/mysql/MySqlClientModule.java | 4 +- .../presto/plugin/mysql/MySqlConfig.java | 2 +- .../plugin/mysql/DockerizedMySqlServer.java | 89 ++ .../presto/plugin/mysql/MySqlQueryRunner.java | 32 +- .../presto/plugin/mysql/TestMySqlConfig.java | 6 +- .../mysql/TestMySqlDistributedQueries.java | 10 +- .../mysql/TestMySqlIntegrationSmokeTest.java | 6 +- .../plugin/mysql/TestMySqlTypeMapping.java | 10 +- presto-orc/pom.xml | 15 +- .../presto/orc/AbstractOrcRecordReader.java | 13 +- .../facebook/presto/orc/ByteArrayUtils.java | 42 +- .../facebook/presto/orc/CacheStatsMBean.java | 53 + .../orc/CachingStripeMetadataSource.java | 111 ++ .../facebook/presto/orc/FilterFunction.java | 69 + .../presto/orc/OrcBatchRecordReader.java | 22 +- .../facebook/presto/orc/OrcDecompressor.java | 8 +- .../com/facebook/presto/orc/OrcReader.java | 233 +-- .../facebook/presto/orc/OrcReaderOptions.java | 54 + .../presto/orc/OrcSelectiveRecordReader.java | 648 +++++++- .../com/facebook/presto/orc/OrcWriter.java | 39 +- .../presto/orc/OrcWriterFlushStats.java | 2 +- .../presto/orc/OrcZstdDecompressor.java | 25 +- .../orc/StorageStripeMetadataSource.java | 55 + .../facebook/presto/orc/StreamDescriptor.java | 22 +- .../presto/orc/StripeMetadataSource.java | 32 + .../com/facebook/presto/orc/StripeReader.java | 149 +- .../presto/orc/TupleDomainFilter.java | 459 +++++- .../presto/orc/TupleDomainFilterUtils.java | 25 +- .../orc/cache/CachingOrcFileTailSource.java | 55 + .../presto/orc/cache/OrcCacheConfig.java | 147 ++ .../presto/orc/cache/OrcFileTailSource.java | 28 + .../orc/cache/StorageOrcFileTailSource.java | 155 ++ .../presto/orc/metadata/OrcFileTail.java | 83 ++ .../AbstractDecimalSelectiveStreamReader.java | 335 +++++ .../AbstractLongSelectiveStreamReader.java | 37 +- .../reader/ApacheHiveTimestampDecoder.java | 53 + .../presto/orc/reader/BatchStreamReader.java | 3 +- .../presto/orc/reader/BatchStreamReaders.java | 34 +- .../orc/reader/BooleanBatchStreamReader.java | 93 +- .../reader/BooleanSelectiveStreamReader.java | 69 +- .../orc/reader/ByteBatchStreamReader.java | 97 +- .../orc/reader/ByteSelectiveStreamReader.java | 442 ++++++ .../orc/reader/DecimalBatchStreamReader.java | 44 +- .../orc/reader/DoubleBatchStreamReader.java | 25 +- .../reader/DoubleSelectiveStreamReader.java | 445 ++++++ .../orc/reader/FloatBatchStreamReader.java | 25 +- .../reader/FloatSelectiveStreamReader.java | 428 ++++++ .../presto/orc/reader/HierarchicalFilter.java | 63 + .../orc/reader/ListBatchStreamReader.java | 46 +- .../presto/orc/reader/ListFilter.java | 436 ++++++ .../orc/reader/ListSelectiveStreamReader.java | 740 +++++++++ .../orc/reader/LongBatchStreamReader.java | 27 +- .../LongDecimalSelectiveStreamReader.java | 202 +++ .../LongDictionaryBatchStreamReader.java | 28 +- .../LongDictionarySelectiveStreamReader.java | 44 +- .../reader/LongDirectBatchStreamReader.java | 175 ++- .../LongDirectSelectiveStreamReader.java | 66 +- .../orc/reader/LongSelectiveStreamReader.java | 6 + .../orc/reader/MapBatchStreamReader.java | 27 +- .../reader/MapDirectBatchStreamReader.java | 53 +- .../MapDirectSelectiveStreamReader.java | 713 +++++++++ .../orc/reader/MapFlatBatchStreamReader.java | 42 +- .../reader/MapFlatSelectiveStreamReader.java | 795 ++++++++++ .../orc/reader/MapSelectiveStreamReader.java | 143 ++ .../presto/orc/reader/ReaderUtils.java | 128 ++ .../orc/reader/SelectiveStreamReader.java | 7 +- .../orc/reader/SelectiveStreamReaders.java | 118 +- .../ShortDecimalSelectiveStreamReader.java | 177 +++ .../orc/reader/SliceBatchStreamReader.java | 31 +- .../SliceDictionaryBatchStreamReader.java | 50 +- .../SliceDictionarySelectiveReader.java | 642 ++++++++ .../reader/SliceDirectBatchStreamReader.java | 126 +- .../SliceDirectSelectiveStreamReader.java | 531 +++++++ .../reader/SliceSelectiveStreamReader.java | 154 ++ .../presto/orc/reader/StreamReader.java | 2 + .../orc/reader/StructBatchStreamReader.java | 82 +- .../reader/StructSelectiveStreamReader.java | 752 ++++++++++ .../reader/TimestampBatchStreamReader.java | 138 +- .../TimestampSelectiveStreamReader.java | 452 ++++++ .../presto/orc/stream/BooleanInputStream.java | 264 +++- .../presto/orc/stream/ByteInputStream.java | 32 +- .../presto/orc/stream/DoubleInputStream.java | 7 +- .../presto/orc/stream/FloatInputStream.java | 7 +- .../presto/orc/stream/LongInputStream.java | 32 +- .../orc/stream/LongInputStreamDwrf.java | 42 +- .../presto/orc/stream/LongInputStreamV1.java | 166 ++- .../presto/orc/stream/LongInputStreamV2.java | 78 +- .../presto/orc/stream/OrcInputStream.java | 289 +++- .../orc/writer/BooleanColumnWriter.java | 9 +- .../presto/orc/writer/ByteColumnWriter.java | 9 +- .../orc/writer/DecimalColumnWriter.java | 9 +- .../presto/orc/writer/DoubleColumnWriter.java | 9 +- .../presto/orc/writer/FloatColumnWriter.java | 9 +- .../presto/orc/writer/ListColumnWriter.java | 9 +- .../presto/orc/writer/LongColumnWriter.java | 9 +- .../presto/orc/writer/MapColumnWriter.java | 9 +- .../writer/SliceDictionaryColumnWriter.java | 16 +- .../orc/writer/SliceDirectColumnWriter.java | 9 +- .../presto/orc/writer/StructColumnWriter.java | 7 +- .../orc/writer/TimestampColumnWriter.java | 10 +- .../presto/orc/AbstractTestOrcReader.java | 101 ++ .../orc/BenchmarkBatchStreamReaders.java | 731 ++------- .../BenchmarkBatchStreamReadersWithZstd.java | 256 ++++ .../orc/BenchmarkSelectiveStreamReaders.java | 137 +- .../orc/BenchmarkZstdJniDecompression.java | 130 ++ .../presto/orc/OrcReaderTestingUtils.java | 37 + .../com/facebook/presto/orc/OrcTester.java | 683 ++++++++- .../presto/orc/TestByteArrayUtils.java | 29 + .../presto/orc/TestCachingOrcDataSource.java | 14 +- .../TestDictionaryCompressionOptimizer.java | 4 +- .../presto/orc/TestFilterFunction.java | 141 ++ .../facebook/presto/orc/TestListFilter.java | 318 ++++ ...java => TestMapFlatBatchStreamReader.java} | 34 +- .../orc/TestMapFlatSelectiveStreamReader.java | 650 ++++++++ .../presto/orc/TestOrcCacheConfig.java | 72 + .../com/facebook/presto/orc/TestOrcLz4.java | 18 +- .../presto/orc/TestOrcReaderMemoryUsage.java | 12 +- .../presto/orc/TestOrcReaderPositions.java | 18 +- .../facebook/presto/orc/TestOrcWriter.java | 90 +- .../presto/orc/TestPositionalFilter.java | 87 ++ .../presto/orc/TestReadBloomFilter.java | 137 ++ .../presto/orc/TestSelectiveOrcReader.java | 899 ++++++++++- .../orc/TestStructBatchStreamReader.java | 16 +- .../presto/orc/TestTupleDomainFilter.java | 27 + .../orc/TestTupleDomainFilterUtils.java | 63 +- .../presto/orc/TestZstdJniDecompression.java | 66 + .../presto/orc/TestingOrcPredicate.java | 48 +- .../presto/orc/TrackingTupleDomainFilter.java | 73 + .../orc/TupleDomainFilterOrderChecker.java | 43 + .../presto/orc/stream/TestBooleanStream.java | 72 +- .../flat_map_all_empty_maps.dwrf | Bin 770 -> 1026 bytes .../test_flat_map/flat_map_all_null_maps.dwrf | Bin 959 -> 1170 bytes .../test_flat_map/flat_map_binary.dwrf | Bin 25941 -> 14219 bytes .../test_flat_map/flat_map_boolean.dwrf | Bin 16503 -> 11097 bytes .../flat_map_boolean_with_null.dwrf | Bin 29852 -> 24379 bytes .../test_flat_map/flat_map_byte.dwrf | Bin 18451 -> 13180 bytes .../flat_map_byte_with_null.dwrf | Bin 31742 -> 26323 bytes .../test_flat_map/flat_map_double.dwrf | Bin 23434 -> 17546 bytes .../flat_map_double_with_null.dwrf | Bin 36769 -> 30744 bytes .../test_flat_map/flat_map_float.dwrf | Bin 37451 -> 17420 bytes .../flat_map_float_with_null.dwrf | Bin 49786 -> 30608 bytes .../resources/test_flat_map/flat_map_int.dwrf | Bin 19749 -> 12318 bytes .../flat_map_int_missing_sequences.dwrf | Bin 10342 -> 6726 bytes .../test_flat_map/flat_map_int_with_null.dwrf | Bin 32987 -> 25486 bytes .../test_flat_map/flat_map_list.dwrf | Bin 38111 -> 20307 bytes .../flat_map_list_with_null.dwrf | Bin 50664 -> 33379 bytes .../test_flat_map/flat_map_long.dwrf | Bin 19749 -> 12318 bytes .../resources/test_flat_map/flat_map_map.dwrf | Bin 56476 -> 42640 bytes .../test_flat_map/flat_map_map_with_null.dwrf | Bin 68753 -> 55057 bytes .../flat_map_mixed_encodings.dwrf | Bin 88125 -> 52812 bytes .../test_flat_map/flat_map_short.dwrf | Bin 19745 -> 12318 bytes .../flat_map_some_empty_maps.dwrf | Bin 19620 -> 12218 bytes .../flat_map_some_null_maps.dwrf | Bin 19050 -> 12452 bytes .../test_flat_map/flat_map_string.dwrf | Bin 21702 -> 16193 bytes .../flat_map_string_with_null.dwrf | Bin 35011 -> 29266 bytes .../test_flat_map/flat_map_struct.dwrf | Bin 47770 -> 34179 bytes .../flat_map_struct_with_null.dwrf | Bin 60665 -> 47351 bytes presto-parquet/pom.xml | 11 +- .../presto/parquet/ParquetTypeUtils.java | 41 +- .../presto/parquet/predicate/Predicate.java | 10 +- .../parquet/predicate/PredicateUtils.java | 20 +- .../TupleDomainParquetPredicate.java | 23 +- .../presto/parquet/reader/ParquetReader.java | 38 +- .../TestTupleDomainParquetPredicate.java | 2 +- presto-parser/pom.xml | 4 +- .../com/facebook/presto/sql/parser/SqlBase.g4 | 91 +- .../presto/sql/ExpressionFormatter.java | 11 +- .../com/facebook/presto/sql/SqlFormatter.java | 82 +- .../presto/sql/parser/AstBuilder.java | 137 +- .../presto/sql/parser/ErrorHandler.java | 2 +- .../facebook/presto/sql/parser/SqlParser.java | 15 +- .../presto/sql/parser/SqlParserOptions.java | 5 + .../presto/sql/tree/ArrayConstructor.java | 2 +- .../facebook/presto/sql/tree/AstVisitor.java | 35 +- .../presto/sql/tree/CreateFunction.java | 143 ++ .../facebook/presto/sql/tree/CurrentPath.java | 65 - .../presto/sql/tree/DropFunction.java | 107 ++ .../presto/sql/tree/ExpressionRewriter.java | 5 - .../sql/tree/ExpressionTreeRewriter.java | 15 +- .../presto/sql/tree/FunctionCall.java | 38 +- .../facebook/presto/sql/tree/PathElement.java | 105 -- .../presto/sql/tree/PathSpecification.java | 84 -- .../sql/tree/RoutineCharacteristics.java | 124 ++ .../com/facebook/presto/sql/tree/SetPath.java | 89 -- .../sql/tree/SqlParameterDeclaration.java | 70 + .../presto/sql/parser/TestSqlParser.java | 163 +- .../parser/TestSqlParserErrorHandling.java | 4 +- .../sql/parser/TestStatementBuilder.java | 2 + presto-password-authenticators/pom.xml | 10 +- .../presto/password/LdapAuthenticator.java | 2 +- .../password/LdapAuthenticatorFactory.java | 4 +- .../facebook/presto/password/LdapConfig.java | 4 +- .../presto/password/TestLdapConfig.java | 10 +- presto-pinot-toolkit/pom.xml | 188 +++ .../com/facebook/presto/pinot/ForPinot.java | 31 + .../presto/pinot/PinotBrokerPageSource.java | 420 ++++++ .../presto/pinot/PinotClusterInfoFetcher.java | 430 ++++++ .../facebook/presto/pinot/PinotColumn.java | 56 +- .../presto/pinot/PinotColumnHandle.java | 112 ++ .../presto/pinot/PinotColumnMetadata.java | 68 + .../presto/pinot/PinotColumnUtils.java | 73 + .../facebook/presto/pinot/PinotConfig.java | 419 ++++++ .../presto/pinot/PinotConnection.java | 112 ++ .../facebook/presto/pinot/PinotConnector.java | 134 ++ .../presto/pinot/PinotConnectorFactory.java | 97 ++ .../pinot/PinotConnectorPlanOptimizer.java | 234 +++ .../facebook/presto/pinot/PinotErrorCode.java | 70 + .../facebook/presto/pinot/PinotException.java | 54 + .../presto/pinot/PinotHandleResolver.java | 55 + .../facebook/presto/pinot/PinotMetadata.java | 181 +++ .../facebook/presto/pinot/PinotMetrics.java | 114 ++ .../presto/pinot/PinotMetricsStats.java | 85 ++ .../facebook/presto/pinot/PinotModule.java | 108 ++ .../pinot/PinotNodePartitioningProvider.java | 59 + .../presto/pinot/PinotPageSourceProvider.java | 100 ++ .../presto/pinot/PinotPushdownUtils.java | 215 +++ .../presto/pinot/PinotSegmentPageSource.java | 434 ++++++ .../presto/pinot/PinotSessionProperties.java | 153 ++ .../com/facebook/presto/pinot/PinotSplit.java | 165 ++ .../presto/pinot/PinotSplitManager.java | 192 +++ .../com/facebook/presto/pinot/PinotTable.java | 62 + .../presto/pinot/PinotTableHandle.java | 129 ++ .../presto/pinot/PinotTableLayoutHandle.java | 66 + .../presto/pinot/PinotTransactionHandle.java | 22 + .../com/facebook/presto/pinot/PinotUtils.java | 53 + .../presto/pinot/RebindSafeMBeanServer.java | 344 +++++ .../PinotAggregationProjectConverter.java | 262 ++++ .../presto/pinot/query/PinotExpression.java | 45 + .../query/PinotFilterExpressionConverter.java | 229 +++ .../PinotProjectExpressionConverter.java | 137 ++ .../pinot/query/PinotQueryGenerator.java | 467 ++++++ .../query/PinotQueryGeneratorContext.java | 468 ++++++ .../facebook/presto/pinot/MetadataUtil.java | 87 ++ .../pinot/MockPinotClusterInfoFetcher.java | 420 ++++++ .../pinot/TestPinotBrokerPageSource.java | 205 +++ .../pinot/TestPinotClusterInfoFetcher.java | 80 + .../presto/pinot/TestPinotColumnHandle.java | 55 + .../presto/pinot/TestPinotConfig.java | 123 ++ .../presto/pinot/TestPinotMetadata.java | 46 + .../presto/pinot/TestPinotQueryBase.java | 225 +++ .../pinot/TestPinotSessionProperties.java | 42 + .../presto/pinot/TestPinotSplitManager.java | 165 ++ .../presto/pinot/TestPinotTableHandle.java | 54 + .../TestPinotConnectorPlanOptimizer.java | 233 +++ .../query/TestPinotExpressionConverters.java | 171 +++ .../pinot/query/TestPinotQueryGenerator.java | 238 +++ presto-pinot/pom.xml | 64 + .../facebook/presto/pinot/PinotPlugin.java | 28 + .../services/com.facebook.presto.spi.Plugin | 2 + presto-plugin-toolkit/pom.xml | 10 +- .../presto/plugin/base/JsonUtils.java | 2 +- .../FileBasedAccessControlConfig.java | 2 +- .../FileBasedAccessControlModule.java | 4 +- .../TestFileBasedAccessControlConfig.java | 8 +- presto-postgresql/pom.xml | 10 +- .../plugin/postgresql/PostgreSqlClient.java | 19 +- .../postgresql/PostgreSqlClientModule.java | 2 +- .../postgresql/PostgreSqlQueryRunner.java | 18 +- .../TestPostgreSqlCaseInsensitiveMapping.java | 190 +++ .../TestPostgreSqlDistributedQueries.java | 5 +- .../TestPostgreSqlIntegrationSmokeTest.java | 2 +- .../postgresql/TestPostgreSqlTypeMapping.java | 6 +- .../docker/files/presto-launcher-wrapper.sh | 8 +- .../conf/docker/multinode/docker-compose.yml | 4 + .../conf/presto/etc/jvm.config | 2 + .../conf/presto/etc/log.properties | 2 +- .../presto/etc/multinode-master-jvm.config | 23 + .../presto/etc/multinode-master.properties | 2 - .../presto/etc/multinode-worker-jvm.config | 23 + presto-product-tests/pom.xml | 10 +- .../presto/tests/AlterTableTests.java | 2 +- .../facebook/presto/tests/SqlCancelTests.java | 20 +- .../presto/tests/cli/PrestoCliTests.java | 2 +- .../presto/tests/hive/TestGrantRevoke.java | 2 +- .../presto/tests/hive/TestHiveCoercion.java | 2 +- .../facebook/presto/tests/jdbc/JdbcTests.java | 2 +- .../presto/tests/jdbc/PreparedStatements.java | 2 +- .../tests/mysql/CreateTableAsSelect.java | 2 +- .../querystats/HttpQueryStatsClient.java | 16 +- .../QueryStatsClientModuleProvider.java | 6 +- .../presto/tests/sqlserver/TestInsert.java | 2 +- .../presto/tests/sqlserver/TestSelect.java | 2 +- .../presto/tests/utils/JdbcDriverUtils.java | 2 +- .../selectInformationSchemaColumns.result | 1 + .../querystats/TestHttpQueryStatsClient.java | 10 +- presto-proxy/pom.xml | 30 +- .../presto/proxy/JsonWebTokenHandler.java | 2 +- .../presto/proxy/JwtHandlerConfig.java | 6 +- .../facebook/presto/proxy/PrestoProxy.java | 20 +- .../facebook/presto/proxy/ProxyConfig.java | 4 +- .../facebook/presto/proxy/ProxyModule.java | 6 +- .../facebook/presto/proxy/ProxyResource.java | 32 +- .../presto/proxy/ProxyResponseHandler.java | 12 +- .../presto/proxy/TestJwtHandlerConfig.java | 6 +- .../presto/proxy/TestProxyConfig.java | 6 +- .../presto/proxy/TestProxyServer.java | 20 +- presto-raptor/pom.xml | 45 +- .../facebook/presto/raptor/PluginInfo.java | 5 + .../presto/raptor/RaptorConnector.java | 18 +- .../presto/raptor/RaptorConnectorFactory.java | 13 +- .../presto/raptor/RaptorErrorCode.java | 4 +- .../presto/raptor/RaptorMetadata.java | 41 +- .../presto/raptor/RaptorMetadataFactory.java | 8 +- .../raptor/RaptorOutputTableHandle.java | 11 +- .../presto/raptor/RaptorPageSink.java | 24 +- .../presto/raptor/RaptorPageSinkProvider.java | 11 +- .../raptor/RaptorPageSourceProvider.java | 10 +- .../facebook/presto/raptor/RaptorPlugin.java | 8 +- .../presto/raptor/RaptorProcedureModule.java | 33 + .../raptor/RaptorSessionProperties.java | 13 +- .../presto/raptor/RaptorSplitManager.java | 2 +- .../presto/raptor/RaptorTableHandle.java | 11 +- .../presto/raptor/RaptorTableProperties.java | 12 + .../TriggerBucketBalancerProcedure.java | 57 + .../presto/raptor/backup/BackupConfig.java | 4 +- .../presto/raptor/backup/BackupManager.java | 27 +- .../presto/raptor/backup/BackupModule.java | 8 +- .../raptor/backup/BackupOperationStats.java | 4 +- .../raptor/backup/FileBackupConfig.java | 4 +- .../raptor/backup/FileBackupModule.java | 2 +- .../presto/raptor/backup/FileBackupStore.java | 2 +- .../raptor/backup/HttpBackupConfig.java | 4 +- .../raptor/backup/HttpBackupModule.java | 6 +- .../presto/raptor/backup/HttpBackupStore.java | 28 +- .../raptor/backup/ManagedBackupStore.java | 2 +- .../raptor/backup/TimeoutBackupStore.java | 6 +- .../raptor/filesystem/FileSystemContext.java | 82 + .../raptor/filesystem/FileSystemModule.java | 59 + .../raptor/filesystem/FileSystemUtil.java | 73 + .../presto/raptor/filesystem/HdfsModule.java | 64 + .../filesystem/HdfsOrcDataEnvironment.java | 77 + .../raptor/filesystem/HdfsOrcDataSource.java | 74 + .../raptor/filesystem/HdfsStorageService.java | 132 ++ .../LocalFileStorageService.java} | 82 +- .../filesystem/LocalFileSystemModule.java | 31 + .../filesystem/LocalOrcDataEnvironment.java | 90 ++ .../RaptorCachingHdfsConfiguration.java | 116 ++ .../raptor/filesystem/RaptorHdfsConfig.java | 197 +++ .../filesystem/RaptorHdfsConfiguration.java | 23 + .../RaptorHiveHdfsConfiguration.java | 190 +++ .../filesystem/RaptorLocalFileSystem.java | 50 + .../raptor/metadata/AssignmentLimiter.java | 4 +- .../raptor/metadata/DatabaseConfig.java | 4 +- .../metadata/DatabaseMetadataModule.java | 18 +- .../raptor/metadata/DatabaseShardManager.java | 54 +- .../metadata/DatabaseShardRecorder.java | 2 +- .../presto/raptor/metadata/Distribution.java | 4 +- .../raptor/metadata/JdbcDatabaseConfig.java | 2 +- .../raptor/metadata/MetadataConfig.java | 4 +- .../presto/raptor/metadata/MetadataDao.java | 11 +- .../presto/raptor/metadata/SchemaDao.java | 20 + .../presto/raptor/metadata/SchemaDaoUtil.java | 48 +- .../presto/raptor/metadata/ShardCleaner.java | 23 +- .../raptor/metadata/ShardCleanerConfig.java | 4 +- .../presto/raptor/metadata/ShardIterator.java | 2 +- .../presto/raptor/metadata/ShardManager.java | 2 +- .../presto/raptor/metadata/Table.java | 14 +- .../raptor/metadata/TableMetadataRow.java | 13 +- .../raptor/security/RaptorSecurityConfig.java | 2 +- .../raptor/security/RaptorSecurityModule.java | 4 +- .../presto/raptor/storage/BackupStats.java | 4 +- .../presto/raptor/storage/BucketBalancer.java | 6 +- .../raptor/storage/BucketBalancerConfig.java | 4 +- .../raptor/storage/OrcDataEnvironment.java | 33 + ...FileRewriter.java => OrcFileRewriter.java} | 57 +- .../presto/raptor/storage/OrcFileWriter.java | 58 +- .../presto/raptor/storage/OrcPageSource.java | 19 +- .../raptor/storage/OrcRecordFileRewriter.java | 214 --- .../raptor/storage/OrcRecordWriter.java | 347 ----- .../raptor/storage/OrcStorageManager.java | 234 +-- .../raptor/storage/ReaderAttributes.java | 18 +- .../presto/raptor/storage/ShardEjector.java | 23 +- .../raptor/storage/ShardRecoveryManager.java | 34 +- .../raptor/storage/ShardRecoveryStats.java | 4 +- .../presto/raptor/storage/ShardStats.java | 45 +- .../presto/raptor/storage/StorageManager.java | 6 +- .../raptor/storage/StorageManagerConfig.java | 65 +- .../presto/raptor/storage/StorageModule.java | 71 +- .../presto/raptor/storage/StorageService.java | 13 +- .../raptor/storage/StorageTypeConverter.java | 8 + .../storage/organization/OrganizationJob.java | 2 +- .../organization/ShardCompactionManager.java | 4 +- .../storage/organization/ShardCompactor.java | 15 +- .../ShardOrganizationManager.java | 9 +- .../storage/organization/ShardOrganizer.java | 8 +- .../TableMetadataSystemTable.java | 6 +- .../presto/raptor/util/ConcatPageSource.java | 9 + .../raptor/util/PrioritizedFifoExecutor.java | 4 +- .../raptor/util/RebindSafeMBeanServer.java | 2 +- .../presto/raptor/util/SyncingFileSystem.java | 119 -- .../presto/raptor/RaptorQueryRunner.java | 38 +- .../presto/raptor/TestRaptorConnector.java | 18 +- .../presto/raptor/TestRaptorPlugin.java | 4 +- .../raptor/backup/TestBackupConfig.java | 6 +- .../raptor/backup/TestBackupManager.java | 25 +- .../raptor/backup/TestFileBackupConfig.java | 6 +- .../raptor/backup/TestFileBackupStore.java | 1 + .../raptor/backup/TestHttpBackupConfig.java | 6 +- .../raptor/backup/TestHttpBackupStore.java | 16 +- .../backup/TestingHttpBackupResource.java | 2 +- .../TestRaptorDistributedQueries.java | 2 +- .../TestRaptorDistributedQueriesBucketed.java | 2 +- .../TestRaptorIntegrationSmokeTest.java | 70 +- ...estRaptorIntegrationSmokeTestBucketed.java | 2 +- ...> TestRaptorIntegrationSmokeTestHdfs.java} | 6 +- .../TestRaptorIntegrationSmokeTestMySql.java | 8 +- .../metadata/TestAssignmentLimiter.java | 2 +- .../raptor/metadata/TestDatabaseConfig.java | 6 +- .../metadata/TestDatabaseShardManager.java | 76 +- .../metadata/TestJdbcDatabaseConfig.java | 6 +- .../raptor/metadata/TestMetadataConfig.java | 6 +- .../raptor/metadata/TestMetadataDao.java | 6 +- .../raptor/metadata/TestRaptorMetadata.java | 107 +- .../metadata/TestRaptorSplitManager.java | 8 +- .../raptor/metadata/TestShardCleaner.java | 21 +- .../metadata/TestShardCleanerConfig.java | 6 +- .../presto/raptor/metadata/TestShardDao.java | 8 +- .../raptor/metadata/TestUpgradeMetadata.java | 164 ++ .../raptor/metadata/TestingShardDao.java | 14 + .../security/TestRaptorFileBasedSecurity.java | 1 + .../security/TestRaptorReadOnlySecurity.java | 2 +- .../security/TestRaptorSecurityConfig.java | 2 +- .../presto/raptor/storage/OrcTestingUtil.java | 56 +- .../raptor/storage/TestBucketBalancer.java | 8 +- .../storage/TestBucketBalancerConfig.java | 6 +- ....java => TestLocalFileStorageService.java} | 25 +- .../raptor/storage/TestOrcFileRewriter.java | 280 ++-- .../raptor/storage/TestOrcStorageManager.java | 120 +- .../raptor/storage/TestRaptorHdfsConfig.java | 77 + .../raptor/storage/TestShardEjector.java | 18 +- .../raptor/storage/TestShardRecovery.java | 34 +- .../raptor/storage/TestShardWriter.java | 45 +- .../storage/TestStorageManagerConfig.java | 35 +- .../TestCompactionSetCreator.java | 8 +- .../organization/TestShardCompactor.java | 13 +- .../TestShardOrganizationManager.java | 16 +- .../organization/TestShardOrganizerUtil.java | 3 +- .../storage/organization/TestTuple.java | 4 +- .../TestShardMetadataRecordCursor.java | 8 +- presto-rcfile/pom.xml | 6 +- .../presto/rcfile/binary/ListEncoding.java | 2 +- .../presto/rcfile/binary/MapEncoding.java | 2 +- .../presto/rcfile/binary/StructEncoding.java | 2 +- .../presto/rcfile/text/ListEncoding.java | 2 +- .../presto/rcfile/text/MapEncoding.java | 2 +- .../presto/rcfile/text/StructEncoding.java | 2 +- .../TestBufferedOutputStreamSliceOutput.java | 2 +- presto-record-decoder/pom.xml | 6 +- .../decoder/json/JsonFieldDecoderTester.java | 2 +- .../TestCustomDateTimeJsonFieldDecoder.java | 2 +- .../presto/decoder/json/TestJsonDecoder.java | 2 +- presto-redis/pom.xml | 12 +- .../presto/redis/RedisConnectorConfig.java | 4 +- .../presto/redis/RedisConnectorFactory.java | 4 +- .../presto/redis/RedisConnectorModule.java | 6 +- .../presto/redis/RedisJedisManager.java | 2 +- .../facebook/presto/redis/RedisMetadata.java | 2 +- .../presto/redis/RedisRecordCursor.java | 2 +- .../redis/RedisTableDescriptionSupplier.java | 4 +- .../presto/redis/RedisQueryRunner.java | 6 +- .../redis/TestRedisConnectorConfig.java | 2 +- .../presto/redis/TestRedisPlugin.java | 2 +- .../presto/redis/util/CodecSupplier.java | 6 +- .../presto/redis/util/RedisTestUtils.java | 2 +- presto-redshift/pom.xml | 4 +- .../plugin/redshift/RedshiftClient.java | 5 +- .../plugin/redshift/RedshiftClientModule.java | 2 +- presto-resource-group-managers/pom.xml | 16 +- .../FileResourceGroupConfig.java | 2 +- ...FileResourceGroupConfigurationManager.java | 6 +- ...ourceGroupConfigurationManagerFactory.java | 4 +- .../FileResourceGroupsModule.java | 2 +- .../db/DbResourceGroupConfig.java | 4 +- .../DbResourceGroupConfigurationManager.java | 41 +- ...ourceGroupConfigurationManagerFactory.java | 4 +- .../db/DbResourceGroupsModule.java | 2 +- .../db/DbSourceExactMatchSelector.java | 4 +- .../resourceGroups/db/SelectorRecord.java | 6 +- .../TestFileResourceGroupConfig.java | 6 +- .../db/TestDbResourceGroupConfig.java | 6 +- ...stDbResourceGroupConfigurationManager.java | 4 +- .../db/TestDbSourceExactMatchSelector.java | 2 +- .../db/TestResourceGroupsDao.java | 6 +- presto-server-rpm/pom.xml | 2 +- presto-server/pom.xml | 14 +- presto-server/src/main/assembly/presto.xml | 4 + presto-session-property-managers/pom.xml | 10 +- .../session/FileSessionPropertyManager.java | 6 +- .../FileSessionPropertyManagerConfig.java | 2 +- .../FileSessionPropertyManagerFactory.java | 4 +- .../FileSessionPropertyManagerModule.java | 2 +- .../TestFileSessionPropertyManager.java | 2 +- .../TestFileSessionPropertyManagerConfig.java | 6 +- presto-spi/pom.xml | 6 +- .../presto/spi/CatalogSchemaName.java | 8 + .../presto/spi/ConnectorPageSource.java | 6 + .../presto/spi/ConnectorPlanOptimizer.java | 4 - .../facebook/presto/spi/ConnectorSession.java | 14 + .../facebook/presto/spi/FixedPageSource.java | 8 + .../presto/spi/InMemoryRecordSet.java | 4 + .../java/com/facebook/presto/spi/Plugin.java | 6 + .../facebook/presto/spi/RecordPageSource.java | 9 + .../presto/spi/StandardErrorCode.java | 2 + .../com/facebook/presto/spi/Subfield.java | 9 + .../presto/spi/SubfieldTokenizer.java | 2 +- .../presto/spi/block/AbstractArrayBlock.java | 7 +- .../presto/spi/block/AbstractMapBlock.java | 29 +- .../presto/spi/block/AbstractRowBlock.java | 7 +- .../spi/block/AbstractSingleArrayBlock.java | 4 +- .../spi/block/AbstractSingleMapBlock.java | 10 +- .../spi/block/AbstractSingleRowBlock.java | 6 +- .../spi/block/AbstractVariableWidthBlock.java | 6 +- .../com/facebook/presto/spi/block/Block.java | 5 +- .../presto/spi/block/BlockFlattener.java | 4 +- .../presto/spi/block/ColumnarArray.java | 20 +- .../presto/spi/block/ColumnarMap.java | 52 +- .../presto/spi/block/ColumnarRow.java | 19 + .../presto/spi/block/DictionaryBlock.java | 11 +- .../facebook/presto/spi/block/LazyBlock.java | 6 +- .../facebook/presto/spi/block/MapBlock.java | 15 +- .../presto/spi/block/MapBlockBuilder.java | 6 +- .../presto/spi/block/MapBlockEncoding.java | 2 +- .../spi/block/RunLengthEncodedBlock.java | 6 +- .../spi/block/SingleMapBlockEncoding.java | 2 +- .../presto/spi/block/VariableWidthBlock.java | 4 +- .../spi/block/VariableWidthBlockBuilder.java | 4 +- .../spi/connector/ConnectorContext.java | 6 + .../spi/connector/ConnectorMetadata.java | 11 +- .../ConnectorPageSourceProvider.java | 15 +- .../ConnectorPlanOptimizerProvider.java | 11 +- .../ClassLoaderSafeConnectorMetadata.java | 17 +- ...LoaderSafeConnectorPageSourceProvider.java | 9 +- .../spi/eventlistener/QueryContext.java | 9 - .../spi/eventlistener/QueryStatistics.java | 22 +- ...ibution.java => ResourceDistribution.java} | 4 +- .../eventlistener/SplitCompletedEvent.java | 8 + .../spi/eventlistener/StageGcStatistics.java | 9 + .../spi/function/CatalogSchemaPrefix.java | 90 ++ .../presto/spi/function/FunctionHandle.java | 2 +- .../spi/function}/FunctionHandleResolver.java | 4 +- .../function/FunctionImplementationType.java | 7 +- .../presto/spi/function/FunctionMetadata.java | 47 +- .../function/FunctionNamespaceManager.java | 68 + .../FunctionNamespaceManagerFactory.java | 15 +- .../FunctionNamespaceTransactionHandle.java | 5 +- .../presto/spi/function/OperatorType.java | 12 +- .../spi/function/QualifiedFunctionName.java | 89 ++ .../spi/function/RoutineCharacteristics.java | 97 ++ .../ScalarFunctionImplementation.java | 10 +- .../presto/spi/function/Signature.java | 18 +- .../presto/spi/function}/SqlFunction.java | 9 +- .../spi/function/SqlFunctionHandle.java | 94 ++ .../presto/spi/function/SqlFunctionId.java | 88 ++ .../spi/function/SqlFunctionProperties.java | 127 ++ .../spi/function/SqlInvokedFunction.java | 215 +++ ...qlInvokedScalarFunctionImplementation.java | 35 + .../presto/spi/function/SqlParameter.java | 75 + .../spi/function/ValueWindowFunction.java | 12 + .../presto/spi}/plan/AggregationNode.java | 94 +- .../presto/spi}/plan/Assignments.java | 24 +- .../facebook/presto/spi}/plan/ExceptNode.java | 17 +- .../plan/FilterStatsCalculatorService.java | 42 + .../presto/spi}/plan/IntersectNode.java | 17 +- .../facebook/presto/spi/plan/LimitNode.java | 126 ++ .../facebook/presto/spi/plan/Ordering.java | 84 ++ .../presto/spi/plan/OrderingScheme.java | 129 ++ .../facebook/presto/spi/plan/PlanVisitor.java | 35 + .../presto/spi}/plan/ProjectNode.java | 21 +- .../presto/spi/plan/SetOperationNode.java | 120 ++ .../presto/spi/plan/TableScanNode.java | 4 +- .../facebook/presto/spi}/plan/TopNNode.java | 49 +- .../facebook/presto/spi}/plan/UnionNode.java | 17 +- .../facebook/presto/spi/predicate/Domain.java | 2 +- .../presto/spi/predicate/Primitives.java | 2 +- .../presto/spi/predicate/SortedRangeSet.java | 21 + .../presto/spi/predicate/TupleDomain.java | 41 + .../spi/relation/ConstantExpression.java | 5 +- .../presto/spi/relation/DomainTranslator.java | 13 +- .../spi/relation/ExpressionOptimizer.java | 14 +- .../spi/relation/LogicalRowExpressions.java | 514 ------- .../spi/relation/PredicateCompiler.java | 3 +- .../spi/relation/RowExpressionService.java | 7 + .../presto/spi/security/Identity.java | 7 +- .../facebook/presto/spi/type/ArrayType.java | 12 +- .../com/facebook/presto/spi/type/MapType.java | 11 +- .../com/facebook/presto/spi/type/RowType.java | 12 +- .../presto/spi/type/StandardTypes.java | 1 + .../presto/spi/TestStandardErrorCode.java | 4 +- .../presto/spi/TestSubfieldTokenizer.java | 9 +- .../spi/block/CountingArrayAllocator.java | 2 +- .../presto/spi/block/TestingSession.java | 13 + .../spi/predicate/TestAllOrNoneValueSet.java | 2 +- .../presto/spi/predicate/TestDomain.java | 2 +- .../spi/predicate/TestEquatableValueSet.java | 2 +- .../presto/spi/predicate/TestMarker.java | 2 +- .../presto/spi/predicate/TestRange.java | 2 +- .../spi/predicate/TestSortedRangeSet.java | 2 +- .../presto/spi/predicate/TestTupleDomain.java | 2 +- .../resourceGroups/TestResourceGroupId.java | 2 +- .../presto/spi/security/TestSelectedRole.java | 4 +- .../presto/spi/type/TestParameterKind.java | 6 +- presto-sql-function/pom.xml | 59 + ...actSqlInvokedFunctionNamespaceManager.java | 206 +++ ...InvokedFunctionNamespaceManagerConfig.java | 53 + ...uidFunctionNamespaceTransactionHandle.java | 71 + .../testing/SqlInvokedFunctionTestUtils.java | 67 + ...InvokedFunctionNamespaceManagerConfig.java | 51 + presto-sqlserver/pom.xml | 4 +- .../plugin/sqlserver/SqlServerClient.java | 9 +- .../sqlserver/SqlServerClientModule.java | 2 +- presto-teradata-functions/pom.xml | 4 +- .../functions/TeradataDateFunctions.java | 2 +- presto-testing-docker/pom.xml | 79 + .../testing/docker/DockerContainer.java | 251 ++++ presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 22 +- .../tests/AbstractTestAggregations.java | 13 + .../tests/AbstractTestDistributedQueries.java | 2 +- .../presto/tests/AbstractTestQueries.java | 92 +- .../tests/AbstractTestQueryFramework.java | 8 +- .../tests/AbstractTestRepartitionQueries.java | 330 ++++ .../tests/AbstractTestingPrestoClient.java | 1 - .../com/facebook/presto/tests/CreateHll.java | 2 +- .../presto/tests/DistributedQueryRunner.java | 8 +- .../facebook/presto/tests/H2QueryRunner.java | 79 +- .../presto/tests/QueryAssertions.java | 2 +- .../presto/tests/StandaloneQueryRunner.java | 4 +- .../presto/tests/StatefulSleepingSum.java | 14 +- .../presto/tests/TestingPrestoClient.java | 9 + .../presto/execution/TestBeginQuery.java | 13 +- .../execution/TestCompletedEventWarnings.java | 5 +- .../presto/execution/TestEventListener.java | 2 +- .../presto/execution/TestFinalQueryInfo.java | 3 +- .../presto/execution/TestQueryRunnerUtil.java | 2 +- .../presto/execution/TestWarnings.java | 30 +- .../execution/TestingSessionContext.java | 12 - .../TestResourceGroupIntegration.java | 2 +- ...ourceGroupConfigurationManagerFactory.java | 6 +- .../db/H2ResourceGroupsModule.java | 2 +- .../resourceGroups/db/H2TestUtil.java | 4 +- .../resourceGroups/db/TestQueuesDb.java | 2 +- .../tests/TestDistributedSpilledQueries.java | 6 + .../presto/tests/TestLocalQueries.java | 4 +- .../tests/TestQueryPlanDeterminism.java | 2 +- .../presto/tests/TestQueryTaskLimit.java | 107 ++ .../presto/tests/TestRepartitionQueries.java | 27 + .../TestRepartitionQueriesWithSmallPages.java | 24 +- .../presto/tests/TestSqlFunctions.java | 62 + .../presto/tests/tpch/TpchQueryRunner.java | 4 +- presto-thrift-connector-api/pom.xml | 4 +- .../connector/thrift/api/TestReadWrite.java | 2 +- presto-thrift-connector/pom.xml | 16 +- .../connector/thrift/ThriftConnector.java | 8 +- .../thrift/ThriftConnectorConfig.java | 2 +- .../thrift/ThriftConnectorFactory.java | 6 +- .../thrift/ThriftConnectorStats.java | 2 +- .../thrift/ThriftIndexPageSource.java | 14 +- .../presto/connector/thrift/ThriftModule.java | 4 +- .../connector/thrift/ThriftPageSource.java | 12 +- .../connector/thrift/ThriftSplitManager.java | 2 +- .../ExtendedSimpleAddressSelectorBinder.java | 2 +- .../thrift/util/RebindSafeMBeanServer.java | 2 +- .../thrift/TestThriftConnectorConfig.java | 2 +- .../connector/thrift/TestThriftPlugin.java | 2 +- .../thrift/integration/ThriftQueryRunner.java | 6 +- presto-thrift-testing-server/pom.xml | 10 +- .../thrift/server/ThriftTpchServer.java | 4 +- .../thrift/server/ThriftTpchService.java | 6 +- presto-tpcds/pom.xml | 8 +- .../facebook/presto/tpcds/TpcdsMetadata.java | 4 +- .../TpcdsTableStatisticsFactory.java | 11 +- .../com/facebook/presto/tpcds/TestTpcds.java | 5 - .../tpcds/TestTpcdsMetadataStatistics.java | 21 +- .../presto/tpcds/TpcdsQueryRunner.java | 4 +- presto-tpch/pom.xml | 2 +- .../facebook/presto/tpch/TpchMetadata.java | 25 +- .../presto/tpch/TestTpchMetadata.java | 9 +- presto-verifier/pom.xml | 62 +- .../presto/verifier/PrestoVerifyCommand.java | 14 +- .../verifier/annotation/ForControl.java | 31 + .../presto/verifier/annotation/ForTest.java | 31 + .../event/DeterminismAnalysisDetails.java | 63 + .../event/DeterminismAnalysisRun.java | 103 ++ .../verifier/event/EventClientModule.java | 4 +- .../event/HumanReadableEventClient.java | 2 +- .../verifier/event/JsonEventClient.java | 4 +- .../{FailureInfo.java => QueryFailure.java} | 35 +- .../presto/verifier/event/QueryInfo.java | 23 +- .../verifier/event/VerifierQueryEvent.java | 81 +- .../framework/AbstractVerification.java | 244 +-- .../framework/AbstractVerifyCommand.java | 8 +- .../verifier/framework/ClusterType.java | 20 + .../presto/verifier/framework/Column.java | 14 +- .../verifier/framework/DataVerification.java | 203 +-- .../framework/DeterminismAnalysis.java | 57 + .../verifier/framework/ForwardingDriver.java | 81 - .../verifier/framework/JdbcDriverUtil.java | 93 -- .../LimitQueryDeterminismAnalysis.java | 22 + .../LimitQueryDeterminismAnalyzer.java | 139 ++ .../verifier/framework/QueryBundle.java | 21 +- .../framework/QueryConfiguration.java | 67 +- .../QueryConfigurationOverrides.java | 39 + .../QueryConfigurationOverridesConfig.java | 131 ++ .../verifier/framework/QueryException.java | 30 +- .../verifier/framework/QueryOrigin.java | 113 -- .../presto/verifier/framework/QueryStage.java | 80 + .../verifier/framework/SkippedReason.java | 9 +- .../framework/VerificationContext.java | 123 +- .../framework/VerificationFactory.java | 68 +- .../framework/VerificationManager.java | 94 +- .../framework/VerificationResult.java | 64 - .../verifier/framework/VerifierConfig.java | 347 +---- .../verifier/framework/VerifierModule.java | 105 +- .../verifier/framework/VerifierUtil.java | 33 + .../verifier/framework/VerifyCommand.java | 5 +- .../prestoaction/HttpNodeResourceClient.java | 87 ++ .../JdbcPrestoAction.java | 124 +- .../prestoaction/NodeResourceClient.java | 19 + .../PrestoAction.java | 17 +- .../prestoaction/PrestoActionFactory.java | 22 + .../verifier/prestoaction/PrestoAddress.java | 44 + .../prestoaction/PrestoClusterConfig.java | 149 ++ .../PrestoExceptionClassifier.java | 25 +- .../prestoaction/RoutingPrestoAction.java | 51 + .../RoutingPrestoActionFactory.java | 71 + .../SqlExceptionClassifier.java | 6 +- .../VerificationPrestoActionModule.java | 62 + .../AbstractPrestoQueryFailureResolver.java | 17 +- ...eededGlobalMemoryLimitFailureResolver.java | 18 +- .../ExceededTimeLimitFailureResolver.java | 18 +- .../verifier/resolver/FailureResolver.java | 3 +- .../resolver/FailureResolverConfig.java | 67 + .../resolver/FailureResolverFactory.java | 19 + .../FailureResolverFactoryContext.java | 68 + .../resolver/FailureResolverManager.java | 44 + .../FailureResolverManagerFactory.java | 47 + .../resolver/FailureResolverModule.java | 44 + .../TooManyOpenPartitionsFailureResolver.java | 118 ++ .../presto/verifier/retry/RetryConfig.java | 2 +- .../presto/verifier/retry/RetryDriver.java | 35 +- .../verifier/rewrite/QueryRewriteConfig.java | 42 + .../{framework => rewrite}/QueryRewriter.java | 61 +- .../rewrite/QueryRewriterFactory.java | 21 + .../VerificationQueryRewriterFactory.java | 63 + .../VerificationQueryRewriterModule.java | 34 + .../source/MySqlSourceQueryConfig.java | 4 +- .../verifier/source/SourceQueryModule.java | 4 +- .../source/StringToStringMapColumnMapper.java | 4 +- .../presto/verifier/source/VerifierDao.java | 23 +- ...estPrestoVerifierIntegrationSmokeTest.java | 23 +- .../presto/verifier/VerifierTestUtil.java | 33 +- .../framework/TestDataVerification.java | 142 +- .../TestLimitQueryDeterminismAnalyzer.java | 169 +++ .../framework/TestQueryConfiguration.java | 115 ++ ...TestQueryConfigurationOverridesConfig.java | 62 + .../framework/TestQueryException.java | 11 +- .../framework/TestVerificationContext.java | 38 +- .../framework/TestVerificationManager.java | 136 +- .../framework/TestVerifierConfig.java | 82 +- .../TestHttpNodeResourceClient.java | 43 + .../TestJdbcPrestoAction.java | 68 +- .../prestoaction/TestPrestoClusterConfig.java | 90 ++ .../TestPrestoExceptionClassifier.java | 27 +- ...bstractTestPrestoQueryFailureResolver.java | 17 +- ...eededGlobalMemoryLimitFailureResolver.java | 9 +- .../TestExceededTimeLimitFailureResolver.java | 6 +- .../resolver/TestFailureResolverConfig.java | 53 + ...tTooManyOpenPartitionsFailureResolver.java | 130 ++ .../verifier/retry/TestRetryConfig.java | 6 +- .../verifier/retry/TestRetryDriver.java | 56 +- .../rewrite/TestQueryRewriteConfig.java | 45 + .../TestQueryRewriter.java | 46 +- .../source/TestMySqlSourceQueryConfig.java | 6 +- .../source/TestMySqlSourceQuerySupplier.java | 8 +- .../source/TestSourceQueryModule.java | 4 +- pull_request_template.md | 5 +- 2603 files changed, 94790 insertions(+), 22430 deletions(-) create mode 100644 .mailmap create mode 100644 presto-array/src/main/java/com/facebook/presto/array/Arrays.java create mode 100644 presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcIdentity.java create mode 100644 presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/RemoteTableNameCacheKey.java create mode 100644 presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java create mode 100644 presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcExpression.java create mode 100644 presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcFilterToSqlTranslator.java create mode 100644 presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java create mode 100644 presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/function/JdbcTranslationUtil.java create mode 100644 presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/function/OperatorTranslators.java create mode 100644 presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java create mode 100644 presto-benchmark-runner/pom.xml create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkQuery.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkRunnerConfig.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkSuite.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkSuiteInfo.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/ConcurrentExecutionPhase.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/MySqlBenchmarkSuiteConfig.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/PhaseSpecification.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/StreamExecutionPhase.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteConfig.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteDao.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteModule.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteSupplier.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/DbBenchmarkSuiteSupplier.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/PhaseSpecificationsColumnMapper.java create mode 100644 presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/StringToStringMapColumnMapper.java create mode 100644 presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/BenchmarkTestUtil.java create mode 100644 presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/framework/TestBenchmarkRunnerConfig.java create mode 100644 presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/source/TestBenchmarkSuiteConfig.java create mode 100644 presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/source/TestBenchmarkSuiteModule.java create mode 100644 presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/source/TestDbBenchmarkSuiteSupplier.java create mode 100644 presto-cache/pom.xml create mode 100644 presto-cache/src/main/java/com/facebook/presto/cache/CacheConfig.java create mode 100644 presto-cache/src/main/java/com/facebook/presto/cache/CacheManager.java create mode 100644 presto-cache/src/main/java/com/facebook/presto/cache/CacheStats.java create mode 100644 presto-cache/src/main/java/com/facebook/presto/cache/CachingFileSystem.java create mode 100644 presto-cache/src/main/java/com/facebook/presto/cache/CachingInputStream.java create mode 100644 presto-cache/src/main/java/com/facebook/presto/cache/FileReadRequest.java create mode 100644 presto-cache/src/main/java/com/facebook/presto/cache/ForCachingFileSystem.java create mode 100644 presto-cache/src/main/java/com/facebook/presto/cache/LocalRangeCacheManager.java rename presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceFactory.java => presto-cache/src/main/java/com/facebook/presto/cache/NoOpCacheManager.java (64%) create mode 100644 presto-cache/src/test/java/com/facebook/presto/cache/TestCacheConfig.java create mode 100644 presto-cache/src/test/java/com/facebook/presto/cache/TestLocalRangeCacheManager.java create mode 100644 presto-docs/src/main/sphinx/connector/pinot.rst create mode 100644 presto-docs/src/main/sphinx/functions/ip.rst create mode 100644 presto-docs/src/main/sphinx/release/release-0.224.rst create mode 100644 presto-docs/src/main/sphinx/release/release-0.225.rst create mode 100644 presto-docs/src/main/sphinx/release/release-0.226.rst create mode 100644 presto-docs/src/main/sphinx/release/release-0.227.rst create mode 100644 presto-docs/src/main/sphinx/release/release-0.228.rst create mode 100644 presto-docs/src/main/sphinx/release/release-0.229.rst create mode 100644 presto-elasticsearch/src/test/resources/queryrunner/nested.data.json create mode 100644 presto-expressions/pom.xml rename {presto-spi/src/main/java/com/facebook/presto/spi/relation => presto-expressions/src/main/java/com/facebook/presto/expressions}/DefaultRowExpressionTraversalVisitor.java (78%) create mode 100644 presto-expressions/src/main/java/com/facebook/presto/expressions/LogicalRowExpressions.java rename {presto-spi/src/main/java/com/facebook/presto/spi/relation => presto-expressions/src/main/java/com/facebook/presto/expressions}/RowExpressionNodeInliner.java (93%) rename {presto-spi/src/main/java/com/facebook/presto/spi/relation => presto-expressions/src/main/java/com/facebook/presto/expressions}/RowExpressionRewriter.java (80%) rename {presto-spi/src/main/java/com/facebook/presto/spi/relation => presto-expressions/src/main/java/com/facebook/presto/expressions}/RowExpressionTreeRewriter.java (92%) create mode 100644 presto-expressions/src/main/java/com/facebook/presto/expressions/translator/FunctionTranslator.java create mode 100644 presto-expressions/src/main/java/com/facebook/presto/expressions/translator/RowExpressionTranslator.java create mode 100644 presto-expressions/src/main/java/com/facebook/presto/expressions/translator/RowExpressionTreeTranslator.java create mode 100644 presto-expressions/src/main/java/com/facebook/presto/expressions/translator/TranslatedExpression.java create mode 100644 presto-expressions/src/main/java/com/facebook/presto/expressions/translator/TranslatorAnnotationParser.java create mode 100644 presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/rtree/Flatbush.java create mode 100644 presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/rtree/HasExtent.java create mode 100644 presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/rtree/HilbertIndex.java rename presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/{GeometrySerde.java => EsriGeometrySerde.java} (92%) create mode 100644 presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/TestGeometryUtils.java create mode 100644 presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkFlatbushBuild.java create mode 100644 presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkFlatbushQuery.java create mode 100644 presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkJtsStrTreeBuild.java create mode 100644 presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkJtsStrTreeQuery.java create mode 100644 presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/RtreeTestUtils.java create mode 100644 presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/TestFlatbush.java create mode 100644 presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/TestHilbertIndex.java create mode 100644 presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestGeoRelations.java create mode 100644 presto-hive-metastore/pom.xml rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/ForCachingHiveMetastore.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/ForRecordingHiveMetastore.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/HdfsConfiguration.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java (93%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/HiveBasicStatistics.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java (98%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java (67%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/HiveType.java (95%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/HiveTypeName.java (93%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/HiveViewNotSupportedException.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/LocationHandle.java (89%) create mode 100644 presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientConfig.java create mode 100644 presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientModule.java create mode 100644 presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreErrorCode.java rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/PartitionNotFoundException.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java (96%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/RetryDriver.java (99%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/SchemaAlreadyExistsException.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/TableAlreadyExistsException.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/TableOfflineException.java (95%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/TypeTranslator.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/authentication/GenericExceptionAction.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/authentication/HdfsAuthentication.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/authentication/HiveMetastoreAuthentication.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/authentication/NoHiveMetastoreAuthentication.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/BooleanStatistics.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java (98%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/Column.java (85%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/Database.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/DateStatistics.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/DecimalStatistics.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/DoubleStatistics.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java (98%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/HiveColumnStatistics.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/HiveMetastoreModule.java (92%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/HivePageSinkMetadata.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/HivePageSinkMetadataProvider.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/HivePartitionName.java (97%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/HivePrivilegeInfo.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/HiveTableName.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/IntegerStatistics.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/MetastoreConfig.java (95%) create mode 100644 presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/Partition.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/PartitionFilter.java (100%) rename {presto-hive/src/main/java/com/facebook/presto/hive => presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore}/PartitionStatistics.java (97%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java (93%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/PrincipalPrivileges.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/RecordingHiveMetastore.java (93%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java (97%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java (97%) rename {presto-hive/src/main/java/com/facebook/presto/hive/util => presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore}/Statistics.java (96%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/Storage.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java (96%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/Table.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/UserDatabaseKey.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/UserTableKey.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/file/DatabaseMetadata.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java (97%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastoreConfig.java (92%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java (96%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java (99%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/file/PermissionMetadata.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java (100%) create mode 100644 presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/ForGlueHiveMetastore.java rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/glue/GlueExpressionUtil.java (97%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java (92%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java (73%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java (64%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueInputConverter.java (99%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java (98%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java (98%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveCluster.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java (97%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreApiStats.java (96%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java (85%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/StaticHiveCluster.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/StaticMetastoreConfig.java (94%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java (99%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreStats.java (100%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java (91%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java (99%) rename {presto-hive => presto-hive-metastore}/src/main/java/com/facebook/presto/hive/metastore/thrift/Transport.java (100%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/TestPartitionOfflineException.java (100%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/TestTableOfflineException.java (100%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java (99%) rename presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreUtil.java => presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestHiveMetastoreUtil.java (74%) create mode 100644 presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreClientConfig.java rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreConfig.java (82%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java (97%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/TestStorage.java (91%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/UnimplementedHiveMetastore.java (99%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueExpressionUtil.java (100%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java (74%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java (99%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClient.java (100%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClientFactory.java (100%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticHiveCluster.java (98%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticMetastoreConfig.java (90%) rename presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftMetastoreUtil.java => presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftHiveMetastoreUtil.java (99%) rename {presto-hive => presto-hive-metastore}/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java (78%) create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/BucketAdaptation.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/DynamicConfigurationProvider.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/FileOpener.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/FilteringPageSource.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/HadoopFileOpener.java rename presto-hive/src/main/java/com/facebook/presto/hive/{HdfsConfigurationUpdater.java => HdfsConfigurationInitializer.java} (89%) create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercer.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/HiveFileInfo.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionObjectBuilder.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/PartitionObjectBuilder.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/SortingFileWriterFactory.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/gcs/GcsAccessTokenProvider.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/gcs/GcsConfigurationInitializer.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/gcs/GcsConfigurationProvider.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/gcs/HiveGcsConfig.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/gcs/HiveGcsConfigurationInitializer.java create mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/gcs/HiveGcsModule.java delete mode 100644 presto-hive/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestCoercingFilters.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientInMemoryMetastoreWithFilterPushdown.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithOptimizedRepartitioning.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestHivePartitionManager.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownDistributedQueries.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownIntegrationSmokeTest.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestParquetDistributedQueries.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java create mode 100644 presto-hive/src/test/java/com/facebook/presto/hive/gcs/TestHiveGcsConfig.java create mode 100644 presto-main/src/main/java/com/facebook/presto/cost/ConnectorFilterStatsCalculatorService.java rename presto-main/src/main/java/com/facebook/presto/execution/{BasicStageStats.java => BasicStageExecutionStats.java} (95%) create mode 100644 presto-main/src/main/java/com/facebook/presto/execution/CreateFunctionTask.java create mode 100644 presto-main/src/main/java/com/facebook/presto/execution/DropFunctionTask.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/execution/SetPathTask.java create mode 100644 presto-main/src/main/java/com/facebook/presto/execution/StageExecutionId.java create mode 100644 presto-main/src/main/java/com/facebook/presto/execution/StageExecutionInfo.java rename presto-main/src/main/java/com/facebook/presto/execution/{StageState.java => StageExecutionState.java} (90%) rename presto-main/src/main/java/com/facebook/presto/execution/{StageStateMachine.java => StageExecutionStateMachine.java} (74%) rename presto-main/src/main/java/com/facebook/presto/execution/{StageStats.java => StageExecutionStats.java} (96%) rename presto-main/src/main/java/com/facebook/presto/execution/{MemoryTrackingRemoteTaskFactory.java => TrackingRemoteTaskFactory.java} (75%) create mode 100644 presto-main/src/main/java/com/facebook/presto/execution/scheduler/ExecutionWriterTarget.java create mode 100644 presto-main/src/main/java/com/facebook/presto/execution/scheduler/TableWriteInfo.java create mode 100644 presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunction.java rename presto-main/src/main/java/com/facebook/presto/metadata/{StaticFunctionHandle.java => BuiltInFunctionHandle.java} (82%) rename presto-main/src/main/java/com/facebook/presto/metadata/{StaticFunctionNamespaceHandleResolver.java => BuiltInFunctionNamespaceHandleResolver.java} (83%) rename presto-main/src/main/java/com/facebook/presto/metadata/{StaticFunctionNamespace.java => BuiltInFunctionNamespaceManager.java} (72%) delete mode 100644 presto-main/src/main/java/com/facebook/presto/metadata/OperatorSignatureUtils.java create mode 100644 presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceStore.java rename presto-main/src/main/java/com/facebook/presto/metadata/{FunctionNamespace.java => StaticFunctionNamespaceStoreConfig.java} (50%) delete mode 100644 presto-main/src/main/java/com/facebook/presto/operator/ArrayOfRowsUnnester.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/operator/ArrayUnnester.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/operator/MapUnnester.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/MoreByteArrays.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/TableWriterMergeInfo.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/TableWriterMergeOperator.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/TableWriterUtils.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/UncheckedByteArrays.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/operator/UnnestOperator.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyAggregation.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyState.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyStateFactory.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyStateSerializer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyStateStrategy.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/EntropyCalculations.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/FixedHistogramJacknifeStateStrategy.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/FixedHistogramMleStateStrategy.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/FixedHistogramStateStrategyUtils.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/UnweightedReservoirSampleStateStrategy.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/WeightedReservoirSampleStateStrategy.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/reservoirsample/UnweightedDoubleReservoirSample.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/aggregation/reservoirsample/WeightedDoubleReservoirSample.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/AbstractBlockEncodingBuffer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/ArrayBlockEncodingBuffer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/BlockEncodingBuffer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/ByteArrayBlockEncodingBuffer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/DecodedBlockNode.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/Int128ArrayBlockEncodingBuffer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/IntArrayBlockEncodingBuffer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/LongArrayBlockEncodingBuffer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/MapBlockEncodingBuffer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/OptimizedPartitionedOutputOperator.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/PartitionedOutputInfo.java rename presto-main/src/main/java/com/facebook/presto/operator/{ => repartition}/PartitionedOutputOperator.java (87%) create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/RowBlockEncodingBuffer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/ShortArrayBlockEncodingBuffer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/repartition/VariableWidthBlockEncodingBuffer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayAllMatchFunction.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayAnyMatchFunction.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayCombinationsFunction.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayNoneMatchFunction.java rename presto-main/src/main/java/com/facebook/presto/operator/{Unnester.java => scalar/BlockToBooleanFunction.java} (69%) create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/scalar/BooleanToBooleanFunction.java rename presto-main/src/main/java/com/facebook/presto/operator/scalar/{ScalarFunctionImplementation.java => BuiltInScalarFunctionImplementation.java} (94%) create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/scalar/DoubleToBooleanFunction.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/scalar/IpPrefixFunctions.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/scalar/LongToBooleanFunction.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/scalar/SliceToBooleanFunction.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/unnest/ArrayOfRowsUnnester.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/unnest/ArrayUnnester.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/unnest/MapUnnester.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/unnest/ReplicatedBlockBuilder.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/unnest/UnnestBlockBuilder.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/unnest/UnnestOperator.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/unnest/UnnestOperatorBlockUtil.java create mode 100644 presto-main/src/main/java/com/facebook/presto/operator/unnest/Unnester.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/sql/SqlPath.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/LookupSymbolResolver.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/NoOpVariableResolver.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/OrderingScheme.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/PartitioningProviderManager.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionEqualityInference.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionPredicateExtractor.java rename presto-main/src/main/java/com/facebook/presto/sql/planner/{NoOpSymbolResolver.java => VariableResolver.java} (70%) delete mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DesugarCurrentPath.java rename presto-main/src/main/java/com/facebook/presto/sql/planner/{optimizations/TranslateExpressions.java => iterative/rule/RowExpressionRewriteRuleSet.java} (53%) create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyArrayOperations.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyArrayOperationsRewriter.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TranslateExpressions.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/BeginTableWrite.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RowExpressionPredicatePushDown.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SetOperationNodeUtils.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/plan/LimitNode.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SetOperationNode.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterMergeNode.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/relational/SqlFunctionUtils.java delete mode 100644 presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java create mode 100644 presto-main/src/main/java/com/facebook/presto/testing/InMemoryFunctionNamespaceManager.java create mode 100644 presto-main/src/main/java/com/facebook/presto/transaction/DelegatingTransactionManager.java create mode 100644 presto-main/src/main/java/com/facebook/presto/type/IpPrefixOperators.java create mode 100644 presto-main/src/main/java/com/facebook/presto/type/IpPrefixType.java create mode 100644 presto-main/src/main/java/com/facebook/presto/util/GcStatusMonitor.java create mode 100644 presto-main/src/main/java/com/facebook/presto/util/StringTableUtils.java delete mode 100644 presto-main/src/test/java/com/facebook/presto/execution/TestSetPathTask.java rename presto-main/src/test/java/com/facebook/presto/execution/{TestStageStateMachine.java => TestStageExecutionStateMachine.java} (65%) rename presto-main/src/test/java/com/facebook/presto/execution/{TestStageStats.java => TestStageExecutionStats.java} (88%) rename presto-main/src/test/java/com/facebook/presto/metadata/{TestStaticFunctionNamespace.java => TestFunctionManager.java} (79%) create mode 100644 presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionNamespaceManager.java create mode 100644 presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespaceStoreConfig.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/BenchmarkCompressToByteBuffer.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/BenchmarkMoreByteArrays.java delete mode 100644 presto-main/src/test/java/com/facebook/presto/operator/BenchmarkPartitionedOutputOperator.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/BenchmarkReadBlock.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/BenchmarkUncheckedByteArrays.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/BenchmarkUnnestOperator.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/TestMoreByteArrays.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/TestPartitionedOutputOperator.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/TestUncheckedByteArrays.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/AbstractTestFixedHistogramAggregation.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/AbstractTestReservoirAggregation.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/AbstractTestStateStrategy.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestEntropyCalculations.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramJacknifeAggregation.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramJacknifeStateStrategy.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramMleAggregation.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramMleStateStrategy.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestIllegalMethodAggregation.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestUnweightedReservoirAggregation.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestUnweightedReservoirSampleStateStrategy.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestWeightedReservoirAggregation.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestWeightedReservoirSampleStateStrategy.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/reservoirsample/TestUnweightedDoubleReservoirSample.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/aggregation/reservoirsample/TestWeightedDoubleReservoirSample.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/repartition/BenchmarkPartitionedOutputOperator.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/repartition/TestBlockEncodingBuffers.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/repartition/TestOptimizedPartitionedOutputOperator.java rename presto-main/src/test/java/com/facebook/presto/operator/scalar/{TestRegexpFunctions.java => AbstractTestRegexpFunctions.java} (94%) create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayCombinationsFunction.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayIntersectFunction.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayMatchFunctions.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArraySortFunction.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/scalar/TestIpPrefixFunctions.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/scalar/TestJoniRegexpFunctions.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/scalar/TestRe2jRegexpFunctions.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/unnest/TestArrayOfRowsUnnester.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/unnest/TestArrayUnnester.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/unnest/TestMapUnnester.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/unnest/TestReplicatedBlockBuilder.java create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/unnest/TestUnnestBlockBuilder.java rename presto-main/src/test/java/com/facebook/presto/operator/{ => unnest}/TestUnnestOperator.java (97%) create mode 100644 presto-main/src/test/java/com/facebook/presto/operator/unnest/TestUnnesterUtil.java create mode 100644 presto-main/src/test/java/com/facebook/presto/sql/planner/TestNullabilityAnalyzer.java create mode 100644 presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionEqualityInference.java create mode 100644 presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionPredicateExtractor.java rename presto-main/src/test/java/com/facebook/presto/sql/planner/{optimizations => iterative/rule}/TestTranslateExpressions.java (64%) create mode 100644 presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestFullOuterJoinWithCoalesce.java rename presto-main/src/test/java/com/facebook/presto/sql/planner/plan/{TestAssingments.java => TestAssignments.java} (77%) rename presto-main/src/test/java/com/facebook/presto/sql/{TestExpressionOptimizer.java => relational/TestRowExpressionOptimizer.java} (85%) create mode 100644 presto-main/src/test/java/com/facebook/presto/sql/relational/TestRowExpressionTranslator.java create mode 100644 presto-main/src/test/java/com/facebook/presto/sql/relational/TestSubExpressions.java create mode 100644 presto-main/src/test/java/com/facebook/presto/tests/LogTestDurationListener.java create mode 100644 presto-main/src/test/java/com/facebook/presto/type/TestIpPrefixOperators.java create mode 100644 presto-main/src/test/java/com/facebook/presto/type/TestIpPrefixType.java create mode 100644 presto-main/src/test/java/com/facebook/presto/util/TestStringTableUtils.java create mode 100644 presto-main/src/test/resources/META-INF/services/org.testng.ITestNGListener create mode 100644 presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoClientConfig.java create mode 100644 presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/DockerizedMySqlServer.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/CacheStatsMBean.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/CachingStripeMetadataSource.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/OrcReaderOptions.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/StorageStripeMetadataSource.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/StripeMetadataSource.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/cache/CachingOrcFileTailSource.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/cache/OrcCacheConfig.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/cache/OrcFileTailSource.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/cache/StorageOrcFileTailSource.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcFileTail.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/AbstractDecimalSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/ApacheHiveTimestampDecoder.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/ByteSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/DoubleSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/FloatSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/HierarchicalFilter.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/ListFilter.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/ListSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDecimalSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/MapDirectSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/MapFlatSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/MapSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/ReaderUtils.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/ShortDecimalSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDictionarySelectiveReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDirectSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/StructSelectiveStreamReader.java create mode 100644 presto-orc/src/main/java/com/facebook/presto/orc/reader/TimestampSelectiveStreamReader.java create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkBatchStreamReadersWithZstd.java create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkZstdJniDecompression.java create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/OrcReaderTestingUtils.java create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/TestByteArrayUtils.java create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/TestFilterFunction.java create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/TestListFilter.java rename presto-orc/src/test/java/com/facebook/presto/orc/{TestFlatMap.java => TestMapFlatBatchStreamReader.java} (94%) create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/TestMapFlatSelectiveStreamReader.java create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/TestOrcCacheConfig.java create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/TestPositionalFilter.java create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/TestReadBloomFilter.java create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/TestZstdJniDecompression.java create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/TrackingTupleDomainFilter.java create mode 100644 presto-orc/src/test/java/com/facebook/presto/orc/TupleDomainFilterOrderChecker.java create mode 100644 presto-parser/src/main/java/com/facebook/presto/sql/tree/CreateFunction.java delete mode 100644 presto-parser/src/main/java/com/facebook/presto/sql/tree/CurrentPath.java create mode 100644 presto-parser/src/main/java/com/facebook/presto/sql/tree/DropFunction.java delete mode 100644 presto-parser/src/main/java/com/facebook/presto/sql/tree/PathElement.java delete mode 100644 presto-parser/src/main/java/com/facebook/presto/sql/tree/PathSpecification.java create mode 100644 presto-parser/src/main/java/com/facebook/presto/sql/tree/RoutineCharacteristics.java delete mode 100644 presto-parser/src/main/java/com/facebook/presto/sql/tree/SetPath.java create mode 100644 presto-parser/src/main/java/com/facebook/presto/sql/tree/SqlParameterDeclaration.java create mode 100644 presto-pinot-toolkit/pom.xml create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/ForPinot.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotBrokerPageSource.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotClusterInfoFetcher.java rename presto-main/src/main/java/com/facebook/presto/sql/SqlPathElement.java => presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumn.java (50%) create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumnHandle.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumnMetadata.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumnUtils.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConfig.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnection.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnector.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnectorFactory.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnectorPlanOptimizer.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotErrorCode.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotException.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotHandleResolver.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotMetadata.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotMetrics.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotMetricsStats.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotModule.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotNodePartitioningProvider.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotPageSourceProvider.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotPushdownUtils.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSegmentPageSource.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSessionProperties.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSplit.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSplitManager.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTable.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTableHandle.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTableLayoutHandle.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTransactionHandle.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotUtils.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/RebindSafeMBeanServer.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotAggregationProjectConverter.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotExpression.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotFilterExpressionConverter.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotProjectExpressionConverter.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotQueryGenerator.java create mode 100644 presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotQueryGeneratorContext.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/MetadataUtil.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/MockPinotClusterInfoFetcher.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotBrokerPageSource.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotClusterInfoFetcher.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotColumnHandle.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotConfig.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotMetadata.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotQueryBase.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSessionProperties.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSplitManager.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotTableHandle.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/query/TestPinotConnectorPlanOptimizer.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/query/TestPinotExpressionConverters.java create mode 100644 presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/query/TestPinotQueryGenerator.java create mode 100644 presto-pinot/pom.xml create mode 100644 presto-pinot/src/main/java/com/facebook/presto/pinot/PinotPlugin.java create mode 100644 presto-pinot/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin create mode 100644 presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlCaseInsensitiveMapping.java create mode 100644 presto-product-tests/conf/presto/etc/multinode-master-jvm.config create mode 100644 presto-product-tests/conf/presto/etc/multinode-worker-jvm.config create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorProcedureModule.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/TriggerBucketBalancerProcedure.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/FileSystemContext.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/FileSystemModule.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/FileSystemUtil.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsModule.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsOrcDataEnvironment.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsOrcDataSource.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsStorageService.java rename presto-raptor/src/main/java/com/facebook/presto/raptor/{storage/FileStorageService.java => filesystem/LocalFileStorageService.java} (65%) create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/LocalFileSystemModule.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/LocalOrcDataEnvironment.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorCachingHdfsConfiguration.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorHdfsConfig.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorHdfsConfiguration.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorHiveHdfsConfiguration.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorLocalFileSystem.java create mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcDataEnvironment.java rename presto-raptor/src/main/java/com/facebook/presto/raptor/storage/{OrcPageFileRewriter.java => OrcFileRewriter.java} (80%) delete mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java delete mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java delete mode 100644 presto-raptor/src/main/java/com/facebook/presto/raptor/util/SyncingFileSystem.java rename presto-raptor/src/test/java/com/facebook/presto/raptor/integration/{TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter.java => TestRaptorIntegrationSmokeTestHdfs.java} (84%) create mode 100644 presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestUpgradeMetadata.java rename presto-raptor/src/test/java/com/facebook/presto/raptor/storage/{TestFileStorageService.java => TestLocalFileStorageService.java} (79%) create mode 100644 presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestRaptorHdfsConfig.java rename presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/{StageCpuDistribution.java => ResourceDistribution.java} (97%) create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/CatalogSchemaPrefix.java rename {presto-main/src/main/java/com/facebook/presto/metadata => presto-spi/src/main/java/com/facebook/presto/spi/function}/FunctionHandleResolver.java (87%) rename presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolResolver.java => presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionImplementationType.java (83%) create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceManager.java rename presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java => presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceManagerFactory.java (65%) rename presto-client/src/main/java/com/facebook/presto/client/ClientCapabilities.java => presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceTransactionHandle.java (85%) create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/QualifiedFunctionName.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/RoutineCharacteristics.java rename presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespaceFactory.java => presto-spi/src/main/java/com/facebook/presto/spi/function/ScalarFunctionImplementation.java (78%) rename {presto-main/src/main/java/com/facebook/presto/metadata => presto-spi/src/main/java/com/facebook/presto/spi/function}/SqlFunction.java (81%) create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionHandle.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionId.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionProperties.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedScalarFunctionImplementation.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/function/SqlParameter.java rename {presto-main/src/main/java/com/facebook/presto/sql/planner => presto-spi/src/main/java/com/facebook/presto/spi}/plan/AggregationNode.java (79%) rename {presto-main/src/main/java/com/facebook/presto/sql/planner => presto-spi/src/main/java/com/facebook/presto/spi}/plan/Assignments.java (88%) rename {presto-main/src/main/java/com/facebook/presto/sql/planner => presto-spi/src/main/java/com/facebook/presto/spi}/plan/ExceptNode.java (67%) create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/plan/FilterStatsCalculatorService.java rename {presto-main/src/main/java/com/facebook/presto/sql/planner => presto-spi/src/main/java/com/facebook/presto/spi}/plan/IntersectNode.java (68%) create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/plan/LimitNode.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/plan/Ordering.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/plan/OrderingScheme.java rename {presto-main/src/main/java/com/facebook/presto/sql/planner => presto-spi/src/main/java/com/facebook/presto/spi}/plan/ProjectNode.java (79%) create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/plan/SetOperationNode.java rename {presto-main/src/main/java/com/facebook/presto/sql/planner => presto-spi/src/main/java/com/facebook/presto/spi}/plan/TopNNode.java (63%) rename {presto-main/src/main/java/com/facebook/presto/sql/planner => presto-spi/src/main/java/com/facebook/presto/spi}/plan/UnionNode.java (68%) delete mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/relation/LogicalRowExpressions.java create mode 100644 presto-sql-function/pom.xml create mode 100644 presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/AbstractSqlInvokedFunctionNamespaceManager.java create mode 100644 presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/SqlInvokedFunctionNamespaceManagerConfig.java create mode 100644 presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/UuidFunctionNamespaceTransactionHandle.java create mode 100644 presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/testing/SqlInvokedFunctionTestUtils.java create mode 100644 presto-sql-function/src/test/java/com/facebook/presto/sqlfunction/TestSqlInvokedFunctionNamespaceManagerConfig.java create mode 100644 presto-testing-docker/pom.xml create mode 100644 presto-testing-docker/src/main/java/com/facebook/presto/testing/docker/DockerContainer.java create mode 100644 presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestRepartitionQueries.java create mode 100644 presto-tests/src/test/java/com/facebook/presto/tests/TestQueryTaskLimit.java create mode 100644 presto-tests/src/test/java/com/facebook/presto/tests/TestRepartitionQueries.java rename presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestWithOptimizedWriter.java => presto-tests/src/test/java/com/facebook/presto/tests/TestRepartitionQueriesWithSmallPages.java (50%) create mode 100644 presto-tests/src/test/java/com/facebook/presto/tests/TestSqlFunctions.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForControl.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForTest.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisDetails.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisRun.java rename presto-verifier/src/main/java/com/facebook/presto/verifier/event/{FailureInfo.java => QueryFailure.java} (62%) create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ClusterType.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalysis.java delete mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ForwardingDriver.java delete mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/framework/JdbcDriverUtil.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalysis.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalyzer.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfigurationOverrides.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfigurationOverridesConfig.java delete mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryOrigin.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryStage.java delete mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationResult.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/HttpNodeResourceClient.java rename presto-verifier/src/main/java/com/facebook/presto/verifier/{framework => prestoaction}/JdbcPrestoAction.java (60%) create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/NodeResourceClient.java rename presto-verifier/src/main/java/com/facebook/presto/verifier/{framework => prestoaction}/PrestoAction.java (73%) create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoActionFactory.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAddress.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoClusterConfig.java rename presto-verifier/src/main/java/com/facebook/presto/verifier/{framework => prestoaction}/PrestoExceptionClassifier.java (87%) create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoAction.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoActionFactory.java rename presto-verifier/src/main/java/com/facebook/presto/verifier/{framework => prestoaction}/SqlExceptionClassifier.java (71%) create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/VerificationPrestoActionModule.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverConfig.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverFactory.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverFactoryContext.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverManager.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverManagerFactory.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverModule.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/TooManyOpenPartitionsFailureResolver.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriteConfig.java rename presto-verifier/src/main/java/com/facebook/presto/verifier/{framework => rewrite}/QueryRewriter.java (83%) create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriterFactory.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java create mode 100644 presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterModule.java create mode 100644 presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestLimitQueryDeterminismAnalyzer.java create mode 100644 presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java create mode 100644 presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfigurationOverridesConfig.java create mode 100644 presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestHttpNodeResourceClient.java rename presto-verifier/src/test/java/com/facebook/presto/verifier/{framework => prestoaction}/TestJdbcPrestoAction.java (65%) create mode 100644 presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoClusterConfig.java rename presto-verifier/src/test/java/com/facebook/presto/verifier/{framework => prestoaction}/TestPrestoExceptionClassifier.java (84%) create mode 100644 presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestFailureResolverConfig.java create mode 100644 presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java create mode 100644 presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriteConfig.java rename presto-verifier/src/test/java/com/facebook/presto/verifier/{framework => rewrite}/TestQueryRewriter.java (82%) diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000000000..c468c759fd5df --- /dev/null +++ b/.mailmap @@ -0,0 +1,25 @@ + + + + + + + <50636602+kewang1024@users.noreply.github.com> + + <34256034+rschlussel2@users.noreply.github.com> + <34256034+rschlussel@users.noreply.github.com> + + + + + <6372365+viczhang861@users.noreply.github.com> + + + + + + + <33299678+yingsu00@users.noreply.github.com> + + + diff --git a/.travis.yml b/.travis.yml index 124b775a07bd5..b784aa1f103bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: java env: global: - - MAVEN_OPTS="-Xmx512M -XX:+ExitOnOutOfMemoryError" + - MAVEN_OPTS="-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_SKIP_CHECKS_AND_DOCS="-Dair.check.skip-all=true -Dmaven.javadoc.skip=true" - MAVEN_FAST_INSTALL="-DskipTests $MAVEN_SKIP_CHECKS_AND_DOCS -B -q -T C1" - ARTIFACTS_UPLOAD_PATH_BRANCH=travis_build_artifacts/${TRAVIS_REPO_SLUG}/${TRAVIS_BRANCH}/${TRAVIS_BUILD_NUMBER} @@ -19,6 +19,10 @@ env: - TEST_SPECIFIC_MODULES=presto-hive - TEST_SPECIFIC_MODULES=presto-hive TEST_FLAGS="-P test-hive-materialized" - TEST_SPECIFIC_MODULES=presto-hive TEST_FLAGS="-P test-hive-recoverable-grouped-execution" + - TEST_SPECIFIC_MODULES=presto-hive TEST_FLAGS="-P test-hive-pushdown-filter-queries-basic" + - TEST_SPECIFIC_MODULES=presto-hive TEST_FLAGS="-P test-hive-pushdown-filter-queries-advanced" + - TEST_SPECIFIC_MODULES=presto-hive TEST_FLAGS="-P test-hive-repartitioning" + - TEST_SPECIFIC_MODULES=presto-hive TEST_FLAGS="-P test-hive-parquet" - TEST_SPECIFIC_MODULES=presto-main - TEST_SPECIFIC_MODULES=presto-mongodb - TEST_OTHER_MODULES=!presto-tests,!presto-raptor,!presto-accumulo,!presto-cassandra,!presto-hive,!presto-kudu,!presto-docs,!presto-server,!presto-server-rpm,!presto-main,!presto-mongodb diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 003eab3281b74..44bb4359d5d11 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,9 +2,7 @@ ## Contributor License Agreement ("CLA") -In order to accept your pull request, we need you to submit a CLA. You only need to do this once, so if you've done this for another Facebook open source project, you're good to go. If you are submitting a pull request for the first time, just let us know that you have completed the CLA and we can cross-check with your GitHub username. - -Complete your CLA here: +In order to accept your pull request, we need you to submit a CLA. You only need to do this once, so if you've done this for one repository in the [prestodb](https://github.com/prestodb) organization, you're good to go. If you are submitting a pull request for the first time, the communitybridge-easycla bot will notify you if you haven't signed, and will provide you with a link. If you are contributing on behalf of a company, you might want to let the person who manages your corporate CLA whitelist know they will be receiving a request from you. ## License diff --git a/README.md b/README.md index 83a1cd560d2ab..0d055aab0e151 100644 --- a/README.md +++ b/README.md @@ -103,26 +103,5 @@ To iterate quickly, simply re-build the project in IntelliJ after packaging is c ## Release Notes -When authoring a pull request, the PR description should include relevant release notes. - -The release notes should be placed at the bottom of PR description. The format of the section will be: - -``` -== RELEASE NOTES == - -General Changes -* ... -* ... - -Hive Changes -* ... -* ... -``` - -Long line items can be wrapped without leading asterisk (*). White-spacing and capitalization are flexible. - -If the pull request does not have a user-facing impact and thus a release note is not required. Place the following line at the bottom of the PR description: - -``` -== NO RELEASE NOTE == -``` +When authoring a pull request, the PR description should include its relevant release notes. +Follow [Release Notes Guidelines](https://github.com/prestodb/presto/wiki/Release-Notes-Guidelines) when authoring release notes. diff --git a/pom.xml b/pom.xml index 0e7cc461f755e..b5566288c1a37 100644 --- a/pom.xml +++ b/pom.xml @@ -3,19 +3,19 @@ 4.0.0 - io.airlift + com.facebook.airlift airbase - 88 + 95 com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT pom presto-root Presto - https://github.com/facebook/presto + https://github.com/prestodb/presto 2012 @@ -28,8 +28,8 @@ - scm:git:git://github.com/facebook/presto.git - https://github.com/facebook/presto + scm:git:git://github.com/prestodb/presto.git + https://github.com/prestodb/presto HEAD @@ -45,13 +45,14 @@ 3.3.9 4.7.1 - 0.178 + 0.187 ${dep.airlift.version} 0.36 + 0.6 1.11.445 3.9.0 3.4.0 - 1.19 + 1.22 2.10 1.50 6.10 @@ -60,6 +61,7 @@ 1.10.0 1.6.8 6.2.1 + 1.9.17 org.javassist javassist 3.22.0-GA + + + com.starburstdata + starburst-spotify-docker-client + 8.11.7-0.6 + + + + net.jodah + failsafe + 2.0.1 + @@ -1181,7 +1344,7 @@ io.airlift.maven.plugins sphinx-maven-plugin - 2.0 + 2.1 @@ -1212,6 +1375,7 @@ org.codehaus.plexus:plexus-utils + com.google.guava:guava diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index 1df9d37b32bc1..de741b608759a 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-accumulo @@ -167,7 +167,7 @@ - io.airlift + com.facebook.airlift bootstrap @@ -178,22 +178,22 @@ - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift configuration - io.airlift + com.facebook.airlift concurrent @@ -232,6 +232,7 @@ org.apache.commons commons-lang3 + 3.4 @@ -240,17 +241,6 @@ 2.4 - - joda-time - joda-time - - - - io.airlift - joda-to-java-time-bridge - runtime - - org.apache.zookeeper zookeeper diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloClient.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloClient.java index d3c9cff3683fa..7726a03165dda 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloClient.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloClient.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.accumulo; +import com.facebook.airlift.log.Logger; import com.facebook.presto.accumulo.conf.AccumuloConfig; import com.facebook.presto.accumulo.conf.AccumuloSessionProperties; import com.facebook.presto.accumulo.conf.AccumuloTableProperties; @@ -39,7 +40,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; -import io.airlift.log.Logger; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloConnector.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloConnector.java index 7a7e2959ad1d1..d0e379ecfee37 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloConnector.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloConnector.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.accumulo; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.accumulo.conf.AccumuloSessionProperties; import com.facebook.presto.accumulo.conf.AccumuloTableProperties; import com.facebook.presto.accumulo.io.AccumuloPageSinkProvider; @@ -25,8 +27,6 @@ import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spi.transaction.IsolationLevel; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloConnectorFactory.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloConnectorFactory.java index 84feb97c84576..4ed6295feff48 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloConnectorFactory.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloConnectorFactory.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.accumulo; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.connector.ConnectorFactory; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloModule.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloModule.java index 7568cb41a4649..6617905f910e4 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloModule.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloModule.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.accumulo; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.facebook.presto.accumulo.conf.AccumuloConfig; import com.facebook.presto.accumulo.conf.AccumuloSessionProperties; import com.facebook.presto.accumulo.conf.AccumuloTableProperties; @@ -30,8 +32,6 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Scopes; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; @@ -45,11 +45,11 @@ import javax.inject.Inject; import javax.inject.Provider; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.presto.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloTableManager.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloTableManager.java index 162d6982d5058..0a7099bf3383d 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloTableManager.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/AccumuloTableManager.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.accumulo; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.PrestoException; -import io.airlift.log.Logger; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/conf/AccumuloConfig.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/conf/AccumuloConfig.java index d3e09b15a27dc..f99f96e79ac03 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/conf/AccumuloConfig.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/conf/AccumuloConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.accumulo.conf; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.ConfigSecuritySensitive; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.ConfigSecuritySensitive; import io.airlift.units.Duration; import javax.validation.constraints.Min; diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/index/ColumnCardinalityCache.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/index/ColumnCardinalityCache.java index 2a7a6c5371ac7..2e14a31ece8fe 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/index/ColumnCardinalityCache.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/index/ColumnCardinalityCache.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.accumulo.index; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.log.Logger; import com.facebook.presto.accumulo.conf.AccumuloConfig; import com.facebook.presto.accumulo.model.AccumuloColumnConstraint; import com.facebook.presto.spi.PrestoException; @@ -25,8 +27,6 @@ import com.google.common.collect.ListMultimap; import com.google.common.collect.Multimap; import com.google.common.collect.MultimapBuilder; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.log.Logger; import io.airlift.units.Duration; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.Connector; @@ -56,6 +56,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR; import static com.facebook.presto.accumulo.index.Indexer.CARDINALITY_CQ_AS_TEXT; import static com.facebook.presto.accumulo.index.Indexer.getIndexColumnFamily; @@ -63,7 +64,6 @@ import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.collect.Streams.stream; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.Long.parseLong; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/index/IndexLookup.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/index/IndexLookup.java index ed6d59e5e888e..f6944af64c266 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/index/IndexLookup.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/index/IndexLookup.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.accumulo.index; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.log.Logger; import com.facebook.presto.accumulo.model.AccumuloColumnConstraint; import com.facebook.presto.accumulo.model.TabletSplitMetadata; import com.facebook.presto.accumulo.serializers.AccumuloRowSerializer; @@ -22,8 +24,6 @@ import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Multimap; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.log.Logger; import io.airlift.units.Duration; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.Connector; @@ -52,6 +52,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.accumulo.AccumuloClient.getRangesFromDomain; import static com.facebook.presto.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR; import static com.facebook.presto.accumulo.conf.AccumuloSessionProperties.getIndexCardinalityCachePollingDuration; @@ -68,7 +69,6 @@ import static com.facebook.presto.accumulo.index.Indexer.getMetricsTableName; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/io/AccumuloPageSink.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/io/AccumuloPageSink.java index aebbb1d613bb9..f248184330574 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/io/AccumuloPageSink.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/io/AccumuloPageSink.java @@ -44,6 +44,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_DNE; import static com.facebook.presto.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; @@ -59,7 +60,6 @@ import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; import static com.facebook.presto.spi.type.TinyintType.TINYINT; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static java.util.Objects.requireNonNull; import static java.util.concurrent.CompletableFuture.completedFuture; diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/io/AccumuloRecordSet.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/io/AccumuloRecordSet.java index 35fb9f623e204..a5652d2e608ec 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/io/AccumuloRecordSet.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/io/AccumuloRecordSet.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.accumulo.io; +import com.facebook.airlift.log.Logger; import com.facebook.presto.accumulo.conf.AccumuloSessionProperties; import com.facebook.presto.accumulo.model.AccumuloColumnConstraint; import com.facebook.presto.accumulo.model.AccumuloColumnHandle; @@ -26,7 +27,6 @@ import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import io.airlift.log.Logger; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchScanner; diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/metadata/ZooKeeperMetadataManager.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/metadata/ZooKeeperMetadataManager.java index 2455f6528e396..fe29b63636d06 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/metadata/ZooKeeperMetadataManager.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/metadata/ZooKeeperMetadataManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.accumulo.metadata; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.accumulo.AccumuloModule; import com.facebook.presto.accumulo.conf.AccumuloConfig; import com.facebook.presto.spi.PrestoException; @@ -22,7 +23,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.json.ObjectMapperProvider; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.RetryForever; diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/model/Field.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/model/Field.java index d890da5e21a7e..1a0d66d219ecf 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/model/Field.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/model/Field.java @@ -254,7 +254,7 @@ private static boolean equals(Block block1, Block block2) boolean retval = block1.getPositionCount() == block2.getPositionCount(); for (int i = 0; i < block1.getPositionCount() && retval; ++i) { if (block1 instanceof ArrayBlock && block2 instanceof ArrayBlock) { - retval = equals(block1.getObject(i, Block.class), block2.getObject(i, Block.class)); + retval = equals(block1.getBlock(i), block2.getBlock(i)); } else { retval = block1.compareTo(i, 0, block1.getSliceLength(i), block2, i, 0, block2.getSliceLength(i)) == 0; diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/model/Row.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/model/Row.java index 045e2d388206c..ff97bb1885d5c 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/model/Row.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/model/Row.java @@ -22,13 +22,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.commons.lang.StringUtils; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.ISODateTimeFormat; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -55,9 +56,9 @@ public class Row { - private static final DateTimeFormatter DATE_PARSER = ISODateTimeFormat.date(); - private static final DateTimeFormatter TIME_PARSER = DateTimeFormat.forPattern("HH:mm:ss"); - private static final DateTimeFormatter TIMESTAMP_PARSER = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS"); + private static final DateTimeFormatter DATE_PARSER = DateTimeFormatter.ISO_LOCAL_DATE; + private static final DateTimeFormatter TIME_PARSER = DateTimeFormatter.ISO_LOCAL_TIME; + private static final DateTimeFormatter TIMESTAMP_PARSER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); private final List fields = new ArrayList<>(); @@ -201,7 +202,7 @@ else if (type.equals(BOOLEAN)) { return Boolean.parseBoolean(str); } else if (type.equals(DATE)) { - return new Date(DATE_PARSER.parseDateTime(str).getMillis()); + return Date.valueOf(LocalDate.parse(str, DATE_PARSER)); } else if (type.equals(DOUBLE)) { return Double.parseDouble(str); @@ -216,10 +217,10 @@ else if (type.equals(SMALLINT)) { return Short.parseShort(str); } else if (type.equals(TIME)) { - return new Time(TIME_PARSER.parseDateTime(str).getMillis()); + return Time.valueOf(LocalTime.parse(str, TIME_PARSER)); } else if (type.equals(TIMESTAMP)) { - return new Timestamp(TIMESTAMP_PARSER.parseDateTime(str).getMillis()); + return Timestamp.valueOf(LocalDateTime.parse(str, TIMESTAMP_PARSER)); } else if (type.equals(TINYINT)) { return Byte.valueOf(str); diff --git a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/serializers/AccumuloRowSerializer.java b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/serializers/AccumuloRowSerializer.java index c99a60c7eaacc..3b2a607f90a7c 100644 --- a/presto-accumulo/src/main/java/com/facebook/presto/accumulo/serializers/AccumuloRowSerializer.java +++ b/presto-accumulo/src/main/java/com/facebook/presto/accumulo/serializers/AccumuloRowSerializer.java @@ -608,10 +608,10 @@ static Object readObject(Type type, Block block, int position) { if (Types.isArrayType(type)) { Type elementType = Types.getElementType(type); - return getArrayFromBlock(elementType, block.getObject(position, Block.class)); + return getArrayFromBlock(elementType, block.getBlock(position)); } else if (Types.isMapType(type)) { - return getMapFromBlock(type, block.getObject(position, Block.class)); + return getMapFromBlock(type, block.getBlock(position)); } else { if (type.getJavaType() == Slice.class) { diff --git a/presto-accumulo/src/test/java/com/facebook/presto/accumulo/AccumuloQueryRunner.java b/presto-accumulo/src/test/java/com/facebook/presto/accumulo/AccumuloQueryRunner.java index 3411e0d422494..a0e32120a254c 100644 --- a/presto-accumulo/src/test/java/com/facebook/presto/accumulo/AccumuloQueryRunner.java +++ b/presto-accumulo/src/test/java/com/facebook/presto/accumulo/AccumuloQueryRunner.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.accumulo; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.accumulo.conf.AccumuloConfig; import com.facebook.presto.accumulo.serializers.LexicoderRowSerializer; @@ -23,7 +24,6 @@ import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedSet; -import io.airlift.log.Logger; import io.airlift.tpch.TpchTable; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; diff --git a/presto-accumulo/src/test/java/com/facebook/presto/accumulo/model/TestAccumuloSplit.java b/presto-accumulo/src/test/java/com/facebook/presto/accumulo/model/TestAccumuloSplit.java index cfcff4b92615e..724987dced244 100644 --- a/presto-accumulo/src/test/java/com/facebook/presto/accumulo/model/TestAccumuloSplit.java +++ b/presto-accumulo/src/test/java/com/facebook/presto/accumulo/model/TestAccumuloSplit.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.accumulo.model; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.accumulo.serializers.LexicoderRowSerializer; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import org.apache.accumulo.core.data.Range; import org.testng.annotations.Test; diff --git a/presto-accumulo/src/test/java/com/facebook/presto/accumulo/model/TestRow.java b/presto-accumulo/src/test/java/com/facebook/presto/accumulo/model/TestRow.java index aec1b192fdb99..61e4f318523bd 100644 --- a/presto-accumulo/src/test/java/com/facebook/presto/accumulo/model/TestRow.java +++ b/presto-accumulo/src/test/java/com/facebook/presto/accumulo/model/TestRow.java @@ -108,7 +108,7 @@ public void testRowFromString() schema.addColumn("m", Optional.of("m"), Optional.of("m"), VARCHAR); schema.addColumn("n", Optional.of("n"), Optional.of("n"), VARCHAR); - Row actual = Row.fromString(schema, "a,b,c|true|1999-01-01|123.45678|123.45678|12345678|12345678|12345|12:30:00|1999-01-01 12:30:00.0|123|O'Leary|O'Leary|", '|'); + Row actual = Row.fromString(schema, "a,b,c|true|1999-01-01|123.45678|123.45678|12345678|12345678|12345|12:30:00|1999-01-01 12:30:00.000|123|O'Leary|O'Leary|", '|'); assertEquals(actual, expected); } } diff --git a/presto-accumulo/src/test/java/com/facebook/presto/accumulo/serializers/AbstractTestAccumuloRowSerializer.java b/presto-accumulo/src/test/java/com/facebook/presto/accumulo/serializers/AbstractTestAccumuloRowSerializer.java index 0b6015be1d06e..8b79b6a6a0846 100644 --- a/presto-accumulo/src/test/java/com/facebook/presto/accumulo/serializers/AbstractTestAccumuloRowSerializer.java +++ b/presto-accumulo/src/test/java/com/facebook/presto/accumulo/serializers/AbstractTestAccumuloRowSerializer.java @@ -27,13 +27,13 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.testng.annotations.Test; import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.AbstractMap.SimpleImmutableEntry; import java.util.List; import java.util.Map; @@ -108,7 +108,7 @@ public void testBoolean() public void testDate() throws Exception { - Date expected = new Date(new DateTime(2001, 2, 3, 4, 5, 6, DateTimeZone.UTC).getMillis()); + Date expected = new Date(ZonedDateTime.of(2001, 2, 3, 4, 5, 6, 0, ZoneId.of("UTC")).toEpochSecond()); AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); byte[] data = serializer.encode(DATE, expected); diff --git a/presto-array/pom.xml b/presto-array/pom.xml index e4153a01de26a..4a93a464120da 100644 --- a/presto-array/pom.xml +++ b/presto-array/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-array diff --git a/presto-array/src/main/java/com/facebook/presto/array/Arrays.java b/presto-array/src/main/java/com/facebook/presto/array/Arrays.java new file mode 100644 index 0000000000000..b720413cccc06 --- /dev/null +++ b/presto-array/src/main/java/com/facebook/presto/array/Arrays.java @@ -0,0 +1,139 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.array; + +import static com.facebook.presto.array.Arrays.ExpansionFactor.SMALL; +import static com.facebook.presto.array.Arrays.ExpansionOption.INITIALIZE; +import static com.facebook.presto.array.Arrays.ExpansionOption.NONE; +import static com.facebook.presto.array.Arrays.ExpansionOption.PRESERVE; + +public class Arrays +{ + private Arrays() {} + + public static int[] ensureCapacity(int[] buffer, int capacity) + { + return ensureCapacity(buffer, capacity, SMALL, NONE); + } + + public static int[] ensureCapacity(int[] buffer, int capacity, ExpansionFactor expansionFactor, ExpansionOption expansionOption) + { + int newCapacity = (int) (capacity * expansionFactor.expansionFactor); + + if (buffer == null) { + buffer = new int[newCapacity]; + } + else if (buffer.length < capacity) { + if (expansionOption == PRESERVE) { + buffer = java.util.Arrays.copyOf(buffer, newCapacity); + } + else { + buffer = new int[newCapacity]; + } + } + else if (expansionOption == INITIALIZE) { + java.util.Arrays.fill(buffer, 0); + } + + return buffer; + } + + public static long[] ensureCapacity(long[] buffer, int capacity) + { + if (buffer == null || buffer.length < capacity) { + return new long[(int) (capacity * SMALL.expansionFactor)]; + } + + return buffer; + } + + public static boolean[] ensureCapacity(boolean[] buffer, int capacity) + { + if (buffer == null || buffer.length < capacity) { + return new boolean[(int) (capacity * SMALL.expansionFactor)]; + } + + return buffer; + } + + public static byte[] ensureCapacity(byte[] buffer, int capacity) + { + if (buffer == null || buffer.length < capacity) { + return new byte[(int) (capacity * SMALL.expansionFactor)]; + } + + return buffer; + } + + public static int[][] ensureCapacity(int[][] buffer, int capacity) + { + if (buffer == null || buffer.length < capacity) { + return new int[capacity][]; + } + + return buffer; + } + + public static boolean[][] ensureCapacity(boolean[][] buffer, int capacity) + { + if (buffer == null || buffer.length < capacity) { + return new boolean[capacity][]; + } + + return buffer; + } + + public static byte[] ensureCapacity(byte[] buffer, int capacity, ExpansionFactor expansionFactor, ExpansionOption expansionOption) + { + int newCapacity = (int) (capacity * expansionFactor.expansionFactor); + + if (buffer == null) { + buffer = new byte[newCapacity]; + } + else if (buffer.length < capacity) { + if (expansionOption == PRESERVE) { + buffer = java.util.Arrays.copyOf(buffer, newCapacity); + } + else { + buffer = new byte[newCapacity]; + } + } + else if (expansionOption == INITIALIZE) { + java.util.Arrays.fill(buffer, (byte) 0); + } + + return buffer; + } + + public enum ExpansionFactor + { + SMALL(1.0), + MEDIUM(1.5), + LARGE(2.0); + + private final double expansionFactor; + + ExpansionFactor(double expansionFactor) + { + this.expansionFactor = expansionFactor; + } + } + + public enum ExpansionOption + { + PRESERVE, + INITIALIZE, + NONE; + } +} diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index ea2f24c2b6117..6877c71607191 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-atop @@ -28,27 +28,27 @@ - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift configuration - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift log @@ -80,7 +80,7 @@ - io.airlift + com.facebook.airlift log-manager runtime @@ -142,7 +142,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-atop/src/main/java/com/facebook/presto/atop/AtopConnector.java b/presto-atop/src/main/java/com/facebook/presto/atop/AtopConnector.java index 561779cf2f707..2898dbec0dcad 100644 --- a/presto-atop/src/main/java/com/facebook/presto/atop/AtopConnector.java +++ b/presto-atop/src/main/java/com/facebook/presto/atop/AtopConnector.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.atop; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorAccessControl; import com.facebook.presto.spi.connector.ConnectorMetadata; @@ -20,8 +22,6 @@ import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.transaction.IsolationLevel; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-atop/src/main/java/com/facebook/presto/atop/AtopConnectorConfig.java b/presto-atop/src/main/java/com/facebook/presto/atop/AtopConnectorConfig.java index 63292cfcd36ea..8ca4a2446b6fc 100644 --- a/presto-atop/src/main/java/com/facebook/presto/atop/AtopConnectorConfig.java +++ b/presto-atop/src/main/java/com/facebook/presto/atop/AtopConnectorConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.atop; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import io.airlift.units.Duration; import io.airlift.units.MinDuration; diff --git a/presto-atop/src/main/java/com/facebook/presto/atop/AtopConnectorFactory.java b/presto-atop/src/main/java/com/facebook/presto/atop/AtopConnectorFactory.java index 1fa6323f4fed2..e653f6cfae146 100644 --- a/presto-atop/src/main/java/com/facebook/presto/atop/AtopConnectorFactory.java +++ b/presto-atop/src/main/java/com/facebook/presto/atop/AtopConnectorFactory.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.atop; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.plugin.base.security.AllowAllAccessControlModule; import com.facebook.presto.plugin.base.security.FileBasedAccessControlModule; import com.facebook.presto.spi.ConnectorHandleResolver; @@ -21,15 +23,13 @@ import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.connector.ConnectorFactory; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; import static com.facebook.presto.atop.AtopConnectorConfig.SECURITY_FILE; import static com.facebook.presto.atop.AtopConnectorConfig.SECURITY_NONE; import static com.google.common.base.Throwables.throwIfUnchecked; -import static io.airlift.configuration.ConditionalModule.installModuleIf; import static java.util.Objects.requireNonNull; public class AtopConnectorFactory diff --git a/presto-atop/src/main/java/com/facebook/presto/atop/AtopModule.java b/presto-atop/src/main/java/com/facebook/presto/atop/AtopModule.java index 36b78baeadd73..87609a88d0acc 100644 --- a/presto-atop/src/main/java/com/facebook/presto/atop/AtopModule.java +++ b/presto-atop/src/main/java/com/facebook/presto/atop/AtopModule.java @@ -19,7 +19,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; public class AtopModule diff --git a/presto-atop/src/main/java/com/facebook/presto/atop/AtopPageSource.java b/presto-atop/src/main/java/com/facebook/presto/atop/AtopPageSource.java index 97b8fcaacd458..1456e8250f5e7 100644 --- a/presto-atop/src/main/java/com/facebook/presto/atop/AtopPageSource.java +++ b/presto-atop/src/main/java/com/facebook/presto/atop/AtopPageSource.java @@ -83,6 +83,12 @@ public long getCompletedBytes() return 0; } + @Override + public long getCompletedPositions() + { + return 0; + } + @Override public long getReadTimeNanos() { diff --git a/presto-atop/src/main/java/com/facebook/presto/atop/AtopProcessFactory.java b/presto-atop/src/main/java/com/facebook/presto/atop/AtopProcessFactory.java index 71678d1d4cf54..4cd5a2478fe6a 100644 --- a/presto-atop/src/main/java/com/facebook/presto/atop/AtopProcessFactory.java +++ b/presto-atop/src/main/java/com/facebook/presto/atop/AtopProcessFactory.java @@ -32,10 +32,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.atop.AtopErrorCode.ATOP_CANNOT_START_PROCESS_ERROR; import static com.facebook.presto.atop.AtopErrorCode.ATOP_READ_TIMEOUT; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newFixedThreadPool; diff --git a/presto-atop/src/test/java/com/facebook/presto/atop/LocalAtopQueryRunner.java b/presto-atop/src/test/java/com/facebook/presto/atop/LocalAtopQueryRunner.java index baeda33fc85e1..024993c4e249e 100644 --- a/presto-atop/src/test/java/com/facebook/presto/atop/LocalAtopQueryRunner.java +++ b/presto-atop/src/test/java/com/facebook/presto/atop/LocalAtopQueryRunner.java @@ -21,8 +21,8 @@ import java.util.Map; import java.util.TimeZone; +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.testing.Closeables.closeAllSuppress; public final class LocalAtopQueryRunner { diff --git a/presto-atop/src/test/java/com/facebook/presto/atop/TestAtopConnectorConfig.java b/presto-atop/src/test/java/com/facebook/presto/atop/TestAtopConnectorConfig.java index e7966a80d9dec..a7b439df37a27 100644 --- a/presto-atop/src/test/java/com/facebook/presto/atop/TestAtopConnectorConfig.java +++ b/presto-atop/src/test/java/com/facebook/presto/atop/TestAtopConnectorConfig.java @@ -20,11 +20,11 @@ import java.util.Map; import java.util.TimeZone; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static com.facebook.presto.atop.AtopConnectorConfig.SECURITY_FILE; import static com.facebook.presto.atop.AtopConnectorConfig.SECURITY_NONE; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static java.util.concurrent.TimeUnit.MINUTES; public class TestAtopConnectorConfig diff --git a/presto-atop/src/test/java/com/facebook/presto/atop/TestAtopPlugin.java b/presto-atop/src/test/java/com/facebook/presto/atop/TestAtopPlugin.java index de65d72863eb2..87b787a4dab0b 100644 --- a/presto-atop/src/test/java/com/facebook/presto/atop/TestAtopPlugin.java +++ b/presto-atop/src/test/java/com/facebook/presto/atop/TestAtopPlugin.java @@ -15,8 +15,8 @@ import org.testng.annotations.Test; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.testing.Assertions.assertInstanceOf; public class TestAtopPlugin { diff --git a/presto-atop/src/test/java/com/facebook/presto/atop/TestAtopSplit.java b/presto-atop/src/test/java/com/facebook/presto/atop/TestAtopSplit.java index 664b077c6bcf2..8bca1b40fe23e 100644 --- a/presto-atop/src/test/java/com/facebook/presto/atop/TestAtopSplit.java +++ b/presto-atop/src/test/java/com/facebook/presto/atop/TestAtopSplit.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.atop; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.HostAddress; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; import java.time.ZoneId; diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 3af3028f234a2..6e122dc73a0f5 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-base-jdbc @@ -18,25 +18,30 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift configuration - io.airlift + com.facebook.airlift concurrent + + io.airlift + units + + com.google.guava guava @@ -85,6 +90,12 @@ presto-spi + + + com.facebook.presto + presto-expressions + + io.airlift slice @@ -103,7 +114,7 @@ - io.airlift + com.facebook.airlift testing test @@ -115,7 +126,7 @@ - io.airlift + com.facebook.airlift json test @@ -138,6 +149,13 @@ test + + com.facebook.presto + presto-main + test + test-jar + + com.facebook.presto presto-tpch @@ -155,6 +173,12 @@ presto-tests test + + + com.facebook.presto + presto-parser + test + diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/BaseJdbcClient.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/BaseJdbcClient.java index f6bce0c289334..fb855c7a8b4f4 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/BaseJdbcClient.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/BaseJdbcClient.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorSession; @@ -28,11 +29,13 @@ import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.VarcharType; +import com.google.common.base.CharMatcher; import com.google.common.base.Joiner; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; import javax.annotation.Nullable; import javax.annotation.PreDestroy; @@ -44,6 +47,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; @@ -68,14 +72,20 @@ import static com.facebook.presto.spi.type.TinyintType.TINYINT; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.spi.type.Varchars.isVarcharType; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.base.Verify.verify; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Iterables.getOnlyElement; import static java.lang.String.format; +import static java.lang.String.join; import static java.sql.ResultSetMetaData.columnNullable; import static java.util.Collections.nCopies; import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; public class BaseJdbcClient implements JdbcClient @@ -101,6 +111,9 @@ public class BaseJdbcClient protected final String connectorId; protected final ConnectionFactory connectionFactory; protected final String identifierQuote; + protected final boolean caseInsensitiveNameMatching; + protected final Cache> remoteSchemaNames; + protected final Cache> remoteTableNames; public BaseJdbcClient(JdbcConnectorId connectorId, BaseJdbcConfig config, String identifierQuote, ConnectionFactory connectionFactory) { @@ -108,6 +121,12 @@ public BaseJdbcClient(JdbcConnectorId connectorId, BaseJdbcConfig config, String requireNonNull(config, "config is null"); // currently unused, retained as parameter for future extensions this.identifierQuote = requireNonNull(identifierQuote, "identifierQuote is null"); this.connectionFactory = requireNonNull(connectionFactory, "connectionFactory is null"); + + this.caseInsensitiveNameMatching = config.isCaseInsensitiveNameMatching(); + CacheBuilder remoteNamesCacheBuilder = CacheBuilder.newBuilder() + .expireAfterWrite(config.getCaseInsensitiveNameMatchingCacheTtl().toMillis(), MILLISECONDS); + this.remoteSchemaNames = remoteNamesCacheBuilder.build(); + this.remoteTableNames = remoteNamesCacheBuilder.build(); } @PreDestroy @@ -118,15 +137,32 @@ public void destroy() } @Override - public Set getSchemaNames() + public String getIdentifierQuote() + { + return identifierQuote; + } + + @Override + public final Set getSchemaNames(JdbcIdentity identity) { - try (Connection connection = connectionFactory.openConnection(); - ResultSet resultSet = connection.getMetaData().getSchemas()) { + try (Connection connection = connectionFactory.openConnection(identity)) { + return listSchemas(connection).stream() + .map(schemaName -> schemaName.toLowerCase(ENGLISH)) + .collect(toImmutableSet()); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + protected Collection listSchemas(Connection connection) + { + try (ResultSet resultSet = connection.getMetaData().getSchemas()) { ImmutableSet.Builder schemaNames = ImmutableSet.builder(); while (resultSet.next()) { - String schemaName = resultSet.getString("TABLE_SCHEM").toLowerCase(ENGLISH); + String schemaName = resultSet.getString("TABLE_SCHEM"); // skip internal schemas - if (!schemaName.equals("information_schema")) { + if (!schemaName.equalsIgnoreCase("information_schema")) { schemaNames.add(schemaName); } } @@ -138,17 +174,16 @@ public Set getSchemaNames() } @Override - public List getTableNames(@Nullable String schema) + public List getTableNames(JdbcIdentity identity, Optional schema) { - try (Connection connection = connectionFactory.openConnection()) { - DatabaseMetaData metadata = connection.getMetaData(); - if (metadata.storesUpperCaseIdentifiers() && (schema != null)) { - schema = schema.toUpperCase(ENGLISH); - } - try (ResultSet resultSet = getTables(connection, schema, null)) { + try (Connection connection = connectionFactory.openConnection(identity)) { + Optional remoteSchema = schema.map(schemaName -> toRemoteSchemaName(identity, connection, schemaName)); + try (ResultSet resultSet = getTables(connection, remoteSchema, Optional.empty())) { ImmutableList.Builder list = ImmutableList.builder(); while (resultSet.next()) { - list.add(getSchemaTableName(resultSet)); + String tableSchema = getTableSchemaName(resultSet); + String tableName = resultSet.getString("TABLE_NAME"); + list.add(new SchemaTableName(tableSchema.toLowerCase(ENGLISH), tableName.toLowerCase(ENGLISH))); } return list.build(); } @@ -160,17 +195,12 @@ public List getTableNames(@Nullable String schema) @Nullable @Override - public JdbcTableHandle getTableHandle(SchemaTableName schemaTableName) + public JdbcTableHandle getTableHandle(JdbcIdentity identity, SchemaTableName schemaTableName) { - try (Connection connection = connectionFactory.openConnection()) { - DatabaseMetaData metadata = connection.getMetaData(); - String jdbcSchemaName = schemaTableName.getSchemaName(); - String jdbcTableName = schemaTableName.getTableName(); - if (metadata.storesUpperCaseIdentifiers()) { - jdbcSchemaName = jdbcSchemaName.toUpperCase(ENGLISH); - jdbcTableName = jdbcTableName.toUpperCase(ENGLISH); - } - try (ResultSet resultSet = getTables(connection, jdbcSchemaName, jdbcTableName)) { + try (Connection connection = connectionFactory.openConnection(identity)) { + String remoteSchema = toRemoteSchemaName(identity, connection, schemaTableName.getSchemaName()); + String remoteTable = toRemoteTableName(identity, connection, remoteSchema, schemaTableName.getTableName()); + try (ResultSet resultSet = getTables(connection, Optional.of(remoteSchema), Optional.of(remoteTable))) { List tableHandles = new ArrayList<>(); while (resultSet.next()) { tableHandles.add(new JdbcTableHandle( @@ -197,7 +227,7 @@ public JdbcTableHandle getTableHandle(SchemaTableName schemaTableName) @Override public List getColumns(ConnectorSession session, JdbcTableHandle tableHandle) { - try (Connection connection = connectionFactory.openConnection()) { + try (Connection connection = connectionFactory.openConnection(JdbcIdentity.from(session))) { try (ResultSet resultSet = getColumns(tableHandle, connection.getMetaData())) { List columns = new ArrayList<>(); while (resultSet.next()) { @@ -232,7 +262,7 @@ public Optional toPrestoType(ConnectorSession session, JdbcTypeHand } @Override - public ConnectorSplitSource getSplits(JdbcTableLayoutHandle layoutHandle) + public ConnectorSplitSource getSplits(JdbcIdentity identity, JdbcTableLayoutHandle layoutHandle) { JdbcTableHandle tableHandle = layoutHandle.getTable(); JdbcSplit jdbcSplit = new JdbcSplit( @@ -241,15 +271,15 @@ public ConnectorSplitSource getSplits(JdbcTableLayoutHandle layoutHandle) tableHandle.getSchemaName(), tableHandle.getTableName(), layoutHandle.getTupleDomain(), - Optional.empty()); + layoutHandle.getAdditionalPredicate()); return new FixedSplitSource(ImmutableList.of(jdbcSplit)); } @Override - public Connection getConnection(JdbcSplit split) + public Connection getConnection(JdbcIdentity identity, JdbcSplit split) throws SQLException { - Connection connection = connectionFactory.openConnection(); + Connection connection = connectionFactory.openConnection(identity); try { connection.setReadOnly(true); } @@ -276,10 +306,10 @@ public PreparedStatement buildSql(Connection connection, JdbcSplit split, List columnNames = ImmutableList.builder(); ImmutableList.Builder columnTypes = ImmutableList.builder(); ImmutableList.Builder columnList = ImmutableList.builder(); @@ -344,16 +368,18 @@ protected JdbcOutputTableHandle createTable(ConnectorTableMetadata tableMetadata columnTypes.add(column.getType()); columnList.add(getColumnString(column, columnName)); } - Joiner.on(", ").appendTo(sql, columnList.build()); - sql.append(")"); - execute(connection, sql.toString()); + String sql = format( + "CREATE TABLE %s (%s)", + quoted(catalog, remoteSchema, tableName), + join(", ", columnList.build())); + execute(connection, sql); return new JdbcOutputTableHandle( connectorId, catalog, - schema, - table, + remoteSchema, + remoteTable, columnNames.build(), columnTypes.build(), tableName); @@ -377,24 +403,26 @@ protected String generateTemporaryTableName() return "tmp_presto_" + UUID.randomUUID().toString().replace("-", ""); } + //todo @Override - public void commitCreateTable(JdbcOutputTableHandle handle) + public void commitCreateTable(JdbcIdentity identity, JdbcOutputTableHandle handle) { renameTable( + identity, handle.getCatalogName(), new SchemaTableName(handle.getSchemaName(), handle.getTemporaryTableName()), new SchemaTableName(handle.getSchemaName(), handle.getTableName())); } @Override - public void renameTable(JdbcTableHandle handle, SchemaTableName newTable) + public void renameTable(JdbcIdentity identity, JdbcTableHandle handle, SchemaTableName newTable) { - renameTable(handle.getCatalogName(), handle.getSchemaTableName(), newTable); + renameTable(identity, handle.getCatalogName(), handle.getSchemaTableName(), newTable); } - protected void renameTable(String catalogName, SchemaTableName oldTable, SchemaTableName newTable) + protected void renameTable(JdbcIdentity identity, String catalogName, SchemaTableName oldTable, SchemaTableName newTable) { - try (Connection connection = connectionFactory.openConnection()) { + try (Connection connection = connectionFactory.openConnection(identity)) { DatabaseMetaData metadata = connection.getMetaData(); String schemaName = oldTable.getSchemaName(); String tableName = oldTable.getTableName(); @@ -418,21 +446,21 @@ protected void renameTable(String catalogName, SchemaTableName oldTable, SchemaT } @Override - public void finishInsertTable(JdbcOutputTableHandle handle) + public void finishInsertTable(JdbcIdentity identity, JdbcOutputTableHandle handle) { String temporaryTable = quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTemporaryTableName()); String targetTable = quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName()); String insertSql = format("INSERT INTO %s SELECT * FROM %s", targetTable, temporaryTable); String cleanupSql = "DROP TABLE " + temporaryTable; - try (Connection connection = getConnection(handle)) { + try (Connection connection = getConnection(identity, handle)) { execute(connection, insertSql); } catch (SQLException e) { throw new PrestoException(JDBC_ERROR, e); } - try (Connection connection = getConnection(handle)) { + try (Connection connection = getConnection(identity, handle)) { execute(connection, cleanupSql); } catch (SQLException e) { @@ -441,9 +469,9 @@ public void finishInsertTable(JdbcOutputTableHandle handle) } @Override - public void addColumn(JdbcTableHandle handle, ColumnMetadata column) + public void addColumn(JdbcIdentity identity, JdbcTableHandle handle, ColumnMetadata column) { - try (Connection connection = connectionFactory.openConnection()) { + try (Connection connection = connectionFactory.openConnection(identity)) { String schema = handle.getSchemaName(); String table = handle.getTableName(); String columnName = column.getName(); @@ -465,9 +493,9 @@ public void addColumn(JdbcTableHandle handle, ColumnMetadata column) } @Override - public void renameColumn(JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName) + public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName) { - try (Connection connection = connectionFactory.openConnection()) { + try (Connection connection = connectionFactory.openConnection(identity)) { DatabaseMetaData metadata = connection.getMetaData(); if (metadata.storesUpperCaseIdentifiers()) { newColumnName = newColumnName.toUpperCase(ENGLISH); @@ -485,9 +513,9 @@ public void renameColumn(JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, St } @Override - public void dropColumn(JdbcTableHandle handle, JdbcColumnHandle column) + public void dropColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle column) { - try (Connection connection = connectionFactory.openConnection()) { + try (Connection connection = connectionFactory.openConnection(identity)) { String sql = format( "ALTER TABLE %s DROP COLUMN %s", quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName()), @@ -500,13 +528,13 @@ public void dropColumn(JdbcTableHandle handle, JdbcColumnHandle column) } @Override - public void dropTable(JdbcTableHandle handle) + public void dropTable(JdbcIdentity identity, JdbcTableHandle handle) { StringBuilder sql = new StringBuilder() .append("DROP TABLE ") .append(quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName())); - try (Connection connection = connectionFactory.openConnection()) { + try (Connection connection = connectionFactory.openConnection(identity)) { execute(connection, sql.toString()); } catch (SQLException e) { @@ -515,9 +543,9 @@ public void dropTable(JdbcTableHandle handle) } @Override - public void rollbackCreateTable(JdbcOutputTableHandle handle) + public void rollbackCreateTable(JdbcIdentity identity, JdbcOutputTableHandle handle) { - dropTable(new JdbcTableHandle( + dropTable(identity, new JdbcTableHandle( handle.getConnectorId(), new SchemaTableName(handle.getSchemaName(), handle.getTemporaryTableName()), handle.getCatalogName(), @@ -537,10 +565,10 @@ public String buildInsertSql(JdbcOutputTableHandle handle) } @Override - public Connection getConnection(JdbcOutputTableHandle handle) + public Connection getConnection(JdbcIdentity identity, JdbcOutputTableHandle handle) throws SQLException { - return connectionFactory.openConnection(); + return connectionFactory.openConnection(identity); } @Override @@ -550,28 +578,125 @@ public PreparedStatement getPreparedStatement(Connection connection, String sql) return connection.prepareStatement(sql); } - protected ResultSet getTables(Connection connection, String schemaName, String tableName) + protected ResultSet getTables(Connection connection, Optional schemaName, Optional tableName) throws SQLException { DatabaseMetaData metadata = connection.getMetaData(); - String escape = metadata.getSearchStringEscape(); + Optional escape = Optional.ofNullable(metadata.getSearchStringEscape()); return metadata.getTables( connection.getCatalog(), - escapeNamePattern(schemaName, escape), - escapeNamePattern(tableName, escape), + escapeNamePattern(schemaName, escape).orElse(null), + escapeNamePattern(tableName, escape).orElse(null), new String[] {"TABLE", "VIEW"}); } - protected SchemaTableName getSchemaTableName(ResultSet resultSet) + protected String getTableSchemaName(ResultSet resultSet) throws SQLException { - return new SchemaTableName( - resultSet.getString("TABLE_SCHEM").toLowerCase(ENGLISH), - resultSet.getString("TABLE_NAME").toLowerCase(ENGLISH)); + return resultSet.getString("TABLE_SCHEM"); + } + + protected String toRemoteSchemaName(JdbcIdentity identity, Connection connection, String schemaName) + { + requireNonNull(schemaName, "schemaName is null"); + verify(CharMatcher.forPredicate(Character::isUpperCase).matchesNoneOf(schemaName), "Expected schema name from internal metadata to be lowercase: %s", schemaName); + + if (caseInsensitiveNameMatching) { + try { + Map mapping = remoteSchemaNames.getIfPresent(identity); + if (mapping != null && !mapping.containsKey(schemaName)) { + // This might be a schema that has just been created. Force reload. + mapping = null; + } + if (mapping == null) { + mapping = listSchemasByLowerCase(connection); + remoteSchemaNames.put(identity, mapping); + } + String remoteSchema = mapping.get(schemaName); + if (remoteSchema != null) { + return remoteSchema; + } + } + catch (RuntimeException e) { + throw new PrestoException(JDBC_ERROR, "Failed to find remote schema name: " + firstNonNull(e.getMessage(), e), e); + } + } + + try { + DatabaseMetaData metadata = connection.getMetaData(); + if (metadata.storesUpperCaseIdentifiers()) { + return schemaName.toUpperCase(ENGLISH); + } + return schemaName; + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + protected Map listSchemasByLowerCase(Connection connection) + { + return listSchemas(connection).stream() + .collect(toImmutableMap(schemaName -> schemaName.toLowerCase(ENGLISH), schemaName -> schemaName)); + } + + protected String toRemoteTableName(JdbcIdentity identity, Connection connection, String remoteSchema, String tableName) + { + requireNonNull(remoteSchema, "remoteSchema is null"); + requireNonNull(tableName, "tableName is null"); + verify(CharMatcher.forPredicate(Character::isUpperCase).matchesNoneOf(tableName), "Expected table name from internal metadata to be lowercase: %s", tableName); + + if (caseInsensitiveNameMatching) { + try { + RemoteTableNameCacheKey cacheKey = new RemoteTableNameCacheKey(identity, remoteSchema); + Map mapping = remoteTableNames.getIfPresent(cacheKey); + if (mapping != null && !mapping.containsKey(tableName)) { + // This might be a table that has just been created. Force reload. + mapping = null; + } + if (mapping == null) { + mapping = listTablesByLowerCase(connection, remoteSchema); + remoteTableNames.put(cacheKey, mapping); + } + String remoteTable = mapping.get(tableName); + if (remoteTable != null) { + return remoteTable; + } + } + catch (RuntimeException e) { + throw new PrestoException(JDBC_ERROR, "Failed to find remote table name: " + firstNonNull(e.getMessage(), e), e); + } + } + + try { + DatabaseMetaData metadata = connection.getMetaData(); + if (metadata.storesUpperCaseIdentifiers()) { + return tableName.toUpperCase(ENGLISH); + } + return tableName; + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + protected Map listTablesByLowerCase(Connection connection, String remoteSchema) + { + try (ResultSet resultSet = getTables(connection, Optional.of(remoteSchema), Optional.empty())) { + ImmutableMap.Builder map = ImmutableMap.builder(); + while (resultSet.next()) { + String tableName = resultSet.getString("TABLE_NAME"); + map.put(tableName.toLowerCase(ENGLISH), tableName); + } + return map.build(); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } } @Override - public TableStatistics getTableStatistics(ConnectorSession session, JdbcTableHandle handle, TupleDomain tupleDomain) + public TableStatistics getTableStatistics(ConnectorSession session, JdbcTableHandle handle, List columnHandles, TupleDomain tupleDomain) { return TableStatistics.empty(); } @@ -630,11 +755,18 @@ protected String quoted(String catalog, String schema, String table) return sb.toString(); } - protected static String escapeNamePattern(String name, String escape) + protected static Optional escapeNamePattern(Optional name, Optional escape) { - if ((name == null) || (escape == null)) { + if (!name.isPresent() || !escape.isPresent()) { return name; } + return Optional.of(escapeNamePattern(name.get(), escape.get())); + } + + private static String escapeNamePattern(String name, String escape) + { + requireNonNull(name, "name is null"); + requireNonNull(escape, "escape is null"); checkArgument(!escape.equals("_"), "Escape string must not be '_'"); checkArgument(!escape.equals("%"), "Escape string must not be '%'"); name = name.replace(escape, escape + escape); @@ -646,11 +778,11 @@ protected static String escapeNamePattern(String name, String escape) private static ResultSet getColumns(JdbcTableHandle tableHandle, DatabaseMetaData metadata) throws SQLException { - String escape = metadata.getSearchStringEscape(); + Optional escape = Optional.ofNullable(metadata.getSearchStringEscape()); return metadata.getColumns( tableHandle.getCatalogName(), - escapeNamePattern(tableHandle.getSchemaName(), escape), - escapeNamePattern(tableHandle.getTableName(), escape), + escapeNamePattern(Optional.ofNullable(tableHandle.getSchemaName()), escape).orElse(null), + escapeNamePattern(Optional.ofNullable(tableHandle.getTableName()), escape).orElse(null), null); } } diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/BaseJdbcConfig.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/BaseJdbcConfig.java index 5725721aab7ad..cae962e51a183 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/BaseJdbcConfig.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/BaseJdbcConfig.java @@ -13,16 +13,22 @@ */ package com.facebook.presto.plugin.jdbc; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigSecuritySensitive; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigSecuritySensitive; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; import javax.validation.constraints.NotNull; +import static java.util.concurrent.TimeUnit.MINUTES; + public class BaseJdbcConfig { private String connectionUrl; private String connectionUser; private String connectionPassword; + private boolean caseInsensitiveNameMatching; + private Duration caseInsensitiveNameMatchingCacheTtl = new Duration(1, MINUTES); @NotNull public String getConnectionUrl() @@ -61,4 +67,30 @@ public BaseJdbcConfig setConnectionPassword(String connectionPassword) this.connectionPassword = connectionPassword; return this; } + + public boolean isCaseInsensitiveNameMatching() + { + return caseInsensitiveNameMatching; + } + + @Config("case-insensitive-name-matching") + public BaseJdbcConfig setCaseInsensitiveNameMatching(boolean caseInsensitiveNameMatching) + { + this.caseInsensitiveNameMatching = caseInsensitiveNameMatching; + return this; + } + + @NotNull + @MinDuration("0ms") + public Duration getCaseInsensitiveNameMatchingCacheTtl() + { + return caseInsensitiveNameMatchingCacheTtl; + } + + @Config("case-insensitive-name-matching.cache-ttl") + public BaseJdbcConfig setCaseInsensitiveNameMatchingCacheTtl(Duration caseInsensitiveNameMatchingCacheTtl) + { + this.caseInsensitiveNameMatchingCacheTtl = caseInsensitiveNameMatchingCacheTtl; + return this; + } } diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/ConnectionFactory.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/ConnectionFactory.java index c3a598b527e4c..bcece27cc25db 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/ConnectionFactory.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/ConnectionFactory.java @@ -20,7 +20,7 @@ public interface ConnectionFactory extends AutoCloseable { - Connection openConnection() + Connection openConnection(JdbcIdentity identity) throws SQLException; @Override diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/DriverConnectionFactory.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/DriverConnectionFactory.java index e55ff263d5c8f..9187cde709832 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/DriverConnectionFactory.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/DriverConnectionFactory.java @@ -54,7 +54,7 @@ public DriverConnectionFactory(Driver driver, String connectionUrl, Properties c } @Override - public Connection openConnection() + public Connection openConnection(JdbcIdentity identity) throws SQLException { Connection connection = driver.connect(connectionUrl, connectionProperties); diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcClient.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcClient.java index b4a77d90228fb..6c4779e411c8d 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcClient.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcClient.java @@ -33,25 +33,27 @@ public interface JdbcClient { - default boolean schemaExists(String schema) + default boolean schemaExists(JdbcIdentity identity, String schema) { - return getSchemaNames().contains(schema); + return getSchemaNames(identity).contains(schema); } - Set getSchemaNames(); + String getIdentifierQuote(); - List getTableNames(@Nullable String schema); + Set getSchemaNames(JdbcIdentity identity); + + List getTableNames(JdbcIdentity identity, Optional schema); @Nullable - JdbcTableHandle getTableHandle(SchemaTableName schemaTableName); + JdbcTableHandle getTableHandle(JdbcIdentity identity, SchemaTableName schemaTableName); List getColumns(ConnectorSession session, JdbcTableHandle tableHandle); Optional toPrestoType(ConnectorSession session, JdbcTypeHandle typeHandle); - ConnectorSplitSource getSplits(JdbcTableLayoutHandle layoutHandle); + ConnectorSplitSource getSplits(JdbcIdentity identity, JdbcTableLayoutHandle layoutHandle); - Connection getConnection(JdbcSplit split) + Connection getConnection(JdbcIdentity identity, JdbcSplit split) throws SQLException; default void abortReadConnection(Connection connection) @@ -63,35 +65,35 @@ default void abortReadConnection(Connection connection) PreparedStatement buildSql(Connection connection, JdbcSplit split, List columnHandles) throws SQLException; - void addColumn(JdbcTableHandle handle, ColumnMetadata column); + void addColumn(JdbcIdentity identity, JdbcTableHandle handle, ColumnMetadata column); - void dropColumn(JdbcTableHandle handle, JdbcColumnHandle column); + void dropColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle column); - void renameColumn(JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName); + void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName); - void renameTable(JdbcTableHandle handle, SchemaTableName newTableName); + void renameTable(JdbcIdentity identity, JdbcTableHandle handle, SchemaTableName newTableName); - void createTable(ConnectorTableMetadata tableMetadata); + void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata); JdbcOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata); - void commitCreateTable(JdbcOutputTableHandle handle); + void commitCreateTable(JdbcIdentity identity, JdbcOutputTableHandle handle); JdbcOutputTableHandle beginInsertTable(ConnectorSession session, ConnectorTableMetadata tableMetadata); - void finishInsertTable(JdbcOutputTableHandle handle); + void finishInsertTable(JdbcIdentity identity, JdbcOutputTableHandle handle); - void dropTable(JdbcTableHandle jdbcTableHandle); + void dropTable(JdbcIdentity identity, JdbcTableHandle jdbcTableHandle); - void rollbackCreateTable(JdbcOutputTableHandle handle); + void rollbackCreateTable(JdbcIdentity identity, JdbcOutputTableHandle handle); String buildInsertSql(JdbcOutputTableHandle handle); - Connection getConnection(JdbcOutputTableHandle handle) + Connection getConnection(JdbcIdentity identity, JdbcOutputTableHandle handle) throws SQLException; PreparedStatement getPreparedStatement(Connection connection, String sql) throws SQLException; - TableStatistics getTableStatistics(ConnectorSession session, JdbcTableHandle handle, TupleDomain tupleDomain); + TableStatistics getTableStatistics(ConnectorSession session, JdbcTableHandle handle, List columnHandles, TupleDomain tupleDomain); } diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java index edea34467dfdd..b14656b4b1b3f 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java @@ -13,19 +13,24 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; +import com.facebook.presto.plugin.jdbc.optimization.JdbcPlanOptimizerProvider; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorAccessControl; import com.facebook.presto.spi.connector.ConnectorCapabilities; import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.connector.ConnectorPageSinkProvider; +import com.facebook.presto.spi.connector.ConnectorPlanOptimizerProvider; import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.procedure.Procedure; +import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.transaction.IsolationLevel; import com.google.common.collect.ImmutableSet; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import javax.inject.Inject; @@ -55,6 +60,10 @@ public class JdbcConnector private final Set procedures; private final ConcurrentMap transactions = new ConcurrentHashMap<>(); + private final FunctionMetadataManager functionManager; + private final StandardFunctionResolution functionResolution; + private final RowExpressionService rowExpressionService; + private final JdbcClient jdbcClient; @Inject public JdbcConnector( @@ -64,7 +73,11 @@ public JdbcConnector( JdbcRecordSetProvider jdbcRecordSetProvider, JdbcPageSinkProvider jdbcPageSinkProvider, Optional accessControl, - Set procedures) + Set procedures, + FunctionMetadataManager functionManager, + StandardFunctionResolution functionResolution, + RowExpressionService rowExpressionService, + JdbcClient jdbcClient) { this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); this.jdbcMetadataFactory = requireNonNull(jdbcMetadataFactory, "jdbcMetadataFactory is null"); @@ -73,6 +86,21 @@ public JdbcConnector( this.jdbcPageSinkProvider = requireNonNull(jdbcPageSinkProvider, "jdbcPageSinkProvider is null"); this.accessControl = requireNonNull(accessControl, "accessControl is null"); this.procedures = ImmutableSet.copyOf(requireNonNull(procedures, "procedures is null")); + this.functionManager = requireNonNull(functionManager, "functionManager is null"); + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); + this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null"); + this.jdbcClient = requireNonNull(jdbcClient, "jdbcClient is null"); + } + + @Override + public ConnectorPlanOptimizerProvider getConnectorPlanOptimizerProvider() + { + return new JdbcPlanOptimizerProvider( + jdbcClient, + functionManager, + functionResolution, + rowExpressionService.getDeterminismEvaluator(), + rowExpressionService.getExpressionOptimizer()); } @Override diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnectorFactory.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnectorFactory.java index 59072f7d1189b..7866025905ef2 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnectorFactory.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnectorFactory.java @@ -13,14 +13,17 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.airlift.bootstrap.Bootstrap; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.connector.ConnectorFactory; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.RowExpressionService; import com.google.inject.Injector; import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; import java.util.Map; @@ -62,7 +65,14 @@ public Connector create(String catalogName, Map requiredConfig, requireNonNull(requiredConfig, "requiredConfig is null"); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { - Bootstrap app = new Bootstrap(new JdbcModule(catalogName), module); + Bootstrap app = new Bootstrap( + binder -> { + binder.bind(FunctionMetadataManager.class).toInstance(context.getFunctionMetadataManager()); + binder.bind(StandardFunctionResolution.class).toInstance(context.getStandardFunctionResolution()); + binder.bind(RowExpressionService.class).toInstance(context.getRowExpressionService()); + }, + new JdbcModule(catalogName), + module); Injector injector = app .strictConfig() diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcIdentity.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcIdentity.java new file mode 100644 index 0000000000000..870463b20b381 --- /dev/null +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcIdentity.java @@ -0,0 +1,79 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.jdbc; + +import com.facebook.presto.spi.ConnectorSession; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class JdbcIdentity +{ + public static JdbcIdentity from(ConnectorSession session) + { + return new JdbcIdentity(session.getIdentity().getUser(), session.getIdentity().getExtraCredentials()); + } + + private final String user; + private final Map extraCredentials; + + public JdbcIdentity(String user, Map extraCredentials) + { + this.user = requireNonNull(user, "user is null"); + this.extraCredentials = ImmutableMap.copyOf(requireNonNull(extraCredentials, "extraCredentials is null")); + } + + public String getUser() + { + return user; + } + + public Map getExtraCredentials() + { + return extraCredentials; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + JdbcIdentity that = (JdbcIdentity) o; + return Objects.equals(user, that.user) && + Objects.equals(extraCredentials, that.extraCredentials); + } + + @Override + public int hashCode() + { + return Objects.hash(user, extraCredentials); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("user", user) + .add("extraCredentials", extraCredentials.keySet()) + .toString(); + } +} diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadata.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadata.java index a478656627134..26ca1c0066872 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadata.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadata.java @@ -43,6 +43,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import static com.facebook.presto.spi.StandardErrorCode.PERMISSION_DENIED; import static com.google.common.base.Preconditions.checkState; @@ -65,26 +66,26 @@ public JdbcMetadata(JdbcClient jdbcClient, boolean allowDropTable) @Override public boolean schemaExists(ConnectorSession session, String schemaName) { - return jdbcClient.schemaExists(schemaName); + return jdbcClient.schemaExists(JdbcIdentity.from(session), schemaName); } @Override public List listSchemaNames(ConnectorSession session) { - return ImmutableList.copyOf(jdbcClient.getSchemaNames()); + return ImmutableList.copyOf(jdbcClient.getSchemaNames(JdbcIdentity.from(session))); } @Override public JdbcTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) { - return jdbcClient.getTableHandle(tableName); + return jdbcClient.getTableHandle(JdbcIdentity.from(session), tableName); } @Override public List getTableLayouts(ConnectorSession session, ConnectorTableHandle table, Constraint constraint, Optional> desiredColumns) { JdbcTableHandle tableHandle = (JdbcTableHandle) table; - ConnectorTableLayout layout = new ConnectorTableLayout(new JdbcTableLayoutHandle(tableHandle, constraint.getSummary())); + ConnectorTableLayout layout = new ConnectorTableLayout(new JdbcTableLayoutHandle(tableHandle, constraint.getSummary(), Optional.empty())); return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary())); } @@ -107,9 +108,9 @@ public ConnectorTableMetadata getTableMetadata(ConnectorSession session, Connect } @Override - public List listTables(ConnectorSession session, String schemaNameOrNull) + public List listTables(ConnectorSession session, Optional schemaName) { - return jdbcClient.getTableNames(schemaNameOrNull); + return jdbcClient.getTableNames(JdbcIdentity.from(session), schemaName); } @Override @@ -137,7 +138,7 @@ public Map> listTableColumns(ConnectorSess } for (SchemaTableName tableName : tables) { try { - JdbcTableHandle tableHandle = jdbcClient.getTableHandle(tableName); + JdbcTableHandle tableHandle = jdbcClient.getTableHandle(JdbcIdentity.from(session), tableName); if (tableHandle == null) { continue; } @@ -163,28 +164,28 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle throw new PrestoException(PERMISSION_DENIED, "DROP TABLE is disabled in this catalog"); } JdbcTableHandle handle = (JdbcTableHandle) tableHandle; - jdbcClient.dropTable(handle); + jdbcClient.dropTable(JdbcIdentity.from(session), handle); } @Override public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional layout) { JdbcOutputTableHandle handle = jdbcClient.beginCreateTable(session, tableMetadata); - setRollback(() -> jdbcClient.rollbackCreateTable(handle)); + setRollback(() -> jdbcClient.rollbackCreateTable(JdbcIdentity.from(session), handle)); return handle; } @Override public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) { - jdbcClient.createTable(tableMetadata); + jdbcClient.createTable(session, tableMetadata); } @Override public Optional finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments, Collection computedStatistics) { JdbcOutputTableHandle handle = (JdbcOutputTableHandle) tableHandle; - jdbcClient.commitCreateTable(handle); + jdbcClient.commitCreateTable(JdbcIdentity.from(session), handle); clearRollback(); return Optional.empty(); } @@ -208,7 +209,7 @@ public void rollback() public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle) { JdbcOutputTableHandle handle = jdbcClient.beginInsertTable(session, getTableMetadata(session, tableHandle)); - setRollback(() -> jdbcClient.rollbackCreateTable(handle)); + setRollback(() -> jdbcClient.rollbackCreateTable(JdbcIdentity.from(session), handle)); return handle; } @@ -216,7 +217,7 @@ public ConnectorInsertTableHandle beginInsert(ConnectorSession session, Connecto public Optional finishInsert(ConnectorSession session, ConnectorInsertTableHandle tableHandle, Collection fragments, Collection computedStatistics) { JdbcOutputTableHandle jdbcInsertHandle = (JdbcOutputTableHandle) tableHandle; - jdbcClient.finishInsertTable(jdbcInsertHandle); + jdbcClient.finishInsertTable(JdbcIdentity.from(session), jdbcInsertHandle); return Optional.empty(); } @@ -224,7 +225,7 @@ public Optional finishInsert(ConnectorSession session, public void addColumn(ConnectorSession session, ConnectorTableHandle table, ColumnMetadata columnMetadata) { JdbcTableHandle tableHandle = (JdbcTableHandle) table; - jdbcClient.addColumn(tableHandle, columnMetadata); + jdbcClient.addColumn(JdbcIdentity.from(session), tableHandle, columnMetadata); } @Override @@ -232,7 +233,7 @@ public void dropColumn(ConnectorSession session, ConnectorTableHandle table, Col { JdbcTableHandle tableHandle = (JdbcTableHandle) table; JdbcColumnHandle columnHandle = (JdbcColumnHandle) column; - jdbcClient.dropColumn(tableHandle, columnHandle); + jdbcClient.dropColumn(JdbcIdentity.from(session), tableHandle, columnHandle); } @Override @@ -240,20 +241,21 @@ public void renameColumn(ConnectorSession session, ConnectorTableHandle table, C { JdbcTableHandle tableHandle = (JdbcTableHandle) table; JdbcColumnHandle columnHandle = (JdbcColumnHandle) column; - jdbcClient.renameColumn(tableHandle, columnHandle, target); + jdbcClient.renameColumn(JdbcIdentity.from(session), tableHandle, columnHandle, target); } @Override public void renameTable(ConnectorSession session, ConnectorTableHandle table, SchemaTableName newTableName) { JdbcTableHandle tableHandle = (JdbcTableHandle) table; - jdbcClient.renameTable(tableHandle, newTableName); + jdbcClient.renameTable(JdbcIdentity.from(session), tableHandle, newTableName); } @Override - public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint) + public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Optional tableLayoutHandle, List columnHandles, Constraint constraint) { JdbcTableHandle handle = (JdbcTableHandle) tableHandle; - return jdbcClient.getTableStatistics(session, handle, constraint.getSummary()); + List columns = columnHandles.stream().map(JdbcColumnHandle.class::cast).collect(Collectors.toList()); + return jdbcClient.getTableStatistics(session, handle, columns, constraint.getSummary()); } } diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataConfig.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataConfig.java index 6b3e5fb427d92..c9be190e41112 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataConfig.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.plugin.jdbc; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; public class JdbcMetadataConfig { diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcModule.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcModule.java index cbea8091c315e..26b53d7a3adba 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcModule.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcModule.java @@ -19,9 +19,9 @@ import com.google.inject.Module; import com.google.inject.Scopes; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static com.google.inject.multibindings.Multibinder.newSetBinder; import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; -import static io.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; public class JdbcModule diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcPageSink.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcPageSink.java index b31c9c83ab5e1..159fd1aae21cd 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcPageSink.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcPageSink.java @@ -13,7 +13,9 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.ConnectorPageSink; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; @@ -22,7 +24,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.primitives.Shorts; import com.google.common.primitives.SignedBytes; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import org.joda.time.DateTimeZone; @@ -67,10 +68,10 @@ public class JdbcPageSink private final List columnTypes; private int batchSize; - public JdbcPageSink(JdbcOutputTableHandle handle, JdbcClient jdbcClient) + public JdbcPageSink(ConnectorSession session, JdbcOutputTableHandle handle, JdbcClient jdbcClient) { try { - connection = jdbcClient.getConnection(handle); + connection = jdbcClient.getConnection(JdbcIdentity.from(session), handle); } catch (SQLException e) { throw new PrestoException(JDBC_ERROR, e); diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcPageSinkProvider.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcPageSinkProvider.java index f2485ecaf924f..8fe13b8d64eee 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcPageSinkProvider.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcPageSinkProvider.java @@ -41,13 +41,13 @@ public JdbcPageSinkProvider(JdbcClient jdbcClient) public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorOutputTableHandle tableHandle, PageSinkProperties pageSinkProperties) { checkArgument(!pageSinkProperties.isPartitionCommitRequired(), "Jdbc connector does not support partition commit"); - return new JdbcPageSink((JdbcOutputTableHandle) tableHandle, jdbcClient); + return new JdbcPageSink(session, (JdbcOutputTableHandle) tableHandle, jdbcClient); } @Override public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorInsertTableHandle tableHandle, PageSinkProperties pageSinkProperties) { checkArgument(!pageSinkProperties.isPartitionCommitRequired(), "Jdbc connector does not support partition commit"); - return new JdbcPageSink((JdbcOutputTableHandle) tableHandle, jdbcClient); + return new JdbcPageSink(session, (JdbcOutputTableHandle) tableHandle, jdbcClient); } } diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcRecordCursor.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcRecordCursor.java index f45545baa5375..9cec1bc74735a 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcRecordCursor.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcRecordCursor.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.RecordCursor; import com.facebook.presto.spi.type.Type; import com.google.common.base.VerifyException; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import java.sql.Connection; @@ -86,7 +86,7 @@ else if (javaType == Slice.class) { } try { - connection = jdbcClient.getConnection(split); + connection = jdbcClient.getConnection(JdbcIdentity.from(session), split); statement = jdbcClient.buildSql(connection, split, columnHandles); log.debug("Executing: %s", statement.toString()); resultSet = statement.executeQuery(); diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcSplit.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcSplit.java index 912ca7f39e11e..f7a1ed702aab3 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcSplit.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcSplit.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.presto.plugin.jdbc.optimization.JdbcExpression; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.HostAddress; @@ -36,7 +37,7 @@ public class JdbcSplit private final String schemaName; private final String tableName; private final TupleDomain tupleDomain; - private final Optional additionalPredicate; + private final Optional additionalPredicate; @JsonCreator public JdbcSplit( @@ -45,7 +46,7 @@ public JdbcSplit( @JsonProperty("schemaName") @Nullable String schemaName, @JsonProperty("tableName") String tableName, @JsonProperty("tupleDomain") TupleDomain tupleDomain, - @JsonProperty("additionalProperty") Optional additionalPredicate) + @JsonProperty("additionalProperty") Optional additionalPredicate) { this.connectorId = requireNonNull(connectorId, "connector id is null"); this.catalogName = catalogName; @@ -88,7 +89,7 @@ public TupleDomain getTupleDomain() } @JsonProperty - public Optional getAdditionalPredicate() + public Optional getAdditionalPredicate() { return additionalPredicate; } diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcSplitManager.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcSplitManager.java index 8b3c3526c5a2b..f25f4f8657cc5 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcSplitManager.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcSplitManager.java @@ -42,6 +42,6 @@ public ConnectorSplitSource getSplits( SplitSchedulingContext splitSchedulingContext) { JdbcTableLayoutHandle layoutHandle = (JdbcTableLayoutHandle) layout; - return jdbcClient.getSplits(layoutHandle); + return jdbcClient.getSplits(JdbcIdentity.from(session), layoutHandle); } } diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcTableHandle.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcTableHandle.java index 82d91ce8892b7..c9ec95841b14f 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcTableHandle.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcTableHandle.java @@ -30,6 +30,8 @@ public final class JdbcTableHandle { private final String connectorId; private final SchemaTableName schemaTableName; + + // catalog, schema and table names are reported by the remote database private final String catalogName; private final String schemaName; private final String tableName; diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcTableLayoutHandle.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcTableLayoutHandle.java index e1c5a9a4446b2..0bea87df5c37c 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcTableLayoutHandle.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcTableLayoutHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.presto.plugin.jdbc.optimization.JdbcExpression; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.predicate.TupleDomain; @@ -20,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Objects; +import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -28,14 +30,23 @@ public class JdbcTableLayoutHandle { private final JdbcTableHandle table; private final TupleDomain tupleDomain; + private final Optional additionalPredicate; @JsonCreator public JdbcTableLayoutHandle( @JsonProperty("table") JdbcTableHandle table, - @JsonProperty("tupleDomain") TupleDomain domain) + @JsonProperty("tupleDomain") TupleDomain domain, + @JsonProperty("additionalPredicate") Optional additionalPredicate) { this.table = requireNonNull(table, "table is null"); this.tupleDomain = requireNonNull(domain, "tupleDomain is null"); + this.additionalPredicate = additionalPredicate; + } + + @JsonProperty + public Optional getAdditionalPredicate() + { + return additionalPredicate; } @JsonProperty @@ -61,13 +72,14 @@ public boolean equals(Object o) } JdbcTableLayoutHandle that = (JdbcTableLayoutHandle) o; return Objects.equals(table, that.table) && - Objects.equals(tupleDomain, that.tupleDomain); + Objects.equals(tupleDomain, that.tupleDomain) && + Objects.equals(additionalPredicate, that.additionalPredicate); } @Override public int hashCode() { - return Objects.hash(table, tupleDomain); + return Objects.hash(table, tupleDomain, additionalPredicate); } @Override diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/QueryBuilder.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/QueryBuilder.java index 8de9f4265e42e..fa27d8c4a1fae 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/QueryBuilder.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/QueryBuilder.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.presto.plugin.jdbc.optimization.JdbcExpression; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.Range; @@ -102,7 +103,7 @@ public PreparedStatement buildSql( String table, List columns, TupleDomain tupleDomain, - Optional additionalPredicate) + Optional additionalPredicate) throws SQLException { StringBuilder sql = new StringBuilder(); @@ -133,8 +134,11 @@ public PreparedStatement buildSql( if (additionalPredicate.isPresent()) { clauses = ImmutableList.builder() .addAll(clauses) - .add(additionalPredicate.get()) + .add(additionalPredicate.get().getExpression()) .build(); + accumulator.addAll(additionalPredicate.get().getBoundConstantValues().stream() + .map(constantExpression -> new TypeAndValue(constantExpression.getType(), constantExpression.getValue())) + .collect(ImmutableList.toImmutableList())); } if (!clauses.isEmpty()) { sql.append(" WHERE ") diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/RemoteTableNameCacheKey.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/RemoteTableNameCacheKey.java new file mode 100644 index 0000000000000..2d70ce2f4ea31 --- /dev/null +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/RemoteTableNameCacheKey.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.jdbc; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +final class RemoteTableNameCacheKey +{ + private final JdbcIdentity identity; + private final String schema; + + RemoteTableNameCacheKey(JdbcIdentity identity, String schema) + { + this.identity = requireNonNull(identity, "identity is null"); + this.schema = requireNonNull(schema, "schema is null"); + } + + JdbcIdentity getIdentity() + { + return identity; + } + + String getSchema() + { + return schema; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RemoteTableNameCacheKey that = (RemoteTableNameCacheKey) o; + return Objects.equals(identity, that.identity) && + Objects.equals(schema, that.schema); + } + + @Override + public int hashCode() + { + return Objects.hash(identity, schema); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("identity", identity) + .add("schema", schema) + .toString(); + } +} diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java new file mode 100644 index 0000000000000..812cfd8512df9 --- /dev/null +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java @@ -0,0 +1,163 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.jdbc.optimization; + +import com.facebook.presto.expressions.LogicalRowExpressions; +import com.facebook.presto.expressions.translator.TranslatedExpression; +import com.facebook.presto.plugin.jdbc.JdbcTableHandle; +import com.facebook.presto.plugin.jdbc.JdbcTableLayoutHandle; +import com.facebook.presto.spi.ConnectorPlanOptimizer; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.VariableAllocator; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.PlanVisitor; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.relation.DeterminismEvaluator; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpression; +import com.google.common.collect.ImmutableList; + +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.expressions.translator.FunctionTranslator.buildFunctionTranslator; +import static com.facebook.presto.expressions.translator.RowExpressionTreeTranslator.translateWith; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; +import static java.util.Objects.requireNonNull; + +public class JdbcComputePushdown + implements ConnectorPlanOptimizer +{ + private final ExpressionOptimizer expressionOptimizer; + private final JdbcFilterToSqlTranslator jdbcFilterToSqlTranslator; + private final LogicalRowExpressions logicalRowExpressions; + + public JdbcComputePushdown( + FunctionMetadataManager functionMetadataManager, + StandardFunctionResolution functionResolution, + DeterminismEvaluator determinismEvaluator, + ExpressionOptimizer expressionOptimizer, + String identifierQuote, + Set> functionTranslators) + { + requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + requireNonNull(identifierQuote, "identifierQuote is null"); + requireNonNull(functionTranslators, "functionTranslators is null"); + requireNonNull(determinismEvaluator, "determinismEvaluator is null"); + requireNonNull(functionResolution, "functionResolution is null"); + + this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.jdbcFilterToSqlTranslator = new JdbcFilterToSqlTranslator( + functionMetadataManager, + buildFunctionTranslator(functionTranslators), + identifierQuote); + this.logicalRowExpressions = new LogicalRowExpressions( + determinismEvaluator, + functionResolution, + functionMetadataManager); + } + + @Override + public PlanNode optimize( + PlanNode maxSubplan, + ConnectorSession session, + VariableAllocator variableAllocator, + PlanNodeIdAllocator idAllocator) + { + return maxSubplan.accept(new Visitor(session, idAllocator), null); + } + + private class Visitor + extends PlanVisitor + { + private final ConnectorSession session; + private final PlanNodeIdAllocator idAllocator; + + public Visitor(ConnectorSession session, PlanNodeIdAllocator idAllocator) + { + this.session = requireNonNull(session, "session is null"); + this.idAllocator = requireNonNull(idAllocator, "idAllocator is null"); + } + + @Override + public PlanNode visitPlan(PlanNode node, Void context) + { + ImmutableList.Builder children = ImmutableList.builder(); + boolean changed = false; + for (PlanNode child : node.getSources()) { + PlanNode newChild = child.accept(this, null); + if (newChild != child) { + changed = true; + } + children.add(newChild); + } + + if (!changed) { + return node; + } + return node.replaceChildren(children.build()); + } + + @Override + public PlanNode visitFilter(FilterNode node, Void context) + { + if (!(node.getSource() instanceof TableScanNode)) { + return node; + } + + TableScanNode oldTableScanNode = (TableScanNode) node.getSource(); + TableHandle oldTableHandle = oldTableScanNode.getTable(); + JdbcTableHandle oldConnectorTable = (JdbcTableHandle) oldTableHandle.getConnectorHandle(); + + RowExpression predicate = expressionOptimizer.optimize(node.getPredicate(), OPTIMIZED, session); + predicate = logicalRowExpressions.convertToConjunctiveNormalForm(predicate); + TranslatedExpression jdbcExpression = translateWith( + predicate, + jdbcFilterToSqlTranslator, + oldTableScanNode.getAssignments()); + + // TODO if jdbcExpression is not present, walk through translated subtree to find out which parts can be pushed down + if (!oldTableHandle.getLayout().isPresent() || !jdbcExpression.getTranslated().isPresent()) { + return node; + } + + JdbcTableLayoutHandle oldTableLayoutHandle = (JdbcTableLayoutHandle) oldTableHandle.getLayout().get(); + JdbcTableLayoutHandle newTableLayoutHandle = new JdbcTableLayoutHandle( + oldConnectorTable, + oldTableLayoutHandle.getTupleDomain(), + jdbcExpression.getTranslated()); + + TableHandle tableHandle = new TableHandle( + oldTableHandle.getConnectorId(), + oldTableHandle.getConnectorHandle(), + oldTableHandle.getTransaction(), + Optional.of(newTableLayoutHandle)); + + TableScanNode newTableScanNode = new TableScanNode( + idAllocator.getNextId(), + tableHandle, + oldTableScanNode.getOutputVariables(), + oldTableScanNode.getAssignments(), + oldTableScanNode.getCurrentConstraint(), + oldTableScanNode.getEnforcedConstraint()); + + return new FilterNode(idAllocator.getNextId(), newTableScanNode, node.getPredicate()); + } + } +} diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcExpression.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcExpression.java new file mode 100644 index 0000000000000..80618e0fa18b7 --- /dev/null +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcExpression.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.jdbc.optimization; + +import com.facebook.presto.spi.relation.ConstantExpression; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class JdbcExpression +{ + private final String expression; + private final List boundConstantValues; + + public JdbcExpression(String expression) + { + this(expression, ImmutableList.of()); + } + + @JsonCreator + public JdbcExpression( + @JsonProperty("translatedString") String expression, + @JsonProperty("boundConstantValues") List constantBindValues) + { + this.expression = requireNonNull(expression, "expression is null"); + this.boundConstantValues = requireNonNull(constantBindValues, "boundConstantValues is null"); + } + + @JsonProperty + public String getExpression() + { + return expression; + } + + /** + * Constant expressions are not added to the expression String. Instead they appear as "?" in the query. + * This is because we would potentially lose precision on double values. Hence when we make a PreparedStatement + * out of the SQL string replacing every "?" by it's corresponding actual bindValue. + * + * @return List of constants to replace in the SQL string. + */ + @JsonProperty + public List getBoundConstantValues() + { + return boundConstantValues; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + JdbcExpression that = (JdbcExpression) o; + return expression.equals(that.expression) && + boundConstantValues.equals(that.boundConstantValues); + } + + @Override + public int hashCode() + { + return Objects.hash(expression, boundConstantValues); + } +} diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcFilterToSqlTranslator.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcFilterToSqlTranslator.java new file mode 100644 index 0000000000000..3be5bd98e5aea --- /dev/null +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcFilterToSqlTranslator.java @@ -0,0 +1,181 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.jdbc.optimization; + +import com.facebook.presto.expressions.translator.FunctionTranslator; +import com.facebook.presto.expressions.translator.RowExpressionTranslator; +import com.facebook.presto.expressions.translator.RowExpressionTreeTranslator; +import com.facebook.presto.expressions.translator.TranslatedExpression; +import com.facebook.presto.plugin.jdbc.JdbcColumnHandle; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.BooleanType; +import com.facebook.presto.spi.type.CharType; +import com.facebook.presto.spi.type.DateType; +import com.facebook.presto.spi.type.DoubleType; +import com.facebook.presto.spi.type.IntegerType; +import com.facebook.presto.spi.type.RealType; +import com.facebook.presto.spi.type.SmallintType; +import com.facebook.presto.spi.type.TimeType; +import com.facebook.presto.spi.type.TimeWithTimeZoneType; +import com.facebook.presto.spi.type.TimestampType; +import com.facebook.presto.spi.type.TimestampWithTimeZoneType; +import com.facebook.presto.spi.type.TinyintType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.VarcharType; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.expressions.translator.TranslatedExpression.untranslated; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class JdbcFilterToSqlTranslator + extends RowExpressionTranslator> +{ + private final FunctionMetadataManager functionMetadataManager; + private final FunctionTranslator functionTranslator; + private final String quote; + + public JdbcFilterToSqlTranslator(FunctionMetadataManager functionMetadataManager, FunctionTranslator functionTranslator, String quote) + { + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + this.functionTranslator = requireNonNull(functionTranslator, "functionTranslator is null"); + this.quote = requireNonNull(quote, "quote is null"); + } + + @Override + public TranslatedExpression translateConstant(ConstantExpression literal, Map context, RowExpressionTreeTranslator> rowExpressionTreeTranslator) + { + if (isSupportedType(literal.getType())) { + return new TranslatedExpression<>( + Optional.of(new JdbcExpression("?", ImmutableList.of(literal))), + literal, + ImmutableList.of()); + } + return untranslated(literal); + } + + @Override + public TranslatedExpression translateVariable(VariableReferenceExpression variable, Map context, RowExpressionTreeTranslator> rowExpressionTreeTranslator) + { + JdbcColumnHandle columnHandle = (JdbcColumnHandle) context.get(variable); + requireNonNull(columnHandle, format("Unrecognized variable %s", variable)); + return new TranslatedExpression<>( + Optional.of(new JdbcExpression(quote + columnHandle.getColumnName().replace(quote, quote + quote) + quote)), + variable, + ImmutableList.of()); + } + + @Override + public TranslatedExpression translateLambda(LambdaDefinitionExpression lambda, Map context, RowExpressionTreeTranslator> rowExpressionTreeTranslator) + { + return untranslated(lambda); + } + + @Override + public TranslatedExpression translateCall(CallExpression call, Map context, RowExpressionTreeTranslator> rowExpressionTreeTranslator) + { + List> translatedExpressions = call.getArguments().stream() + .map(expression -> rowExpressionTreeTranslator.rewrite(expression, context)) + .collect(toImmutableList()); + + FunctionMetadata functionMetadata = functionMetadataManager.getFunctionMetadata(call.getFunctionHandle()); + + try { + return functionTranslator.translate(functionMetadata, call, translatedExpressions); + } + catch (Throwable t) { + // no-op + } + return untranslated(call, translatedExpressions); + } + + @Override + public TranslatedExpression translateSpecialForm(SpecialFormExpression specialForm, Map context, RowExpressionTreeTranslator> rowExpressionTreeTranslator) + { + List> translatedExpressions = specialForm.getArguments().stream() + .map(expression -> rowExpressionTreeTranslator.rewrite(expression, context)) + .collect(toImmutableList()); + + List jdbcExpressions = translatedExpressions.stream() + .map(TranslatedExpression::getTranslated) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(toImmutableList()); + + if (jdbcExpressions.size() < translatedExpressions.size()) { + return untranslated(specialForm, translatedExpressions); + } + + List sqlBodies = jdbcExpressions.stream() + .map(JdbcExpression::getExpression) + .map(sql -> '(' + sql + ')') + .collect(toImmutableList()); + List variableBindings = jdbcExpressions.stream() + .map(JdbcExpression::getBoundConstantValues) + .flatMap(List::stream) + .collect(toImmutableList()); + + switch (specialForm.getForm()) { + case AND: + return new TranslatedExpression<>( + Optional.of(new JdbcExpression(format("(%s)", Joiner.on(" AND ").join(sqlBodies)), variableBindings)), + specialForm, + translatedExpressions); + case OR: + return new TranslatedExpression<>( + Optional.of(new JdbcExpression(format("(%s)", Joiner.on(" OR ").join(sqlBodies)), variableBindings)), + specialForm, + translatedExpressions); + case IN: + return new TranslatedExpression<>( + Optional.of(new JdbcExpression(format("(%s IN (%s))", sqlBodies.get(0), Joiner.on(" , ").join(sqlBodies.subList(1, sqlBodies.size()))), variableBindings)), + specialForm, + translatedExpressions); + } + return untranslated(specialForm, translatedExpressions); + } + + private static boolean isSupportedType(Type type) + { + Type validType = requireNonNull(type, "type is null"); + return validType.equals(BigintType.BIGINT) || + validType.equals(TinyintType.TINYINT) || + validType.equals(SmallintType.SMALLINT) || + validType.equals(IntegerType.INTEGER) || + validType.equals(DoubleType.DOUBLE) || + validType.equals(RealType.REAL) || + validType.equals(BooleanType.BOOLEAN) || + validType.equals(DateType.DATE) || + validType.equals(TimeType.TIME) || + validType.equals(TimeWithTimeZoneType.TIME_WITH_TIME_ZONE) || + validType.equals(TimestampType.TIMESTAMP) || + validType.equals(TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE) || + validType instanceof VarcharType || + validType instanceof CharType; + } +} diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java new file mode 100644 index 0000000000000..9dcd399b66472 --- /dev/null +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.jdbc.optimization; + +import com.facebook.presto.plugin.jdbc.JdbcClient; +import com.facebook.presto.plugin.jdbc.optimization.function.OperatorTranslators; +import com.facebook.presto.spi.ConnectorPlanOptimizer; +import com.facebook.presto.spi.connector.ConnectorPlanOptimizerProvider; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.DeterminismEvaluator; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.google.common.collect.ImmutableSet; +import com.google.inject.Inject; + +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public class JdbcPlanOptimizerProvider + implements ConnectorPlanOptimizerProvider +{ + private final FunctionMetadataManager functionManager; + private final StandardFunctionResolution functionResolution; + private final DeterminismEvaluator determinismEvaluator; + private final ExpressionOptimizer expressionOptimizer; + private final String identifierQuote; + + @Inject + public JdbcPlanOptimizerProvider( + JdbcClient jdbcClient, + FunctionMetadataManager functionManager, + StandardFunctionResolution functionResolution, + DeterminismEvaluator determinismEvaluator, + ExpressionOptimizer expressionOptimizer) + { + this.functionManager = requireNonNull(functionManager, "functionManager is null"); + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); + this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); + this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.identifierQuote = jdbcClient.getIdentifierQuote(); + } + + @Override + public Set getLogicalPlanOptimizers() + { + return ImmutableSet.of(); + } + + @Override + public Set getPhysicalPlanOptimizers() + { + return ImmutableSet.of(new JdbcComputePushdown( + functionManager, + functionResolution, + determinismEvaluator, + expressionOptimizer, + identifierQuote, + getFunctionTranslators())); + } + + private Set> getFunctionTranslators() + { + return ImmutableSet.of(OperatorTranslators.class); + } +} diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/function/JdbcTranslationUtil.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/function/JdbcTranslationUtil.java new file mode 100644 index 0000000000000..f0feef55129f8 --- /dev/null +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/function/JdbcTranslationUtil.java @@ -0,0 +1,41 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.jdbc.optimization.function; + +import com.facebook.presto.plugin.jdbc.optimization.JdbcExpression; +import com.facebook.presto.spi.relation.ConstantExpression; + +import java.util.Arrays; +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +public class JdbcTranslationUtil +{ + private JdbcTranslationUtil() + { + } + + public static String infixOperation(String operator, JdbcExpression left, JdbcExpression right) + { + return String.format("(%s %s %s)", left.getExpression(), operator, right.getExpression()); + } + + public static List forwardBindVariables(JdbcExpression... jdbcExpressions) + { + return Arrays.stream(jdbcExpressions).map(JdbcExpression::getBoundConstantValues) + .flatMap(List::stream) + .collect(toImmutableList()); + } +} diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/function/OperatorTranslators.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/function/OperatorTranslators.java new file mode 100644 index 0000000000000..ff47b33767df5 --- /dev/null +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/function/OperatorTranslators.java @@ -0,0 +1,69 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.jdbc.optimization.function; + +import com.facebook.presto.plugin.jdbc.optimization.JdbcExpression; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.ScalarOperator; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.type.StandardTypes; + +import static com.facebook.presto.plugin.jdbc.optimization.function.JdbcTranslationUtil.forwardBindVariables; +import static com.facebook.presto.plugin.jdbc.optimization.function.JdbcTranslationUtil.infixOperation; +import static com.facebook.presto.spi.function.OperatorType.ADD; +import static com.facebook.presto.spi.function.OperatorType.EQUAL; +import static com.facebook.presto.spi.function.OperatorType.NOT_EQUAL; +import static com.facebook.presto.spi.function.OperatorType.SUBTRACT; + +public class OperatorTranslators +{ + private OperatorTranslators() + { + } + + @ScalarOperator(ADD) + @SqlType(StandardTypes.BIGINT) + public static JdbcExpression add(@SqlType(StandardTypes.BIGINT) JdbcExpression left, @SqlType(StandardTypes.BIGINT) JdbcExpression right) + { + return new JdbcExpression(infixOperation("+", left, right), forwardBindVariables(left, right)); + } + + @ScalarOperator(SUBTRACT) + @SqlType(StandardTypes.BIGINT) + public static JdbcExpression subtract(@SqlType(StandardTypes.BIGINT) JdbcExpression left, @SqlType(StandardTypes.BIGINT) JdbcExpression right) + { + return new JdbcExpression(infixOperation("-", left, right), forwardBindVariables(left, right)); + } + + @ScalarOperator(EQUAL) + @SqlType(StandardTypes.BOOLEAN) + public static JdbcExpression equal(@SqlType(StandardTypes.BIGINT) JdbcExpression left, @SqlType(StandardTypes.BIGINT) JdbcExpression right) + { + return new JdbcExpression(infixOperation("=", left, right), forwardBindVariables(left, right)); + } + + @ScalarOperator(NOT_EQUAL) + @SqlType(StandardTypes.BOOLEAN) + public static JdbcExpression notEqual(@SqlType(StandardTypes.BIGINT) JdbcExpression left, @SqlType(StandardTypes.BIGINT) JdbcExpression right) + { + return new JdbcExpression(infixOperation("<>", left, right), forwardBindVariables(left, right)); + } + + @ScalarFunction("not") + @SqlType(StandardTypes.BOOLEAN) + public static JdbcExpression not(@SqlType(StandardTypes.BOOLEAN) JdbcExpression expression) + { + return new JdbcExpression(String.format("(NOT(%s))", expression.getExpression()), expression.getBoundConstantValues()); + } +} diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/JdbcQueryRunner.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/JdbcQueryRunner.java index 605dc02bf51fc..e4862c26a2f2b 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/JdbcQueryRunner.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/JdbcQueryRunner.java @@ -25,10 +25,10 @@ import java.sql.Statement; import java.util.Map; +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tests.QueryAssertions.copyTpchTables; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static io.airlift.testing.Closeables.closeAllSuppress; public final class JdbcQueryRunner { diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/MetadataUtil.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/MetadataUtil.java index 82d6c3fc7d722..074f60a1258af 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/MetadataUtil.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/MetadataUtil.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; import java.util.Map; diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestBaseJdbcConfig.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestBaseJdbcConfig.java index bf8508a5130ef..1b34371a024a2 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestBaseJdbcConfig.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestBaseJdbcConfig.java @@ -13,12 +13,16 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; +import io.airlift.units.Duration; import org.testng.annotations.Test; import java.util.Map; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; + public class TestBaseJdbcConfig { @Test @@ -27,7 +31,9 @@ public void testDefaults() ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(BaseJdbcConfig.class) .setConnectionUrl(null) .setConnectionUser(null) - .setConnectionPassword(null)); + .setConnectionPassword(null) + .setCaseInsensitiveNameMatching(false) + .setCaseInsensitiveNameMatchingCacheTtl(new Duration(1, MINUTES))); } @Test @@ -37,12 +43,16 @@ public void testExplicitPropertyMappings() .put("connection-url", "jdbc:h2:mem:config") .put("connection-user", "user") .put("connection-password", "password") + .put("case-insensitive-name-matching", "true") + .put("case-insensitive-name-matching.cache-ttl", "1s") .build(); BaseJdbcConfig expected = new BaseJdbcConfig() .setConnectionUrl("jdbc:h2:mem:config") .setConnectionUser("user") - .setConnectionPassword("password"); + .setConnectionPassword("password") + .setCaseInsensitiveNameMatching(true) + .setCaseInsensitiveNameMatchingCacheTtl(new Duration(1, SECONDS)); ConfigAssertions.assertFullMapping(properties, expected); } diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcClient.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcClient.java index 9f857da493288..5553dd5ede5cf 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcClient.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcClient.java @@ -27,6 +27,7 @@ import org.testng.annotations.Test; import java.util.List; +import java.util.Optional; import static com.facebook.presto.plugin.jdbc.TestingDatabase.CONNECTOR_ID; import static com.facebook.presto.plugin.jdbc.TestingJdbcTypeHandle.JDBC_BIGINT; @@ -75,17 +76,18 @@ public void tearDown() @Test public void testMetadata() { - assertTrue(jdbcClient.getSchemaNames().containsAll(ImmutableSet.of("example", "tpch"))); - assertEquals(jdbcClient.getTableNames("example"), ImmutableList.of( + JdbcIdentity identity = JdbcIdentity.from(session); + assertTrue(jdbcClient.getSchemaNames(identity).containsAll(ImmutableSet.of("example", "tpch"))); + assertEquals(jdbcClient.getTableNames(identity, Optional.of("example")), ImmutableList.of( new SchemaTableName("example", "numbers"), new SchemaTableName("example", "view_source"), new SchemaTableName("example", "view"))); - assertEquals(jdbcClient.getTableNames("tpch"), ImmutableList.of( + assertEquals(jdbcClient.getTableNames(identity, Optional.of("tpch")), ImmutableList.of( new SchemaTableName("tpch", "lineitem"), new SchemaTableName("tpch", "orders"))); SchemaTableName schemaTableName = new SchemaTableName("example", "numbers"); - JdbcTableHandle table = jdbcClient.getTableHandle(schemaTableName); + JdbcTableHandle table = jdbcClient.getTableHandle(identity, schemaTableName); assertNotNull(table, "table is null"); assertEquals(table.getCatalogName(), catalogName.toUpperCase(ENGLISH)); assertEquals(table.getSchemaName(), "EXAMPLE"); @@ -101,7 +103,7 @@ public void testMetadata() public void testMetadataWithSchemaPattern() { SchemaTableName schemaTableName = new SchemaTableName("exa_ple", "num_ers"); - JdbcTableHandle table = jdbcClient.getTableHandle(schemaTableName); + JdbcTableHandle table = jdbcClient.getTableHandle(JdbcIdentity.from(session), schemaTableName); assertNotNull(table, "table is null"); assertEquals(jdbcClient.getColumns(session, table), ImmutableList.of( new JdbcColumnHandle(CONNECTOR_ID, "TE_T", JDBC_VARCHAR, VARCHAR, true), @@ -112,7 +114,7 @@ public void testMetadataWithSchemaPattern() public void testMetadataWithFloatAndDoubleCol() { SchemaTableName schemaTableName = new SchemaTableName("exa_ple", "table_with_float_col"); - JdbcTableHandle table = jdbcClient.getTableHandle(schemaTableName); + JdbcTableHandle table = jdbcClient.getTableHandle(JdbcIdentity.from(session), schemaTableName); assertNotNull(table, "table is null"); assertEquals(jdbcClient.getColumns(session, table), ImmutableList.of( new JdbcColumnHandle(CONNECTOR_ID, "COL1", JDBC_BIGINT, BIGINT, true), @@ -132,9 +134,9 @@ public void testCreateWithNullableColumns() new ColumnMetadata("columnC", BigintType.BIGINT, false, null, null, false, emptyMap()), new ColumnMetadata("columnD", DateType.DATE, false, null, null, false, emptyMap())); - jdbcClient.createTable(new ConnectorTableMetadata(schemaTableName, expectedColumns)); + jdbcClient.createTable(session, new ConnectorTableMetadata(schemaTableName, expectedColumns)); - JdbcTableHandle tableHandle = jdbcClient.getTableHandle(schemaTableName); + JdbcTableHandle tableHandle = jdbcClient.getTableHandle(JdbcIdentity.from(session), schemaTableName); try { assertEquals(tableHandle.getTableName(), tableName); @@ -145,7 +147,7 @@ public void testCreateWithNullableColumns() new JdbcColumnHandle(CONNECTOR_ID, "COLUMND", JDBC_DATE, DateType.DATE, false))); } finally { - jdbcClient.dropTable(tableHandle); + jdbcClient.dropTable(JdbcIdentity.from(session), tableHandle); } } @@ -157,26 +159,26 @@ public void testAlterColumns() List expectedColumns = ImmutableList.of( new ColumnMetadata("columnA", BigintType.BIGINT, null, null, false)); - jdbcClient.createTable(new ConnectorTableMetadata(schemaTableName, expectedColumns)); + jdbcClient.createTable(session, new ConnectorTableMetadata(schemaTableName, expectedColumns)); - JdbcTableHandle tableHandle = jdbcClient.getTableHandle(schemaTableName); + JdbcTableHandle tableHandle = jdbcClient.getTableHandle(JdbcIdentity.from(session), schemaTableName); try { assertEquals(tableHandle.getTableName(), tableName); assertEquals(jdbcClient.getColumns(session, tableHandle), ImmutableList.of( new JdbcColumnHandle(CONNECTOR_ID, "COLUMNA", JDBC_BIGINT, BigintType.BIGINT, true))); - jdbcClient.addColumn(tableHandle, new ColumnMetadata("columnB", DoubleType.DOUBLE, null, null, false)); + jdbcClient.addColumn(JdbcIdentity.from(session), tableHandle, new ColumnMetadata("columnB", DoubleType.DOUBLE, null, null, false)); assertEquals(jdbcClient.getColumns(session, tableHandle), ImmutableList.of( new JdbcColumnHandle(CONNECTOR_ID, "COLUMNA", JDBC_BIGINT, BigintType.BIGINT, true), new JdbcColumnHandle(CONNECTOR_ID, "COLUMNB", JDBC_DOUBLE, DoubleType.DOUBLE, true))); - jdbcClient.dropColumn(tableHandle, new JdbcColumnHandle(CONNECTOR_ID, "COLUMNB", JDBC_BIGINT, BigintType.BIGINT, true)); + jdbcClient.dropColumn(JdbcIdentity.from(session), tableHandle, new JdbcColumnHandle(CONNECTOR_ID, "COLUMNB", JDBC_BIGINT, BigintType.BIGINT, true)); assertEquals(jdbcClient.getColumns(session, tableHandle), ImmutableList.of( new JdbcColumnHandle(CONNECTOR_ID, "COLUMNA", JDBC_BIGINT, BigintType.BIGINT, true))); } finally { - jdbcClient.dropTable(tableHandle); + jdbcClient.dropTable(JdbcIdentity.from(session), tableHandle); } } } diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcColumnHandle.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcColumnHandle.java index a35337139c0fa..17e63da1d9d55 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcColumnHandle.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcColumnHandle.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.plugin.jdbc; -import io.airlift.testing.EquivalenceTester; +import com.facebook.airlift.testing.EquivalenceTester; import org.testng.annotations.Test; import static com.facebook.presto.plugin.jdbc.MetadataUtil.COLUMN_CODEC; diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcIntegrationSmokeTest.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcIntegrationSmokeTest.java index d0af6a8310fbf..df45f5de91445 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcIntegrationSmokeTest.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcIntegrationSmokeTest.java @@ -14,6 +14,7 @@ package com.facebook.presto.plugin.jdbc; import com.facebook.presto.tests.AbstractTestIntegrationSmokeTest; +import org.testng.annotations.Test; import static com.facebook.presto.plugin.jdbc.JdbcQueryRunner.createJdbcQueryRunner; import static io.airlift.tpch.TpchTable.ORDERS; @@ -25,4 +26,47 @@ public TestJdbcIntegrationSmokeTest() { super(() -> createJdbcQueryRunner(ORDERS)); } + + @Test + public void testPlanOptimizerFilterPushdownCorrectness() + { + assertUpdate("CREATE TABLE test_pushdown1(col1 bigint, col2 bigint)"); + assertUpdate("INSERT INTO test_pushdown1 VALUES (5, 4), (2, 4), (4, 4)", "VALUES(3)"); + // PickTableLayout#pushPredicateIntoTableScan cannot push down this predicate however, JdbcPlanOptimizer can. + assertQuery("SELECT * FROM test_pushdown1 WHERE col1 + col2 = 8", "VALUES(4, 4)"); + } + + @Test + public void testPlanOptimizerAndDefaultFilterPushdownCorrectness() + { + assertUpdate("CREATE TABLE test_pushdown2(col1 bigint, col2 bigint)"); + assertUpdate("INSERT INTO test_pushdown2 VALUES (5, 4), (2, 4), (4, 4)", "VALUES(3)"); + // PickTableLayout#pushPredicateIntoTableScan and JdbcPlanOptimizer are both capable of pushing down these predicates. + assertQuery("SELECT * FROM test_pushdown2 WHERE col1 = 5 AND col2 = 4", "VALUES(5, 4)"); + } + + @Test + public void testIncompatibleFilterPushdownCorrectness() + { + assertUpdate("CREATE TABLE test_pushdown3(col1 bigint, col2 bigint)"); + assertUpdate("INSERT INTO test_pushdown3 VALUES (5, 4), (2, 4), (4, 4)", "VALUES(3)"); + // Neither PickTableLayout#pushPredicateIntoTableScan or JdbcPlanOptimizer are capable of pushing down these predicates. + assertQuery("SELECT * FROM test_pushdown3 WHERE NOT(col1 + col2 <= 6)", "VALUES (5, 4), (4, 4)"); + } + + @Test + public void testArithmetic() + { + assertUpdate("CREATE TABLE test_arithmetic(col1 bigint, col2 bigint)"); + assertUpdate("INSERT INTO test_arithmetic VALUES (5, 4), (2, 4), (4, 4)", "VALUES(3)"); + assertQuery("SELECT * FROM test_arithmetic WHERE NOT(col1 + col2 - 6 = 0)", "VALUES (5, 4), (4, 4)"); + } + + @Test + public void testBooleanNot() + { + assertUpdate("CREATE TABLE test_not(col1 bigint, col2 boolean)"); + assertUpdate("INSERT INTO test_not VALUES (5, true), (2, true), (4, false)", "VALUES(3)"); + assertQuery("SELECT col1 FROM test_not WHERE NOT(col2)", "VALUES(4)"); + } } diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadataConfig.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadataConfig.java index da079f32904da..53e09b44ffabc 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadataConfig.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadataConfig.java @@ -18,9 +18,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestJdbcMetadataConfig { diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcQueryBuilder.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcQueryBuilder.java index f590f019e23ed..f5fdd51d1a500 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcQueryBuilder.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcQueryBuilder.java @@ -39,6 +39,7 @@ import java.util.Locale; import java.util.Optional; +import static com.facebook.airlift.testing.Assertions.assertContains; import static com.facebook.presto.plugin.jdbc.TestingJdbcTypeHandle.JDBC_BIGINT; import static com.facebook.presto.plugin.jdbc.TestingJdbcTypeHandle.JDBC_BOOLEAN; import static com.facebook.presto.plugin.jdbc.TestingJdbcTypeHandle.JDBC_CHAR; @@ -63,7 +64,6 @@ import static com.facebook.presto.spi.type.TinyintType.TINYINT; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static io.airlift.slice.Slices.utf8Slice; -import static io.airlift.testing.Assertions.assertContains; import static java.lang.Float.floatToRawIntBits; import static java.lang.String.format; import static java.time.temporal.ChronoUnit.DAYS; diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcRecordSetProvider.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcRecordSetProvider.java index 8c395b260e413..3e13a96ec4457 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcRecordSetProvider.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcRecordSetProvider.java @@ -32,14 +32,15 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.spi.type.VarcharType.createVarcharType; import static com.facebook.presto.testing.TestingConnectorSession.SESSION; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static io.airlift.slice.Slices.utf8Slice; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; @@ -47,6 +48,8 @@ @Test public class TestJdbcRecordSetProvider { + private static final JdbcIdentity IDENTITY = new JdbcIdentity("user", ImmutableMap.of()); + private TestingDatabase database; private JdbcClient jdbcClient; private JdbcSplit split; @@ -64,7 +67,7 @@ public void setUp() jdbcClient = database.getJdbcClient(); split = database.getSplit("example", "numbers"); - table = jdbcClient.getTableHandle(new SchemaTableName("example", "numbers")); + table = jdbcClient.getTableHandle(IDENTITY, new SchemaTableName("example", "numbers")); Map columns = database.getColumnHandles("example", "numbers"); textColumn = columns.get("text"); @@ -178,8 +181,8 @@ public void testTupleDomain() private RecordCursor getCursor(JdbcTableHandle jdbcTableHandle, List columns, TupleDomain domain) { - JdbcTableLayoutHandle layoutHandle = new JdbcTableLayoutHandle(jdbcTableHandle, domain); - ConnectorSplitSource splits = jdbcClient.getSplits(layoutHandle); + JdbcTableLayoutHandle layoutHandle = new JdbcTableLayoutHandle(jdbcTableHandle, domain, Optional.empty()); + ConnectorSplitSource splits = jdbcClient.getSplits(IDENTITY, layoutHandle); JdbcSplit split = (JdbcSplit) getOnlyElement(getFutureValue(splits.getNextBatch(NOT_PARTITIONED, 1000)).getSplits()); ConnectorTransactionHandle transaction = new JdbcTransactionHandle(); diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcSplit.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcSplit.java index 1d4b3fa010ae7..4f741ca965304 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcSplit.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcSplit.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.predicate.TupleDomain; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; import java.util.Optional; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; public class TestJdbcSplit diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcTableHandle.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcTableHandle.java index ab854c5c70408..3c3393d361c45 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcTableHandle.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcTableHandle.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.airlift.testing.EquivalenceTester; import com.facebook.presto.spi.SchemaTableName; -import io.airlift.testing.EquivalenceTester; import org.testng.annotations.Test; import static com.facebook.presto.plugin.jdbc.MetadataUtil.TABLE_CODEC; diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestingDatabase.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestingDatabase.java index b94c501d4e377..e18a101177e91 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestingDatabase.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestingDatabase.java @@ -25,13 +25,14 @@ import java.sql.SQLException; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Properties; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.concurrent.MoreFutures.getFutureValue; final class TestingDatabase implements AutoCloseable @@ -97,15 +98,16 @@ public JdbcClient getJdbcClient() public JdbcSplit getSplit(String schemaName, String tableName) { - JdbcTableHandle jdbcTableHandle = jdbcClient.getTableHandle(new SchemaTableName(schemaName, tableName)); - JdbcTableLayoutHandle jdbcLayoutHandle = new JdbcTableLayoutHandle(jdbcTableHandle, TupleDomain.all()); - ConnectorSplitSource splits = jdbcClient.getSplits(jdbcLayoutHandle); + JdbcIdentity identity = JdbcIdentity.from(session); + JdbcTableHandle jdbcTableHandle = jdbcClient.getTableHandle(identity, new SchemaTableName(schemaName, tableName)); + JdbcTableLayoutHandle jdbcLayoutHandle = new JdbcTableLayoutHandle(jdbcTableHandle, TupleDomain.all(), Optional.empty()); + ConnectorSplitSource splits = jdbcClient.getSplits(identity, jdbcLayoutHandle); return (JdbcSplit) getOnlyElement(getFutureValue(splits.getNextBatch(NOT_PARTITIONED, 1000)).getSplits()); } public Map getColumnHandles(String schemaName, String tableName) { - JdbcTableHandle tableHandle = jdbcClient.getTableHandle(new SchemaTableName(schemaName, tableName)); + JdbcTableHandle tableHandle = jdbcClient.getTableHandle(JdbcIdentity.from(session), new SchemaTableName(schemaName, tableName)); List columns = jdbcClient.getColumns(session, tableHandle); checkArgument(columns != null, "table not found: %s.%s", schemaName, tableName); diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestingH2JdbcModule.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestingH2JdbcModule.java index 95a7a3104945e..585638b7eb405 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestingH2JdbcModule.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestingH2JdbcModule.java @@ -21,7 +21,7 @@ import java.util.Map; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static java.lang.String.format; class TestingH2JdbcModule diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java new file mode 100644 index 0000000000000..4c5966175e096 --- /dev/null +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java @@ -0,0 +1,334 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.jdbc.optimization; + +import com.facebook.presto.Session; +import com.facebook.presto.cost.PlanNodeStatsEstimate; +import com.facebook.presto.cost.StatsAndCosts; +import com.facebook.presto.cost.StatsProvider; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.plugin.jdbc.JdbcColumnHandle; +import com.facebook.presto.plugin.jdbc.JdbcTableHandle; +import com.facebook.presto.plugin.jdbc.JdbcTableLayoutHandle; +import com.facebook.presto.plugin.jdbc.JdbcTypeHandle; +import com.facebook.presto.plugin.jdbc.optimization.function.OperatorTranslators; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.DeterminismEvaluator; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.TestingRowExpressionTranslator; +import com.facebook.presto.sql.planner.Plan; +import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.planner.assertions.MatchResult; +import com.facebook.presto.sql.planner.assertions.Matcher; +import com.facebook.presto.sql.planner.assertions.PlanAssert; +import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; +import com.facebook.presto.sql.planner.assertions.SymbolAliases; +import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; +import com.facebook.presto.sql.tree.SymbolReference; +import com.facebook.presto.testing.TestingConnectorSession; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.testng.annotations.Test; + +import java.sql.Types; +import java.util.Arrays; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; + +public class TestJdbcComputePushdown +{ + private static final Metadata METADATA = MetadataManager.createTestMetadataManager(); + private static final PlanBuilder PLAN_BUILDER = new PlanBuilder(TEST_SESSION, new PlanNodeIdAllocator(), METADATA); + private static final PlanNodeIdAllocator ID_ALLOCATOR = new PlanNodeIdAllocator(); + private static final String CATALOG_NAME = "Jdbc"; + private static final String CONNECTOR_ID = new ConnectorId(CATALOG_NAME).toString(); + + private final TestingRowExpressionTranslator sqlToRowExpressionTranslator; + + private final JdbcComputePushdown jdbcComputePushdown; + + public TestJdbcComputePushdown() + { + this.sqlToRowExpressionTranslator = new TestingRowExpressionTranslator(METADATA); + FunctionManager functionManager = METADATA.getFunctionManager(); + StandardFunctionResolution functionResolution = new FunctionResolution(functionManager); + DeterminismEvaluator determinismEvaluator = new RowExpressionDeterminismEvaluator(functionManager); + + this.jdbcComputePushdown = new JdbcComputePushdown( + functionManager, + functionResolution, + determinismEvaluator, + new RowExpressionOptimizer(METADATA), + "'", + getFunctionTranslators()); + } + + @Test + public void testJdbcComputePushdownAll() + { + String table = "test_table"; + String schema = "test_schema"; + + String expression = "(c1 + c2) - c2"; + TypeProvider typeProvider = TypeProvider.copyOf(ImmutableMap.of("c1", BIGINT, "c2", BIGINT)); + RowExpression rowExpression = sqlToRowExpressionTranslator.translateAndOptimize(expression(expression), typeProvider); + Set columns = Stream.of("c1", "c2").map(TestJdbcComputePushdown::integerJdbcColumnHandle).collect(Collectors.toSet()); + PlanNode original = filter(jdbcTableScan(schema, table, BIGINT, "c1", "c2"), rowExpression); + + JdbcTableHandle jdbcTableHandle = new JdbcTableHandle(CONNECTOR_ID, new SchemaTableName(schema, table), CATALOG_NAME, schema, table); + JdbcTableLayoutHandle jdbcTableLayoutHandle = new JdbcTableLayoutHandle(jdbcTableHandle, TupleDomain.none(), Optional.of(new JdbcExpression("(('c1' + 'c2') - 'c2')"))); + + ConnectorSession session = new TestingConnectorSession(ImmutableList.of()); + PlanNode actual = this.jdbcComputePushdown.optimize(original, session, null, ID_ALLOCATOR); + assertPlanMatch( + actual, + PlanMatchPattern.filter(expression, JdbcTableScanMatcher.jdbcTableScanPattern(jdbcTableLayoutHandle, columns))); + } + + @Test + public void testJdbcComputePushdownBooleanOperations() + { + String table = "test_table"; + String schema = "test_schema"; + + String expression = "(((c1 + c2) - c2 <> c2) OR c2 = c1) AND c1 <> c2"; + TypeProvider typeProvider = TypeProvider.copyOf(ImmutableMap.of("c1", BIGINT, "c2", BIGINT)); + RowExpression rowExpression = sqlToRowExpressionTranslator.translateAndOptimize(expression(expression), typeProvider); + Set columns = Stream.of("c1", "c2").map(TestJdbcComputePushdown::integerJdbcColumnHandle).collect(Collectors.toSet()); + PlanNode original = filter(jdbcTableScan(schema, table, BIGINT, "c1", "c2"), rowExpression); + + JdbcTableHandle jdbcTableHandle = new JdbcTableHandle(CONNECTOR_ID, new SchemaTableName(schema, table), CATALOG_NAME, schema, table); + JdbcTableLayoutHandle jdbcTableLayoutHandle = new JdbcTableLayoutHandle( + jdbcTableHandle, + TupleDomain.none(), + Optional.of(new JdbcExpression("((((((('c1' + 'c2') - 'c2') <> 'c2')) OR (('c2' = 'c1')))) AND (('c1' <> 'c2')))"))); + + ConnectorSession session = new TestingConnectorSession(ImmutableList.of()); + PlanNode actual = this.jdbcComputePushdown.optimize(original, session, null, ID_ALLOCATOR); + assertPlanMatch( + actual, + PlanMatchPattern.filter(expression, JdbcTableScanMatcher.jdbcTableScanPattern(jdbcTableLayoutHandle, columns))); + } + + @Test + public void testJdbcComputePushdownUnsupported() + { + String table = "test_table"; + String schema = "test_schema"; + + String expression = "(c1 + c2) > c2"; + TypeProvider typeProvider = TypeProvider.copyOf(ImmutableMap.of("c1", BIGINT, "c2", BIGINT)); + RowExpression rowExpression = sqlToRowExpressionTranslator.translateAndOptimize(expression(expression), typeProvider); + Set columns = Stream.of("c1", "c2").map(TestJdbcComputePushdown::integerJdbcColumnHandle).collect(Collectors.toSet()); + PlanNode original = filter(jdbcTableScan(schema, table, BIGINT, "c1", "c2"), rowExpression); + + JdbcTableHandle jdbcTableHandle = new JdbcTableHandle(CONNECTOR_ID, new SchemaTableName(schema, table), CATALOG_NAME, schema, table); + // Test should expect an empty entry for translatedSql since > is an unsupported function currently in the optimizer + JdbcTableLayoutHandle jdbcTableLayoutHandle = new JdbcTableLayoutHandle(jdbcTableHandle, TupleDomain.none(), Optional.empty()); + + ConnectorSession session = new TestingConnectorSession(ImmutableList.of()); + PlanNode actual = this.jdbcComputePushdown.optimize(original, session, null, ID_ALLOCATOR); + assertPlanMatch(actual, PlanMatchPattern.filter( + expression, + JdbcTableScanMatcher.jdbcTableScanPattern(jdbcTableLayoutHandle, columns))); + } + + @Test + public void testJdbcComputePushdownWithConstants() + { + String table = "test_table"; + String schema = "test_schema"; + + String expression = "(c1 + c2) = 3"; + TypeProvider typeProvider = TypeProvider.copyOf(ImmutableMap.of("c1", BIGINT, "c2", BIGINT)); + RowExpression rowExpression = sqlToRowExpressionTranslator.translateAndOptimize(expression(expression), typeProvider); + Set columns = Stream.of("c1", "c2").map(TestJdbcComputePushdown::integerJdbcColumnHandle).collect(Collectors.toSet()); + PlanNode original = filter(jdbcTableScan(schema, table, BIGINT, "c1", "c2"), rowExpression); + + JdbcTableHandle jdbcTableHandle = new JdbcTableHandle(CONNECTOR_ID, new SchemaTableName(schema, table), CATALOG_NAME, schema, table); + JdbcTableLayoutHandle jdbcTableLayoutHandle = new JdbcTableLayoutHandle( + jdbcTableHandle, + TupleDomain.none(), + Optional.of(new JdbcExpression("(('c1' + 'c2') = ?)", ImmutableList.of(new ConstantExpression(Long.valueOf(3), INTEGER))))); + + ConnectorSession session = new TestingConnectorSession(ImmutableList.of()); + PlanNode actual = this.jdbcComputePushdown.optimize(original, session, null, ID_ALLOCATOR); + assertPlanMatch(actual, PlanMatchPattern.filter( + expression, + JdbcTableScanMatcher.jdbcTableScanPattern(jdbcTableLayoutHandle, columns))); + } + + @Test + public void testJdbcComputePushdownNotOperator() + { + String table = "test_table"; + String schema = "test_schema"; + + String expression = "c1 AND NOT(c2)"; + TypeProvider typeProvider = TypeProvider.copyOf(ImmutableMap.of("c1", BOOLEAN, "c2", BOOLEAN)); + RowExpression rowExpression = sqlToRowExpressionTranslator.translateAndOptimize(expression(expression), typeProvider); + PlanNode original = filter(jdbcTableScan(schema, table, BOOLEAN, "c1", "c2"), rowExpression); + + Set columns = Stream.of("c1", "c2").map(TestJdbcComputePushdown::booleanJdbcColumnHandle).collect(Collectors.toSet()); + JdbcTableHandle jdbcTableHandle = new JdbcTableHandle(CONNECTOR_ID, new SchemaTableName(schema, table), CATALOG_NAME, schema, table); + JdbcTableLayoutHandle jdbcTableLayoutHandle = new JdbcTableLayoutHandle( + jdbcTableHandle, + TupleDomain.none(), + Optional.of(new JdbcExpression("(('c1') AND ((NOT('c2'))))"))); + + ConnectorSession session = new TestingConnectorSession(ImmutableList.of()); + PlanNode actual = this.jdbcComputePushdown.optimize(original, session, null, ID_ALLOCATOR); + assertPlanMatch(actual, PlanMatchPattern.filter( + expression, + JdbcTableScanMatcher.jdbcTableScanPattern(jdbcTableLayoutHandle, columns))); + } + + private Set> getFunctionTranslators() + { + return ImmutableSet.of(OperatorTranslators.class); + } + + private static VariableReferenceExpression newVariable(String name, Type type) + { + return new VariableReferenceExpression(name, type); + } + + private static JdbcColumnHandle integerJdbcColumnHandle(String name) + { + return new JdbcColumnHandle(CONNECTOR_ID, name, new JdbcTypeHandle(Types.BIGINT, 10, 0), BIGINT, false); + } + + private static JdbcColumnHandle booleanJdbcColumnHandle(String name) + { + return new JdbcColumnHandle(CONNECTOR_ID, name, new JdbcTypeHandle(Types.BOOLEAN, 1, 0), BOOLEAN, false); + } + + private static JdbcColumnHandle getColumnHandleForVariable(String name, Type type) + { + return type.equals(BOOLEAN) ? booleanJdbcColumnHandle(name) : integerJdbcColumnHandle(name); + } + + private static void assertPlanMatch(PlanNode actual, PlanMatchPattern expected) + { + assertPlanMatch(actual, expected, TypeProvider.empty()); + } + + private static void assertPlanMatch(PlanNode actual, PlanMatchPattern expected, TypeProvider typeProvider) + { + PlanAssert.assertPlan( + TEST_SESSION, + METADATA, + (node, sourceStats, lookup, session, types) -> PlanNodeStatsEstimate.unknown(), + new Plan(actual, typeProvider, StatsAndCosts.empty()), + expected); + } + + private TableScanNode jdbcTableScan(String schema, String table, Type type, String... columnNames) + { + JdbcTableHandle jdbcTableHandle = new JdbcTableHandle(CONNECTOR_ID, new SchemaTableName(schema, table), CATALOG_NAME, schema, table); + JdbcTableLayoutHandle jdbcTableLayoutHandle = new JdbcTableLayoutHandle(jdbcTableHandle, TupleDomain.none(), Optional.empty()); + TableHandle tableHandle = new TableHandle(new ConnectorId(CATALOG_NAME), jdbcTableHandle, new ConnectorTransactionHandle() {}, Optional.of(jdbcTableLayoutHandle)); + + return PLAN_BUILDER.tableScan( + tableHandle, + Arrays.stream(columnNames).map(column -> newVariable(column, type)).collect(toImmutableList()), + Arrays.stream(columnNames) + .map(column -> newVariable(column, type)) + .collect(toMap(identity(), entry -> getColumnHandleForVariable(entry.getName(), type)))); + } + + private FilterNode filter(PlanNode source, RowExpression predicate) + { + return PLAN_BUILDER.filter(predicate, source); + } + + private static final class JdbcTableScanMatcher + implements Matcher + { + private final JdbcTableLayoutHandle jdbcTableLayoutHandle; + private final Set columns; + + static PlanMatchPattern jdbcTableScanPattern(JdbcTableLayoutHandle jdbcTableLayoutHandle, Set columns) + { + return node(TableScanNode.class).with(new JdbcTableScanMatcher(jdbcTableLayoutHandle, columns)); + } + + private JdbcTableScanMatcher(JdbcTableLayoutHandle jdbcTableLayoutHandle, Set columns) + { + this.jdbcTableLayoutHandle = jdbcTableLayoutHandle; + this.columns = columns; + } + + @Override + public boolean shapeMatches(PlanNode node) + { + return node instanceof TableScanNode; + } + + @Override + public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) + { + checkState(shapeMatches(node), "Plan testing framework error: shapeMatches returned false in detailMatches in %s", this.getClass().getName()); + + TableScanNode tableScanNode = (TableScanNode) node; + JdbcTableLayoutHandle layoutHandle = (JdbcTableLayoutHandle) tableScanNode.getTable().getLayout().get(); + if (jdbcTableLayoutHandle.getTable().equals(layoutHandle.getTable()) + && jdbcTableLayoutHandle.getTupleDomain().equals(layoutHandle.getTupleDomain()) + && ((!jdbcTableLayoutHandle.getAdditionalPredicate().isPresent() && !layoutHandle.getAdditionalPredicate().isPresent()) + || jdbcTableLayoutHandle.getAdditionalPredicate().get().getExpression().equals(layoutHandle.getAdditionalPredicate().get().getExpression()))) { + return MatchResult.match( + SymbolAliases.builder().putAll( + columns.stream() + .map(column -> ((JdbcColumnHandle) column).getColumnName()) + .collect(toMap(identity(), SymbolReference::new))) + .build()); + } + + return MatchResult.NO_MATCH; + } + } +} diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index c5b0b3f1b87df..c27f69edacd00 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-benchmark-driver @@ -28,17 +28,17 @@ - io.airlift + com.facebook.airlift discovery - io.airlift + com.facebook.airlift http-client - io.airlift + com.facebook.airlift json @@ -48,7 +48,7 @@ - io.airlift + com.facebook.airlift log-manager diff --git a/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkDriverOptions.java b/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkDriverOptions.java index b318a85524f26..c54d4b0a030f8 100644 --- a/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkDriverOptions.java +++ b/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkDriverOptions.java @@ -96,7 +96,6 @@ public ClientSession getClientSession() null, catalog, schema, - null, TimeZone.getDefault().getID(), Locale.getDefault(), ImmutableMap.of(), diff --git a/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkQueryRunner.java b/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkQueryRunner.java index 71e04c2fd3709..2f478e4388280 100644 --- a/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkQueryRunner.java +++ b/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkQueryRunner.java @@ -13,6 +13,13 @@ */ package com.facebook.presto.benchmark.driver; +import com.facebook.airlift.discovery.client.ServiceDescriptor; +import com.facebook.airlift.discovery.client.ServiceDescriptorsRepresentation; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpClientConfig; +import com.facebook.airlift.http.client.JsonResponseHandler; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.jetty.JettyHttpClient; import com.facebook.presto.client.ClientSession; import com.facebook.presto.client.QueryData; import com.facebook.presto.client.QueryError; @@ -20,13 +27,6 @@ import com.facebook.presto.client.StatementStats; import com.google.common.collect.ImmutableList; import com.google.common.net.HostAndPort; -import io.airlift.discovery.client.ServiceDescriptor; -import io.airlift.discovery.client.ServiceDescriptorsRepresentation; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpClientConfig; -import io.airlift.http.client.JsonResponseHandler; -import io.airlift.http.client.Request; -import io.airlift.http.client.jetty.JettyHttpClient; import io.airlift.units.Duration; import okhttp3.OkHttpClient; @@ -38,6 +38,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.airlift.http.client.StringResponseHandler.createStringResponseHandler; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.benchmark.driver.BenchmarkQueryResult.failResult; import static com.facebook.presto.benchmark.driver.BenchmarkQueryResult.passResult; import static com.facebook.presto.client.OkHttpUtil.setupCookieJar; @@ -45,11 +50,6 @@ import static com.facebook.presto.client.StatementClientFactory.newStatementClient; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Verify.verify; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.http.client.StringResponseHandler.createStringResponseHandler; -import static io.airlift.json.JsonCodec.jsonCodec; import static java.lang.Long.parseLong; import static java.lang.String.format; import static java.util.Objects.requireNonNull; diff --git a/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/PrestoBenchmarkDriver.java b/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/PrestoBenchmarkDriver.java index a0558b80f2c78..e69dd85139a16 100644 --- a/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/PrestoBenchmarkDriver.java +++ b/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/PrestoBenchmarkDriver.java @@ -13,20 +13,19 @@ */ package com.facebook.presto.benchmark.driver; +import com.facebook.airlift.log.Level; +import com.facebook.airlift.log.Logging; +import com.facebook.airlift.log.LoggingConfiguration; import com.facebook.presto.client.ClientSession; import com.google.common.collect.ImmutableList; import io.airlift.airline.Command; import io.airlift.airline.HelpOption; -import io.airlift.log.Level; -import io.airlift.log.Logging; -import io.airlift.log.LoggingConfiguration; import javax.inject.Inject; import java.io.File; import java.io.IOException; import java.io.PrintStream; -import java.io.UncheckedIOException; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -161,9 +160,6 @@ public static void initializeLogging(boolean debug) logging.disableConsole(); } } - catch (IOException e) { - throw new UncheckedIOException(e); - } finally { System.setOut(out); System.setErr(err); diff --git a/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/Suite.java b/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/Suite.java index 282236e91ab7b..19b917f0e1b74 100644 --- a/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/Suite.java +++ b/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/Suite.java @@ -27,11 +27,11 @@ import java.util.Optional; import java.util.regex.Pattern; +import static com.facebook.airlift.json.JsonCodec.mapJsonCodec; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Streams.stream; -import static io.airlift.json.JsonCodec.mapJsonCodec; import static java.util.Objects.requireNonNull; public class Suite diff --git a/presto-benchmark-runner/pom.xml b/presto-benchmark-runner/pom.xml new file mode 100644 index 0000000000000..0c34849d9fd1d --- /dev/null +++ b/presto-benchmark-runner/pom.xml @@ -0,0 +1,96 @@ + + + 4.0.0 + + + presto-root + com.facebook.presto + 0.231-SNAPSHOT + + + presto-benchmark-runner + presto-benchmark-runner + + + ${project.parent.basedir} + + + + + com.google.inject + guice + + + + com.google.guava + guava + + + + org.jdbi + jdbi3-core + + + + org.jdbi + jdbi3-sqlobject + + + + com.facebook.airlift + bootstrap + + + + com.facebook.airlift + configuration + + + + com.facebook.airlift + json + + + + io.airlift + units + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + javax.validation + validation-api + + + + + org.testng + testng + test + + + + com.facebook.airlift + testing + test + + + + com.facebook.presto + testing-mysql-server-5 + test + + + + com.facebook.presto + testing-mysql-server-base + test + + + diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkQuery.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkQuery.java new file mode 100644 index 0000000000000..49b204e8f6f74 --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkQuery.java @@ -0,0 +1,104 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.framework; + +import org.jdbi.v3.core.mapper.reflect.ColumnName; +import org.jdbi.v3.core.mapper.reflect.JdbiConstructor; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class BenchmarkQuery +{ + private final String querySet; + private final String name; + private final String query; + private final String catalog; + private final String schema; + + @JdbiConstructor + public BenchmarkQuery( + @ColumnName("query_set") String querySet, + @ColumnName("name") String name, + @ColumnName("query") String query, + @ColumnName("catalog") String catalog, + @ColumnName("schema") String schema) + { + this.querySet = requireNonNull(querySet, "querySet is null"); + this.name = requireNonNull(name, "name is null"); + this.query = clean(query); + this.catalog = requireNonNull(catalog, "catalog is null"); + this.schema = requireNonNull(schema, "schema is null"); + } + + public String getQuerySet() + { + return querySet; + } + + public String getName() + { + return name; + } + + public String getQuery() + { + return query; + } + + public String getCatalog() + { + return catalog; + } + + public String getSchema() + { + return schema; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + BenchmarkQuery o = (BenchmarkQuery) obj; + return Objects.equals(querySet, o.querySet) && + Objects.equals(name, o.name) && + Objects.equals(query, o.query) && + Objects.equals(catalog, o.catalog) && + Objects.equals(schema, o.schema); + } + + @Override + public int hashCode() + { + return Objects.hash(querySet, name, query, catalog, schema); + } + + private static String clean(String sql) + { + sql = sql.replaceAll("\t", " "); + sql = sql.replaceAll("\n+", "\n"); + sql = sql.trim(); + while (sql.endsWith(";")) { + sql = sql.substring(0, sql.length() - 1).trim(); + } + return sql; + } +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkRunnerConfig.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkRunnerConfig.java new file mode 100644 index 0000000000000..11abafc668360 --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkRunnerConfig.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.framework; + +import com.facebook.airlift.configuration.Config; + +import javax.validation.constraints.NotNull; + +import static com.facebook.presto.benchmark.source.DbBenchmarkSuiteSupplier.BENCHMARK_SUITE_SUPPLIER; + +public class BenchmarkRunnerConfig +{ + private String testId; + private String benchmarkSuiteSupplier = BENCHMARK_SUITE_SUPPLIER; + + @NotNull + public String getTestId() + { + return testId; + } + + @Config("test-id") + public BenchmarkRunnerConfig setTestId(String testId) + { + this.testId = testId; + return this; + } + + @NotNull + public String getBenchmarkSuiteSupplier() + { + return benchmarkSuiteSupplier; + } + + @Config("benchmark-suite-supplier") + public BenchmarkRunnerConfig setBenchmarkSuiteSupplier(String benchmarkSuiteSupplier) + { + this.benchmarkSuiteSupplier = benchmarkSuiteSupplier; + return this; + } +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkSuite.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkSuite.java new file mode 100644 index 0000000000000..6be29042dcae6 --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkSuite.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.framework; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Objects.requireNonNull; +import static java.util.function.UnaryOperator.identity; + +public class BenchmarkSuite +{ + private final BenchmarkSuiteInfo suiteInfo; + private final Map queries; + + public BenchmarkSuite( + BenchmarkSuiteInfo suiteInfo, + List queries) + { + this.suiteInfo = requireNonNull(suiteInfo, "SuiteInfo is null"); + this.queries = queries.stream() + .collect(toImmutableMap(BenchmarkQuery::getName, identity())); + } + + public BenchmarkSuiteInfo getSuiteInfo() + { + return suiteInfo; + } + + public Map getQueryMap() + { + return queries; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + BenchmarkSuite o = (BenchmarkSuite) obj; + return Objects.equals(suiteInfo, o.suiteInfo) && + Objects.equals(queries, o.queries); + } + + @Override + public int hashCode() + { + return Objects.hash(suiteInfo, queries); + } +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkSuiteInfo.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkSuiteInfo.java new file mode 100644 index 0000000000000..b57bd8c17db16 --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/BenchmarkSuiteInfo.java @@ -0,0 +1,87 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.framework; + +import com.google.common.collect.ImmutableMap; +import org.jdbi.v3.core.mapper.reflect.ColumnName; +import org.jdbi.v3.core.mapper.reflect.JdbiConstructor; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class BenchmarkSuiteInfo +{ + private final String suite; + private final String querySet; + private final List phases; + private final Map sessionProperties; + + @JdbiConstructor + public BenchmarkSuiteInfo( + @ColumnName("suite") String suite, + @ColumnName("query_set") String querySet, + @ColumnName("phases") List phases, + @ColumnName("session_properties") Map sessionProperties) + { + this.suite = requireNonNull(suite, "suite is null"); + this.querySet = requireNonNull(querySet, "querySet is null"); + this.phases = requireNonNull(phases, "phases is null"); + this.sessionProperties = ImmutableMap.copyOf(sessionProperties); + } + + public String getSuite() + { + return suite; + } + + public String getQuerySet() + { + return querySet; + } + + public List getPhases() + { + return phases; + } + + public Map getSessionProperties() + { + return sessionProperties; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + BenchmarkSuiteInfo o = (BenchmarkSuiteInfo) obj; + return Objects.equals(suite, o.getSuite()) && + Objects.equals(querySet, o.querySet) && + Objects.equals(phases, o.phases) && + Objects.equals(sessionProperties, o.sessionProperties); + } + + @Override + public int hashCode() + { + return Objects.hash(suite, querySet, phases, sessionProperties); + } +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/ConcurrentExecutionPhase.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/ConcurrentExecutionPhase.java new file mode 100644 index 0000000000000..a6967b8793a86 --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/ConcurrentExecutionPhase.java @@ -0,0 +1,72 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.framework; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class ConcurrentExecutionPhase + extends PhaseSpecification +{ + private final ExecutionStrategy executionStrategy; + private final List queries; + + @JsonCreator + public ConcurrentExecutionPhase(String name, ExecutionStrategy executionStrategy, List queries) + { + super(name); + this.executionStrategy = requireNonNull(executionStrategy, "executionStrategy is null"); + this.queries = requireNonNull(ImmutableList.copyOf(queries), "queries is null"); + } + + @JsonProperty + @Override + public ExecutionStrategy getExecutionStrategy() + { + return executionStrategy; + } + + @JsonProperty + public List getQueries() + { + return queries; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ConcurrentExecutionPhase o = (ConcurrentExecutionPhase) obj; + return Objects.equals(getExecutionStrategy(), o.getExecutionStrategy()) && + Objects.equals(getName(), o.getName()) && + Objects.equals(queries, o.queries); + } + + @Override + public int hashCode() + { + return Objects.hash(getExecutionStrategy(), getName(), queries); + } +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/MySqlBenchmarkSuiteConfig.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/MySqlBenchmarkSuiteConfig.java new file mode 100644 index 0000000000000..7d32722615e47 --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/MySqlBenchmarkSuiteConfig.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.framework; + +import com.facebook.airlift.configuration.Config; + +import javax.validation.constraints.NotNull; + +public class MySqlBenchmarkSuiteConfig +{ + private String databaseUrl; + + @NotNull + public String getDatabaseUrl() + { + return databaseUrl; + } + + @Config("database-url") + public MySqlBenchmarkSuiteConfig setDatabaseUrl(String databaseUrl) + { + this.databaseUrl = databaseUrl; + return this; + } +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/PhaseSpecification.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/PhaseSpecification.java new file mode 100644 index 0000000000000..5d4abe5cf9515 --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/PhaseSpecification.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.framework; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import static java.util.Objects.requireNonNull; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "@type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = StreamExecutionPhase.class, name = "stream"), + @JsonSubTypes.Type(value = ConcurrentExecutionPhase.class, name = "concurrent")}) +public abstract class PhaseSpecification +{ + public enum ExecutionStrategy + { + STREAM, + CONCURRENT + } + + private final String name; + + public abstract ExecutionStrategy getExecutionStrategy(); + + public PhaseSpecification(String name) + { + this.name = requireNonNull(name, "name is null"); + } + + @JsonProperty + public String getName() + { + return name; + } +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/StreamExecutionPhase.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/StreamExecutionPhase.java new file mode 100644 index 0000000000000..e8f01f0682702 --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/framework/StreamExecutionPhase.java @@ -0,0 +1,75 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.framework; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +public class StreamExecutionPhase + extends PhaseSpecification +{ + private final ExecutionStrategy executionStrategy; + private final List> streams; + + @JsonCreator + public StreamExecutionPhase(String name, ExecutionStrategy executionStrategy, List> streams) + { + super(name); + this.executionStrategy = requireNonNull(executionStrategy, "executionStrategy is null"); + this.streams = streams.stream() + .map(ImmutableList::copyOf) + .collect(toImmutableList()); + } + + @JsonProperty + @Override + public ExecutionStrategy getExecutionStrategy() + { + return executionStrategy; + } + + @JsonProperty + public List> getStreams() + { + return streams; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + StreamExecutionPhase o = (StreamExecutionPhase) obj; + return Objects.equals(getExecutionStrategy(), o.getExecutionStrategy()) && + Objects.equals(getName(), o.getName()) && + Objects.equals(streams, o.streams); + } + + @Override + public int hashCode() + { + return Objects.hash(getExecutionStrategy(), getName(), streams); + } +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteConfig.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteConfig.java new file mode 100644 index 0000000000000..71502fb785104 --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteConfig.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.source; + +import com.facebook.airlift.configuration.Config; + +import javax.validation.constraints.NotNull; + +public class BenchmarkSuiteConfig +{ + private String suite; + private String suitesTableName = "benchmark_suites"; + private String queriesTableName = "benchmark_queries"; + + @NotNull + public String getSuite() + { + return suite; + } + + @Config("suite") + public BenchmarkSuiteConfig setSuite(String suite) + { + this.suite = suite; + return this; + } + + @NotNull + public String getSuitesTableName() + { + return suitesTableName; + } + + @Config("suites-table-name") + public BenchmarkSuiteConfig setSuitesTableName(String suitesTableName) + { + this.suitesTableName = suitesTableName; + return this; + } + + @NotNull + public String getQueriesTableName() + { + return queriesTableName; + } + + @Config("queries-table-name") + public BenchmarkSuiteConfig setQueriesTableName(String queriesTableName) + { + this.queriesTableName = queriesTableName; + return this; + } +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteDao.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteDao.java new file mode 100644 index 0000000000000..dfe6b5710778e --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteDao.java @@ -0,0 +1,87 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.source; + +import com.facebook.presto.benchmark.framework.BenchmarkQuery; +import com.facebook.presto.benchmark.framework.BenchmarkSuiteInfo; +import org.jdbi.v3.sqlobject.config.RegisterColumnMapper; +import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; + +import java.util.List; + +@RegisterColumnMapper(StringToStringMapColumnMapper.class) +@RegisterColumnMapper(PhaseSpecificationsColumnMapper.class) +public interface BenchmarkSuiteDao +{ + @SqlUpdate("CREATE TABLE (\n" + + " `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,\n" + + " `suite` varchar(256) NOT NULL,\n" + + " `query_set` varchar(256) NOT NULL,\n" + + " `phases` mediumtext NOT NULL,\n" + + " `session_properties` mediumtext NOT NULL,\n" + + " `created_by` varchar(256) NOT NULL,\n" + + " `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n" + + " `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n" + + " PRIMARY KEY (`id`),\n" + + " UNIQUE KEY suite (`suite`))") + void createBenchmarkSuitesTable( + @Define("table_name") String tableName); + + @SqlUpdate("CREATE TABLE (\n" + + " `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,\n" + + " `query_set` varchar(256) NOT NULL,\n" + + " `name` varchar(256) NOT NULL,\n" + + " `catalog` varchar(256) NOT NULL,\n" + + " `schema` varchar(256) NOT NULL,\n" + + " `query` mediumtext NOT NULL,\n" + + " `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n" + + " `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n" + + " PRIMARY KEY (`id`),\n" + + " UNIQUE KEY `query_set_name` (`query_set`,`name`))") + void createBenchmarkQueriesTable( + @Define("table_name") String tableName); + + @SqlQuery("SELECT\n" + + " `suite`,\n" + + " `query_set`,\n" + + " `phases`,\n" + + " `session_properties`\n" + + "FROM\n" + + " \n" + + "WHERE\n" + + " `suite` = :suite\n") + @RegisterConstructorMapper(BenchmarkSuiteInfo.class) + BenchmarkSuiteInfo getBenchmarkSuiteInfo( + @Define("table_name") String tableName, + @Bind("suite") String suite); + + @SqlQuery("SELECT\n" + + " `name`,\n" + + " `query_set`,\n" + + " `catalog`,\n" + + " `schema`,\n" + + " `query`\n" + + "FROM\n" + + " \n" + + "WHERE\n" + + " `query_set` = :query_set\n") + @RegisterConstructorMapper(BenchmarkQuery.class) + List getBenchmarkQueries( + @Define("table_name") String tableName, + @Bind("query_set") String querySet); +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteModule.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteModule.java new file mode 100644 index 0000000000000..149b8d39effe4 --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteModule.java @@ -0,0 +1,58 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.source; + +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.presto.benchmark.framework.BenchmarkRunnerConfig; +import com.facebook.presto.benchmark.framework.MySqlBenchmarkSuiteConfig; +import com.google.common.collect.ImmutableSet; +import com.google.inject.Binder; +import org.jdbi.v3.core.Jdbi; +import org.jdbi.v3.sqlobject.SqlObjectPlugin; + +import java.sql.DriverManager; +import java.util.Set; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.presto.benchmark.source.DbBenchmarkSuiteSupplier.BENCHMARK_SUITE_SUPPLIER; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.inject.Scopes.SINGLETON; + +public class BenchmarkSuiteModule + extends AbstractConfigurationAwareModule +{ + private final Set supportedBenchmarkSuiteSuppliers; + + public BenchmarkSuiteModule(Set customBenchmarkSuiteSuppliers) + { + this.supportedBenchmarkSuiteSuppliers = ImmutableSet.builder() + .add(BENCHMARK_SUITE_SUPPLIER) + .addAll(customBenchmarkSuiteSuppliers) + .build(); + } + + @Override + protected void setup(Binder binder) + { + String benchmarkSuiteSupplier = buildConfigObject(BenchmarkRunnerConfig.class).getBenchmarkSuiteSupplier(); + checkArgument(supportedBenchmarkSuiteSuppliers.contains(benchmarkSuiteSupplier), "Unsupported BenchmarkSuiteSupplier: %s", benchmarkSuiteSupplier); + + if (BENCHMARK_SUITE_SUPPLIER.equals(benchmarkSuiteSupplier)) { + configBinder(binder).bindConfig(BenchmarkSuiteConfig.class); + binder.bind(BenchmarkSuiteSupplier.class).to(DbBenchmarkSuiteSupplier.class).in(SINGLETON); + String database = buildConfigObject(MySqlBenchmarkSuiteConfig.class).getDatabaseUrl(); + binder.bind(Jdbi.class).toInstance(Jdbi.create(() -> DriverManager.getConnection(database)).installPlugin(new SqlObjectPlugin())); + } + } +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteSupplier.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteSupplier.java new file mode 100644 index 0000000000000..4fc1589006701 --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/BenchmarkSuiteSupplier.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.source; + +import com.facebook.presto.benchmark.framework.BenchmarkSuite; + +import java.util.function.Supplier; + +public interface BenchmarkSuiteSupplier + extends Supplier +{ +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/DbBenchmarkSuiteSupplier.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/DbBenchmarkSuiteSupplier.java new file mode 100644 index 0000000000000..60934febd314e --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/DbBenchmarkSuiteSupplier.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.source; + +import com.facebook.presto.benchmark.framework.BenchmarkSuite; +import com.facebook.presto.benchmark.framework.BenchmarkSuiteInfo; +import com.google.inject.Inject; +import org.jdbi.v3.core.Handle; +import org.jdbi.v3.core.Jdbi; + +import static java.util.Objects.requireNonNull; + +public class DbBenchmarkSuiteSupplier + implements BenchmarkSuiteSupplier +{ + public static final String BENCHMARK_SUITE_SUPPLIER = "mysql"; + + private final Jdbi jdbi; + private final String suitesTableName; + private final String queriesTableName; + private final String suite; + + @Inject + public DbBenchmarkSuiteSupplier(Jdbi jdbi, BenchmarkSuiteConfig config) + { + this.jdbi = requireNonNull(jdbi, "jdbi is null"); + this.suitesTableName = requireNonNull(config.getSuitesTableName(), "suites-table-name is null"); + this.queriesTableName = requireNonNull(config.getQueriesTableName(), "queries-table-name is null"); + this.suite = requireNonNull(config.getSuite(), "suite name is null"); + } + + @Override + public BenchmarkSuite get() + { + BenchmarkSuite benchmarkSuite; + try (Handle handle = jdbi.open()) { + BenchmarkSuiteDao benchmarkDao = handle.attach(BenchmarkSuiteDao.class); + BenchmarkSuiteInfo benchmarkSuiteInfo = benchmarkDao.getBenchmarkSuiteInfo(suitesTableName, suite); + benchmarkSuite = new BenchmarkSuite(benchmarkSuiteInfo, benchmarkDao.getBenchmarkQueries(queriesTableName, benchmarkSuiteInfo.getQuerySet())); + } + return benchmarkSuite; + } +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/PhaseSpecificationsColumnMapper.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/PhaseSpecificationsColumnMapper.java new file mode 100644 index 0000000000000..47e9919a080f2 --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/PhaseSpecificationsColumnMapper.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.source; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.benchmark.framework.PhaseSpecification; +import org.jdbi.v3.core.mapper.ColumnMapper; +import org.jdbi.v3.core.statement.StatementContext; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; + +public class PhaseSpecificationsColumnMapper + implements ColumnMapper> +{ + protected static final JsonCodec> PHASE_SPECIFICATION_LIST_CODEC = listJsonCodec(PhaseSpecification.class); + + @Override + public List map(ResultSet resultSet, int columnNumber, StatementContext ctx) + throws SQLException + { + String columnValue = resultSet.getString(columnNumber); + return columnValue == null ? null : PHASE_SPECIFICATION_LIST_CODEC.fromJson(columnValue); + } +} diff --git a/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/StringToStringMapColumnMapper.java b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/StringToStringMapColumnMapper.java new file mode 100644 index 0000000000000..5154a86ecc1fd --- /dev/null +++ b/presto-benchmark-runner/src/main/java/com/facebook/presto/benchmark/source/StringToStringMapColumnMapper.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.source; + +import com.facebook.airlift.json.JsonCodec; +import org.jdbi.v3.core.mapper.ColumnMapper; +import org.jdbi.v3.core.statement.StatementContext; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; + +import static com.facebook.airlift.json.JsonCodec.mapJsonCodec; + +public class StringToStringMapColumnMapper + implements ColumnMapper> +{ + protected static final JsonCodec> MAP_CODEC = mapJsonCodec(String.class, String.class); + + @Override + public Map map(ResultSet resultSet, int columnNumber, StatementContext ctx) + throws SQLException + { + String columnValue = resultSet.getString(columnNumber); + return columnValue == null ? null : MAP_CODEC.fromJson(columnValue); + } +} diff --git a/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/BenchmarkTestUtil.java b/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/BenchmarkTestUtil.java new file mode 100644 index 0000000000000..14992297f4618 --- /dev/null +++ b/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/BenchmarkTestUtil.java @@ -0,0 +1,122 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark; + +import com.facebook.presto.benchmark.framework.BenchmarkQuery; +import com.facebook.presto.benchmark.framework.BenchmarkSuite; +import com.facebook.presto.benchmark.framework.BenchmarkSuiteInfo; +import com.facebook.presto.benchmark.framework.ConcurrentExecutionPhase; +import com.facebook.presto.benchmark.framework.PhaseSpecification; +import com.facebook.presto.benchmark.framework.StreamExecutionPhase; +import com.facebook.presto.benchmark.source.BenchmarkSuiteDao; +import com.facebook.presto.testing.mysql.TestingMySqlServer; +import com.google.common.collect.ImmutableList; +import org.jdbi.v3.core.Handle; +import org.jdbi.v3.core.Jdbi; +import org.jdbi.v3.sqlobject.SqlObjectPlugin; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.facebook.presto.benchmark.framework.PhaseSpecification.ExecutionStrategy.CONCURRENT; +import static com.facebook.presto.benchmark.framework.PhaseSpecification.ExecutionStrategy.STREAM; + +public class BenchmarkTestUtil +{ + public static final String CATALOG = "benchmark"; + public static final String SCHEMA = "default"; + public static final String XDB = "presto"; + + private BenchmarkTestUtil() + { + } + + public static TestingMySqlServer setupMySql() + throws Exception + { + TestingMySqlServer mySqlServer = new TestingMySqlServer("testuser", "testpass", ImmutableList.of(XDB)); + Handle handle = getJdbi(mySqlServer).open(); + BenchmarkSuiteDao benchmarkDao = handle.attach(BenchmarkSuiteDao.class); + benchmarkDao.createBenchmarkSuitesTable("benchmark_suites"); + benchmarkDao.createBenchmarkQueriesTable("benchmark_queries"); + return mySqlServer; + } + + public static Jdbi getJdbi(TestingMySqlServer mySqlServer) + { + return Jdbi.create(mySqlServer.getJdbcUrl(XDB)).installPlugin(new SqlObjectPlugin()); + } + + public static void insertBenchmarkQuery(Handle handle, String querySet, String name, String query) + { + handle.execute( + "INSERT INTO benchmark_queries(\n" + + " `query_set`, `name`, `catalog`, `schema`, `query`)\n" + + "SELECT\n" + + " ?,\n" + + " ?,\n" + + " 'benchmark',\n" + + " 'default',\n" + + " ?\n", + querySet, + name, + query); + } + + public static void insertBenchmarkSuite(Handle handle, String suite, String querySet, String phases, String sessionProperties) + { + handle.execute( + "INSERT INTO benchmark_suites(\n" + + " `suite`, `query_set`, `phases`, `session_properties`, `created_by`)\n" + + "SELECT\n" + + " ?,\n" + + " ?,\n" + + " ?,\n" + + " ?,\n" + + " 'benchmark'\n", + suite, + querySet, + phases, + sessionProperties); + } + + public static List getBenchmarkSuitePhases() + { + List> streams = ImmutableList.of(ImmutableList.of("Q1", "Q2"), ImmutableList.of("Q2", "Q3")); + PhaseSpecification streamExecutionPhase = new StreamExecutionPhase("Phase-1", STREAM, streams); + + List queries = ImmutableList.of("Q1", "Q2", "Q3"); + PhaseSpecification concurrentExecutionPhase = new ConcurrentExecutionPhase("Phase-2", CONCURRENT, queries); + + return ImmutableList.of(streamExecutionPhase, concurrentExecutionPhase); + } + + public static Map getBenchmarkSuiteSessionProperties() + { + Map sessionProperties = new HashMap(); + sessionProperties.put("max", "5"); + return sessionProperties; + } + + public static BenchmarkSuite getBenchmarkSuiteObject(String suite, String querySet) + { + BenchmarkQuery benchmarkQuery1 = new BenchmarkQuery(querySet, "Q1", "SELECT 1", CATALOG, SCHEMA); + BenchmarkQuery benchmarkQuery2 = new BenchmarkQuery(querySet, "Q2", "SELECT 2", CATALOG, SCHEMA); + BenchmarkQuery benchmarkQuery3 = new BenchmarkQuery(querySet, "Q3", "SELECT 3", CATALOG, SCHEMA); + + return new BenchmarkSuite(new BenchmarkSuiteInfo(suite, querySet, getBenchmarkSuitePhases(), getBenchmarkSuiteSessionProperties()), + ImmutableList.of(benchmarkQuery1, benchmarkQuery2, benchmarkQuery3)); + } +} diff --git a/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/framework/TestBenchmarkRunnerConfig.java b/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/framework/TestBenchmarkRunnerConfig.java new file mode 100644 index 0000000000000..42405ba77d9eb --- /dev/null +++ b/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/framework/TestBenchmarkRunnerConfig.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.framework; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestBenchmarkRunnerConfig +{ + @Test + public void testDefault() + { + assertRecordedDefaults(recordDefaults(BenchmarkRunnerConfig.class) + .setTestId(null) + .setBenchmarkSuiteSupplier("mysql")); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("test-id", "12345") + .put("benchmark-suite-supplier", "custom-supplier") + .build(); + BenchmarkRunnerConfig expected = new BenchmarkRunnerConfig() + .setTestId("12345") + .setBenchmarkSuiteSupplier("custom-supplier"); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/source/TestBenchmarkSuiteConfig.java b/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/source/TestBenchmarkSuiteConfig.java new file mode 100644 index 0000000000000..29a255e136a49 --- /dev/null +++ b/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/source/TestBenchmarkSuiteConfig.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.source; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestBenchmarkSuiteConfig +{ + @Test + public void testDefault() + { + assertRecordedDefaults(recordDefaults(BenchmarkSuiteConfig.class) + .setSuite(null) + .setSuitesTableName("benchmark_suites") + .setQueriesTableName("benchmark_queries")); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("suite", "test1") + .put("suites-table-name", "suite_source") + .put("queries-table-name", "query_source") + .build(); + BenchmarkSuiteConfig expected = new BenchmarkSuiteConfig() + .setSuite("test1") + .setSuitesTableName("suite_source") + .setQueriesTableName("query_source"); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/source/TestBenchmarkSuiteModule.java b/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/source/TestBenchmarkSuiteModule.java new file mode 100644 index 0000000000000..6693abbc56ecb --- /dev/null +++ b/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/source/TestBenchmarkSuiteModule.java @@ -0,0 +1,103 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.source; + +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.inject.CreationException; +import com.google.inject.Injector; +import com.google.inject.Module; +import org.testng.annotations.Test; + +import java.util.Map; + +import static org.testng.Assert.assertTrue; + +public class TestBenchmarkSuiteModule +{ + private static final Map DEFAULT_CONFIGURATION_PROPERTIES = ImmutableMap.builder() + .put("test-id", "test") + .build(); + + @Test + public void testDbBenchmarkSuiteSupplier() + { + Bootstrap app = new Bootstrap(ImmutableList.builder() + .add(new BenchmarkSuiteModule(ImmutableSet.of())) + .build()); + Injector injector = null; + try { + injector = app.strictConfig() + .setRequiredConfigurationProperties(ImmutableMap.builder() + .putAll(DEFAULT_CONFIGURATION_PROPERTIES) + .put("database-url", "jdbc://localhost:1080") + .put("suite", "test") + .build()) + .initialize(); + assertTrue(injector.getInstance(BenchmarkSuiteSupplier.class) instanceof DbBenchmarkSuiteSupplier); + } + finally { + if (injector != null) { + injector.getInstance(LifeCycleManager.class).stop(); + } + } + } + + @Test + public void testCustomBenchmarkSuiteSupplier() + { + Bootstrap app = new Bootstrap(ImmutableList.builder() + .add(new BenchmarkSuiteModule(ImmutableSet.of("test-supplier"))) + .build()); + Injector injector = null; + try { + injector = app.strictConfig() + .setRequiredConfigurationProperties(ImmutableMap.builder() + .putAll(DEFAULT_CONFIGURATION_PROPERTIES) + .put("benchmark-suite-supplier", "test-supplier") + .build()) + .initialize(); + } + finally { + if (injector != null) { + injector.getInstance(LifeCycleManager.class).stop(); + } + } + } + + @Test(expectedExceptions = CreationException.class, expectedExceptionsMessageRegExp = "Unable to create injector.*") + public void testInvalidBenchmarkSuiteSupplier() + { + Bootstrap app = new Bootstrap(ImmutableList.builder() + .add(new BenchmarkSuiteModule(ImmutableSet.of("test-supplier"))) + .build()); + Injector injector = null; + try { + injector = app.strictConfig() + .setRequiredConfigurationProperties(ImmutableMap.builder() + .putAll(DEFAULT_CONFIGURATION_PROPERTIES) + .put("benchmark-query-supplier", "unknown-supplier") + .build()) + .initialize(); + } + finally { + if (injector != null) { + injector.getInstance(LifeCycleManager.class).stop(); + } + } + } +} diff --git a/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/source/TestDbBenchmarkSuiteSupplier.java b/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/source/TestDbBenchmarkSuiteSupplier.java new file mode 100644 index 0000000000000..31d96cdbd1b93 --- /dev/null +++ b/presto-benchmark-runner/src/test/java/com/facebook/presto/benchmark/source/TestDbBenchmarkSuiteSupplier.java @@ -0,0 +1,78 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark.source; + +import com.facebook.presto.testing.mysql.TestingMySqlServer; +import org.jdbi.v3.core.Handle; +import org.jdbi.v3.core.Jdbi; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.facebook.airlift.testing.Closeables.closeQuietly; +import static com.facebook.presto.benchmark.BenchmarkTestUtil.getBenchmarkSuiteObject; +import static com.facebook.presto.benchmark.BenchmarkTestUtil.getBenchmarkSuitePhases; +import static com.facebook.presto.benchmark.BenchmarkTestUtil.getBenchmarkSuiteSessionProperties; +import static com.facebook.presto.benchmark.BenchmarkTestUtil.getJdbi; +import static com.facebook.presto.benchmark.BenchmarkTestUtil.insertBenchmarkQuery; +import static com.facebook.presto.benchmark.BenchmarkTestUtil.insertBenchmarkSuite; +import static com.facebook.presto.benchmark.BenchmarkTestUtil.setupMySql; +import static com.facebook.presto.benchmark.source.PhaseSpecificationsColumnMapper.PHASE_SPECIFICATION_LIST_CODEC; +import static com.facebook.presto.benchmark.source.StringToStringMapColumnMapper.MAP_CODEC; +import static org.testng.Assert.assertEquals; + +@Test(singleThreaded = true) +public class TestDbBenchmarkSuiteSupplier +{ + private static final String SUITE = "test_suite"; + private static final String QUERY_SET = "test_set"; + private static TestingMySqlServer mySqlServer; + private static Handle handle; + private static Jdbi jdbi; + + @BeforeClass + public void setup() + throws Exception + { + mySqlServer = setupMySql(); + jdbi = getJdbi(mySqlServer); + handle = jdbi.open(); + } + + @AfterClass(alwaysRun = true) + public void destroy() + { + closeQuietly(mySqlServer, handle); + } + + @AfterMethod + public void cleanup() + { + handle.execute("DELETE FROM benchmark_suites"); + handle.execute("DELETE FROM benchmark_queries"); + } + + @Test + public void testSupplySuite() + { + insertBenchmarkQuery(handle, QUERY_SET, "Q1", "SELECT 1"); + insertBenchmarkQuery(handle, QUERY_SET, "Q2", "SELECT 2"); + insertBenchmarkQuery(handle, QUERY_SET, "Q3", "SELECT 3"); + + insertBenchmarkSuite(handle, SUITE, QUERY_SET, PHASE_SPECIFICATION_LIST_CODEC.toJson(getBenchmarkSuitePhases()), MAP_CODEC.toJson(getBenchmarkSuiteSessionProperties())); + + assertEquals(new DbBenchmarkSuiteSupplier(jdbi, new BenchmarkSuiteConfig().setSuite(SUITE)).get(), getBenchmarkSuiteObject(SUITE, QUERY_SET)); + } +} diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 1adc1e4b225b3..c1f9112cfdce8 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-benchmark @@ -43,17 +43,17 @@ - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift log @@ -78,7 +78,7 @@ - io.airlift + com.facebook.airlift stats @@ -90,7 +90,7 @@ - io.airlift + com.facebook.airlift bootstrap runtime diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/AbstractOperatorBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/AbstractOperatorBenchmark.java index 35bd358c1f40c..65d13eba63bf2 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/AbstractOperatorBenchmark.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/AbstractOperatorBenchmark.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.benchmark; +import com.facebook.airlift.stats.CpuTimer; +import com.facebook.airlift.stats.TestingGcMonitor; import com.facebook.presto.Session; import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.TaskId; @@ -53,8 +55,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; -import io.airlift.stats.CpuTimer; -import io.airlift.stats.TestingGcMonitor; import io.airlift.units.DataSize; import java.util.ArrayList; @@ -64,6 +64,8 @@ import java.util.Optional; import java.util.OptionalInt; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.stats.CpuTimer.CpuDuration; import static com.facebook.presto.SystemSessionProperties.getFilterAndProjectMinOutputPageRowCount; import static com.facebook.presto.SystemSessionProperties.getFilterAndProjectMinOutputPageSize; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING; @@ -75,8 +77,6 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.stats.CpuTimer.CpuDuration; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; @@ -174,7 +174,7 @@ protected final OperatorFactory createTableScanOperator(int operatorId, PlanNode public Operator createOperator(DriverContext driverContext) { OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, "BenchmarkSource"); - ConnectorPageSource pageSource = localQueryRunner.getPageSourceManager().createPageSource(session, split, columnHandles); + ConnectorPageSource pageSource = localQueryRunner.getPageSourceManager().createPageSource(session, split, tableHandle, columnHandles); return new PageSourceOperator(pageSource, operatorContext); } @@ -224,7 +224,7 @@ protected final OperatorFactory createHashProjectOperator(int operatorId, PlanNo RowExpression translatedHashExpression = translate(hashExpression.get(), variableToInputMapping.build()); PageFunctionCompiler functionCompiler = new PageFunctionCompiler(localQueryRunner.getMetadata(), 0); - projections.add(functionCompiler.compileProjection(translatedHashExpression, Optional.empty()).get()); + projections.add(functionCompiler.compileProjection(session.getSqlFunctionProperties(), translatedHashExpression, Optional.empty()).get()); return new FilterAndProjectOperator.FilterAndProjectOperatorFactory( operatorId, @@ -280,7 +280,7 @@ protected Map runOnce() localQueryRunner.getScheduler(), new DataSize(256, MEGABYTE), spillSpaceTracker) - .addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0), localQueryRunner.getExecutor()), + .addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0, 0), localQueryRunner.getExecutor()), session, false, false, diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/BenchmarkSuite.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/BenchmarkSuite.java index 904af3d7c5db6..ee6e1defbd153 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/BenchmarkSuite.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/BenchmarkSuite.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.benchmark; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.testing.LocalQueryRunner; import com.google.common.collect.ImmutableList; import com.google.common.io.Files; -import io.airlift.log.Logger; import java.io.File; import java.io.FileOutputStream; diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/CountAggregationBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/CountAggregationBenchmark.java index ba9382944d22b..2d34b2eede407 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/CountAggregationBenchmark.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/CountAggregationBenchmark.java @@ -17,8 +17,8 @@ import com.facebook.presto.operator.AggregationOperator.AggregationOperatorFactory; import com.facebook.presto.operator.OperatorFactory; import com.facebook.presto.operator.aggregation.InternalAggregationFunction; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.PlanNodeId; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.testing.LocalQueryRunner; import com.google.common.collect.ImmutableList; diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/DoubleSumAggregationBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/DoubleSumAggregationBenchmark.java index feced1520ccc7..c541bd99c8e4e 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/DoubleSumAggregationBenchmark.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/DoubleSumAggregationBenchmark.java @@ -18,8 +18,8 @@ import com.facebook.presto.operator.AggregationOperator.AggregationOperatorFactory; import com.facebook.presto.operator.OperatorFactory; import com.facebook.presto.operator.aggregation.InternalAggregationFunction; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.PlanNodeId; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.testing.LocalQueryRunner; import com.google.common.collect.ImmutableList; diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery1.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery1.java index c1acd1baa9e5c..8cbbb861c45df 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery1.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery1.java @@ -24,9 +24,9 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.util.DateTimeUtils; import com.google.common.collect.ImmutableList; diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery6.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery6.java index 57e8d75d50e3a..1b189a2d8bb5f 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery6.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HandTpchQuery6.java @@ -26,9 +26,9 @@ import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.util.DateTimeUtils; import com.google.common.collect.ImmutableList; @@ -71,7 +71,7 @@ protected List createOperatorFactories() // and quantity < 24; OperatorFactory tableScanOperator = createTableScanOperator(0, new PlanNodeId("test"), "lineitem", "extendedprice", "discount", "shipdate", "quantity"); - Supplier projection = new PageFunctionCompiler(localQueryRunner.getMetadata(), 0).compileProjection(field(0, BIGINT), Optional.empty()); + Supplier projection = new PageFunctionCompiler(localQueryRunner.getMetadata(), 0).compileProjection(session.getSqlFunctionProperties(), field(0, BIGINT), Optional.empty()); FilterAndProjectOperator.FilterAndProjectOperatorFactory tpchQuery6Operator = new FilterAndProjectOperator.FilterAndProjectOperatorFactory( 1, diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashAggregationBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashAggregationBenchmark.java index 61d1d3c9590fa..0b5157c2c88e5 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashAggregationBenchmark.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashAggregationBenchmark.java @@ -17,9 +17,9 @@ import com.facebook.presto.operator.HashAggregationOperator.HashAggregationOperatorFactory; import com.facebook.presto.operator.OperatorFactory; import com.facebook.presto.operator.aggregation.InternalAggregationFunction; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.testing.LocalQueryRunner; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashJoinBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashJoinBenchmark.java index f84e884ed470a..1461d6239d3b1 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashJoinBenchmark.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/HashJoinBenchmark.java @@ -39,11 +39,11 @@ import java.util.OptionalInt; import java.util.concurrent.Future; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.benchmark.BenchmarkQueryRunner.createLocalQueryRunner; import static com.facebook.presto.operator.PipelineExecutionStrategy.UNGROUPED_EXECUTION; import static com.facebook.presto.spiller.PartitioningSpillerFactory.unsupportedPartitioningSpillerFactory; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static java.util.Objects.requireNonNull; public class HashJoinBenchmark diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/JsonAvgBenchmarkResultWriter.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/JsonAvgBenchmarkResultWriter.java index b00c08d7c1344..55de2b16cc349 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/JsonAvgBenchmarkResultWriter.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/JsonAvgBenchmarkResultWriter.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.benchmark; +import com.facebook.airlift.json.JsonCodec; import com.fasterxml.jackson.annotation.JsonProperty; -import io.airlift.json.JsonCodec; import java.io.IOException; import java.io.OutputStream; diff --git a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/PredicateFilterBenchmark.java b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/PredicateFilterBenchmark.java index c7f28dcf7153b..7a28e70b6ea99 100644 --- a/presto-benchmark/src/main/java/com/facebook/presto/benchmark/PredicateFilterBenchmark.java +++ b/presto-benchmark/src/main/java/com/facebook/presto/benchmark/PredicateFilterBenchmark.java @@ -59,7 +59,7 @@ protected List createOperatorFactories() field(0, DOUBLE), constant(50000.0, DOUBLE)); ExpressionCompiler expressionCompiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); - Supplier pageProcessor = expressionCompiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field(0, DOUBLE))); + Supplier pageProcessor = expressionCompiler.compilePageProcessor(localQueryRunner.getDefaultSession().getSqlFunctionProperties(), Optional.of(filter), ImmutableList.of(field(0, DOUBLE))); FilterAndProjectOperator.FilterAndProjectOperatorFactory filterAndProjectOperator = new FilterAndProjectOperator.FilterAndProjectOperatorFactory( 1, diff --git a/presto-benchmark/src/test/java/com/facebook/presto/benchmark/MemoryLocalQueryRunner.java b/presto-benchmark/src/test/java/com/facebook/presto/benchmark/MemoryLocalQueryRunner.java index 08519fc807744..f88480753fe17 100644 --- a/presto-benchmark/src/test/java/com/facebook/presto/benchmark/MemoryLocalQueryRunner.java +++ b/presto-benchmark/src/test/java/com/facebook/presto/benchmark/MemoryLocalQueryRunner.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.benchmark; +import com.facebook.airlift.stats.TestingGcMonitor; import com.facebook.presto.Session; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.TaskStateMachine; @@ -34,7 +35,6 @@ import com.facebook.presto.tpch.TpchConnectorFactory; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.stats.TestingGcMonitor; import io.airlift.units.DataSize; import org.intellij.lang.annotations.Language; @@ -88,7 +88,7 @@ public List execute(@Language("SQL") String query) spillSpaceTracker); TaskContext taskContext = queryContext - .addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0), localQueryRunner.getExecutor()), + .addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0, 0), localQueryRunner.getExecutor()), localQueryRunner.getDefaultSession(), false, false, diff --git a/presto-benchto-benchmarks/pom.xml b/presto-benchto-benchmarks/pom.xml index e09f1b26033ad..155b623b3a581 100644 --- a/presto-benchto-benchmarks/pom.xml +++ b/presto-benchto-benchmarks/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-benchto-benchmarks @@ -54,7 +54,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java b/presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java index 887fb56b44765..b32d06784941d 100644 --- a/presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java +++ b/presto-benchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/AbstractCostBasedPlanTest.java @@ -15,10 +15,10 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.assertions.BasePlanTest; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q01.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q01.plan.txt index 89d5d56a431df..bab4967be65d3 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q01.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q01.plan.txt @@ -3,14 +3,14 @@ local exchange (GATHER, SINGLE, []) cross join: join (LEFT, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["sr_customer_sk"]) + remote exchange (REPARTITION, HASH, [sr_customer_sk]) join (INNER, REPLICATED): final aggregation over (sr_customer_sk, sr_store_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["sr_customer_sk", "sr_store_sk"]) + remote exchange (REPARTITION, HASH, [sr_customer_sk, sr_store_sk]) partial aggregation over (sr_customer_sk, sr_store_sk) join (INNER, REPLICATED): scan store_returns @@ -24,11 +24,11 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) final aggregation over (sr_store_sk_24) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["sr_store_sk_24"]) + remote exchange (REPARTITION, HASH, [sr_store_sk_24]) partial aggregation over (sr_store_sk_24) final aggregation over (sr_customer_sk_20, sr_store_sk_24) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["sr_customer_sk_20", "sr_store_sk_24"]) + remote exchange (REPARTITION, HASH, [sr_customer_sk_20, sr_store_sk_24]) partial aggregation over (sr_customer_sk_20, sr_store_sk_24) join (INNER, REPLICATED): scan store_returns diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q02.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q02.plan.txt index 0d0a5fabd8a8d..5c416ecc313a7 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q02.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q02.plan.txt @@ -5,7 +5,7 @@ remote exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): final aggregation over (d_week_seq) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_week_seq"]) + remote exchange (REPARTITION, HASH, [d_week_seq]) partial aggregation over (d_week_seq) join (INNER, REPLICATED): remote exchange (REPARTITION, ROUND_ROBIN, []) @@ -15,14 +15,14 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_week_seq_83"]) + remote exchange (REPARTITION, HASH, [d_week_seq_83]) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["expr_395"]) + remote exchange (REPARTITION, HASH, [expr_395]) join (INNER, PARTITIONED): final aggregation over (d_week_seq_232) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_week_seq_232"]) + remote exchange (REPARTITION, HASH, [d_week_seq_232]) partial aggregation over (d_week_seq_232) join (INNER, REPLICATED): remote exchange (REPARTITION, ROUND_ROBIN, []) @@ -32,5 +32,5 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_week_seq_316"]) + remote exchange (REPARTITION, HASH, [d_week_seq_316]) scan date_dim diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q03.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q03.plan.txt index 70aae3fa1bdde..13645e2a10bbe 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q03.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q03.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (d_year, i_brand, i_brand_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_year", "i_brand", "i_brand_id"]) + remote exchange (REPARTITION, HASH, [d_year, i_brand, i_brand_id]) partial aggregation over (d_year, i_brand, i_brand_id) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q04.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q04.plan.txt index d9edffeffad42..84378b4549cf3 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q04.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q04.plan.txt @@ -2,15 +2,15 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_customer_id_206"]) + remote exchange (REPARTITION, HASH, [c_customer_id_206]) join (INNER, PARTITIONED): join (INNER, PARTITIONED): join (INNER, PARTITIONED): local exchange (REPARTITION, ROUND_ROBIN, []) - remote exchange (REPARTITION, HASH, ["c_customer_id"]) + remote exchange (REPARTITION, HASH, [c_customer_id]) final aggregation over (c_birth_country, c_customer_id, c_email_address, c_first_name, c_last_name, c_login, c_preferred_cust_flag, d_year) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_birth_country", "c_customer_id", "c_email_address", "c_first_name", "c_last_name", "c_login", "c_preferred_cust_flag", "d_year"]) + remote exchange (REPARTITION, HASH, [c_birth_country, c_customer_id, c_email_address, c_first_name, c_last_name, c_login, c_preferred_cust_flag, d_year]) partial aggregation over (c_birth_country, c_customer_id, c_email_address, c_first_name, c_last_name, c_login, c_preferred_cust_flag, d_year) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -21,14 +21,14 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer - remote exchange (REPARTITION, HASH, ["c_customer_id_29"]) + remote exchange (REPARTITION, HASH, [c_customer_id_29]) single aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52) join (INNER, REPLICATED): join (INNER, REPLICATED): values (0 rows) values (0 rows) values (0 rows) - remote exchange (REPARTITION, HASH, ["c_customer_id_123"]) + remote exchange (REPARTITION, HASH, [c_customer_id_123]) single aggregation over (c_birth_country_136, c_customer_id_123, c_email_address_138, c_first_name_130, c_last_name_131, c_login_137, c_preferred_cust_flag_132, d_year_146) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -36,10 +36,10 @@ local exchange (GATHER, SINGLE, []) values (0 rows) values (0 rows) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_247"]) + remote exchange (REPARTITION, HASH, [c_customer_id_247]) final aggregation over (c_birth_country_260, c_customer_id_247, c_email_address_262, c_first_name_254, c_last_name_255, c_login_261, c_preferred_cust_flag_256, d_year_293) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_birth_country_260", "c_customer_id_247", "c_email_address_262", "c_first_name_254", "c_last_name_255", "c_login_261", "c_preferred_cust_flag_256", "d_year_293"]) + remote exchange (REPARTITION, HASH, [c_birth_country_260, c_customer_id_247, c_email_address_262, c_first_name_254, c_last_name_255, c_login_261, c_preferred_cust_flag_256, d_year_293]) partial aggregation over (c_birth_country_260, c_customer_id_247, c_email_address_262, c_first_name_254, c_last_name_255, c_login_261, c_preferred_cust_flag_256, d_year_293) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -50,14 +50,14 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer - remote exchange (REPARTITION, HASH, ["c_customer_id_354"]) + remote exchange (REPARTITION, HASH, [c_customer_id_354]) single aggregation over (c_birth_country_367, c_customer_id_354, c_email_address_369, c_first_name_361, c_last_name_362, c_login_368, c_preferred_cust_flag_363, d_year_411) join (INNER, REPLICATED): join (INNER, REPLICATED): values (0 rows) values (0 rows) values (0 rows) - remote exchange (REPARTITION, HASH, ["c_customer_id_482"]) + remote exchange (REPARTITION, HASH, [c_customer_id_482]) single aggregation over (c_birth_country_495, c_customer_id_482, c_email_address_497, c_first_name_489, c_last_name_490, c_login_496, c_preferred_cust_flag_491, d_year_539) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -65,17 +65,17 @@ local exchange (GATHER, SINGLE, []) values (0 rows) values (0 rows) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_640"]) + remote exchange (REPARTITION, HASH, [c_customer_id_640]) single aggregation over (c_birth_country_653, c_customer_id_640, c_email_address_655, c_first_name_647, c_last_name_648, c_login_654, c_preferred_cust_flag_649, d_year_686) join (INNER, REPLICATED): join (INNER, REPLICATED): values (0 rows) values (0 rows) values (0 rows) - remote exchange (REPARTITION, HASH, ["c_customer_id_747"]) + remote exchange (REPARTITION, HASH, [c_customer_id_747]) final aggregation over (c_birth_country_760, c_customer_id_747, c_email_address_762, c_first_name_754, c_last_name_755, c_login_761, c_preferred_cust_flag_756, d_year_804) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_birth_country_760", "c_customer_id_747", "c_email_address_762", "c_first_name_754", "c_last_name_755", "c_login_761", "c_preferred_cust_flag_756", "d_year_804"]) + remote exchange (REPARTITION, HASH, [c_birth_country_760, c_customer_id_747, c_email_address_762, c_first_name_754, c_last_name_755, c_login_761, c_preferred_cust_flag_756, d_year_804]) partial aggregation over (c_birth_country_760, c_customer_id_747, c_email_address_762, c_first_name_754, c_last_name_755, c_login_761, c_preferred_cust_flag_756, d_year_804) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -86,7 +86,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer - remote exchange (REPARTITION, HASH, ["c_customer_id_875"]) + remote exchange (REPARTITION, HASH, [c_customer_id_875]) single aggregation over (c_birth_country_888, c_customer_id_875, c_email_address_890, c_first_name_882, c_last_name_883, c_login_889, c_preferred_cust_flag_884, d_year_932) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -94,17 +94,17 @@ local exchange (GATHER, SINGLE, []) values (0 rows) values (0 rows) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_1033"]) + remote exchange (REPARTITION, HASH, [c_customer_id_1033]) single aggregation over (c_birth_country_1046, c_customer_id_1033, c_email_address_1048, c_first_name_1040, c_last_name_1041, c_login_1047, c_preferred_cust_flag_1042, d_year_1079) join (INNER, REPLICATED): join (INNER, REPLICATED): values (0 rows) values (0 rows) values (0 rows) - remote exchange (REPARTITION, HASH, ["c_customer_id_1140"]) + remote exchange (REPARTITION, HASH, [c_customer_id_1140]) final aggregation over (c_birth_country_1153, c_customer_id_1140, c_email_address_1155, c_first_name_1147, c_last_name_1148, c_login_1154, c_preferred_cust_flag_1149, d_year_1197) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_birth_country_1153", "c_customer_id_1140", "c_email_address_1155", "c_first_name_1147", "c_last_name_1148", "c_login_1154", "c_preferred_cust_flag_1149", "d_year_1197"]) + remote exchange (REPARTITION, HASH, [c_birth_country_1153, c_customer_id_1140, c_email_address_1155, c_first_name_1147, c_last_name_1148, c_login_1154, c_preferred_cust_flag_1149, d_year_1197]) partial aggregation over (c_birth_country_1153, c_customer_id_1140, c_email_address_1155, c_first_name_1147, c_last_name_1148, c_login_1154, c_preferred_cust_flag_1149, d_year_1197) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -115,7 +115,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer - remote exchange (REPARTITION, HASH, ["c_customer_id_1268"]) + remote exchange (REPARTITION, HASH, [c_customer_id_1268]) single aggregation over (c_birth_country_1281, c_customer_id_1268, c_email_address_1283, c_first_name_1275, c_last_name_1276, c_login_1282, c_preferred_cust_flag_1277, d_year_1325) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -123,62 +123,62 @@ local exchange (GATHER, SINGLE, []) values (0 rows) values (0 rows) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_1426"]) + remote exchange (REPARTITION, HASH, [c_customer_id_1426]) single aggregation over (c_birth_country_1439, c_customer_id_1426, c_email_address_1441, c_first_name_1433, c_last_name_1434, c_login_1440, c_preferred_cust_flag_1435, d_year_1472) join (INNER, REPLICATED): join (INNER, REPLICATED): values (0 rows) values (0 rows) values (0 rows) - remote exchange (REPARTITION, HASH, ["c_customer_id_1533"]) + remote exchange (REPARTITION, HASH, [c_customer_id_1533]) single aggregation over (c_birth_country_1546, c_customer_id_1533, c_email_address_1548, c_first_name_1540, c_last_name_1541, c_login_1547, c_preferred_cust_flag_1542, d_year_1590) join (INNER, REPLICATED): join (INNER, REPLICATED): values (0 rows) values (0 rows) values (0 rows) - remote exchange (REPARTITION, HASH, ["c_customer_id_1661"]) + remote exchange (REPARTITION, HASH, [c_customer_id_1661]) final aggregation over (c_birth_country_1674, c_customer_id_1661, c_email_address_1676, c_first_name_1668, c_last_name_1669, c_login_1675, c_preferred_cust_flag_1670, d_year_1718) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_birth_country_1674", "c_customer_id_1661", "c_email_address_1676", "c_first_name_1668", "c_last_name_1669", "c_login_1675", "c_preferred_cust_flag_1670", "d_year_1718"]) + remote exchange (REPARTITION, HASH, [c_birth_country_1674, c_customer_id_1661, c_email_address_1676, c_first_name_1668, c_last_name_1669, c_login_1675, c_preferred_cust_flag_1670, d_year_1718]) partial aggregation over (c_birth_country_1674, c_customer_id_1661, c_email_address_1676, c_first_name_1668, c_last_name_1669, c_login_1675, c_preferred_cust_flag_1670, d_year_1718) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_1682"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk_1682]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_1660"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_1660]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_1819"]) + remote exchange (REPARTITION, HASH, [c_customer_id_1819]) single aggregation over (c_birth_country_1832, c_customer_id_1819, c_email_address_1834, c_first_name_1826, c_last_name_1827, c_login_1833, c_preferred_cust_flag_1828, d_year_1865) join (INNER, REPLICATED): join (INNER, REPLICATED): values (0 rows) values (0 rows) values (0 rows) - remote exchange (REPARTITION, HASH, ["c_customer_id_1926"]) + remote exchange (REPARTITION, HASH, [c_customer_id_1926]) single aggregation over (c_birth_country_1939, c_customer_id_1926, c_email_address_1941, c_first_name_1933, c_last_name_1934, c_login_1940, c_preferred_cust_flag_1935, d_year_1983) join (INNER, REPLICATED): join (INNER, REPLICATED): values (0 rows) values (0 rows) values (0 rows) - remote exchange (REPARTITION, HASH, ["c_customer_id_2054"]) + remote exchange (REPARTITION, HASH, [c_customer_id_2054]) final aggregation over (c_birth_country_2067, c_customer_id_2054, c_email_address_2069, c_first_name_2061, c_last_name_2062, c_login_2068, c_preferred_cust_flag_2063, d_year_2111) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_birth_country_2067", "c_customer_id_2054", "c_email_address_2069", "c_first_name_2061", "c_last_name_2062", "c_login_2068", "c_preferred_cust_flag_2063", "d_year_2111"]) + remote exchange (REPARTITION, HASH, [c_birth_country_2067, c_customer_id_2054, c_email_address_2069, c_first_name_2061, c_last_name_2062, c_login_2068, c_preferred_cust_flag_2063, d_year_2111]) partial aggregation over (c_birth_country_2067, c_customer_id_2054, c_email_address_2069, c_first_name_2061, c_last_name_2062, c_login_2068, c_preferred_cust_flag_2063, d_year_2111) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_2075"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk_2075]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_2053"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_2053]) scan customer diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q05.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q05.plan.txt index 9577f71abc72a..ebed85d1de978 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q05.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q05.plan.txt @@ -1,13 +1,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, HASH, ["channel$gid", "groupid", "id$gid"]) - remote exchange (REPARTITION, HASH, ["channel$gid", "groupid", "id$gid"]) + local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) partial aggregation over (channel$gid, groupid, id$gid) local exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (s_store_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["s_store_id"]) + remote exchange (REPARTITION, HASH, [s_store_id]) partial aggregation over (s_store_id) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -22,7 +22,7 @@ local exchange (GATHER, SINGLE, []) scan store final aggregation over (cp_catalog_page_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cp_catalog_page_id"]) + remote exchange (REPARTITION, HASH, [cp_catalog_page_id]) partial aggregation over (cp_catalog_page_id) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -37,17 +37,17 @@ local exchange (GATHER, SINGLE, []) scan catalog_page final aggregation over (web_site_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["web_site_id"]) + remote exchange (REPARTITION, HASH, [web_site_id]) partial aggregation over (web_site_id) join (INNER, REPLICATED): join (INNER, REPLICATED): remote exchange (REPARTITION, ROUND_ROBIN, []) scan web_sales join (RIGHT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_item_sk_216", "ws_order_number_230"]) + remote exchange (REPARTITION, HASH, [ws_item_sk_216, ws_order_number_230]) scan web_sales local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["wr_item_sk", "wr_order_number"]) + remote exchange (REPARTITION, HASH, [wr_item_sk, wr_order_number]) scan web_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q06.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q06.plan.txt index 1fc16834e6e44..9611a61dadd6d 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q06.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q06.plan.txt @@ -2,13 +2,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (ca_state) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_state"]) + remote exchange (REPARTITION, HASH, [ca_state]) partial aggregation over (ca_state) cross join: join (LEFT, REPLICATED): join (INNER, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) join (INNER, REPLICATED): scan store_sales local exchange (GATHER, SINGLE, []) @@ -21,16 +21,16 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (d_month_seq_3) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_month_seq_3"]) + remote exchange (REPARTITION, HASH, [d_month_seq_3]) partial aggregation over (d_month_seq_3) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_addr_sk"]) + remote exchange (REPARTITION, HASH, [c_current_addr_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk"]) + remote exchange (REPARTITION, HASH, [ca_address_sk]) scan customer_address local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) @@ -39,7 +39,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) final aggregation over (i_category_43) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_category_43"]) + remote exchange (REPARTITION, HASH, [i_category_43]) partial aggregation over (i_category_43) scan item local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q07.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q07.plan.txt index 32f4763b443da..bbdd31eb49672 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q07.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q07.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (i_item_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id"]) + remote exchange (REPARTITION, HASH, [i_item_id]) partial aggregation over (i_item_id) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q08.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q08.plan.txt index 059fe336d437f..44e1dde57dde3 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q08.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q08.plan.txt @@ -2,10 +2,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (s_store_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["s_store_name"]) + remote exchange (REPARTITION, HASH, [s_store_name]) partial aggregation over (s_store_name) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["substr_55"]) + remote exchange (REPARTITION, HASH, [substr_55]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan store_sales @@ -16,21 +16,21 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan store local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["substr_56"]) + remote exchange (REPARTITION, HASH, [substr_56]) final aggregation over (expr_29) - local exchange (REPARTITION, HASH, ["expr_29"]) - remote exchange (REPARTITION, HASH, ["expr_47"]) + local exchange (REPARTITION, HASH, [expr_29]) + remote exchange (REPARTITION, HASH, [expr_47]) partial aggregation over (expr_47) scan customer_address - remote exchange (REPARTITION, HASH, ["expr_50"]) + remote exchange (REPARTITION, HASH, [expr_50]) partial aggregation over (expr_50) final aggregation over (ca_zip_11) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_zip_11"]) + remote exchange (REPARTITION, HASH, [ca_zip_11]) partial aggregation over (ca_zip_11) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ca_address_sk_2"]) + remote exchange (REPARTITION, HASH, [ca_address_sk_2]) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_current_addr_sk"]) + remote exchange (REPARTITION, HASH, [c_current_addr_sk]) scan customer diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q10.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q10.plan.txt index 50e17a7b593a1..ecd28c6bb8dc6 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q10.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q10.plan.txt @@ -2,12 +2,12 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (cd_credit_rating, cd_dep_college_count, cd_dep_count, cd_dep_employed_count, cd_education_status, cd_gender, cd_marital_status, cd_purchase_estimate) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cd_credit_rating", "cd_dep_college_count", "cd_dep_count", "cd_dep_employed_count", "cd_education_status", "cd_gender", "cd_marital_status", "cd_purchase_estimate"]) + remote exchange (REPARTITION, HASH, [cd_credit_rating, cd_dep_college_count, cd_dep_count, cd_dep_employed_count, cd_education_status, cd_gender, cd_marital_status, cd_purchase_estimate]) partial aggregation over (cd_credit_rating, cd_dep_college_count, cd_dep_count, cd_dep_employed_count, cd_education_status, cd_gender, cd_marital_status, cd_purchase_estimate) join (RIGHT, PARTITIONED): final aggregation over (cs_ship_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_ship_customer_sk"]) + remote exchange (REPARTITION, HASH, [cs_ship_customer_sk]) partial aggregation over (cs_ship_customer_sk) join (INNER, REPLICATED): scan catalog_sales @@ -18,7 +18,7 @@ local exchange (GATHER, SINGLE, []) join (RIGHT, PARTITIONED): final aggregation over (ws_bill_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk]) partial aggregation over (ws_bill_customer_sk) join (INNER, REPLICATED): scan web_sales @@ -26,7 +26,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) join (INNER, REPLICATED): scan customer_demographics local exchange (GATHER, SINGLE, []) @@ -34,7 +34,7 @@ local exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): final aggregation over (ss_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) partial aggregation over (ss_customer_sk) join (INNER, REPLICATED): scan store_sales @@ -42,7 +42,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) join (INNER, REPLICATED): scan customer local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q11.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q11.plan.txt index d6d71fad2e5b6..37e54178b332f 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q11.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q11.plan.txt @@ -4,10 +4,10 @@ local exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): join (INNER, PARTITIONED): local exchange (REPARTITION, ROUND_ROBIN, []) - remote exchange (REPARTITION, HASH, ["c_customer_id"]) + remote exchange (REPARTITION, HASH, [c_customer_id]) final aggregation over (c_birth_country, c_customer_id, c_email_address, c_first_name, c_last_name, c_login, c_preferred_cust_flag, d_year) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_birth_country", "c_customer_id", "c_email_address", "c_first_name", "c_last_name", "c_login", "c_preferred_cust_flag", "d_year"]) + remote exchange (REPARTITION, HASH, [c_birth_country, c_customer_id, c_email_address, c_first_name, c_last_name, c_login, c_preferred_cust_flag, d_year]) partial aggregation over (c_birth_country, c_customer_id, c_email_address, c_first_name, c_last_name, c_login, c_preferred_cust_flag, d_year) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -18,7 +18,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer - remote exchange (REPARTITION, HASH, ["c_customer_id_29"]) + remote exchange (REPARTITION, HASH, [c_customer_id_29]) single aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -26,10 +26,10 @@ local exchange (GATHER, SINGLE, []) values (0 rows) values (0 rows) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_153"]) + remote exchange (REPARTITION, HASH, [c_customer_id_153]) final aggregation over (c_birth_country_166, c_customer_id_153, c_email_address_168, c_first_name_160, c_last_name_161, c_login_167, c_preferred_cust_flag_162, d_year_199) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_birth_country_166", "c_customer_id_153", "c_email_address_168", "c_first_name_160", "c_last_name_161", "c_login_167", "c_preferred_cust_flag_162", "d_year_199"]) + remote exchange (REPARTITION, HASH, [c_birth_country_166, c_customer_id_153, c_email_address_168, c_first_name_160, c_last_name_161, c_login_167, c_preferred_cust_flag_162, d_year_199]) partial aggregation over (c_birth_country_166, c_customer_id_153, c_email_address_168, c_first_name_160, c_last_name_161, c_login_167, c_preferred_cust_flag_162, d_year_199) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -40,7 +40,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer - remote exchange (REPARTITION, HASH, ["c_customer_id_260"]) + remote exchange (REPARTITION, HASH, [c_customer_id_260]) single aggregation over (c_birth_country_273, c_customer_id_260, c_email_address_275, c_first_name_267, c_last_name_268, c_login_274, c_preferred_cust_flag_269, d_year_317) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -48,48 +48,48 @@ local exchange (GATHER, SINGLE, []) values (0 rows) values (0 rows) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_418"]) + remote exchange (REPARTITION, HASH, [c_customer_id_418]) single aggregation over (c_birth_country_431, c_customer_id_418, c_email_address_433, c_first_name_425, c_last_name_426, c_login_432, c_preferred_cust_flag_427, d_year_464) join (INNER, REPLICATED): join (INNER, REPLICATED): values (0 rows) values (0 rows) values (0 rows) - remote exchange (REPARTITION, HASH, ["c_customer_id_525"]) + remote exchange (REPARTITION, HASH, [c_customer_id_525]) final aggregation over (c_birth_country_538, c_customer_id_525, c_email_address_540, c_first_name_532, c_last_name_533, c_login_539, c_preferred_cust_flag_534, d_year_582) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_birth_country_538", "c_customer_id_525", "c_email_address_540", "c_first_name_532", "c_last_name_533", "c_login_539", "c_preferred_cust_flag_534", "d_year_582"]) + remote exchange (REPARTITION, HASH, [c_birth_country_538, c_customer_id_525, c_email_address_540, c_first_name_532, c_last_name_533, c_login_539, c_preferred_cust_flag_534, d_year_582]) partial aggregation over (c_birth_country_538, c_customer_id_525, c_email_address_540, c_first_name_532, c_last_name_533, c_login_539, c_preferred_cust_flag_534, d_year_582) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_546"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk_546]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_524"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_524]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_683"]) + remote exchange (REPARTITION, HASH, [c_customer_id_683]) single aggregation over (c_birth_country_696, c_customer_id_683, c_email_address_698, c_first_name_690, c_last_name_691, c_login_697, c_preferred_cust_flag_692, d_year_729) join (INNER, REPLICATED): join (INNER, REPLICATED): values (0 rows) values (0 rows) values (0 rows) - remote exchange (REPARTITION, HASH, ["c_customer_id_790"]) + remote exchange (REPARTITION, HASH, [c_customer_id_790]) final aggregation over (c_birth_country_803, c_customer_id_790, c_email_address_805, c_first_name_797, c_last_name_798, c_login_804, c_preferred_cust_flag_799, d_year_847) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_birth_country_803", "c_customer_id_790", "c_email_address_805", "c_first_name_797", "c_last_name_798", "c_login_804", "c_preferred_cust_flag_799", "d_year_847"]) + remote exchange (REPARTITION, HASH, [c_birth_country_803, c_customer_id_790, c_email_address_805, c_first_name_797, c_last_name_798, c_login_804, c_preferred_cust_flag_799, d_year_847]) partial aggregation over (c_birth_country_803, c_customer_id_790, c_email_address_805, c_first_name_797, c_last_name_798, c_login_804, c_preferred_cust_flag_799, d_year_847) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_811"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk_811]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_789"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_789]) scan customer diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q12.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q12.plan.txt index 6f94c98c52243..45440b8540f78 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q12.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q12.plan.txt @@ -1,10 +1,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_class"]) + remote exchange (REPARTITION, HASH, [i_class]) final aggregation over (i_category, i_class, i_current_price, i_item_desc, i_item_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_category", "i_class", "i_current_price", "i_item_desc", "i_item_id"]) + remote exchange (REPARTITION, HASH, [i_category, i_class, i_current_price, i_item_desc, i_item_id]) partial aggregation over (i_category, i_class, i_current_price, i_item_desc, i_item_id) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_1.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_1.plan.txt index ec4c1ac8a9016..9d5966e6652b2 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_1.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_1.plan.txt @@ -1,17 +1,17 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743) - local exchange (REPARTITION, HASH, ["channel$gid", "groupid", "i_brand_id$gid_1742", "i_category_id$gid_1744", "i_class_id$gid_1743"]) - remote exchange (REPARTITION, HASH, ["channel$gid", "groupid", "i_brand_id$gid_1742", "i_category_id$gid_1744", "i_class_id$gid_1743"]) + local exchange (REPARTITION, HASH, [channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743]) + remote exchange (REPARTITION, HASH, [channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743]) partial aggregation over (channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743) local exchange (REPARTITION, ROUND_ROBIN, []) cross join: final aggregation over (i_brand_id, i_category_id, i_class_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_id", "i_category_id", "i_class_id"]) + remote exchange (REPARTITION, HASH, [i_brand_id, i_category_id, i_class_id]) partial aggregation over (i_brand_id, i_category_id, i_class_id) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_item_sk"]) + remote exchange (REPARTITION, HASH, [ss_item_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan store_sales @@ -22,13 +22,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk_1"]) + remote exchange (REPARTITION, HASH, [i_item_sk_1]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_brand_id_8", "i_category_id_12", "i_class_id_10"]) + remote exchange (REPARTITION, HASH, [i_brand_id_8, i_category_id_12, i_class_id_10]) scan item final aggregation over (expr_216, expr_217, expr_218) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_id_53", "i_category_id_57", "i_class_id_55"]) + remote exchange (REPARTITION, HASH, [i_brand_id_53, i_category_id_57, i_class_id_55]) partial aggregation over (i_brand_id_53, i_category_id_57, i_class_id_55) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -39,7 +39,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item - remote exchange (REPARTITION, HASH, ["i_brand_id_108", "i_category_id_112", "i_class_id_110"]) + remote exchange (REPARTITION, HASH, [i_brand_id_108, i_category_id_112, i_class_id_110]) partial aggregation over (i_brand_id_108, i_category_id_112, i_class_id_110) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -50,7 +50,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item - remote exchange (REPARTITION, HASH, ["i_brand_id_167", "i_category_id_171", "i_class_id_169"]) + remote exchange (REPARTITION, HASH, [i_brand_id_167, i_category_id_171, i_class_id_169]) partial aggregation over (i_brand_id_167, i_category_id_171, i_class_id_169) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -87,10 +87,10 @@ local exchange (GATHER, SINGLE, []) cross join: final aggregation over (i_brand_id_508, i_category_id_512, i_class_id_510) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_id_508", "i_category_id_512", "i_class_id_510"]) + remote exchange (REPARTITION, HASH, [i_brand_id_508, i_category_id_512, i_class_id_510]) partial aggregation over (i_brand_id_508, i_category_id_512, i_class_id_510) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_item_sk_482"]) + remote exchange (REPARTITION, HASH, [cs_item_sk_482]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan catalog_sales @@ -101,13 +101,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk_552"]) + remote exchange (REPARTITION, HASH, [i_item_sk_552]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_brand_id_559", "i_category_id_563", "i_class_id_561"]) + remote exchange (REPARTITION, HASH, [i_brand_id_559, i_category_id_563, i_class_id_561]) scan item final aggregation over (expr_836, expr_837, expr_838) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_id_604", "i_category_id_608", "i_class_id_606"]) + remote exchange (REPARTITION, HASH, [i_brand_id_604, i_category_id_608, i_class_id_606]) partial aggregation over (i_brand_id_604, i_category_id_608, i_class_id_606) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -118,7 +118,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item - remote exchange (REPARTITION, HASH, ["i_brand_id_694", "i_category_id_698", "i_class_id_696"]) + remote exchange (REPARTITION, HASH, [i_brand_id_694, i_category_id_698, i_class_id_696]) partial aggregation over (i_brand_id_694, i_category_id_698, i_class_id_696) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -129,7 +129,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item - remote exchange (REPARTITION, HASH, ["i_brand_id_787", "i_category_id_791", "i_class_id_789"]) + remote exchange (REPARTITION, HASH, [i_brand_id_787, i_category_id_791, i_class_id_789]) partial aggregation over (i_brand_id_787, i_category_id_791, i_class_id_789) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -166,10 +166,10 @@ local exchange (GATHER, SINGLE, []) cross join: final aggregation over (i_brand_id_1135, i_category_id_1139, i_class_id_1137) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_id_1135", "i_category_id_1139", "i_class_id_1137"]) + remote exchange (REPARTITION, HASH, [i_brand_id_1135, i_category_id_1139, i_class_id_1137]) partial aggregation over (i_brand_id_1135, i_category_id_1139, i_class_id_1137) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_item_sk_1097"]) + remote exchange (REPARTITION, HASH, [ws_item_sk_1097]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan web_sales @@ -180,13 +180,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk_1179"]) + remote exchange (REPARTITION, HASH, [i_item_sk_1179]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_brand_id_1186", "i_category_id_1190", "i_class_id_1188"]) + remote exchange (REPARTITION, HASH, [i_brand_id_1186, i_category_id_1190, i_class_id_1188]) scan item final aggregation over (expr_1463, expr_1464, expr_1465) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_id_1231", "i_category_id_1235", "i_class_id_1233"]) + remote exchange (REPARTITION, HASH, [i_brand_id_1231, i_category_id_1235, i_class_id_1233]) partial aggregation over (i_brand_id_1231, i_category_id_1235, i_class_id_1233) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -197,7 +197,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item - remote exchange (REPARTITION, HASH, ["i_brand_id_1321", "i_category_id_1325", "i_class_id_1323"]) + remote exchange (REPARTITION, HASH, [i_brand_id_1321, i_category_id_1325, i_class_id_1323]) partial aggregation over (i_brand_id_1321, i_category_id_1325, i_class_id_1323) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -208,7 +208,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item - remote exchange (REPARTITION, HASH, ["i_brand_id_1414", "i_category_id_1418", "i_class_id_1416"]) + remote exchange (REPARTITION, HASH, [i_brand_id_1414, i_category_id_1418, i_class_id_1416]) partial aggregation over (i_brand_id_1414, i_category_id_1418, i_class_id_1416) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_2.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_2.plan.txt index 6b3da8b0b0a43..8dcae8f2d4aaa 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_2.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_2.plan.txt @@ -4,11 +4,11 @@ local exchange (GATHER, SINGLE, []) cross join: final aggregation over (i_brand_id, i_category_id, i_class_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_id", "i_category_id", "i_class_id"]) + remote exchange (REPARTITION, HASH, [i_brand_id, i_category_id, i_class_id]) partial aggregation over (i_brand_id, i_category_id, i_class_id) join (INNER, REPLICATED): semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_item_sk"]) + remote exchange (REPARTITION, HASH, [ss_item_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan store_sales @@ -19,13 +19,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk_1"]) + remote exchange (REPARTITION, HASH, [i_item_sk_1]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_brand_id_8", "i_category_id_12", "i_class_id_10"]) + remote exchange (REPARTITION, HASH, [i_brand_id_8, i_category_id_12, i_class_id_10]) scan item final aggregation over (expr_216, expr_217, expr_218) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_id_53", "i_category_id_57", "i_class_id_55"]) + remote exchange (REPARTITION, HASH, [i_brand_id_53, i_category_id_57, i_class_id_55]) partial aggregation over (i_brand_id_53, i_category_id_57, i_class_id_55) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -36,7 +36,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item - remote exchange (REPARTITION, HASH, ["i_brand_id_108", "i_category_id_112", "i_class_id_110"]) + remote exchange (REPARTITION, HASH, [i_brand_id_108, i_category_id_112, i_class_id_110]) partial aggregation over (i_brand_id_108, i_category_id_112, i_class_id_110) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -47,7 +47,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item - remote exchange (REPARTITION, HASH, ["i_brand_id_167", "i_category_id_171", "i_class_id_169"]) + remote exchange (REPARTITION, HASH, [i_brand_id_167, i_category_id_171, i_class_id_169]) partial aggregation over (i_brand_id_167, i_category_id_171, i_class_id_169) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -89,11 +89,11 @@ local exchange (GATHER, SINGLE, []) cross join: final aggregation over (i_brand_id_534, i_category_id_538, i_class_id_536) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_id_534", "i_category_id_538", "i_class_id_536"]) + remote exchange (REPARTITION, HASH, [i_brand_id_534, i_category_id_538, i_class_id_536]) partial aggregation over (i_brand_id_534, i_category_id_538, i_class_id_536) join (INNER, REPLICATED): semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_item_sk_506"]) + remote exchange (REPARTITION, HASH, [ss_item_sk_506]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan store_sales @@ -104,13 +104,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk_578"]) + remote exchange (REPARTITION, HASH, [i_item_sk_578]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_brand_id_585", "i_category_id_589", "i_class_id_587"]) + remote exchange (REPARTITION, HASH, [i_brand_id_585, i_category_id_589, i_class_id_587]) scan item final aggregation over (expr_862, expr_863, expr_864) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_id_630", "i_category_id_634", "i_class_id_632"]) + remote exchange (REPARTITION, HASH, [i_brand_id_630, i_category_id_634, i_class_id_632]) partial aggregation over (i_brand_id_630, i_category_id_634, i_class_id_632) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -121,7 +121,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item - remote exchange (REPARTITION, HASH, ["i_brand_id_720", "i_category_id_724", "i_class_id_722"]) + remote exchange (REPARTITION, HASH, [i_brand_id_720, i_category_id_724, i_class_id_722]) partial aggregation over (i_brand_id_720, i_category_id_724, i_class_id_722) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -132,7 +132,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item - remote exchange (REPARTITION, HASH, ["i_brand_id_813", "i_category_id_817", "i_class_id_815"]) + remote exchange (REPARTITION, HASH, [i_brand_id_813, i_category_id_817, i_class_id_815]) partial aggregation over (i_brand_id_813, i_category_id_817, i_class_id_815) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q15.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q15.plan.txt index f359fbefb40f2..9c08a57a7e584 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q15.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q15.plan.txt @@ -2,20 +2,20 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (ca_zip) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_zip"]) + remote exchange (REPARTITION, HASH, [ca_zip]) partial aggregation over (ca_zip) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [cs_bill_customer_sk]) join (INNER, REPLICATED): scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_addr_sk"]) + remote exchange (REPARTITION, HASH, [c_current_addr_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk"]) + remote exchange (REPARTITION, HASH, [ca_address_sk]) scan customer_address diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q16.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q16.plan.txt index 8fc57e09f6a8f..f0df98483720b 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q16.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q16.plan.txt @@ -7,10 +7,10 @@ final aggregation over () local exchange (GATHER, SINGLE, []) partial aggregation over (ca_state, cc_county, cs_call_center_sk, cs_ext_ship_cost, cs_net_profit, cs_order_number, cs_ship_addr_sk, cs_ship_date_sk, cs_warehouse_sk, d_date, unique) join (RIGHT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_order_number_17"]) + remote exchange (REPARTITION, HASH, [cs_order_number_17]) scan catalog_sales local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_order_number"]) + remote exchange (REPARTITION, HASH, [cs_order_number]) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -26,6 +26,6 @@ final aggregation over () scan call_center final aggregation over (cr_order_number) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cr_order_number"]) + remote exchange (REPARTITION, HASH, [cr_order_number]) partial aggregation over (cr_order_number) scan catalog_returns diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q17.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q17.plan.txt index 2736834ce145a..4c96de2af2d82 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q17.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q17.plan.txt @@ -2,21 +2,21 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (i_item_desc, i_item_id, s_state) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_desc", "i_item_id", "s_state"]) + remote exchange (REPARTITION, HASH, [i_item_desc, i_item_id, s_state]) partial aggregation over (i_item_desc, i_item_id, s_state) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_item_sk", "sr_customer_sk"]) + remote exchange (REPARTITION, HASH, [i_item_sk, sr_customer_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk", "ss_item_sk", "ss_ticket_number"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk, ss_item_sk, ss_ticket_number]) join (INNER, REPLICATED): scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["sr_customer_sk", "sr_item_sk", "sr_ticket_number"]) + remote exchange (REPARTITION, HASH, [sr_customer_sk, sr_item_sk, sr_ticket_number]) join (INNER, REPLICATED): scan store_returns local exchange (GATHER, SINGLE, []) @@ -29,7 +29,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk", "cs_item_sk"]) + remote exchange (REPARTITION, HASH, [cs_bill_customer_sk, cs_item_sk]) join (INNER, REPLICATED): scan catalog_sales local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q18.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q18.plan.txt index 70547d9eecf53..7d4c99ffd8b9d 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q18.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q18.plan.txt @@ -1,8 +1,8 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid) - local exchange (REPARTITION, HASH, ["ca_country$gid", "ca_county$gid", "ca_state$gid", "groupid", "i_item_id$gid"]) - remote exchange (REPARTITION, HASH, ["ca_country$gid", "ca_county$gid", "ca_state$gid", "groupid", "i_item_id$gid"]) + local exchange (REPARTITION, HASH, [ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid]) + remote exchange (REPARTITION, HASH, [ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid]) partial aggregation over (ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -15,15 +15,15 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_cdemo_sk"]) + remote exchange (REPARTITION, HASH, [c_current_cdemo_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_addr_sk"]) + remote exchange (REPARTITION, HASH, [c_current_addr_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk"]) + remote exchange (REPARTITION, HASH, [ca_address_sk]) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cd_demo_sk_0"]) + remote exchange (REPARTITION, HASH, [cd_demo_sk_0]) scan customer_demographics local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q19.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q19.plan.txt index c500acf6fca3d..cb55b0a0f10ef 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q19.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q19.plan.txt @@ -2,19 +2,19 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (i_brand, i_brand_id, i_manufact, i_manufact_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand", "i_brand_id", "i_manufact", "i_manufact_id"]) + remote exchange (REPARTITION, HASH, [i_brand, i_brand_id, i_manufact, i_manufact_id]) partial aggregation over (i_brand, i_brand_id, i_manufact, i_manufact_id) join (INNER, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ca_address_sk"]) + remote exchange (REPARTITION, HASH, [ca_address_sk]) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_current_addr_sk"]) + remote exchange (REPARTITION, HASH, [c_current_addr_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan store_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q20.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q20.plan.txt index 852a9d2a3e5c1..7e579a4f08a0b 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q20.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q20.plan.txt @@ -1,10 +1,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_class"]) + remote exchange (REPARTITION, HASH, [i_class]) final aggregation over (i_category, i_class, i_current_price, i_item_desc, i_item_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_category", "i_class", "i_current_price", "i_item_desc", "i_item_id"]) + remote exchange (REPARTITION, HASH, [i_category, i_class, i_current_price, i_item_desc, i_item_id]) partial aggregation over (i_category, i_class, i_current_price, i_item_desc, i_item_id) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q21.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q21.plan.txt index 50e40917c430b..81838260c85d6 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q21.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q21.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (i_item_id, w_warehouse_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id", "w_warehouse_name"]) + remote exchange (REPARTITION, HASH, [i_item_id, w_warehouse_name]) partial aggregation over (i_item_id, w_warehouse_name) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q22.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q22.plan.txt index d323affadfc06..38c414f6d58fb 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q22.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q22.plan.txt @@ -1,8 +1,8 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid) - local exchange (REPARTITION, HASH, ["groupid", "i_brand$gid", "i_category$gid", "i_class$gid", "i_product_name$gid"]) - remote exchange (REPARTITION, HASH, ["groupid", "i_brand$gid", "i_category$gid", "i_class$gid", "i_product_name$gid"]) + local exchange (REPARTITION, HASH, [groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid]) + remote exchange (REPARTITION, HASH, [groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid]) partial aggregation over (groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q23_1.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q23_1.plan.txt index bb4fd6610c702..604e01f8b2f7c 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q23_1.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q23_1.plan.txt @@ -3,19 +3,19 @@ final aggregation over () remote exchange (GATHER, SINGLE, []) partial aggregation over () semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [cs_bill_customer_sk]) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_item_sk"]) + remote exchange (REPARTITION, HASH, [cs_item_sk]) join (INNER, REPLICATED): scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk"]) + remote exchange (REPARTITION, HASH, [i_item_sk]) final aggregation over (d_date_3, i_item_sk, substr) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date_3", "i_item_sk", "substr"]) + remote exchange (REPARTITION, HASH, [d_date_3, i_item_sk, substr]) partial aggregation over (d_date_3, i_item_sk, substr) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -27,11 +27,11 @@ final aggregation over () remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) cross join: final aggregation over (c_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) partial aggregation over (c_customer_sk) join (INNER, REPLICATED): scan store_sales @@ -46,7 +46,7 @@ final aggregation over () partial aggregation over () final aggregation over (c_customer_sk_110) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_110"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_110]) partial aggregation over (c_customer_sk_110) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -59,19 +59,19 @@ final aggregation over () scan customer partial aggregation over () semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk]) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_item_sk"]) + remote exchange (REPARTITION, HASH, [ws_item_sk]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk_275"]) + remote exchange (REPARTITION, HASH, [i_item_sk_275]) final aggregation over (d_date_249, i_item_sk_275, substr_297) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date_249", "i_item_sk_275", "substr_297"]) + remote exchange (REPARTITION, HASH, [d_date_249, i_item_sk_275, substr_297]) partial aggregation over (d_date_249, i_item_sk_275, substr_297) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -83,11 +83,11 @@ final aggregation over () remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_355"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_355]) cross join: final aggregation over (c_customer_sk_355) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_355"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_355]) partial aggregation over (c_customer_sk_355) join (INNER, REPLICATED): scan store_sales @@ -102,7 +102,7 @@ final aggregation over () partial aggregation over () final aggregation over (c_customer_sk_405) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_405"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_405]) partial aggregation over (c_customer_sk_405) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q23_2.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q23_2.plan.txt index a4ede20f017fa..fb7ad017ee3d2 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q23_2.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q23_2.plan.txt @@ -2,27 +2,27 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (c_first_name, c_last_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name", "c_last_name"]) + remote exchange (REPARTITION, HASH, [c_first_name, c_last_name]) partial aggregation over (c_first_name, c_last_name) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [cs_bill_customer_sk]) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_item_sk"]) + remote exchange (REPARTITION, HASH, [cs_item_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [cs_bill_customer_sk]) join (INNER, REPLICATED): scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk"]) + remote exchange (REPARTITION, HASH, [i_item_sk]) final aggregation over (d_date_3, i_item_sk, substr) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date_3", "i_item_sk", "substr"]) + remote exchange (REPARTITION, HASH, [d_date_3, i_item_sk, substr]) partial aggregation over (d_date_3, i_item_sk, substr) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -34,11 +34,11 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_80"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_80]) cross join: final aggregation over (c_customer_sk_80) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_80"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_80]) partial aggregation over (c_customer_sk_80) join (INNER, REPLICATED): scan store_sales @@ -53,7 +53,7 @@ local exchange (GATHER, SINGLE, []) partial aggregation over () final aggregation over (c_customer_sk_128) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_128"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_128]) partial aggregation over (c_customer_sk_128) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -66,27 +66,27 @@ local exchange (GATHER, SINGLE, []) scan customer final aggregation over (c_first_name_229, c_last_name_230) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name_229", "c_last_name_230"]) + remote exchange (REPARTITION, HASH, [c_first_name_229, c_last_name_230]) partial aggregation over (c_first_name_229, c_last_name_230) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk]) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_item_sk"]) + remote exchange (REPARTITION, HASH, [ws_item_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_221"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_221]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk_319"]) + remote exchange (REPARTITION, HASH, [i_item_sk_319]) final aggregation over (d_date_293, i_item_sk_319, substr_341) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date_293", "i_item_sk_319", "substr_341"]) + remote exchange (REPARTITION, HASH, [d_date_293, i_item_sk_319, substr_341]) partial aggregation over (d_date_293, i_item_sk_319, substr_341) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -98,11 +98,11 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_399"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_399]) cross join: final aggregation over (c_customer_sk_399) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_399"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_399]) partial aggregation over (c_customer_sk_399) join (INNER, REPLICATED): scan store_sales @@ -117,7 +117,7 @@ local exchange (GATHER, SINGLE, []) partial aggregation over () final aggregation over (c_customer_sk_449) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_449"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_449]) partial aggregation over (c_customer_sk_449) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q24_1.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q24_1.plan.txt index cfd0fbc60c469..014b277549bf3 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q24_1.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q24_1.plan.txt @@ -2,16 +2,16 @@ remote exchange (GATHER, SINGLE, []) cross join: final aggregation over (c_first_name, c_last_name, s_store_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name", "c_last_name", "s_store_name"]) + remote exchange (REPARTITION, HASH, [c_first_name, c_last_name, s_store_name]) partial aggregation over (c_first_name, c_last_name, s_store_name) final aggregation over (c_first_name, c_last_name, ca_state, i_color, i_current_price, i_manager_id, i_size, i_units, s_state, s_store_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name", "c_last_name", "ca_state", "i_color", "i_current_price", "i_manager_id", "i_size", "i_units", "s_state", "s_store_name"]) + remote exchange (REPARTITION, HASH, [c_first_name, c_last_name, ca_state, i_color, i_current_price, i_manager_id, i_size, i_units, s_state, s_store_name]) partial aggregation over (c_first_name, c_last_name, ca_state, i_color, i_current_price, i_manager_id, i_size, i_units, s_state, s_store_name) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_birth_country", "s_zip"]) + remote exchange (REPARTITION, HASH, [c_birth_country, s_zip]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan store_sales @@ -26,10 +26,10 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan store local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_zip", "upper"]) + remote exchange (REPARTITION, HASH, [ca_zip, upper]) scan customer_address local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) @@ -39,20 +39,20 @@ remote exchange (GATHER, SINGLE, []) partial aggregation over () final aggregation over (c_first_name_181, c_last_name_182, ca_state_199, i_color_168, i_current_price_156, i_manager_id_171, i_size_166, i_units_169, s_state_146, s_store_name_127) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name_181", "c_last_name_182", "ca_state_199", "i_color_168", "i_current_price_156", "i_manager_id_171", "i_size_166", "i_units_169", "s_state_146", "s_store_name_127"]) + remote exchange (REPARTITION, HASH, [c_first_name_181, c_last_name_182, ca_state_199, i_color_168, i_current_price_156, i_manager_id_171, i_size_166, i_units_169, s_state_146, s_store_name_127]) partial aggregation over (c_first_name_181, c_last_name_182, ca_state_199, i_color_168, i_current_price_156, i_manager_id_171, i_size_166, i_units_169, s_state_146, s_store_name_127) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_item_sk_81", "ss_ticket_number_88"]) + remote exchange (REPARTITION, HASH, [ss_item_sk_81, ss_ticket_number_88]) join (INNER, REPLICATED): scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan store local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["sr_item_sk_104", "sr_ticket_number_111"]) + remote exchange (REPARTITION, HASH, [sr_item_sk_104, sr_ticket_number_111]) scan store_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q24_2.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q24_2.plan.txt index cfd0fbc60c469..014b277549bf3 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q24_2.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q24_2.plan.txt @@ -2,16 +2,16 @@ remote exchange (GATHER, SINGLE, []) cross join: final aggregation over (c_first_name, c_last_name, s_store_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name", "c_last_name", "s_store_name"]) + remote exchange (REPARTITION, HASH, [c_first_name, c_last_name, s_store_name]) partial aggregation over (c_first_name, c_last_name, s_store_name) final aggregation over (c_first_name, c_last_name, ca_state, i_color, i_current_price, i_manager_id, i_size, i_units, s_state, s_store_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name", "c_last_name", "ca_state", "i_color", "i_current_price", "i_manager_id", "i_size", "i_units", "s_state", "s_store_name"]) + remote exchange (REPARTITION, HASH, [c_first_name, c_last_name, ca_state, i_color, i_current_price, i_manager_id, i_size, i_units, s_state, s_store_name]) partial aggregation over (c_first_name, c_last_name, ca_state, i_color, i_current_price, i_manager_id, i_size, i_units, s_state, s_store_name) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_birth_country", "s_zip"]) + remote exchange (REPARTITION, HASH, [c_birth_country, s_zip]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan store_sales @@ -26,10 +26,10 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan store local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_zip", "upper"]) + remote exchange (REPARTITION, HASH, [ca_zip, upper]) scan customer_address local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) @@ -39,20 +39,20 @@ remote exchange (GATHER, SINGLE, []) partial aggregation over () final aggregation over (c_first_name_181, c_last_name_182, ca_state_199, i_color_168, i_current_price_156, i_manager_id_171, i_size_166, i_units_169, s_state_146, s_store_name_127) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name_181", "c_last_name_182", "ca_state_199", "i_color_168", "i_current_price_156", "i_manager_id_171", "i_size_166", "i_units_169", "s_state_146", "s_store_name_127"]) + remote exchange (REPARTITION, HASH, [c_first_name_181, c_last_name_182, ca_state_199, i_color_168, i_current_price_156, i_manager_id_171, i_size_166, i_units_169, s_state_146, s_store_name_127]) partial aggregation over (c_first_name_181, c_last_name_182, ca_state_199, i_color_168, i_current_price_156, i_manager_id_171, i_size_166, i_units_169, s_state_146, s_store_name_127) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_item_sk_81", "ss_ticket_number_88"]) + remote exchange (REPARTITION, HASH, [ss_item_sk_81, ss_ticket_number_88]) join (INNER, REPLICATED): scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan store local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["sr_item_sk_104", "sr_ticket_number_111"]) + remote exchange (REPARTITION, HASH, [sr_item_sk_104, sr_ticket_number_111]) scan store_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q25.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q25.plan.txt index f7d2512d33bd4..6ac8246ba6c84 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q25.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q25.plan.txt @@ -2,21 +2,21 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (i_item_desc, i_item_id, s_store_id, s_store_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_desc", "i_item_id", "s_store_id", "s_store_name"]) + remote exchange (REPARTITION, HASH, [i_item_desc, i_item_id, s_store_id, s_store_name]) partial aggregation over (i_item_desc, i_item_id, s_store_id, s_store_name) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_item_sk", "sr_customer_sk"]) + remote exchange (REPARTITION, HASH, [i_item_sk, sr_customer_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk", "ss_item_sk", "ss_ticket_number"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk, ss_item_sk, ss_ticket_number]) join (INNER, REPLICATED): scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["sr_customer_sk", "sr_item_sk", "sr_ticket_number"]) + remote exchange (REPARTITION, HASH, [sr_customer_sk, sr_item_sk, sr_ticket_number]) join (INNER, REPLICATED): scan store_returns local exchange (GATHER, SINGLE, []) @@ -29,7 +29,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk", "cs_item_sk"]) + remote exchange (REPARTITION, HASH, [cs_bill_customer_sk, cs_item_sk]) join (INNER, REPLICATED): scan catalog_sales local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q26.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q26.plan.txt index 555bb67d3baa4..b6a6e9555e2ac 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q26.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q26.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (i_item_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id"]) + remote exchange (REPARTITION, HASH, [i_item_id]) partial aggregation over (i_item_id) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q27.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q27.plan.txt index ed0e2e73e9f3c..042bd1c805181 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q27.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q27.plan.txt @@ -1,11 +1,11 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (groupid, i_item_id$gid, s_state$gid) - local exchange (REPARTITION, HASH, ["groupid", "i_item_id$gid", "s_state$gid"]) - remote exchange (REPARTITION, HASH, ["groupid", "i_item_id$gid", "s_state$gid"]) + local exchange (REPARTITION, HASH, [groupid, i_item_id$gid, s_state$gid]) + remote exchange (REPARTITION, HASH, [groupid, i_item_id$gid, s_state$gid]) partial aggregation over (groupid, i_item_id$gid, s_state$gid) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_item_sk"]) + remote exchange (REPARTITION, HASH, [ss_item_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -20,5 +20,5 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk"]) + remote exchange (REPARTITION, HASH, [i_item_sk]) scan item diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q28.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q28.plan.txt index f8b139b7041c9..015d88f877e99 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q28.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q28.plan.txt @@ -8,40 +8,40 @@ cross join: remote exchange (GATHER, SINGLE, []) partial aggregation over () local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_list_price"]) + remote exchange (REPARTITION, HASH, [ss_list_price]) scan store_sales final aggregation over () local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) partial aggregation over () local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_list_price_27"]) + remote exchange (REPARTITION, HASH, [ss_list_price_27]) scan store_sales final aggregation over () local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) partial aggregation over () local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_list_price_68"]) + remote exchange (REPARTITION, HASH, [ss_list_price_68]) scan store_sales final aggregation over () local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) partial aggregation over () local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_list_price_109"]) + remote exchange (REPARTITION, HASH, [ss_list_price_109]) scan store_sales final aggregation over () local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) partial aggregation over () local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_list_price_150"]) + remote exchange (REPARTITION, HASH, [ss_list_price_150]) scan store_sales final aggregation over () local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) partial aggregation over () local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_list_price_191"]) + remote exchange (REPARTITION, HASH, [ss_list_price_191]) scan store_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q29.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q29.plan.txt index f81f367cc0b29..79a64c4811768 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q29.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q29.plan.txt @@ -2,28 +2,28 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (i_item_desc, i_item_id, s_store_id, s_store_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_desc", "i_item_id", "s_store_id", "s_store_name"]) + remote exchange (REPARTITION, HASH, [i_item_desc, i_item_id, s_store_id, s_store_name]) partial aggregation over (i_item_desc, i_item_id, s_store_id, s_store_name) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk", "cs_item_sk"]) + remote exchange (REPARTITION, HASH, [cs_bill_customer_sk, cs_item_sk]) join (INNER, REPLICATED): scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk", "sr_customer_sk"]) + remote exchange (REPARTITION, HASH, [i_item_sk, sr_customer_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk", "ss_item_sk", "ss_ticket_number"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk, ss_item_sk, ss_ticket_number]) join (INNER, REPLICATED): scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["sr_customer_sk", "sr_item_sk", "sr_ticket_number"]) + remote exchange (REPARTITION, HASH, [sr_customer_sk, sr_item_sk, sr_ticket_number]) join (INNER, REPLICATED): scan store_returns local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q30.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q30.plan.txt index 8953e9982f55b..754a0ef612d24 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q30.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q30.plan.txt @@ -5,17 +5,17 @@ local exchange (GATHER, SINGLE, []) join (INNER, REPLICATED): final aggregation over (ca_state, wr_returning_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_state", "wr_returning_customer_sk"]) + remote exchange (REPARTITION, HASH, [ca_state, wr_returning_customer_sk]) partial aggregation over (ca_state, wr_returning_customer_sk) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["wr_returning_addr_sk"]) + remote exchange (REPARTITION, HASH, [wr_returning_addr_sk]) join (INNER, REPLICATED): scan web_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk"]) + remote exchange (REPARTITION, HASH, [ca_address_sk]) scan customer_address local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) @@ -28,21 +28,21 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) final aggregation over (ca_state_90) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_state_90"]) + remote exchange (REPARTITION, HASH, [ca_state_90]) partial aggregation over (ca_state_90) final aggregation over (ca_state_90, wr_returning_customer_sk_37) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_state_90", "wr_returning_customer_sk_37"]) + remote exchange (REPARTITION, HASH, [ca_state_90, wr_returning_customer_sk_37]) partial aggregation over (ca_state_90, wr_returning_customer_sk_37) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["wr_returning_addr_sk_40"]) + remote exchange (REPARTITION, HASH, [wr_returning_addr_sk_40]) join (INNER, REPLICATED): scan web_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk_82"]) + remote exchange (REPARTITION, HASH, [ca_address_sk_82]) scan customer_address local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q31.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q31.plan.txt index 2ac018bdac173..a47cffe6ff45a 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q31.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q31.plan.txt @@ -6,7 +6,7 @@ remote exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): final aggregation over (ca_county_81, d_qoy_56, d_year_52) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_county_81", "d_qoy_56", "d_year_52"]) + remote exchange (REPARTITION, HASH, [ca_county_81, d_qoy_56, d_year_52]) partial aggregation over (ca_county_81, d_qoy_56, d_year_52) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -18,10 +18,10 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_county_173", 2, 2000]) + remote exchange (REPARTITION, HASH, [2, 2000, ca_county_173]) final aggregation over (ca_county_173, d_qoy_148, d_year_144) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_county_173", "d_qoy_148", "d_year_144"]) + remote exchange (REPARTITION, HASH, [ca_county_173, d_qoy_148, d_year_144]) partial aggregation over (ca_county_173, d_qoy_148, d_year_144) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -35,7 +35,7 @@ remote exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): final aggregation over (ca_county_345, d_qoy_320, d_year_316) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_county_345", "d_qoy_320", "d_year_316"]) + remote exchange (REPARTITION, HASH, [ca_county_345, d_qoy_320, d_year_316]) partial aggregation over (ca_county_345, d_qoy_320, d_year_316) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -47,10 +47,10 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_county_448", 2, 2000]) + remote exchange (REPARTITION, HASH, [2, 2000, ca_county_448]) final aggregation over (ca_county_448, d_qoy_423, d_year_419) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_county_448", "d_qoy_423", "d_year_419"]) + remote exchange (REPARTITION, HASH, [ca_county_448, d_qoy_423, d_year_419]) partial aggregation over (ca_county_448, d_qoy_423, d_year_419) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -62,11 +62,11 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_county", 2, 2000]) + remote exchange (REPARTITION, HASH, [2, 2000, ca_county]) join (INNER, PARTITIONED): final aggregation over (ca_county, d_qoy, d_year) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_county", "d_qoy", "d_year"]) + remote exchange (REPARTITION, HASH, [ca_county, d_qoy, d_year]) partial aggregation over (ca_county, d_qoy, d_year) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -79,7 +79,7 @@ remote exchange (GATHER, SINGLE, []) scan customer_address final aggregation over (ca_county_242, d_qoy_217, d_year_213) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_county_242", "d_qoy_217", "d_year_213"]) + remote exchange (REPARTITION, HASH, [ca_county_242, d_qoy_217, d_year_213]) partial aggregation over (ca_county_242, d_qoy_217, d_year_213) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q32.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q32.plan.txt index bcf3e2fd621a0..001df6a6930d4 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q32.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q32.plan.txt @@ -6,7 +6,7 @@ final aggregation over () join (RIGHT, PARTITIONED): final aggregation over (cs_item_sk_15) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_item_sk_15"]) + remote exchange (REPARTITION, HASH, [cs_item_sk_15]) partial aggregation over (cs_item_sk_15) join (INNER, REPLICATED): scan catalog_sales @@ -14,7 +14,7 @@ final aggregation over () remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_item_sk"]) + remote exchange (REPARTITION, HASH, [cs_item_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan catalog_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q33.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q33.plan.txt index df84a9229a5c9..1b39d5635f024 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q33.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q33.plan.txt @@ -1,13 +1,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (expr_266) - local exchange (REPARTITION, HASH, ["expr_266"]) + local exchange (REPARTITION, HASH, [expr_266]) partial aggregation over (i_manufact_id) final aggregation over (i_manufact_id) local exchange (GATHER, SINGLE, []) partial aggregation over (i_manufact_id) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_manufact_id"]) + remote exchange (REPARTITION, HASH, [i_manufact_id]) join (INNER, REPLICATED): scan item local exchange (GATHER, SINGLE, []) @@ -22,14 +22,14 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_manufact_id_14"]) + remote exchange (REPARTITION, HASH, [i_manufact_id_14]) scan item partial aggregation over (i_manufact_id_98) final aggregation over (i_manufact_id_98) local exchange (GATHER, SINGLE, []) partial aggregation over (i_manufact_id_98) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_manufact_id_98"]) + remote exchange (REPARTITION, HASH, [i_manufact_id_98]) join (INNER, REPLICATED): scan item local exchange (GATHER, SINGLE, []) @@ -44,14 +44,14 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_manufact_id_121"]) + remote exchange (REPARTITION, HASH, [i_manufact_id_121]) scan item partial aggregation over (i_manufact_id_210) final aggregation over (i_manufact_id_210) local exchange (GATHER, SINGLE, []) partial aggregation over (i_manufact_id_210) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_manufact_id_210"]) + remote exchange (REPARTITION, HASH, [i_manufact_id_210]) join (INNER, REPLICATED): scan item local exchange (GATHER, SINGLE, []) @@ -66,5 +66,5 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_manufact_id_233"]) + remote exchange (REPARTITION, HASH, [i_manufact_id_233]) scan item diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q34.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q34.plan.txt index 580389d9f5469..0d67d18200eb2 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q34.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q34.plan.txt @@ -2,10 +2,10 @@ remote exchange (GATHER, SINGLE, []) local exchange (GATHER, UNKNOWN, []) remote exchange (REPARTITION, ROUND_ROBIN, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) final aggregation over (ss_customer_sk, ss_ticket_number) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_customer_sk", "ss_ticket_number"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk, ss_ticket_number]) partial aggregation over (ss_customer_sk, ss_ticket_number) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -21,5 +21,5 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan household_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) scan customer diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q35.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q35.plan.txt index 2bfeeeb541e2d..3358d5882a3ee 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q35.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q35.plan.txt @@ -2,24 +2,24 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (ca_state, cd_dep_college_count, cd_dep_count, cd_dep_employed_count, cd_gender, cd_marital_status) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_state", "cd_dep_college_count", "cd_dep_count", "cd_dep_employed_count", "cd_gender", "cd_marital_status"]) + remote exchange (REPARTITION, HASH, [ca_state, cd_dep_college_count, cd_dep_count, cd_dep_employed_count, cd_gender, cd_marital_status]) partial aggregation over (ca_state, cd_dep_college_count, cd_dep_count, cd_dep_employed_count, cd_gender, cd_marital_status) join (LEFT, PARTITIONED): join (LEFT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_cdemo_sk"]) + remote exchange (REPARTITION, HASH, [c_current_cdemo_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_addr_sk"]) + remote exchange (REPARTITION, HASH, [c_current_addr_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk"]) + remote exchange (REPARTITION, HASH, [ca_address_sk]) scan customer_address final aggregation over (ss_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) partial aggregation over (ss_customer_sk) join (INNER, REPLICATED): scan store_sales @@ -27,11 +27,11 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cd_demo_sk"]) + remote exchange (REPARTITION, HASH, [cd_demo_sk]) scan customer_demographics final aggregation over (ws_bill_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk]) partial aggregation over (ws_bill_customer_sk) join (INNER, REPLICATED): scan web_sales @@ -40,7 +40,7 @@ local exchange (GATHER, SINGLE, []) scan date_dim final aggregation over (cs_ship_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_ship_customer_sk"]) + remote exchange (REPARTITION, HASH, [cs_ship_customer_sk]) partial aggregation over (cs_ship_customer_sk) join (INNER, REPLICATED): scan catalog_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q36.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q36.plan.txt index 5fab99375b4ae..3504408a0aeb3 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q36.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q36.plan.txt @@ -1,10 +1,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["expr_13", "expr_14"]) + remote exchange (REPARTITION, HASH, [expr_13, expr_14]) final aggregation over (groupid, i_category$gid, i_class$gid) - local exchange (REPARTITION, HASH, ["groupid", "i_category$gid", "i_class$gid"]) - remote exchange (REPARTITION, HASH, ["groupid", "i_category$gid", "i_class$gid"]) + local exchange (REPARTITION, HASH, [groupid, i_category$gid, i_class$gid]) + remote exchange (REPARTITION, HASH, [groupid, i_category$gid, i_class$gid]) partial aggregation over (groupid, i_category$gid, i_class$gid) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q37.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q37.plan.txt index e1157d3a5d0f9..903bbfb13bbcb 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q37.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q37.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (i_current_price, i_item_desc, i_item_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_current_price", "i_item_desc", "i_item_id"]) + remote exchange (REPARTITION, HASH, [i_current_price, i_item_desc, i_item_id]) partial aggregation over (i_current_price, i_item_desc, i_item_id) join (INNER, REPLICATED): scan catalog_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q38.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q38.plan.txt index ae3a9a0e34d0b..870a8bec935e4 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q38.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q38.plan.txt @@ -3,11 +3,11 @@ final aggregation over () remote exchange (GATHER, SINGLE, []) partial aggregation over () final aggregation over (c_first_name_114, c_last_name_113, d_date_115) - local exchange (REPARTITION, HASH, ["c_first_name_114", "c_last_name_113", "d_date_115"]) + local exchange (REPARTITION, HASH, [c_first_name_114, c_last_name_113, d_date_115]) partial aggregation over (c_first_name, c_last_name, d_date) final aggregation over (c_first_name, c_last_name, d_date) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name", "c_last_name", "d_date"]) + remote exchange (REPARTITION, HASH, [c_first_name, c_last_name, d_date]) partial aggregation over (c_first_name, c_last_name, d_date) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -21,30 +21,30 @@ final aggregation over () partial aggregation over (c_first_name_42, c_last_name_43, d_date_8) final aggregation over (c_first_name_42, c_last_name_43, d_date_8) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name_42", "c_last_name_43", "d_date_8"]) + remote exchange (REPARTITION, HASH, [c_first_name_42, c_last_name_43, d_date_8]) partial aggregation over (c_first_name_42, c_last_name_43, d_date_8) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [cs_bill_customer_sk]) join (INNER, REPLICATED): scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_34"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_34]) scan customer partial aggregation over (c_first_name_97, c_last_name_98, d_date_63) final aggregation over (c_first_name_97, c_last_name_98, d_date_63) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name_97", "c_last_name_98", "d_date_63"]) + remote exchange (REPARTITION, HASH, [c_first_name_97, c_last_name_98, d_date_63]) partial aggregation over (c_first_name_97, c_last_name_98, d_date_63) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_89"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_89]) scan customer diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q39_1.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q39_1.plan.txt index 3626d2d2403be..f4d76592bd53e 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q39_1.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q39_1.plan.txt @@ -2,10 +2,10 @@ remote exchange (GATHER, SINGLE, []) local exchange (GATHER, UNKNOWN, []) remote exchange (REPARTITION, ROUND_ROBIN, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_item_sk", "w_warehouse_sk"]) + remote exchange (REPARTITION, HASH, [i_item_sk, w_warehouse_sk]) final aggregation over (d_moy, i_item_sk, w_warehouse_name, w_warehouse_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_moy", "i_item_sk", "w_warehouse_name", "w_warehouse_sk"]) + remote exchange (REPARTITION, HASH, [d_moy, i_item_sk, w_warehouse_name, w_warehouse_sk]) partial aggregation over (d_moy, i_item_sk, w_warehouse_name, w_warehouse_sk) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -21,10 +21,10 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan warehouse local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk_66", "w_warehouse_sk_88"]) + remote exchange (REPARTITION, HASH, [i_item_sk_66, w_warehouse_sk_88]) final aggregation over (d_moy_110, i_item_sk_66, w_warehouse_name_90, w_warehouse_sk_88) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_moy_110", "i_item_sk_66", "w_warehouse_name_90", "w_warehouse_sk_88"]) + remote exchange (REPARTITION, HASH, [d_moy_110, i_item_sk_66, w_warehouse_name_90, w_warehouse_sk_88]) partial aggregation over (d_moy_110, i_item_sk_66, w_warehouse_name_90, w_warehouse_sk_88) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q39_2.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q39_2.plan.txt index 3626d2d2403be..f4d76592bd53e 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q39_2.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q39_2.plan.txt @@ -2,10 +2,10 @@ remote exchange (GATHER, SINGLE, []) local exchange (GATHER, UNKNOWN, []) remote exchange (REPARTITION, ROUND_ROBIN, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_item_sk", "w_warehouse_sk"]) + remote exchange (REPARTITION, HASH, [i_item_sk, w_warehouse_sk]) final aggregation over (d_moy, i_item_sk, w_warehouse_name, w_warehouse_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_moy", "i_item_sk", "w_warehouse_name", "w_warehouse_sk"]) + remote exchange (REPARTITION, HASH, [d_moy, i_item_sk, w_warehouse_name, w_warehouse_sk]) partial aggregation over (d_moy, i_item_sk, w_warehouse_name, w_warehouse_sk) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -21,10 +21,10 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan warehouse local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk_66", "w_warehouse_sk_88"]) + remote exchange (REPARTITION, HASH, [i_item_sk_66, w_warehouse_sk_88]) final aggregation over (d_moy_110, i_item_sk_66, w_warehouse_name_90, w_warehouse_sk_88) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_moy_110", "i_item_sk_66", "w_warehouse_name_90", "w_warehouse_sk_88"]) + remote exchange (REPARTITION, HASH, [d_moy_110, i_item_sk_66, w_warehouse_name_90, w_warehouse_sk_88]) partial aggregation over (d_moy_110, i_item_sk_66, w_warehouse_name_90, w_warehouse_sk_88) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q40.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q40.plan.txt index da765df658906..8b8b98c8f152a 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q40.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q40.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (i_item_id, w_state) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id", "w_state"]) + remote exchange (REPARTITION, HASH, [i_item_id, w_state]) partial aggregation over (i_item_id, w_state) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q41.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q41.plan.txt index 6dcbeb51be654..9bc7f11922678 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q41.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q41.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (i_product_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_product_name"]) + remote exchange (REPARTITION, HASH, [i_product_name]) partial aggregation over (i_product_name) cross join: join (LEFT, REPLICATED): @@ -11,7 +11,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) final aggregation over (i_manufact_14) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_manufact_14"]) + remote exchange (REPARTITION, HASH, [i_manufact_14]) partial aggregation over (i_manufact_14) scan item local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q42.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q42.plan.txt index 87792b40cd988..93ddd972513a8 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q42.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q42.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (d_year, i_category, i_category_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_year", "i_category", "i_category_id"]) + remote exchange (REPARTITION, HASH, [d_year, i_category, i_category_id]) partial aggregation over (d_year, i_category, i_category_id) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q43.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q43.plan.txt index 001dbca108273..b70c79a1284fb 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q43.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q43.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (s_store_id, s_store_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["s_store_id", "s_store_name"]) + remote exchange (REPARTITION, HASH, [s_store_id, s_store_name]) partial aggregation over (s_store_id, s_store_name) join (INNER, REPLICATED): scan date_dim diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q44.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q44.plan.txt index d7f796253b756..f12b13c5d2d70 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q44.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q44.plan.txt @@ -1,17 +1,17 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_item_sk_61"]) + remote exchange (REPARTITION, HASH, [ss_item_sk_61]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_item_sk"]) + remote exchange (REPARTITION, HASH, [ss_item_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["rank"]) + remote exchange (REPARTITION, HASH, [rank]) local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) cross join: final aggregation over (ss_item_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_item_sk"]) + remote exchange (REPARTITION, HASH, [ss_item_sk]) partial aggregation over (ss_item_sk) scan store_sales local exchange (GATHER, SINGLE, []) @@ -20,17 +20,17 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (ss_store_sk_13) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_store_sk_13"]) + remote exchange (REPARTITION, HASH, [ss_store_sk_13]) partial aggregation over (ss_store_sk_13) scan store_sales local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["rank_131"]) + remote exchange (REPARTITION, HASH, [rank_131]) local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) cross join: final aggregation over (ss_item_sk_61) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_item_sk_61"]) + remote exchange (REPARTITION, HASH, [ss_item_sk_61]) partial aggregation over (ss_item_sk_61) scan store_sales local exchange (GATHER, SINGLE, []) @@ -39,12 +39,12 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (ss_store_sk_98) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_store_sk_98"]) + remote exchange (REPARTITION, HASH, [ss_store_sk_98]) partial aggregation over (ss_store_sk_98) scan store_sales local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk"]) + remote exchange (REPARTITION, HASH, [i_item_sk]) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk_148"]) + remote exchange (REPARTITION, HASH, [i_item_sk_148]) scan item diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q45.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q45.plan.txt index 6af76139314e2..c76190bd1d738 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q45.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q45.plan.txt @@ -2,24 +2,24 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (ca_city, ca_zip) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_city", "ca_zip"]) + remote exchange (REPARTITION, HASH, [ca_city, ca_zip]) partial aggregation over (ca_city, ca_zip) semijoin (REPLICATED): join (INNER, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_addr_sk"]) + remote exchange (REPARTITION, HASH, [c_current_addr_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk"]) + remote exchange (REPARTITION, HASH, [ca_address_sk]) scan customer_address local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q46.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q46.plan.txt index cfaebe1d13199..251f3587c6efb 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q46.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q46.plan.txt @@ -1,12 +1,12 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) final aggregation over (ca_city, ss_addr_sk, ss_customer_sk, ss_ticket_number) local exchange (GATHER, SINGLE, []) partial aggregation over (ca_city, ss_addr_sk, ss_customer_sk, ss_ticket_number) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_addr_sk"]) + remote exchange (REPARTITION, HASH, [ss_addr_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -21,13 +21,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk"]) + remote exchange (REPARTITION, HASH, [ca_address_sk]) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_addr_sk"]) + remote exchange (REPARTITION, HASH, [c_current_addr_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk_26"]) + remote exchange (REPARTITION, HASH, [ca_address_sk_26]) scan customer_address diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q47.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q47.plan.txt index 52a0683174a91..f4d9df826eee3 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q47.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q47.plan.txt @@ -3,10 +3,10 @@ local exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): join (INNER, PARTITIONED): local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand", "i_category", "s_company_name", "s_store_name"]) + remote exchange (REPARTITION, HASH, [i_brand, i_category, s_company_name, s_store_name]) final aggregation over (d_moy, d_year, i_brand, i_category, s_company_name, s_store_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_moy", "d_year", "i_brand", "i_category", "s_company_name", "s_store_name"]) + remote exchange (REPARTITION, HASH, [d_moy, d_year, i_brand, i_category, s_company_name, s_store_name]) partial aggregation over (d_moy, d_year, i_brand, i_category, s_company_name, s_store_name) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -22,10 +22,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_73", "i_category_77", "s_company_name_155", "s_store_name_143"]) + remote exchange (REPARTITION, HASH, [i_brand_73, i_category_77, s_company_name_155, s_store_name_143]) final aggregation over (d_moy_118, d_year_116, i_brand_73, i_category_77, s_company_name_155, s_store_name_143) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_moy_118", "d_year_116", "i_brand_73", "i_category_77", "s_company_name_155", "s_store_name_143"]) + remote exchange (REPARTITION, HASH, [d_moy_118, d_year_116, i_brand_73, i_category_77, s_company_name_155, s_store_name_143]) partial aggregation over (d_moy_118, d_year_116, i_brand_73, i_category_77, s_company_name_155, s_store_name_143) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -41,10 +41,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_250", "i_category_254", "s_company_name_332", "s_store_name_320"]) + remote exchange (REPARTITION, HASH, [i_brand_250, i_category_254, s_company_name_332, s_store_name_320]) final aggregation over (d_moy_295, d_year_293, i_brand_250, i_category_254, s_company_name_332, s_store_name_320) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_moy_295", "d_year_293", "i_brand_250", "i_category_254", "s_company_name_332", "s_store_name_320"]) + remote exchange (REPARTITION, HASH, [d_moy_295, d_year_293, i_brand_250, i_category_254, s_company_name_332, s_store_name_320]) partial aggregation over (d_moy_295, d_year_293, i_brand_250, i_category_254, s_company_name_332, s_store_name_320) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q48.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q48.plan.txt index a9e080182b8bf..409a033a02359 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q48.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q48.plan.txt @@ -4,7 +4,7 @@ final aggregation over () partial aggregation over () join (INNER, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_addr_sk"]) + remote exchange (REPARTITION, HASH, [ss_addr_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan store_sales @@ -15,7 +15,7 @@ final aggregation over () remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk"]) + remote exchange (REPARTITION, HASH, [ca_address_sk]) scan customer_address local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q49.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q49.plan.txt index 55f768fe4c5e6..046010473e5b4 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q49.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q49.plan.txt @@ -1,56 +1,56 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (expr_244, expr_245, expr_246, expr_247, expr_248) - local exchange (REPARTITION, HASH, ["expr_244", "expr_245", "expr_246", "expr_247", "expr_248"]) - remote exchange (REPARTITION, HASH, ["expr_12", "expr_53", "rank", "rank_23", "ws_item_sk"]) + local exchange (REPARTITION, HASH, [expr_244, expr_245, expr_246, expr_247, expr_248]) + remote exchange (REPARTITION, HASH, [expr_12, expr_53, rank, rank_23, ws_item_sk]) partial aggregation over (expr_12, expr_53, rank, rank_23, ws_item_sk) local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (ws_item_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_item_sk"]) + remote exchange (REPARTITION, HASH, [ws_item_sk]) partial aggregation over (ws_item_sk) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["wr_item_sk", "wr_order_number"]) + remote exchange (REPARTITION, HASH, [wr_item_sk, wr_order_number]) scan web_returns local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_item_sk", "ws_order_number"]) + remote exchange (REPARTITION, HASH, [ws_item_sk, ws_order_number]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim - remote exchange (REPARTITION, HASH, ["cs_item_sk", "expr_101", "expr_136", "rank_113", "rank_115"]) + remote exchange (REPARTITION, HASH, [cs_item_sk, expr_101, expr_136, rank_113, rank_115]) partial aggregation over (cs_item_sk, expr_101, expr_136, rank_113, rank_115) local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (cs_item_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_item_sk"]) + remote exchange (REPARTITION, HASH, [cs_item_sk]) partial aggregation over (cs_item_sk) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cr_item_sk", "cr_order_number"]) + remote exchange (REPARTITION, HASH, [cr_item_sk, cr_order_number]) scan catalog_returns local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_item_sk", "cs_order_number"]) + remote exchange (REPARTITION, HASH, [cs_item_sk, cs_order_number]) join (INNER, REPLICATED): scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim - remote exchange (REPARTITION, HASH, ["expr_194", "expr_239", "rank_206", "rank_208", "ss_item_sk"]) + remote exchange (REPARTITION, HASH, [expr_194, expr_239, rank_206, rank_208, ss_item_sk]) partial aggregation over (expr_194, expr_239, rank_206, rank_208, ss_item_sk) local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (ss_item_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_item_sk"]) + remote exchange (REPARTITION, HASH, [ss_item_sk]) partial aggregation over (ss_item_sk) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["sr_item_sk", "sr_ticket_number"]) + remote exchange (REPARTITION, HASH, [sr_item_sk, sr_ticket_number]) scan store_returns local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_item_sk", "ss_ticket_number"]) + remote exchange (REPARTITION, HASH, [ss_item_sk, ss_ticket_number]) join (INNER, REPLICATED): scan store_sales local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q50.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q50.plan.txt index 9f5732a36de79..fbb14edeff7f1 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q50.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q50.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (s_city, s_company_id, s_county, s_state, s_store_name, s_street_name, s_street_number, s_street_type, s_suite_number, s_zip) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["s_city", "s_company_id", "s_county", "s_state", "s_store_name", "s_street_name", "s_street_number", "s_street_type", "s_suite_number", "s_zip"]) + remote exchange (REPARTITION, HASH, [s_city, s_company_id, s_county, s_state, s_store_name, s_street_name, s_street_number, s_street_type, s_suite_number, s_zip]) partial aggregation over (s_city, s_company_id, s_county, s_state, s_store_name, s_street_name, s_street_number, s_street_type, s_suite_number, s_zip) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q51.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q51.plan.txt index cd54861265c93..16909f269a17e 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q51.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q51.plan.txt @@ -1,13 +1,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["expr_77"]) + remote exchange (REPARTITION, HASH, [expr_77]) join (FULL, PARTITIONED): local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_item_sk"]) + remote exchange (REPARTITION, HASH, [ws_item_sk]) final aggregation over (d_date, ws_item_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date", "ws_item_sk"]) + remote exchange (REPARTITION, HASH, [d_date, ws_item_sk]) partial aggregation over (d_date, ws_item_sk) join (INNER, REPLICATED): scan web_sales @@ -15,10 +15,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_item_sk"]) + remote exchange (REPARTITION, HASH, [ss_item_sk]) final aggregation over (d_date_23, ss_item_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date_23", "ss_item_sk"]) + remote exchange (REPARTITION, HASH, [d_date_23, ss_item_sk]) partial aggregation over (d_date_23, ss_item_sk) join (INNER, REPLICATED): scan store_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q52.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q52.plan.txt index 33752e693e6ce..fdfd5c1e30f09 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q52.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q52.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (d_year, i_brand, i_brand_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_year", "i_brand", "i_brand_id"]) + remote exchange (REPARTITION, HASH, [d_year, i_brand, i_brand_id]) partial aggregation over (d_year, i_brand, i_brand_id) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q53.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q53.plan.txt index cd10ab3698e65..442e2ab36b248 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q53.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q53.plan.txt @@ -1,10 +1,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_manufact_id"]) + remote exchange (REPARTITION, HASH, [i_manufact_id]) final aggregation over (d_qoy, i_manufact_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_qoy", "i_manufact_id"]) + remote exchange (REPARTITION, HASH, [d_qoy, i_manufact_id]) partial aggregation over (d_qoy, i_manufact_id) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q54.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q54.plan.txt index 20e57837636eb..de7c62c6a3c2b 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q54.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q54.plan.txt @@ -2,11 +2,11 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (expr_134) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["expr_134"]) + remote exchange (REPARTITION, HASH, [expr_134]) partial aggregation over (expr_134) final aggregation over (c_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) partial aggregation over (c_customer_sk) cross join: cross join: @@ -22,7 +22,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) final aggregation over (c_current_addr_sk, c_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_current_addr_sk", "c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_current_addr_sk, c_customer_sk]) partial aggregation over (c_current_addr_sk, c_customer_sk) join (INNER, REPLICATED): scan customer @@ -51,7 +51,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (expr_86) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["expr_86"]) + remote exchange (REPARTITION, HASH, [expr_86]) partial aggregation over (expr_86) scan date_dim local exchange (GATHER, SINGLE, []) @@ -60,6 +60,6 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (expr_118) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["expr_118"]) + remote exchange (REPARTITION, HASH, [expr_118]) partial aggregation over (expr_118) scan date_dim diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q55.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q55.plan.txt index 0322599f01e50..ad81649cf1df4 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q55.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q55.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (i_brand, i_brand_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand", "i_brand_id"]) + remote exchange (REPARTITION, HASH, [i_brand, i_brand_id]) partial aggregation over (i_brand, i_brand_id) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q56.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q56.plan.txt index 05da2df474848..d6a55327309d1 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q56.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q56.plan.txt @@ -1,13 +1,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (expr_266) - local exchange (REPARTITION, HASH, ["expr_266"]) + local exchange (REPARTITION, HASH, [expr_266]) partial aggregation over (i_item_id) final aggregation over (i_item_id) local exchange (GATHER, SINGLE, []) partial aggregation over (i_item_id) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_item_id"]) + remote exchange (REPARTITION, HASH, [i_item_id]) join (INNER, REPLICATED): scan item local exchange (GATHER, SINGLE, []) @@ -22,14 +22,14 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id_2"]) + remote exchange (REPARTITION, HASH, [i_item_id_2]) scan item partial aggregation over (i_item_id_86) final aggregation over (i_item_id_86) local exchange (GATHER, SINGLE, []) partial aggregation over (i_item_id_86) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_item_id_86"]) + remote exchange (REPARTITION, HASH, [i_item_id_86]) join (INNER, REPLICATED): scan item local exchange (GATHER, SINGLE, []) @@ -44,14 +44,14 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id_109"]) + remote exchange (REPARTITION, HASH, [i_item_id_109]) scan item partial aggregation over (i_item_id_198) final aggregation over (i_item_id_198) local exchange (GATHER, SINGLE, []) partial aggregation over (i_item_id_198) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_item_id_198"]) + remote exchange (REPARTITION, HASH, [i_item_id_198]) join (INNER, REPLICATED): scan item local exchange (GATHER, SINGLE, []) @@ -66,5 +66,5 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id_221"]) + remote exchange (REPARTITION, HASH, [i_item_id_221]) scan item diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q57.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q57.plan.txt index 22d2cb70c38d2..e6a7895fefbf6 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q57.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q57.plan.txt @@ -3,10 +3,10 @@ local exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): join (INNER, PARTITIONED): local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cc_name", "i_brand", "i_category"]) + remote exchange (REPARTITION, HASH, [cc_name, i_brand, i_category]) final aggregation over (cc_name, d_moy, d_year, i_brand, i_category) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cc_name", "d_moy", "d_year", "i_brand", "i_category"]) + remote exchange (REPARTITION, HASH, [cc_name, d_moy, d_year, i_brand, i_category]) partial aggregation over (cc_name, d_moy, d_year, i_brand, i_category) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -22,10 +22,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan call_center local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cc_name_147", "i_brand_65", "i_category_69"]) + remote exchange (REPARTITION, HASH, [cc_name_147, i_brand_65, i_category_69]) final aggregation over (cc_name_147, d_moy_121, d_year_119, i_brand_65, i_category_69) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cc_name_147", "d_moy_121", "d_year_119", "i_brand_65", "i_category_69"]) + remote exchange (REPARTITION, HASH, [cc_name_147, d_moy_121, d_year_119, i_brand_65, i_category_69]) partial aggregation over (cc_name_147, d_moy_121, d_year_119, i_brand_65, i_category_69) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -41,10 +41,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan call_center local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cc_name_328", "i_brand_246", "i_category_250"]) + remote exchange (REPARTITION, HASH, [cc_name_328, i_brand_246, i_category_250]) final aggregation over (cc_name_328, d_moy_302, d_year_300, i_brand_246, i_category_250) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cc_name_328", "d_moy_302", "d_year_300", "i_brand_246", "i_category_250"]) + remote exchange (REPARTITION, HASH, [cc_name_328, d_moy_302, d_year_300, i_brand_246, i_category_250]) partial aggregation over (cc_name_328, d_moy_302, d_year_300, i_brand_246, i_category_250) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q58.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q58.plan.txt index b918ac3d2fb42..8ff13170a1c3b 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q58.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q58.plan.txt @@ -4,7 +4,7 @@ local exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): final aggregation over (i_item_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id"]) + remote exchange (REPARTITION, HASH, [i_item_id]) partial aggregation over (i_item_id) semijoin (REPLICATED): join (INNER, REPLICATED): @@ -27,7 +27,7 @@ local exchange (GATHER, SINGLE, []) scan date_dim final aggregation over (i_item_id_79) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id_79"]) + remote exchange (REPARTITION, HASH, [i_item_id_79]) partial aggregation over (i_item_id_79) semijoin (REPLICATED): join (INNER, REPLICATED): @@ -50,7 +50,7 @@ local exchange (GATHER, SINGLE, []) scan date_dim final aggregation over (i_item_id_210) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id_210"]) + remote exchange (REPARTITION, HASH, [i_item_id_210]) partial aggregation over (i_item_id_210) semijoin (REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q59.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q59.plan.txt index 37c1bbaf2d1b8..0b3cf288cda09 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q59.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q59.plan.txt @@ -1,12 +1,12 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["d_week_seq", "s_store_id"]) + remote exchange (REPARTITION, HASH, [d_week_seq, s_store_id]) join (INNER, REPLICATED): join (INNER, REPLICATED): final aggregation over (d_week_seq, ss_store_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_week_seq", "ss_store_sk"]) + remote exchange (REPARTITION, HASH, [d_week_seq, ss_store_sk]) partial aggregation over (d_week_seq, ss_store_sk) join (INNER, REPLICATED): scan store_sales @@ -20,12 +20,12 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan store local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["expr_361", "s_store_id_235"]) + remote exchange (REPARTITION, HASH, [expr_361, s_store_id_235]) join (INNER, REPLICATED): join (INNER, REPLICATED): final aggregation over (d_week_seq_147, ss_store_sk_127) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_week_seq_147", "ss_store_sk_127"]) + remote exchange (REPARTITION, HASH, [d_week_seq_147, ss_store_sk_127]) partial aggregation over (d_week_seq_147, ss_store_sk_127) join (INNER, REPLICATED): scan store_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q60.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q60.plan.txt index 05da2df474848..d6a55327309d1 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q60.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q60.plan.txt @@ -1,13 +1,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (expr_266) - local exchange (REPARTITION, HASH, ["expr_266"]) + local exchange (REPARTITION, HASH, [expr_266]) partial aggregation over (i_item_id) final aggregation over (i_item_id) local exchange (GATHER, SINGLE, []) partial aggregation over (i_item_id) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_item_id"]) + remote exchange (REPARTITION, HASH, [i_item_id]) join (INNER, REPLICATED): scan item local exchange (GATHER, SINGLE, []) @@ -22,14 +22,14 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id_2"]) + remote exchange (REPARTITION, HASH, [i_item_id_2]) scan item partial aggregation over (i_item_id_86) final aggregation over (i_item_id_86) local exchange (GATHER, SINGLE, []) partial aggregation over (i_item_id_86) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_item_id_86"]) + remote exchange (REPARTITION, HASH, [i_item_id_86]) join (INNER, REPLICATED): scan item local exchange (GATHER, SINGLE, []) @@ -44,14 +44,14 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id_109"]) + remote exchange (REPARTITION, HASH, [i_item_id_109]) scan item partial aggregation over (i_item_id_198) final aggregation over (i_item_id_198) local exchange (GATHER, SINGLE, []) partial aggregation over (i_item_id_198) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["i_item_id_198"]) + remote exchange (REPARTITION, HASH, [i_item_id_198]) join (INNER, REPLICATED): scan item local exchange (GATHER, SINGLE, []) @@ -66,5 +66,5 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id_221"]) + remote exchange (REPARTITION, HASH, [i_item_id_221]) scan item diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q61.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q61.plan.txt index f9f0b76869567..6881342073d59 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q61.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q61.plan.txt @@ -8,7 +8,7 @@ cross join: local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) join (INNER, REPLICATED): scan date_dim local exchange (GATHER, SINGLE, []) @@ -23,7 +23,7 @@ cross join: remote exchange (REPLICATE, BROADCAST, []) scan store local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) join (INNER, REPLICATED): scan customer local exchange (GATHER, SINGLE, []) @@ -38,7 +38,7 @@ cross join: local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk_7"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk_7]) join (INNER, REPLICATED): scan date_dim local exchange (GATHER, SINGLE, []) @@ -49,7 +49,7 @@ cross join: remote exchange (REPLICATE, BROADCAST, []) scan store local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_84"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_84]) join (INNER, REPLICATED): scan customer local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q62.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q62.plan.txt index c91381b40a0a2..318d6c63dfc3b 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q62.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q62.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (sm_type, substr, web_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["sm_type", "substr", "web_name"]) + remote exchange (REPARTITION, HASH, [sm_type, substr, web_name]) partial aggregation over (sm_type, substr, web_name) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q63.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q63.plan.txt index 23e28eddc4b7d..b89397ce6c5c9 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q63.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q63.plan.txt @@ -1,10 +1,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_manager_id"]) + remote exchange (REPARTITION, HASH, [i_manager_id]) final aggregation over (d_moy, i_manager_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_moy", "i_manager_id"]) + remote exchange (REPARTITION, HASH, [d_moy, i_manager_id]) partial aggregation over (d_moy, i_manager_id) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q64.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q64.plan.txt index b974738680e61..ab6bd93311604 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q64.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q64.plan.txt @@ -6,37 +6,37 @@ remote exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) partial aggregation over (ca_city, ca_city_98, ca_street_name, ca_street_name_95, ca_street_number, ca_street_number_94, ca_zip, ca_zip_101, d_year, d_year_28, d_year_56, i_item_sk, i_product_name, s_store_name, s_zip) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_item_sk"]) + remote exchange (REPARTITION, HASH, [cs_item_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["hd_income_band_sk_88"]) + remote exchange (REPARTITION, HASH, [hd_income_band_sk_88]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["hd_income_band_sk"]) + remote exchange (REPARTITION, HASH, [hd_income_band_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_addr_sk"]) + remote exchange (REPARTITION, HASH, [c_current_addr_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_addr_sk"]) + remote exchange (REPARTITION, HASH, [ss_addr_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_hdemo_sk"]) + remote exchange (REPARTITION, HASH, [c_current_hdemo_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_hdemo_sk"]) + remote exchange (REPARTITION, HASH, [ss_hdemo_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_promo_sk"]) + remote exchange (REPARTITION, HASH, [ss_promo_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_cdemo_sk"]) + remote exchange (REPARTITION, HASH, [c_current_cdemo_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_cdemo_sk"]) + remote exchange (REPARTITION, HASH, [ss_cdemo_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_first_shipto_date_sk"]) + remote exchange (REPARTITION, HASH, [c_first_shipto_date_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_first_sales_date_sk"]) + remote exchange (REPARTITION, HASH, [c_first_sales_date_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_store_sk"]) + remote exchange (REPARTITION, HASH, [ss_store_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_sold_date_sk"]) + remote exchange (REPARTITION, HASH, [ss_sold_date_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["sr_item_sk"]) + remote exchange (REPARTITION, HASH, [sr_item_sk]) join (INNER, REPLICATED): scan store_sales local exchange (GATHER, SINGLE, []) @@ -44,94 +44,94 @@ remote exchange (GATHER, SINGLE, []) scan store_returns final aggregation over (cs_item_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_item_sk"]) + remote exchange (REPARTITION, HASH, [cs_item_sk]) partial aggregation over (cs_item_sk) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_item_sk", "cs_order_number"]) + remote exchange (REPARTITION, HASH, [cs_item_sk, cs_order_number]) scan catalog_sales local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cr_item_sk", "cr_order_number"]) + remote exchange (REPARTITION, HASH, [cr_item_sk, cr_order_number]) scan catalog_returns local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date_sk"]) + remote exchange (REPARTITION, HASH, [d_date_sk]) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["s_store_sk"]) + remote exchange (REPARTITION, HASH, [s_store_sk]) scan store local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date_sk_22"]) + remote exchange (REPARTITION, HASH, [d_date_sk_22]) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date_sk_50"]) + remote exchange (REPARTITION, HASH, [d_date_sk_50]) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cd_demo_sk"]) + remote exchange (REPARTITION, HASH, [cd_demo_sk]) scan customer_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cd_demo_sk_78"]) + remote exchange (REPARTITION, HASH, [cd_demo_sk_78]) scan customer_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["p_promo_sk"]) + remote exchange (REPARTITION, HASH, [p_promo_sk]) scan promotion local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["hd_demo_sk"]) + remote exchange (REPARTITION, HASH, [hd_demo_sk]) scan household_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["hd_demo_sk_87"]) + remote exchange (REPARTITION, HASH, [hd_demo_sk_87]) scan household_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk"]) + remote exchange (REPARTITION, HASH, [ca_address_sk]) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk_92"]) + remote exchange (REPARTITION, HASH, [ca_address_sk_92]) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ib_income_band_sk"]) + remote exchange (REPARTITION, HASH, [ib_income_band_sk]) scan income_band local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ib_income_band_sk_105"]) + remote exchange (REPARTITION, HASH, [ib_income_band_sk_105]) scan income_band local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk"]) + remote exchange (REPARTITION, HASH, [i_item_sk]) scan item final aggregation over (ca_city_547, ca_city_560, ca_street_name_544, ca_street_name_557, ca_street_number_543, ca_street_number_556, ca_zip_550, ca_zip_563, d_year_369, d_year_397, d_year_425, i_item_sk_573, i_product_name_594, s_store_name_452, s_zip_472) local exchange (GATHER, SINGLE, []) partial aggregation over (ca_city_547, ca_city_560, ca_street_name_544, ca_street_name_557, ca_street_number_543, ca_street_number_556, ca_zip_550, ca_zip_563, d_year_369, d_year_397, d_year_425, i_item_sk_573, i_product_name_594, s_store_name_452, s_zip_472) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_item_sk_292"]) + remote exchange (REPARTITION, HASH, [cs_item_sk_292]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["hd_income_band_sk_537"]) + remote exchange (REPARTITION, HASH, [hd_income_band_sk_537]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["hd_income_band_sk_532"]) + remote exchange (REPARTITION, HASH, [hd_income_band_sk_532]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_addr_sk_480"]) + remote exchange (REPARTITION, HASH, [c_current_addr_sk_480]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_addr_sk_240"]) + remote exchange (REPARTITION, HASH, [ss_addr_sk_240]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_hdemo_sk_479"]) + remote exchange (REPARTITION, HASH, [c_current_hdemo_sk_479]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_hdemo_sk_239"]) + remote exchange (REPARTITION, HASH, [ss_hdemo_sk_239]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_promo_sk_242"]) + remote exchange (REPARTITION, HASH, [ss_promo_sk_242]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_cdemo_sk_478"]) + remote exchange (REPARTITION, HASH, [c_current_cdemo_sk_478]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_cdemo_sk_238"]) + remote exchange (REPARTITION, HASH, [ss_cdemo_sk_238]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_first_shipto_date_sk_481"]) + remote exchange (REPARTITION, HASH, [c_first_shipto_date_sk_481]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_first_sales_date_sk_482"]) + remote exchange (REPARTITION, HASH, [c_first_sales_date_sk_482]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk_237"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk_237]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_store_sk_241"]) + remote exchange (REPARTITION, HASH, [ss_store_sk_241]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_234"]) + remote exchange (REPARTITION, HASH, [ss_sold_date_sk_234]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["sr_item_sk_259"]) + remote exchange (REPARTITION, HASH, [sr_item_sk_259]) join (INNER, REPLICATED): scan store_sales local exchange (GATHER, SINGLE, []) @@ -139,56 +139,56 @@ remote exchange (GATHER, SINGLE, []) scan store_returns final aggregation over (cs_item_sk_292) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_item_sk_292"]) + remote exchange (REPARTITION, HASH, [cs_item_sk_292]) partial aggregation over (cs_item_sk_292) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_item_sk_292", "cs_order_number_294"]) + remote exchange (REPARTITION, HASH, [cs_item_sk_292, cs_order_number_294]) scan catalog_sales local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cr_item_sk_313", "cr_order_number_327"]) + remote exchange (REPARTITION, HASH, [cr_item_sk_313, cr_order_number_327]) scan catalog_returns local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date_sk_363"]) + remote exchange (REPARTITION, HASH, [d_date_sk_363]) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["s_store_sk_447"]) + remote exchange (REPARTITION, HASH, [s_store_sk_447]) scan store local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_476"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_476]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date_sk_391"]) + remote exchange (REPARTITION, HASH, [d_date_sk_391]) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date_sk_419"]) + remote exchange (REPARTITION, HASH, [d_date_sk_419]) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cd_demo_sk_494"]) + remote exchange (REPARTITION, HASH, [cd_demo_sk_494]) scan customer_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cd_demo_sk_503"]) + remote exchange (REPARTITION, HASH, [cd_demo_sk_503]) scan customer_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["p_promo_sk_512"]) + remote exchange (REPARTITION, HASH, [p_promo_sk_512]) scan promotion local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["hd_demo_sk_531"]) + remote exchange (REPARTITION, HASH, [hd_demo_sk_531]) scan household_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["hd_demo_sk_536"]) + remote exchange (REPARTITION, HASH, [hd_demo_sk_536]) scan household_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk_541"]) + remote exchange (REPARTITION, HASH, [ca_address_sk_541]) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk_554"]) + remote exchange (REPARTITION, HASH, [ca_address_sk_554]) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ib_income_band_sk_567"]) + remote exchange (REPARTITION, HASH, [ib_income_band_sk_567]) scan income_band local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ib_income_band_sk_570"]) + remote exchange (REPARTITION, HASH, [ib_income_band_sk_570]) scan income_band local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk_573"]) + remote exchange (REPARTITION, HASH, [i_item_sk_573]) scan item diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q65.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q65.plan.txt index f09082ea78cc4..b3e0ee4b00ae1 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q65.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q65.plan.txt @@ -4,7 +4,7 @@ local exchange (GATHER, SINGLE, []) join (INNER, REPLICATED): final aggregation over (ss_item_sk_26, ss_store_sk_31) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_item_sk_26", "ss_store_sk_31"]) + remote exchange (REPARTITION, HASH, [ss_item_sk_26, ss_store_sk_31]) partial aggregation over (ss_item_sk_26, ss_store_sk_31) join (INNER, REPLICATED): scan store_sales @@ -14,15 +14,15 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["s_store_sk"]) + remote exchange (REPARTITION, HASH, [s_store_sk]) scan store final aggregation over (ss_store_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_store_sk"]) + remote exchange (REPARTITION, HASH, [ss_store_sk]) partial aggregation over (ss_store_sk) final aggregation over (ss_item_sk, ss_store_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_item_sk", "ss_store_sk"]) + remote exchange (REPARTITION, HASH, [ss_item_sk, ss_store_sk]) partial aggregation over (ss_item_sk, ss_store_sk) join (INNER, REPLICATED): scan store_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q66.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q66.plan.txt index a9b44132630ab..313752790b117 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q66.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q66.plan.txt @@ -1,11 +1,11 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (concat_306, d_year_307, w_city_302, w_country_305, w_county_303, w_state_304, w_warehouse_name_300, w_warehouse_sq_ft_301) - local exchange (REPARTITION, HASH, ["concat_306", "d_year_307", "w_city_302", "w_country_305", "w_county_303", "w_state_304", "w_warehouse_name_300", "w_warehouse_sq_ft_301"]) + local exchange (REPARTITION, HASH, [concat_306, d_year_307, w_city_302, w_country_305, w_county_303, w_state_304, w_warehouse_name_300, w_warehouse_sq_ft_301]) partial aggregation over (concat, d_year, w_city, w_country, w_county, w_state, w_warehouse_name, w_warehouse_sq_ft) final aggregation over (d_year, w_city, w_country, w_county, w_state, w_warehouse_name, w_warehouse_sq_ft) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_year", "w_city", "w_country", "w_county", "w_state", "w_warehouse_name", "w_warehouse_sq_ft"]) + remote exchange (REPARTITION, HASH, [d_year, w_city, w_country, w_county, w_state, w_warehouse_name, w_warehouse_sq_ft]) partial aggregation over (d_year, w_city, w_country, w_county, w_state, w_warehouse_name, w_warehouse_sq_ft) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -27,7 +27,7 @@ local exchange (GATHER, SINGLE, []) partial aggregation over (concat_242, d_year_136, w_city_124, w_country_128, w_county_125, w_state_126, w_warehouse_name_118, w_warehouse_sq_ft_119) final aggregation over (d_year_136, w_city_124, w_country_128, w_county_125, w_state_126, w_warehouse_name_118, w_warehouse_sq_ft_119) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_year_136", "w_city_124", "w_country_128", "w_county_125", "w_state_126", "w_warehouse_name_118", "w_warehouse_sq_ft_119"]) + remote exchange (REPARTITION, HASH, [d_year_136, w_city_124, w_country_128, w_county_125, w_state_126, w_warehouse_name_118, w_warehouse_sq_ft_119]) partial aggregation over (d_year_136, w_city_124, w_country_128, w_county_125, w_state_126, w_warehouse_name_118, w_warehouse_sq_ft_119) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q67.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q67.plan.txt index 6b438220e7039..c620dffa1ff81 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q67.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q67.plan.txt @@ -1,10 +1,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_category$gid"]) + remote exchange (REPARTITION, HASH, [i_category$gid]) final aggregation over (d_moy$gid, d_qoy$gid, d_year$gid, groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid, s_store_id$gid) - local exchange (REPARTITION, HASH, ["d_moy$gid", "d_qoy$gid", "d_year$gid", "groupid", "i_brand$gid", "i_category$gid", "i_class$gid", "i_product_name$gid", "s_store_id$gid"]) - remote exchange (REPARTITION, HASH, ["d_moy$gid", "d_qoy$gid", "d_year$gid", "groupid", "i_brand$gid", "i_category$gid", "i_class$gid", "i_product_name$gid", "s_store_id$gid"]) + local exchange (REPARTITION, HASH, [d_moy$gid, d_qoy$gid, d_year$gid, groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid, s_store_id$gid]) + remote exchange (REPARTITION, HASH, [d_moy$gid, d_qoy$gid, d_year$gid, groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid, s_store_id$gid]) partial aggregation over (d_moy$gid, d_qoy$gid, d_year$gid, groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid, s_store_id$gid) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q68.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q68.plan.txt index ac2fd19b2cb05..e62b0e3a65eb5 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q68.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q68.plan.txt @@ -1,20 +1,20 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_current_addr_sk"]) + remote exchange (REPARTITION, HASH, [c_current_addr_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) final aggregation over (ca_city, ss_addr_sk, ss_customer_sk, ss_ticket_number) local exchange (GATHER, SINGLE, []) partial aggregation over (ca_city, ss_addr_sk, ss_customer_sk, ss_ticket_number) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ca_address_sk"]) + remote exchange (REPARTITION, HASH, [ca_address_sk]) scan customer_address local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_addr_sk"]) + remote exchange (REPARTITION, HASH, [ss_addr_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -29,5 +29,5 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan household_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_address_sk_32"]) + remote exchange (REPARTITION, HASH, [ca_address_sk_32]) scan customer_address diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q69.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q69.plan.txt index b4e12b0f4a331..27e5ae2f2ad92 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q69.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q69.plan.txt @@ -2,13 +2,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (cd_credit_rating, cd_education_status, cd_gender, cd_marital_status, cd_purchase_estimate) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cd_credit_rating", "cd_education_status", "cd_gender", "cd_marital_status", "cd_purchase_estimate"]) + remote exchange (REPARTITION, HASH, [cd_credit_rating, cd_education_status, cd_gender, cd_marital_status, cd_purchase_estimate]) partial aggregation over (cd_credit_rating, cd_education_status, cd_gender, cd_marital_status, cd_purchase_estimate) join (LEFT, PARTITIONED): join (RIGHT, PARTITIONED): final aggregation over (ws_bill_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk]) partial aggregation over (ws_bill_customer_sk) join (INNER, REPLICATED): scan web_sales @@ -16,16 +16,16 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cd_demo_sk"]) + remote exchange (REPARTITION, HASH, [cd_demo_sk]) scan customer_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_current_cdemo_sk"]) + remote exchange (REPARTITION, HASH, [c_current_cdemo_sk]) join (INNER, PARTITIONED): final aggregation over (ss_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) partial aggregation over (ss_customer_sk) join (INNER, REPLICATED): scan store_sales @@ -33,7 +33,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) join (INNER, REPLICATED): scan customer local exchange (GATHER, SINGLE, []) @@ -41,7 +41,7 @@ local exchange (GATHER, SINGLE, []) scan customer_address final aggregation over (cs_ship_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_ship_customer_sk"]) + remote exchange (REPARTITION, HASH, [cs_ship_customer_sk]) partial aggregation over (cs_ship_customer_sk) join (INNER, REPLICATED): scan catalog_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q70.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q70.plan.txt index 20609c3c276ed..7bc40dab01316 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q70.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q70.plan.txt @@ -1,13 +1,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["expr_110", "expr_111"]) + remote exchange (REPARTITION, HASH, [expr_110, expr_111]) final aggregation over (groupid, s_county$gid, s_state$gid_102) - local exchange (REPARTITION, HASH, ["groupid", "s_county$gid", "s_state$gid_102"]) - remote exchange (REPARTITION, HASH, ["groupid", "s_county$gid", "s_state$gid_102"]) + local exchange (REPARTITION, HASH, [groupid, s_county$gid, s_state$gid_102]) + remote exchange (REPARTITION, HASH, [groupid, s_county$gid, s_state$gid_102]) partial aggregation over (groupid, s_county$gid, s_state$gid_102) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["s_state"]) + remote exchange (REPARTITION, HASH, [s_state]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan store_sales @@ -18,10 +18,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan store local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["s_state_48"]) + remote exchange (REPARTITION, HASH, [s_state_48]) final aggregation over (s_state_48) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["s_state_48"]) + remote exchange (REPARTITION, HASH, [s_state_48]) partial aggregation over (s_state_48) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q71.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q71.plan.txt index 599dd0ab9f9c0..d304b1ae70633 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q71.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q71.plan.txt @@ -3,7 +3,7 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (i_brand, i_brand_id, t_hour, t_minute) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand", "i_brand_id", "t_hour", "t_minute"]) + remote exchange (REPARTITION, HASH, [i_brand, i_brand_id, t_hour, t_minute]) partial aggregation over (i_brand, i_brand_id, t_hour, t_minute) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q72.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q72.plan.txt index bcda1a8ede5a2..9c00eb85c99f8 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q72.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q72.plan.txt @@ -2,12 +2,12 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (d_week_seq, i_item_desc, w_warehouse_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_week_seq", "i_item_desc", "w_warehouse_name"]) + remote exchange (REPARTITION, HASH, [d_week_seq, i_item_desc, w_warehouse_name]) partial aggregation over (d_week_seq, i_item_desc, w_warehouse_name) join (LEFT, REPLICATED): join (LEFT, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["d_week_seq_15", "inv_item_sk"]) + remote exchange (REPARTITION, HASH, [d_week_seq_15, inv_item_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan inventory @@ -18,7 +18,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan warehouse local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_item_sk", "d_week_seq"]) + remote exchange (REPARTITION, HASH, [cs_item_sk, d_week_seq]) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q73.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q73.plan.txt index 9fa2321b10235..b749d8b68db7e 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q73.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q73.plan.txt @@ -2,10 +2,10 @@ remote exchange (GATHER, SINGLE, []) local exchange (GATHER, UNKNOWN, []) remote exchange (REPARTITION, ROUND_ROBIN, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) final aggregation over (ss_customer_sk, ss_ticket_number) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_customer_sk", "ss_ticket_number"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk, ss_ticket_number]) partial aggregation over (ss_customer_sk, ss_ticket_number) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -21,5 +21,5 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan household_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk"]) + remote exchange (REPARTITION, HASH, [c_customer_sk]) scan customer diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q74.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q74.plan.txt index a78033681f1f1..c49dc2b0217ad 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q74.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q74.plan.txt @@ -4,10 +4,10 @@ local exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): join (INNER, PARTITIONED): local exchange (REPARTITION, ROUND_ROBIN, []) - remote exchange (REPARTITION, HASH, ["c_customer_id"]) + remote exchange (REPARTITION, HASH, [c_customer_id]) final aggregation over (c_customer_id, c_first_name, c_last_name, d_year) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id", "c_first_name", "c_last_name", "d_year"]) + remote exchange (REPARTITION, HASH, [c_customer_id, c_first_name, c_last_name, d_year]) partial aggregation over (c_customer_id, c_first_name, c_last_name, d_year) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -18,7 +18,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer - remote exchange (REPARTITION, HASH, ["c_customer_id_17"]) + remote exchange (REPARTITION, HASH, [c_customer_id_17]) single aggregation over (c_customer_id_17, c_first_name_24, c_last_name_25, d_year_40) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -26,10 +26,10 @@ local exchange (GATHER, SINGLE, []) values (0 rows) values (0 rows) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_109"]) + remote exchange (REPARTITION, HASH, [c_customer_id_109]) final aggregation over (c_customer_id_109, c_first_name_116, c_last_name_117, d_year_155) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_109", "c_first_name_116", "c_last_name_117", "d_year_155"]) + remote exchange (REPARTITION, HASH, [c_customer_id_109, c_first_name_116, c_last_name_117, d_year_155]) partial aggregation over (c_customer_id_109, c_first_name_116, c_last_name_117, d_year_155) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -40,7 +40,7 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer - remote exchange (REPARTITION, HASH, ["c_customer_id_200"]) + remote exchange (REPARTITION, HASH, [c_customer_id_200]) single aggregation over (c_customer_id_200, c_first_name_207, c_last_name_208, d_year_257) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -48,48 +48,48 @@ local exchange (GATHER, SINGLE, []) values (0 rows) values (0 rows) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_326"]) + remote exchange (REPARTITION, HASH, [c_customer_id_326]) single aggregation over (c_customer_id_326, c_first_name_333, c_last_name_334, d_year_372) join (INNER, REPLICATED): join (INNER, REPLICATED): values (0 rows) values (0 rows) values (0 rows) - remote exchange (REPARTITION, HASH, ["c_customer_id_417"]) + remote exchange (REPARTITION, HASH, [c_customer_id_417]) final aggregation over (c_customer_id_417, c_first_name_424, c_last_name_425, d_year_474) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_417", "c_first_name_424", "c_last_name_425", "d_year_474"]) + remote exchange (REPARTITION, HASH, [c_customer_id_417, c_first_name_424, c_last_name_425, d_year_474]) partial aggregation over (c_customer_id_417, c_first_name_424, c_last_name_425, d_year_474) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_438"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk_438]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_416"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_416]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_543"]) + remote exchange (REPARTITION, HASH, [c_customer_id_543]) single aggregation over (c_customer_id_543, c_first_name_550, c_last_name_551, d_year_589) join (INNER, REPLICATED): join (INNER, REPLICATED): values (0 rows) values (0 rows) values (0 rows) - remote exchange (REPARTITION, HASH, ["c_customer_id_634"]) + remote exchange (REPARTITION, HASH, [c_customer_id_634]) final aggregation over (c_customer_id_634, c_first_name_641, c_last_name_642, d_year_691) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_id_634", "c_first_name_641", "c_last_name_642", "d_year_691"]) + remote exchange (REPARTITION, HASH, [c_customer_id_634, c_first_name_641, c_last_name_642, d_year_691]) partial aggregation over (c_customer_id_634, c_first_name_641, c_last_name_642, d_year_691) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_655"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk_655]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_633"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_633]) scan customer diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q75.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q75.plan.txt index b036635321401..76c66a43c9e27 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q75.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q75.plan.txt @@ -3,14 +3,14 @@ local exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): single aggregation over (d_year_172, i_brand_id_173, i_category_id_175, i_class_id_174, i_manufact_id_176) final aggregation over (d_year_172, expr_177, expr_178, i_brand_id_173, i_category_id_175, i_class_id_174, i_manufact_id_176) - local exchange (REPARTITION, HASH, ["d_year_172", "i_brand_id_173", "i_category_id_175", "i_class_id_174", "i_manufact_id_176"]) - remote exchange (REPARTITION, HASH, ["i_brand_id", "i_category_id", "i_class_id", "i_manufact_id"]) + local exchange (REPARTITION, HASH, [d_year_172, i_brand_id_173, i_category_id_175, i_class_id_174, i_manufact_id_176]) + remote exchange (REPARTITION, HASH, [i_brand_id, i_category_id, i_class_id, i_manufact_id]) partial aggregation over (d_year, expr, expr_13, i_brand_id, i_category_id, i_class_id, i_manufact_id) join (RIGHT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cr_item_sk", "cr_order_number"]) + remote exchange (REPARTITION, HASH, [cr_item_sk, cr_order_number]) scan catalog_returns local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_item_sk", "cs_order_number"]) + remote exchange (REPARTITION, HASH, [cs_item_sk, cs_order_number]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan catalog_sales @@ -20,13 +20,13 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim - remote exchange (REPARTITION, HASH, ["i_brand_id_28", "i_category_id_32", "i_class_id_30", "i_manufact_id_34"]) + remote exchange (REPARTITION, HASH, [i_brand_id_28, i_category_id_32, i_class_id_30, i_manufact_id_34]) partial aggregation over (d_year_51, expr_84, expr_85, i_brand_id_28, i_category_id_32, i_class_id_30, i_manufact_id_34) join (RIGHT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["sr_item_sk", "sr_ticket_number"]) + remote exchange (REPARTITION, HASH, [sr_item_sk, sr_ticket_number]) scan store_returns local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_item_sk", "ss_ticket_number"]) + remote exchange (REPARTITION, HASH, [ss_item_sk, ss_ticket_number]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan store_sales @@ -36,13 +36,13 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim - remote exchange (REPARTITION, HASH, ["i_brand_id_107", "i_category_id_111", "i_class_id_109", "i_manufact_id_113"]) + remote exchange (REPARTITION, HASH, [i_brand_id_107, i_category_id_111, i_class_id_109, i_manufact_id_113]) partial aggregation over (d_year_130, expr_163, expr_164, i_brand_id_107, i_category_id_111, i_class_id_109, i_manufact_id_113) join (RIGHT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["wr_item_sk", "wr_order_number"]) + remote exchange (REPARTITION, HASH, [wr_item_sk, wr_order_number]) scan web_returns local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_item_sk", "ws_order_number"]) + remote exchange (REPARTITION, HASH, [ws_item_sk, ws_order_number]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan web_sales @@ -55,13 +55,13 @@ local exchange (GATHER, SINGLE, []) single aggregation over (d_year_619, i_brand_id_620, i_category_id_622, i_class_id_621, i_manufact_id_623) final aggregation over (d_year_619, expr_624, expr_625, i_brand_id_620, i_category_id_622, i_class_id_621, i_manufact_id_623) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand_id_275", "i_category_id_279", "i_class_id_277", "i_manufact_id_281"]) + remote exchange (REPARTITION, HASH, [i_brand_id_275, i_category_id_279, i_class_id_277, i_manufact_id_281]) partial aggregation over (d_year_298, expr_358, expr_359, i_brand_id_275, i_category_id_279, i_class_id_277, i_manufact_id_281) join (RIGHT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cr_item_sk_324", "cr_order_number_338"]) + remote exchange (REPARTITION, HASH, [cr_item_sk_324, cr_order_number_338]) scan catalog_returns local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_item_sk_249", "cs_order_number_251"]) + remote exchange (REPARTITION, HASH, [cs_item_sk_249, cs_order_number_251]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan catalog_sales @@ -71,13 +71,13 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim - remote exchange (REPARTITION, HASH, ["i_brand_id_397", "i_category_id_401", "i_class_id_399", "i_manufact_id_403"]) + remote exchange (REPARTITION, HASH, [i_brand_id_397, i_category_id_401, i_class_id_399, i_manufact_id_403]) partial aggregation over (d_year_420, expr_473, expr_474, i_brand_id_397, i_category_id_401, i_class_id_399, i_manufact_id_403) join (RIGHT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["sr_item_sk_446", "sr_ticket_number_453"]) + remote exchange (REPARTITION, HASH, [sr_item_sk_446, sr_ticket_number_453]) scan store_returns local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_item_sk_369", "ss_ticket_number_376"]) + remote exchange (REPARTITION, HASH, [ss_item_sk_369, ss_ticket_number_376]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan store_sales @@ -87,13 +87,13 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim - remote exchange (REPARTITION, HASH, ["i_brand_id_530", "i_category_id_534", "i_class_id_532", "i_manufact_id_536"]) + remote exchange (REPARTITION, HASH, [i_brand_id_530, i_category_id_534, i_class_id_532, i_manufact_id_536]) partial aggregation over (d_year_553, expr_610, expr_611, i_brand_id_530, i_category_id_534, i_class_id_532, i_manufact_id_536) join (RIGHT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["wr_item_sk_579", "wr_order_number_590"]) + remote exchange (REPARTITION, HASH, [wr_item_sk_579, wr_order_number_590]) scan web_returns local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_item_sk_492", "ws_order_number_506"]) + remote exchange (REPARTITION, HASH, [ws_item_sk_492, ws_order_number_506]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan web_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q76.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q76.plan.txt index 8e6f704d4410b..eab23a30206e6 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q76.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q76.plan.txt @@ -1,9 +1,9 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (d_qoy_168, d_year_167, expr_165, expr_166, i_category_169) - local exchange (REPARTITION, HASH, ["d_qoy_168", "d_year_167", "expr_165", "expr_166", "i_category_169"]) + local exchange (REPARTITION, HASH, [d_qoy_168, d_year_167, expr_165, expr_166, i_category_169]) local exchange (REPARTITION, ROUND_ROBIN, []) - remote exchange (REPARTITION, HASH, ["d_qoy", "d_year", "expr_12", "expr_217", "i_category"]) + remote exchange (REPARTITION, HASH, [d_qoy, d_year, expr_12, expr_217, i_category]) partial aggregation over (d_qoy, d_year, expr_12, expr_217, i_category) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -14,20 +14,20 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim - remote exchange (REPARTITION, HASH, ["d_qoy_49", "d_year_45", "expr_223", "expr_68", "i_category_29"]) + remote exchange (REPARTITION, HASH, [d_qoy_49, d_year_45, expr_223, expr_68, i_category_29]) partial aggregation over (d_qoy_49, d_year_45, expr_223, expr_68, i_category_29) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_sold_date_sk"]) + remote exchange (REPARTITION, HASH, [ws_sold_date_sk]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_item_sk"]) + remote exchange (REPARTITION, HASH, [ws_item_sk]) scan web_sales local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_sk_17"]) + remote exchange (REPARTITION, HASH, [i_item_sk_17]) scan item local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_date_sk_39"]) + remote exchange (REPARTITION, HASH, [d_date_sk_39]) scan date_dim - remote exchange (REPARTITION, HASH, ["d_qoy_129", "d_year_125", "expr_147", "expr_160", "i_category_109"]) + remote exchange (REPARTITION, HASH, [d_qoy_129, d_year_125, expr_147, expr_160, i_category_109]) partial aggregation over (d_qoy_129, d_year_125, expr_147, expr_160, i_category_109) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q77.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q77.plan.txt index 32474cf29af00..765e1f82b3284 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q77.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q77.plan.txt @@ -1,14 +1,14 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, HASH, ["channel$gid", "groupid", "id$gid"]) - remote exchange (REPARTITION, HASH, ["channel$gid", "groupid", "id$gid"]) + local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) partial aggregation over (channel$gid, groupid, id$gid) local exchange (REPARTITION, ROUND_ROBIN, []) join (LEFT, PARTITIONED): final aggregation over (s_store_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["s_store_sk"]) + remote exchange (REPARTITION, HASH, [s_store_sk]) partial aggregation over (s_store_sk) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -21,7 +21,7 @@ local exchange (GATHER, SINGLE, []) scan store final aggregation over (s_store_sk_46) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["s_store_sk_46"]) + remote exchange (REPARTITION, HASH, [s_store_sk_46]) partial aggregation over (s_store_sk_46) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -35,7 +35,7 @@ local exchange (GATHER, SINGLE, []) cross join: final aggregation over (cs_call_center_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_call_center_sk"]) + remote exchange (REPARTITION, HASH, [cs_call_center_sk]) partial aggregation over (cs_call_center_sk) join (INNER, REPLICATED): scan catalog_sales @@ -46,7 +46,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) final aggregation over (cr_call_center_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cr_call_center_sk"]) + remote exchange (REPARTITION, HASH, [cr_call_center_sk]) partial aggregation over (cr_call_center_sk) join (INNER, REPLICATED): scan catalog_returns @@ -56,7 +56,7 @@ local exchange (GATHER, SINGLE, []) join (LEFT, PARTITIONED): final aggregation over (wp_web_page_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["wp_web_page_sk"]) + remote exchange (REPARTITION, HASH, [wp_web_page_sk]) partial aggregation over (wp_web_page_sk) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -69,7 +69,7 @@ local exchange (GATHER, SINGLE, []) scan web_page final aggregation over (wp_web_page_sk_298) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["wp_web_page_sk_298"]) + remote exchange (REPARTITION, HASH, [wp_web_page_sk_298]) partial aggregation over (wp_web_page_sk_298) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q78.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q78.plan.txt index cba938a811083..fc43ced46f4a9 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q78.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q78.plan.txt @@ -1,11 +1,11 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) join (INNER, PARTITIONED): final aggregation over (d_year, ss_customer_sk, ss_item_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_year", "ss_customer_sk", "ss_item_sk"]) + remote exchange (REPARTITION, HASH, [d_year, ss_customer_sk, ss_item_sk]) partial aggregation over (d_year, ss_customer_sk, ss_item_sk) join (INNER, REPLICATED): join (LEFT, REPLICATED): @@ -18,7 +18,7 @@ local exchange (GATHER, SINGLE, []) scan date_dim final aggregation over (d_year_53, ws_bill_customer_sk, ws_item_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_year_53", "ws_bill_customer_sk", "ws_item_sk"]) + remote exchange (REPARTITION, HASH, [d_year_53, ws_bill_customer_sk, ws_item_sk]) partial aggregation over (d_year_53, ws_bill_customer_sk, ws_item_sk) join (INNER, REPLICATED): join (LEFT, REPLICATED): @@ -30,10 +30,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [cs_bill_customer_sk]) final aggregation over (cs_bill_customer_sk, cs_item_sk, d_year_130) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk", "cs_item_sk", "d_year_130"]) + remote exchange (REPARTITION, HASH, [cs_bill_customer_sk, cs_item_sk, d_year_130]) partial aggregation over (cs_bill_customer_sk, cs_item_sk, d_year_130) join (INNER, REPLICATED): join (LEFT, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q79.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q79.plan.txt index 1541432df999d..e65a70aa03f4e 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q79.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q79.plan.txt @@ -3,7 +3,7 @@ local exchange (GATHER, SINGLE, []) join (INNER, REPLICATED): final aggregation over (s_city, ss_addr_sk, ss_customer_sk, ss_ticket_number) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["s_city", "ss_addr_sk", "ss_customer_sk", "ss_ticket_number"]) + remote exchange (REPARTITION, HASH, [s_city, ss_addr_sk, ss_customer_sk, ss_ticket_number]) partial aggregation over (s_city, ss_addr_sk, ss_customer_sk, ss_ticket_number) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q80.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q80.plan.txt index 9fbd54c9adae9..99c9b4a65e450 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q80.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q80.plan.txt @@ -1,13 +1,13 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, HASH, ["channel$gid", "groupid", "id$gid"]) - remote exchange (REPARTITION, HASH, ["channel$gid", "groupid", "id$gid"]) + local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) partial aggregation over (channel$gid, groupid, id$gid) local exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (s_store_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["s_store_id"]) + remote exchange (REPARTITION, HASH, [s_store_id]) partial aggregation over (s_store_id) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -32,7 +32,7 @@ local exchange (GATHER, SINGLE, []) scan store final aggregation over (cp_catalog_page_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cp_catalog_page_id"]) + remote exchange (REPARTITION, HASH, [cp_catalog_page_id]) partial aggregation over (cp_catalog_page_id) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -57,7 +57,7 @@ local exchange (GATHER, SINGLE, []) scan catalog_page final aggregation over (web_site_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["web_site_id"]) + remote exchange (REPARTITION, HASH, [web_site_id]) partial aggregation over (web_site_id) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q81.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q81.plan.txt index c32e34721947a..2018e21c9530d 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q81.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q81.plan.txt @@ -5,7 +5,7 @@ local exchange (GATHER, SINGLE, []) join (INNER, REPLICATED): final aggregation over (ca_state, cr_returning_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_state", "cr_returning_customer_sk"]) + remote exchange (REPARTITION, HASH, [ca_state, cr_returning_customer_sk]) partial aggregation over (ca_state, cr_returning_customer_sk) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -27,11 +27,11 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) final aggregation over (ca_state_93) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_state_93"]) + remote exchange (REPARTITION, HASH, [ca_state_93]) partial aggregation over (ca_state_93) final aggregation over (ca_state_93, cr_returning_customer_sk_37) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ca_state_93", "cr_returning_customer_sk_37"]) + remote exchange (REPARTITION, HASH, [ca_state_93, cr_returning_customer_sk_37]) partial aggregation over (ca_state_93, cr_returning_customer_sk_37) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q82.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q82.plan.txt index 0c8eb3bf442d7..ada6da13dc2a6 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q82.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q82.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (i_current_price, i_item_desc, i_item_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_current_price", "i_item_desc", "i_item_id"]) + remote exchange (REPARTITION, HASH, [i_current_price, i_item_desc, i_item_id]) partial aggregation over (i_current_price, i_item_desc, i_item_id) join (INNER, REPLICATED): scan store_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q83.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q83.plan.txt index 8dbec1ac0658c..21406c5ebc4fe 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q83.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q83.plan.txt @@ -3,7 +3,7 @@ local exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): final aggregation over (i_item_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id"]) + remote exchange (REPARTITION, HASH, [i_item_id]) partial aggregation over (i_item_id) semijoin (REPLICATED): join (INNER, REPLICATED): @@ -25,7 +25,7 @@ local exchange (GATHER, SINGLE, []) join (INNER, PARTITIONED): final aggregation over (i_item_id_82) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id_82"]) + remote exchange (REPARTITION, HASH, [i_item_id_82]) partial aggregation over (i_item_id_82) semijoin (REPLICATED): join (INNER, REPLICATED): @@ -46,7 +46,7 @@ local exchange (GATHER, SINGLE, []) scan date_dim final aggregation over (i_item_id_216) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_item_id_216"]) + remote exchange (REPARTITION, HASH, [i_item_id_216]) partial aggregation over (i_item_id_216) semijoin (REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q85.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q85.plan.txt index 77a2fc4bdef06..4f08fb63ae4ad 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q85.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q85.plan.txt @@ -2,25 +2,25 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (r_reason_desc) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["r_reason_desc"]) + remote exchange (REPARTITION, HASH, [r_reason_desc]) partial aggregation over (r_reason_desc) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cd_demo_sk_0", "cd_education_status_3", "cd_marital_status_2"]) + remote exchange (REPARTITION, HASH, [cd_demo_sk_0, cd_education_status_3, cd_marital_status_2]) scan customer_demographics local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cd_education_status", "cd_marital_status", "wr_returning_cdemo_sk"]) + remote exchange (REPARTITION, HASH, [cd_education_status, cd_marital_status, wr_returning_cdemo_sk]) join (INNER, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_item_sk", "ws_order_number"]) + remote exchange (REPARTITION, HASH, [ws_item_sk, ws_order_number]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["wr_item_sk", "wr_order_number"]) + remote exchange (REPARTITION, HASH, [wr_item_sk, wr_order_number]) join (INNER, REPLICATED): scan web_returns local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q86.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q86.plan.txt index 5f7832a4e2f6c..2b02f0f8ed0c7 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q86.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q86.plan.txt @@ -1,10 +1,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["expr_10", "expr_9"]) + remote exchange (REPARTITION, HASH, [expr_10, expr_9]) final aggregation over (groupid, i_category$gid, i_class$gid) - local exchange (REPARTITION, HASH, ["groupid", "i_category$gid", "i_class$gid"]) - remote exchange (REPARTITION, HASH, ["groupid", "i_category$gid", "i_class$gid"]) + local exchange (REPARTITION, HASH, [groupid, i_category$gid, i_class$gid]) + remote exchange (REPARTITION, HASH, [groupid, i_category$gid, i_class$gid]) partial aggregation over (groupid, i_category$gid, i_class$gid) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q87.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q87.plan.txt index 8f1baf197b0ad..6d4a277e3d2e7 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q87.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q87.plan.txt @@ -3,11 +3,11 @@ final aggregation over () remote exchange (GATHER, SINGLE, []) partial aggregation over () final aggregation over (expr_130, expr_131, expr_132) - local exchange (REPARTITION, HASH, ["expr_130", "expr_131", "expr_132"]) + local exchange (REPARTITION, HASH, [expr_130, expr_131, expr_132]) partial aggregation over (c_first_name, c_last_name, d_date) final aggregation over (c_first_name, c_last_name, d_date) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name", "c_last_name", "d_date"]) + remote exchange (REPARTITION, HASH, [c_first_name, c_last_name, d_date]) partial aggregation over (c_first_name, c_last_name, d_date) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -21,30 +21,30 @@ final aggregation over () partial aggregation over (c_first_name_47, c_last_name_48, d_date_13) final aggregation over (c_first_name_47, c_last_name_48, d_date_13) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name_47", "c_last_name_48", "d_date_13"]) + remote exchange (REPARTITION, HASH, [c_first_name_47, c_last_name_48, d_date_13]) partial aggregation over (c_first_name_47, c_last_name_48, d_date_13) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [cs_bill_customer_sk]) join (INNER, REPLICATED): scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_39"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_39]) scan customer partial aggregation over (c_first_name_108, c_last_name_109, d_date_74) final aggregation over (c_first_name_108, c_last_name_109, d_date_74) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_first_name_108", "c_last_name_109", "d_date_74"]) + remote exchange (REPARTITION, HASH, [c_first_name_108, c_last_name_109, d_date_74]) partial aggregation over (c_first_name_108, c_last_name_109, d_date_74) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"]) + remote exchange (REPARTITION, HASH, [ws_bill_customer_sk]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["c_customer_sk_100"]) + remote exchange (REPARTITION, HASH, [c_customer_sk_100]) scan customer diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q89.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q89.plan.txt index 396d705bbc386..fd614d326b07f 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q89.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q89.plan.txt @@ -1,10 +1,10 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_brand", "i_category", "s_company_name", "s_store_name"]) + remote exchange (REPARTITION, HASH, [i_brand, i_category, s_company_name, s_store_name]) final aggregation over (d_moy, i_brand, i_category, i_class, s_company_name, s_store_name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["d_moy", "i_brand", "i_category", "i_class", "s_company_name", "s_store_name"]) + remote exchange (REPARTITION, HASH, [d_moy, i_brand, i_category, i_class, s_company_name, s_store_name]) partial aggregation over (d_moy, i_brand, i_category, i_class, s_company_name, s_store_name) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q91.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q91.plan.txt index a8367e55cdb02..1b6ce4e1fec2d 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q91.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q91.plan.txt @@ -3,7 +3,7 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (cc_call_center_id, cc_manager, cc_name, cd_education_status, cd_marital_status) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cc_call_center_id", "cc_manager", "cc_name", "cd_education_status", "cd_marital_status"]) + remote exchange (REPARTITION, HASH, [cc_call_center_id, cc_manager, cc_name, cd_education_status, cd_marital_status]) partial aggregation over (cc_call_center_id, cc_manager, cc_name, cd_education_status, cd_marital_status) join (INNER, REPLICATED): scan call_center diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q92.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q92.plan.txt index fe73a232a885b..2de88a134fe7c 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q92.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q92.plan.txt @@ -6,7 +6,7 @@ final aggregation over () join (RIGHT, PARTITIONED): final aggregation over (ws_item_sk_3) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_item_sk_3"]) + remote exchange (REPARTITION, HASH, [ws_item_sk_3]) partial aggregation over (ws_item_sk_3) join (INNER, REPLICATED): scan web_sales @@ -14,7 +14,7 @@ final aggregation over () remote exchange (REPLICATE, BROADCAST, []) scan date_dim local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_item_sk"]) + remote exchange (REPARTITION, HASH, [ws_item_sk]) join (INNER, REPLICATED): join (INNER, REPLICATED): scan web_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q93.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q93.plan.txt index 01324ef4f25f5..9d3941bbd74a1 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q93.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q93.plan.txt @@ -2,14 +2,14 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (ss_customer_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_customer_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk]) partial aggregation over (ss_customer_sk) join (INNER, REPLICATED): join (LEFT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ss_item_sk", "ss_ticket_number"]) + remote exchange (REPARTITION, HASH, [ss_item_sk, ss_ticket_number]) scan store_sales local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["sr_item_sk", "sr_ticket_number"]) + remote exchange (REPARTITION, HASH, [sr_item_sk, sr_ticket_number]) scan store_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q94.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q94.plan.txt index c134240654765..d5190c5028a91 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q94.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q94.plan.txt @@ -7,10 +7,10 @@ final aggregation over () local exchange (GATHER, SINGLE, []) partial aggregation over (ca_state, d_date, unique, web_company_name, ws_ext_ship_cost, ws_net_profit, ws_order_number, ws_ship_addr_sk, ws_ship_date_sk, ws_warehouse_sk, ws_web_site_sk) join (RIGHT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_order_number_17"]) + remote exchange (REPARTITION, HASH, [ws_order_number_17]) scan web_sales local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_order_number"]) + remote exchange (REPARTITION, HASH, [ws_order_number]) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -26,6 +26,6 @@ final aggregation over () scan web_site final aggregation over (wr_order_number) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["wr_order_number"]) + remote exchange (REPARTITION, HASH, [wr_order_number]) partial aggregation over (wr_order_number) scan web_returns diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q95.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q95.plan.txt index 1f190927bfd80..d53d7877f5d3f 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q95.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q95.plan.txt @@ -5,7 +5,7 @@ final aggregation over () local exchange (GATHER, SINGLE, []) semijoin (PARTITIONED): semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_order_number"]) + remote exchange (REPARTITION, HASH, [ws_order_number]) join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -20,22 +20,22 @@ final aggregation over () remote exchange (REPLICATE, BROADCAST, []) scan web_site local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_order_number_17"]) + remote exchange (REPARTITION, HASH, [ws_order_number_17]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["ws_order_number_17"]) + remote exchange (REPARTITION, HASH, [ws_order_number_17]) scan web_sales local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_order_number_51"]) + remote exchange (REPARTITION, HASH, [ws_order_number_51]) scan web_sales local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["wr_order_number"]) + remote exchange (REPARTITION, HASH, [wr_order_number]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["wr_order_number"]) + remote exchange (REPARTITION, HASH, [wr_order_number]) join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan web_returns local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ws_order_number_141"]) + remote exchange (REPARTITION, HASH, [ws_order_number_141]) scan web_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q97.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q97.plan.txt index 28b301c990947..2b345e65c251e 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q97.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q97.plan.txt @@ -5,7 +5,7 @@ final aggregation over () join (FULL, PARTITIONED): final aggregation over (ss_customer_sk, ss_item_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["ss_customer_sk", "ss_item_sk"]) + remote exchange (REPARTITION, HASH, [ss_customer_sk, ss_item_sk]) partial aggregation over (ss_customer_sk, ss_item_sk) join (INNER, REPLICATED): scan store_sales @@ -14,7 +14,7 @@ final aggregation over () scan date_dim final aggregation over (cs_bill_customer_sk, cs_item_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk", "cs_item_sk"]) + remote exchange (REPARTITION, HASH, [cs_bill_customer_sk, cs_item_sk]) partial aggregation over (cs_bill_customer_sk, cs_item_sk) join (INNER, REPLICATED): scan catalog_sales diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q98.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q98.plan.txt index 9775b6c511f56..bab8ffb8ac143 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q98.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q98.plan.txt @@ -2,10 +2,10 @@ remote exchange (GATHER, SINGLE, []) local exchange (GATHER, UNKNOWN, []) remote exchange (REPARTITION, ROUND_ROBIN, []) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_class"]) + remote exchange (REPARTITION, HASH, [i_class]) final aggregation over (i_category, i_class, i_current_price, i_item_desc, i_item_id) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["i_category", "i_class", "i_current_price", "i_item_desc", "i_item_id"]) + remote exchange (REPARTITION, HASH, [i_category, i_class, i_current_price, i_item_desc, i_item_id]) partial aggregation over (i_category, i_class, i_current_price, i_item_desc, i_item_id) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q99.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q99.plan.txt index e077b55eaab26..2d6870f92b832 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q99.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q99.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (cc_name, sm_type, substr) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["cc_name", "sm_type", "substr"]) + remote exchange (REPARTITION, HASH, [cc_name, sm_type, substr]) partial aggregation over (cc_name, sm_type, substr) join (INNER, REPLICATED): join (INNER, REPLICATED): diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q01.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q01.plan.txt index d632b41b395aa..649baba23ae08 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q01.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q01.plan.txt @@ -3,6 +3,6 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (linestatus, returnflag) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["linestatus", "returnflag"]) + remote exchange (REPARTITION, HASH, [linestatus, returnflag]) partial aggregation over (linestatus, returnflag) scan lineitem diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q02.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q02.plan.txt index 98d7dfd9ce458..afceedbe57dfa 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q02.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q02.plan.txt @@ -5,7 +5,7 @@ remote exchange (GATHER, SINGLE, []) join (RIGHT, PARTITIONED): final aggregation over (partkey_15) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["partkey_15"]) + remote exchange (REPARTITION, HASH, [partkey_15]) partial aggregation over (partkey_15) join (INNER, REPLICATED): scan partsupp @@ -21,16 +21,16 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan region local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["partkey"]) + remote exchange (REPARTITION, HASH, [partkey]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["suppkey_4"]) + remote exchange (REPARTITION, HASH, [suppkey_4]) join (INNER, REPLICATED): scan partsupp local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan part local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["suppkey"]) + remote exchange (REPARTITION, HASH, [suppkey]) join (INNER, REPLICATED): scan supplier local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q03.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q03.plan.txt index 054e207ee6b27..b0ae8a7837a13 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q03.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q03.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (orderdate, orderkey_3, shippriority) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderdate", "orderkey_3", "shippriority"]) + remote exchange (REPARTITION, HASH, [orderdate, orderkey_3, shippriority]) partial aggregation over (orderdate, orderkey_3, shippriority) join (INNER, REPLICATED): scan lineitem diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q04.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q04.plan.txt index f1a033c8f1dea..f387a01c3d8a6 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q04.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q04.plan.txt @@ -3,13 +3,13 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (orderpriority) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderpriority"]) + remote exchange (REPARTITION, HASH, [orderpriority]) partial aggregation over (orderpriority) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["orderkey"]) + remote exchange (REPARTITION, HASH, [orderkey]) scan orders final aggregation over (orderkey_0) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderkey_0"]) + remote exchange (REPARTITION, HASH, [orderkey_0]) partial aggregation over (orderkey_0) scan lineitem diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q05.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q05.plan.txt index 7583600ca6080..224ecece5dde2 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q05.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q05.plan.txt @@ -3,7 +3,7 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (name_15) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["name_15"]) + remote exchange (REPARTITION, HASH, [name_15]) partial aggregation over (name_15) join (INNER, REPLICATED): join (INNER, REPLICATED): @@ -11,10 +11,10 @@ remote exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["custkey_0"]) + remote exchange (REPARTITION, HASH, [custkey_0]) scan orders local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["custkey"]) + remote exchange (REPARTITION, HASH, [custkey]) join (INNER, REPLICATED): scan customer local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q07.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q07.plan.txt index 375d69c002424..1c96c70ad9581 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q07.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q07.plan.txt @@ -3,10 +3,10 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (expr_24, name_15, name_19) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["expr_24", "name_15", "name_19"]) + remote exchange (REPARTITION, HASH, [expr_24, name_15, name_19]) partial aggregation over (expr_24, name_15, name_19) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["orderkey"]) + remote exchange (REPARTITION, HASH, [orderkey]) join (INNER, REPLICATED): scan lineitem local exchange (GATHER, SINGLE, []) @@ -17,7 +17,7 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan nation local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderkey_3"]) + remote exchange (REPARTITION, HASH, [orderkey_3]) join (INNER, REPLICATED): scan orders local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q08.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q08.plan.txt index 0f2d764aa8708..f8f92b5edabe1 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q08.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q08.plan.txt @@ -3,12 +3,12 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (expr) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["expr"]) + remote exchange (REPARTITION, HASH, [expr]) partial aggregation over (expr) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["suppkey_4"]) + remote exchange (REPARTITION, HASH, [suppkey_4]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["orderkey_7"]) + remote exchange (REPARTITION, HASH, [orderkey_7]) join (INNER, REPLICATED): scan orders local exchange (GATHER, SINGLE, []) @@ -23,14 +23,14 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan region local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderkey"]) + remote exchange (REPARTITION, HASH, [orderkey]) join (INNER, REPLICATED): scan lineitem local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan part local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["suppkey"]) + remote exchange (REPARTITION, HASH, [suppkey]) join (INNER, REPLICATED): scan supplier local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q09.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q09.plan.txt index 1dc7e904716e7..9a36538c96eb6 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q09.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q09.plan.txt @@ -3,30 +3,30 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (expr_18, name_15) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["expr_18", "name_15"]) + remote exchange (REPARTITION, HASH, [expr_18, name_15]) partial aggregation over (expr_18, name_15) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["nationkey"]) + remote exchange (REPARTITION, HASH, [nationkey]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["orderkey"]) + remote exchange (REPARTITION, HASH, [orderkey]) join (INNER, PARTITIONED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["suppkey_4"]) + remote exchange (REPARTITION, HASH, [suppkey_4]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["partkey"]) + remote exchange (REPARTITION, HASH, [partkey]) scan part local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["partkey_3"]) + remote exchange (REPARTITION, HASH, [partkey_3]) scan lineitem local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["suppkey"]) + remote exchange (REPARTITION, HASH, [suppkey]) scan supplier local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["suppkey_8"]) + remote exchange (REPARTITION, HASH, [suppkey_8]) scan partsupp local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderkey_11"]) + remote exchange (REPARTITION, HASH, [orderkey_11]) scan orders local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["nationkey_14"]) + remote exchange (REPARTITION, HASH, [nationkey_14]) scan nation diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q10.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q10.plan.txt index 45310667f8724..f6cf1e4b0da1b 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q10.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q10.plan.txt @@ -4,14 +4,14 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) partial aggregation over (acctbal, address, comment_4, custkey_3, name, name_7, phone) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["custkey_3"]) + remote exchange (REPARTITION, HASH, [custkey_3]) join (INNER, REPLICATED): scan customer local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan nation local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["custkey"]) + remote exchange (REPARTITION, HASH, [custkey]) join (INNER, REPLICATED): scan lineitem local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q11.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q11.plan.txt index bfc5af4b5bb23..6c783db4ea76b 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q11.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q11.plan.txt @@ -4,7 +4,7 @@ remote exchange (GATHER, SINGLE, []) cross join: final aggregation over (partkey) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["partkey"]) + remote exchange (REPARTITION, HASH, [partkey]) partial aggregation over (partkey) join (INNER, REPLICATED): scan partsupp diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q12.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q12.plan.txt index 7fb5b836dfa90..4a002edd5c696 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q12.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q12.plan.txt @@ -3,11 +3,11 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (shipmode) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["shipmode"]) + remote exchange (REPARTITION, HASH, [shipmode]) partial aggregation over (shipmode) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["orderkey"]) + remote exchange (REPARTITION, HASH, [orderkey]) scan orders local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderkey_0"]) + remote exchange (REPARTITION, HASH, [orderkey_0]) scan lineitem diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q13.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q13.plan.txt index 96af9081b5217..99812af1a6922 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q13.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q13.plan.txt @@ -3,14 +3,14 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (count) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["count"]) + remote exchange (REPARTITION, HASH, [count]) partial aggregation over (count) final aggregation over (custkey) local exchange (GATHER, SINGLE, []) partial aggregation over (custkey) join (LEFT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["custkey"]) + remote exchange (REPARTITION, HASH, [custkey]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["custkey_0"]) + remote exchange (REPARTITION, HASH, [custkey_0]) scan orders diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q14.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q14.plan.txt index e46c0418a7596..564bd361bfe07 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q14.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q14.plan.txt @@ -3,8 +3,8 @@ final aggregation over () remote exchange (GATHER, SINGLE, []) partial aggregation over () join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["partkey_0"]) + remote exchange (REPARTITION, HASH, [partkey_0]) scan part local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["partkey"]) + remote exchange (REPARTITION, HASH, [partkey]) scan lineitem diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q15.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q15.plan.txt index 66ec494a2bf4c..7e241a4656918 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q15.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q15.plan.txt @@ -3,11 +3,11 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) join (INNER, REPLICATED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["suppkey"]) + remote exchange (REPARTITION, HASH, [suppkey]) scan supplier final aggregation over (suppkey_0) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["suppkey_0"]) + remote exchange (REPARTITION, HASH, [suppkey_0]) partial aggregation over (suppkey_0) scan lineitem local exchange (GATHER, SINGLE, []) @@ -18,6 +18,6 @@ remote exchange (GATHER, SINGLE, []) partial aggregation over () final aggregation over (suppkey_16) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["suppkey_16"]) + remote exchange (REPARTITION, HASH, [suppkey_16]) partial aggregation over (suppkey_16) scan lineitem diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q16.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q16.plan.txt index ef581b90df64a..7ed9174fceeb2 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q16.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q16.plan.txt @@ -3,19 +3,19 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (brand, size, type) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["brand", "size", "type"]) + remote exchange (REPARTITION, HASH, [brand, size, type]) partial aggregation over (brand, size, type) final aggregation over (brand, size, suppkey, type) local exchange (GATHER, SINGLE, []) partial aggregation over (brand, size, suppkey, type) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["suppkey"]) + remote exchange (REPARTITION, HASH, [suppkey]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["partkey"]) + remote exchange (REPARTITION, HASH, [partkey]) scan partsupp local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["partkey_0"]) + remote exchange (REPARTITION, HASH, [partkey_0]) scan part local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["suppkey_3"]) + remote exchange (REPARTITION, HASH, [suppkey_3]) scan supplier diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q17.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q17.plan.txt index bbcdb95d961dd..ed19d20bdf2da 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q17.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q17.plan.txt @@ -6,11 +6,11 @@ final aggregation over () join (RIGHT, PARTITIONED): final aggregation over (partkey_4) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["partkey_4"]) + remote exchange (REPARTITION, HASH, [partkey_4]) partial aggregation over (partkey_4) scan lineitem local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["partkey"]) + remote exchange (REPARTITION, HASH, [partkey]) join (INNER, REPLICATED): scan lineitem local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q18.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q18.plan.txt index a10b26c531c08..90afa8ba9623b 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q18.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q18.plan.txt @@ -5,20 +5,20 @@ local exchange (GATHER, SINGLE, []) partial aggregation over (custkey, name, orderdate, orderkey, totalprice) semijoin (PARTITIONED): join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["orderkey_3"]) + remote exchange (REPARTITION, HASH, [orderkey_3]) scan lineitem local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderkey"]) + remote exchange (REPARTITION, HASH, [orderkey]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["custkey_0"]) + remote exchange (REPARTITION, HASH, [custkey_0]) scan orders local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["custkey"]) + remote exchange (REPARTITION, HASH, [custkey]) scan customer local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderkey_6"]) + remote exchange (REPARTITION, HASH, [orderkey_6]) final aggregation over (orderkey_6) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderkey_6"]) + remote exchange (REPARTITION, HASH, [orderkey_6]) partial aggregation over (orderkey_6) scan lineitem diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q19.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q19.plan.txt index fa00639798fe2..d37d1ad12acee 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q19.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q19.plan.txt @@ -3,8 +3,8 @@ final aggregation over () remote exchange (GATHER, SINGLE, []) partial aggregation over () join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["partkey"]) + remote exchange (REPARTITION, HASH, [partkey]) scan lineitem local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["partkey_0"]) + remote exchange (REPARTITION, HASH, [partkey_0]) scan part diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q20.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q20.plan.txt index 70c9c82a66708..88c366255133c 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q20.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q20.plan.txt @@ -2,27 +2,27 @@ remote exchange (GATHER, SINGLE, []) local exchange (GATHER, UNKNOWN, []) remote exchange (REPARTITION, ROUND_ROBIN, []) semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["suppkey"]) + remote exchange (REPARTITION, HASH, [suppkey]) join (INNER, REPLICATED): scan supplier local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan nation local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["suppkey_4"]) + remote exchange (REPARTITION, HASH, [suppkey_4]) cross join: join (LEFT, PARTITIONED): semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, ["partkey"]) + remote exchange (REPARTITION, HASH, [partkey]) scan partsupp local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["partkey_8"]) + remote exchange (REPARTITION, HASH, [partkey_8]) scan part local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["partkey_18"]) + remote exchange (REPARTITION, HASH, [partkey_18]) final aggregation over (partkey_18, suppkey_19) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["partkey_18", "suppkey_19"]) + remote exchange (REPARTITION, HASH, [partkey_18, suppkey_19]) partial aggregation over (partkey_18, suppkey_19) scan lineitem local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q21.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q21.plan.txt index 26c25bcf6b2e2..15945f860d3b6 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q21.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q21.plan.txt @@ -2,7 +2,7 @@ local exchange (GATHER, SINGLE, []) remote exchange (GATHER, SINGLE, []) final aggregation over (name) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["name"]) + remote exchange (REPARTITION, HASH, [name]) partial aggregation over (name) single aggregation over (commitdate, exists, name, name_7, nationkey, orderkey, orderstatus, receiptdate, suppkey, unique) join (LEFT, PARTITIONED): @@ -10,26 +10,26 @@ local exchange (GATHER, SINGLE, []) local exchange (GATHER, SINGLE, []) partial aggregation over (commitdate, name, name_7, nationkey, orderkey, orderstatus, receiptdate, suppkey, unique_189) join (LEFT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["orderkey"]) + remote exchange (REPARTITION, HASH, [orderkey]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["nationkey"]) + remote exchange (REPARTITION, HASH, [nationkey]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["orderkey"]) + remote exchange (REPARTITION, HASH, [orderkey]) join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, ["suppkey"]) + remote exchange (REPARTITION, HASH, [suppkey]) scan supplier local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["suppkey_0"]) + remote exchange (REPARTITION, HASH, [suppkey_0]) scan lineitem local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderkey_3"]) + remote exchange (REPARTITION, HASH, [orderkey_3]) scan orders local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["nationkey_6"]) + remote exchange (REPARTITION, HASH, [nationkey_6]) scan nation local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderkey_10"]) + remote exchange (REPARTITION, HASH, [orderkey_10]) scan lineitem local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["orderkey_92"]) + remote exchange (REPARTITION, HASH, [orderkey_92]) scan lineitem diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q22.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q22.plan.txt index da536bc1f9df2..51309d0399ae6 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q22.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpch/q22.plan.txt @@ -3,10 +3,10 @@ remote exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, ROUND_ROBIN, []) final aggregation over (substr) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["substr"]) + remote exchange (REPARTITION, HASH, [substr]) partial aggregation over (substr) join (LEFT, PARTITIONED): - remote exchange (REPARTITION, HASH, ["custkey"]) + remote exchange (REPARTITION, HASH, [custkey]) cross join: scan customer local exchange (GATHER, SINGLE, []) @@ -18,6 +18,6 @@ remote exchange (GATHER, SINGLE, []) scan customer final aggregation over (custkey_13) local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, ["custkey_13"]) + remote exchange (REPARTITION, HASH, [custkey_13]) partial aggregation over (custkey_13) scan orders diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index 77c3107ba5b85..57dd9f4872100 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-blackhole @@ -23,13 +23,13 @@ - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift log runtime @@ -79,7 +79,7 @@ - io.airlift + com.facebook.airlift log-manager test @@ -97,7 +97,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-blackhole/src/main/java/com/facebook/presto/plugin/blackhole/BlackHoleConnectorFactory.java b/presto-blackhole/src/main/java/com/facebook/presto/plugin/blackhole/BlackHoleConnectorFactory.java index cd90c24895213..1c3c0c3c9efc9 100644 --- a/presto-blackhole/src/main/java/com/facebook/presto/plugin/blackhole/BlackHoleConnectorFactory.java +++ b/presto-blackhole/src/main/java/com/facebook/presto/plugin/blackhole/BlackHoleConnectorFactory.java @@ -21,8 +21,8 @@ import java.util.Map; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; public class BlackHoleConnectorFactory diff --git a/presto-blackhole/src/main/java/com/facebook/presto/plugin/blackhole/BlackHolePageSink.java b/presto-blackhole/src/main/java/com/facebook/presto/plugin/blackhole/BlackHolePageSink.java index 26fef01b1c949..80d0a82489b2e 100644 --- a/presto-blackhole/src/main/java/com/facebook/presto/plugin/blackhole/BlackHolePageSink.java +++ b/presto-blackhole/src/main/java/com/facebook/presto/plugin/blackhole/BlackHolePageSink.java @@ -23,7 +23,7 @@ import java.util.Collection; import java.util.concurrent.CompletableFuture; -import static io.airlift.concurrent.MoreFutures.toCompletableFuture; +import static com.facebook.airlift.concurrent.MoreFutures.toCompletableFuture; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; diff --git a/presto-blackhole/src/main/java/com/facebook/presto/plugin/blackhole/BlackHolePageSource.java b/presto-blackhole/src/main/java/com/facebook/presto/plugin/blackhole/BlackHolePageSource.java index 66d0b110dc29e..536bbe7a8fc7d 100644 --- a/presto-blackhole/src/main/java/com/facebook/presto/plugin/blackhole/BlackHolePageSource.java +++ b/presto-blackhole/src/main/java/com/facebook/presto/plugin/blackhole/BlackHolePageSource.java @@ -20,9 +20,9 @@ import java.util.concurrent.CompletableFuture; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.MoreFutures.toCompletableFuture; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.MoreFutures.toCompletableFuture; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -34,6 +34,7 @@ class BlackHolePageSource private final ListeningScheduledExecutorService executorService; private final long pageProcessingDelayInMillis; private long completedBytes; + private long completedPositions; private final long memoryUsageBytes; private boolean closed; private CompletableFuture currentPage; @@ -63,6 +64,7 @@ public Page getNextPage() pagesLeft--; completedBytes += page.getSizeInBytes(); + completedPositions += page.getPositionCount(); if (pageProcessingDelayInMillis == 0) { return page; @@ -100,6 +102,12 @@ public long getCompletedBytes() return completedBytes; } + @Override + public long getCompletedPositions() + { + return completedPositions; + } + @Override public long getReadTimeNanos() { diff --git a/presto-blackhole/src/test/java/com/facebook/presto/plugin/blackhole/BlackHoleQueryRunner.java b/presto-blackhole/src/test/java/com/facebook/presto/plugin/blackhole/BlackHoleQueryRunner.java index 5835c5920536c..d2c8284eaf533 100644 --- a/presto-blackhole/src/test/java/com/facebook/presto/plugin/blackhole/BlackHoleQueryRunner.java +++ b/presto-blackhole/src/test/java/com/facebook/presto/plugin/blackhole/BlackHoleQueryRunner.java @@ -13,17 +13,17 @@ */ package com.facebook.presto.plugin.blackhole; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.log.Logging; import com.facebook.presto.Session; import com.facebook.presto.tests.DistributedQueryRunner; import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import io.airlift.log.Logging; import java.util.Map; +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.testing.Closeables.closeAllSuppress; public final class BlackHoleQueryRunner { diff --git a/presto-blackhole/src/test/java/com/facebook/presto/plugin/blackhole/TestBlackHoleSmoke.java b/presto-blackhole/src/test/java/com/facebook/presto/plugin/blackhole/TestBlackHoleSmoke.java index 3bebd68baf4e0..df95e886c1749 100644 --- a/presto-blackhole/src/test/java/com/facebook/presto/plugin/blackhole/TestBlackHoleSmoke.java +++ b/presto-blackhole/src/test/java/com/facebook/presto/plugin/blackhole/TestBlackHoleSmoke.java @@ -30,6 +30,7 @@ import java.time.LocalDateTime; import java.util.List; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; import static com.facebook.presto.plugin.blackhole.BlackHoleConnector.FIELD_LENGTH_PROPERTY; import static com.facebook.presto.plugin.blackhole.BlackHoleConnector.PAGES_PER_SPLIT_PROPERTY; import static com.facebook.presto.plugin.blackhole.BlackHoleConnector.PAGE_PROCESSING_DELAY; @@ -37,7 +38,6 @@ import static com.facebook.presto.plugin.blackhole.BlackHoleConnector.SPLIT_COUNT_PROPERTY; import static com.facebook.presto.plugin.blackhole.BlackHoleQueryRunner.createQueryRunner; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.testing.Assertions.assertGreaterThan; import static java.lang.String.format; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 7e6fabbcc5a72..52c6a4e5d2c42 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-bytecode diff --git a/presto-cache/pom.xml b/presto-cache/pom.xml new file mode 100644 index 0000000000000..06881c19a71d6 --- /dev/null +++ b/presto-cache/pom.xml @@ -0,0 +1,105 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.231-SNAPSHOT + + + presto-cache + Presto cache library + + + ${project.parent.basedir} + + + + + com.facebook.presto.hadoop + hadoop-apache2 + + + + javax.validation + validation-api + + + + com.facebook.airlift + configuration + + + + com.facebook.airlift + log + + + + com.google.guava + guava + + + + org.weakref + jmxutils + + + + javax.inject + javax.inject + + + + com.google.code.findbugs + jsr305 + true + + + + + com.facebook.presto + presto-spi + provided + + + + io.airlift + slice + provided + + + + io.airlift + units + provided + + + + org.openjdk.jol + jol-core + provided + + + + + com.facebook.airlift + log-manager + runtime + + + + + org.testng + testng + test + + + + com.facebook.airlift + concurrent + test + + + diff --git a/presto-cache/src/main/java/com/facebook/presto/cache/CacheConfig.java b/presto-cache/src/main/java/com/facebook/presto/cache/CacheConfig.java new file mode 100644 index 0000000000000..65ab25d98ee0c --- /dev/null +++ b/presto-cache/src/main/java/com/facebook/presto/cache/CacheConfig.java @@ -0,0 +1,103 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.cache; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import io.airlift.units.DataSize; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; + +import javax.validation.constraints.Min; + +import java.net.URI; + +import static io.airlift.units.DataSize.Unit.GIGABYTE; +import static java.util.concurrent.TimeUnit.DAYS; + +public class CacheConfig +{ + private URI baseDirectory; + private boolean validationEnabled; + private DataSize maxInMemoryCacheSize = new DataSize(2, GIGABYTE); + private int maxCachedEntries = 1_000; + private Duration cacheTtl = new Duration(2, DAYS); + + public URI getBaseDirectory() + { + return baseDirectory; + } + + @Config("cache.base-directory") + @ConfigDescription("Base URI to cache data") + public CacheConfig setBaseDirectory(URI dataURI) + { + this.baseDirectory = dataURI; + return this; + } + + public boolean isValidationEnabled() + { + return validationEnabled; + } + + @Config("cache.validation-enabled") + @ConfigDescription("Enable cache validation by comparing with actual data with cached data") + public CacheConfig setValidationEnabled(boolean validationEnabled) + { + this.validationEnabled = validationEnabled; + return this; + } + + public DataSize getMaxInMemoryCacheSize() + { + return maxInMemoryCacheSize; + } + + @Config("cache.max-in-memory-cache-size") + @ConfigDescription("The maximum cache size allowed in memory") + public CacheConfig setMaxInMemoryCacheSize(DataSize maxInMemoryCacheSize) + { + this.maxInMemoryCacheSize = maxInMemoryCacheSize; + return this; + } + + @Min(1) + public int getMaxCachedEntries() + { + return maxCachedEntries; + } + + @Config("cache.max-cached-entries") + @ConfigDescription("Number of entries allowed in the cache") + public CacheConfig setMaxCachedEntries(int maxCachedEntries) + { + this.maxCachedEntries = maxCachedEntries; + return this; + } + + @MinDuration("0s") + public Duration getCacheTtl() + { + return cacheTtl; + } + + @Config("cache.ttl") + @ConfigDescription("Time-to-live for a cache entry") + public CacheConfig setCacheTtl(Duration cacheTtl) + { + this.cacheTtl = cacheTtl; + return this; + } +} diff --git a/presto-cache/src/main/java/com/facebook/presto/cache/CacheManager.java b/presto-cache/src/main/java/com/facebook/presto/cache/CacheManager.java new file mode 100644 index 0000000000000..9a43ea3b5e19a --- /dev/null +++ b/presto-cache/src/main/java/com/facebook/presto/cache/CacheManager.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.cache; + +import io.airlift.slice.Slice; + +import javax.annotation.concurrent.ThreadSafe; + +@ThreadSafe +public interface CacheManager +{ + /** + * Given {@param request}, check if the data is in cache. + * If it is not in cache, return false. + * Otherwise, save the data in {@param buffer} starting at {@param offset} and return true. + */ + boolean get(FileReadRequest request, byte[] buffer, int offset); + + /** + * Save data in cache + */ + void put(FileReadRequest request, Slice data); +} diff --git a/presto-cache/src/main/java/com/facebook/presto/cache/CacheStats.java b/presto-cache/src/main/java/com/facebook/presto/cache/CacheStats.java new file mode 100644 index 0000000000000..2d7c3b9dfac91 --- /dev/null +++ b/presto-cache/src/main/java/com/facebook/presto/cache/CacheStats.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.cache; + +import org.weakref.jmx.Managed; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.concurrent.atomic.AtomicLong; + +@ThreadSafe +public class CacheStats +{ + private final AtomicLong inMemoryRetainedBytes = new AtomicLong(); + private final AtomicLong hit = new AtomicLong(); + private final AtomicLong miss = new AtomicLong(); + + public void incrementCacheHit() + { + hit.getAndIncrement(); + } + + public void incrementCacheMiss() + { + miss.getAndIncrement(); + } + + public void addInMemoryRetainedBytes(long bytes) + { + inMemoryRetainedBytes.addAndGet(bytes); + } + + @Managed + public long getInMemoryRetainedBytes() + { + return inMemoryRetainedBytes.get(); + } + + @Managed + public long getCacheHit() + { + return hit.get(); + } + + @Managed + public long getCacheMiss() + { + return miss.get(); + } +} diff --git a/presto-cache/src/main/java/com/facebook/presto/cache/CachingFileSystem.java b/presto-cache/src/main/java/com/facebook/presto/cache/CachingFileSystem.java new file mode 100644 index 0000000000000..08ae98f69d437 --- /dev/null +++ b/presto-cache/src/main/java/com/facebook/presto/cache/CachingFileSystem.java @@ -0,0 +1,139 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.cache; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Progressable; + +import java.io.IOException; +import java.net.URI; + +import static java.util.Objects.requireNonNull; + +public final class CachingFileSystem + extends FileSystem +{ + private final URI workingDirectory; + private final CacheManager cacheManager; + private final FileSystem dataTier; + private final boolean cacheValidationEnabled; + + public CachingFileSystem( + URI workingDirectory, + Configuration configuration, + CacheManager cacheManager, + FileSystem dataTier, + boolean cacheValidationEnabled) + { + requireNonNull(workingDirectory, "uri is null"); + requireNonNull(configuration, "configuration is null"); + + this.workingDirectory = URI.create(workingDirectory.getScheme() + "://" + workingDirectory.getAuthority()); + this.cacheManager = requireNonNull(cacheManager, "cacheManager is null"); + this.dataTier = requireNonNull(dataTier, "dataTier is null"); + this.cacheValidationEnabled = cacheValidationEnabled; + + setConf(configuration); + + //noinspection AssignmentToSuperclassField + statistics = getStatistics(workingDirectory.getScheme(), getClass()); + } + + @Override + public void initialize(URI uri, Configuration configuration) + { + // make unique instances + throw new UnsupportedOperationException(); + } + + @Override + public URI getUri() + { + return workingDirectory; + } + + @Override + public void setWorkingDirectory(Path workingDirectory) + { + dataTier.setWorkingDirectory(workingDirectory); + } + + @Override + public Path getWorkingDirectory() + { + return dataTier.getWorkingDirectory(); + } + + @Override + public boolean mkdirs(Path path, FsPermission permission) + throws IOException + { + return dataTier.mkdirs(path, permission); + } + + @Override + public FSDataInputStream open(Path path, int bufferSize) + throws IOException + { + return new CachingInputStream(dataTier.open(path, bufferSize), cacheManager, path, cacheValidationEnabled); + } + + @Override + public FSDataOutputStream create(Path path, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) + throws IOException + { + return dataTier.create(path, permission, overwrite, bufferSize, replication, blockSize, progress); + } + + @Override + public FSDataOutputStream append(Path path, int bufferSize, Progressable progress) + throws IOException + { + return dataTier.append(path, bufferSize, progress); + } + + @Override + public boolean rename(Path source, Path destination) + throws IOException + { + return dataTier.rename(source, destination); + } + + @Override + public boolean delete(Path path, boolean recursive) + throws IOException + { + return dataTier.delete(path, recursive); + } + + @Override + public FileStatus[] listStatus(Path path) + throws IOException + { + return dataTier.listStatus(path); + } + + @Override + public FileStatus getFileStatus(Path path) + throws IOException + { + return dataTier.getFileStatus(path); + } +} diff --git a/presto-cache/src/main/java/com/facebook/presto/cache/CachingInputStream.java b/presto-cache/src/main/java/com/facebook/presto/cache/CachingInputStream.java new file mode 100644 index 0000000000000..e0caf3fad2c69 --- /dev/null +++ b/presto-cache/src/main/java/com/facebook/presto/cache/CachingInputStream.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.cache; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; + +import static com.google.common.base.Verify.verify; +import static io.airlift.slice.Slices.wrappedBuffer; +import static java.util.Objects.requireNonNull; + +final class CachingInputStream + extends FSDataInputStream +{ + private final FSDataInputStream inputStream; + private final CacheManager cacheManager; + private final Path path; + private final boolean cacheValidationEnabled; + + public CachingInputStream( + FSDataInputStream inputStream, + CacheManager cacheManager, + Path path, + boolean cacheValidationEnabled) + { + super(inputStream); + this.inputStream = requireNonNull(inputStream, "inputStream is null"); + this.cacheManager = requireNonNull(cacheManager, "cacheManager is null"); + this.path = requireNonNull(path, "path is null"); + this.cacheValidationEnabled = cacheValidationEnabled; + } + + @Override + public void readFully(long position, byte[] buffer, int offset, int length) + throws IOException + { + FileReadRequest key = new FileReadRequest(path, position, length); + + if (!cacheManager.get(key, buffer, offset)) { + inputStream.readFully(position, buffer, offset, length); + cacheManager.put(key, wrappedBuffer(buffer, offset, length)); + return; + } + + if (cacheValidationEnabled) { + byte[] validationBuffer = new byte[length]; + inputStream.readFully(position, validationBuffer, 0, length); + for (int i = 0; i < length; i++) { + verify(buffer[offset + i] == validationBuffer[i], "corrupted buffer at position " + i); + } + } + } +} diff --git a/presto-cache/src/main/java/com/facebook/presto/cache/FileReadRequest.java b/presto-cache/src/main/java/com/facebook/presto/cache/FileReadRequest.java new file mode 100644 index 0000000000000..410f15da17543 --- /dev/null +++ b/presto-cache/src/main/java/com/facebook/presto/cache/FileReadRequest.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.cache; + +import org.apache.hadoop.fs.Path; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class FileReadRequest +{ + private final Path path; + private final long offset; + private final int length; + + public FileReadRequest(Path path, long offset, int length) + { + this.path = requireNonNull(path, "path is null"); + this.offset = requireNonNull(offset, "offset is null"); + this.length = requireNonNull(length, "length is null"); + } + + public Path getPath() + { + return path; + } + + public long getOffset() + { + return offset; + } + + public int getLength() + { + return length; + } + + @Override + public int hashCode() + { + return Objects.hash(path, offset, length); + } + + @Override + public boolean equals(Object object) + { + if (this == object) { + return true; + } + if (!(object instanceof FileReadRequest)) { + return false; + } + FileReadRequest other = (FileReadRequest) object; + return Objects.equals(this.path, other.path) && + Objects.equals(this.offset, other.offset) && + Objects.equals(this.length, other.length); + } +} diff --git a/presto-cache/src/main/java/com/facebook/presto/cache/ForCachingFileSystem.java b/presto-cache/src/main/java/com/facebook/presto/cache/ForCachingFileSystem.java new file mode 100644 index 0000000000000..54781e30b3527 --- /dev/null +++ b/presto-cache/src/main/java/com/facebook/presto/cache/ForCachingFileSystem.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.cache; + +import javax.inject.Qualifier; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@Qualifier +public @interface ForCachingFileSystem +{ +} diff --git a/presto-cache/src/main/java/com/facebook/presto/cache/LocalRangeCacheManager.java b/presto-cache/src/main/java/com/facebook/presto/cache/LocalRangeCacheManager.java new file mode 100644 index 0000000000000..0b7ffe7dc673a --- /dev/null +++ b/presto-cache/src/main/java/com/facebook/presto/cache/LocalRangeCacheManager.java @@ -0,0 +1,498 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.cache; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.spi.PrestoException; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Range; +import com.google.common.collect.RangeMap; +import com.google.common.collect.TreeRangeMap; +import io.airlift.slice.Slice; +import io.airlift.units.DataSize; +import org.apache.hadoop.fs.Path; + +import javax.annotation.PreDestroy; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; + +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.Iterators.getOnlyElement; +import static io.airlift.units.DataSize.Unit.MEGABYTE; +import static java.lang.StrictMath.toIntExact; +import static java.nio.file.StandardOpenOption.APPEND; +import static java.nio.file.StandardOpenOption.CREATE_NEW; +import static java.util.Objects.requireNonNull; +import static java.util.UUID.randomUUID; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +// 3 major TODOs for this class: +// TODO: Make cache eviction based on cache size rather than file count; add evict count stats to CacheStats as well. +// TODO: Make cache state persistent on disk so we do not need to wipe out cache every time we reboot a server. +@SuppressWarnings("UnstableApiUsage") +public class LocalRangeCacheManager + implements CacheManager +{ + private static final Logger log = Logger.get(LocalRangeCacheManager.class); + + private static final String EXTENSION = ".cache"; + + private static final int FILE_MERGE_BUFFER_SIZE = toIntExact(new DataSize(8, MEGABYTE).toBytes()); + + private final ThreadLocal buffers = ThreadLocal.withInitial(() -> new byte[FILE_MERGE_BUFFER_SIZE]); + + private final ExecutorService cacheFlushExecutor; + private final ExecutorService cacheRemovalExecutor; + + // a mapping from remote file `F` to a range map `M`; the corresponding local cache file for each range in `M` represents the cached chunk of `F` + private final Map persistedRanges = new ConcurrentHashMap<>(); + // a local cache only to control the lifecycle of persisted + private final Cache cache; + + // stats + private final CacheStats stats; + + // config + private final Path baseDirectory; + private final long maxInflightBytes; + + public LocalRangeCacheManager(CacheConfig cacheConfig, CacheStats stats, ExecutorService cacheFlushExecutor, ExecutorService cacheRemovalExecutor) + { + requireNonNull(cacheConfig, "directory is null"); + this.cacheFlushExecutor = cacheFlushExecutor; + this.cacheRemovalExecutor = cacheRemovalExecutor; + this.cache = CacheBuilder.newBuilder() + .maximumSize(cacheConfig.getMaxCachedEntries()) + .expireAfterAccess(cacheConfig.getCacheTtl().toMillis(), MILLISECONDS) + .removalListener(new CacheRemovalListener()) + .recordStats() + .build(); + this.stats = requireNonNull(stats, "stats is null"); + this.baseDirectory = new Path(cacheConfig.getBaseDirectory()); + checkArgument(cacheConfig.getMaxInMemoryCacheSize().toBytes() >= 0, "maxInflightBytes is negative"); + this.maxInflightBytes = cacheConfig.getMaxInMemoryCacheSize().toBytes(); + + File target = new File(baseDirectory.toUri()); + if (!target.exists()) { + try { + Files.createDirectories(target.toPath()); + } + catch (IOException e) { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "cannot create cache directory " + target, e); + } + } + else { + File[] files = target.listFiles(); + if (files == null) { + return; + } + + this.cacheRemovalExecutor.submit(() -> Arrays.stream(files).forEach(file -> { + try { + Files.delete(file.toPath()); + } + catch (IOException e) { + // ignore + } + })); + } + } + + @PreDestroy + public void destroy() + { + cacheFlushExecutor.shutdownNow(); + cacheRemovalExecutor.shutdownNow(); + buffers.remove(); + } + + @Override + public boolean get(FileReadRequest request, byte[] buffer, int offset) + { + boolean result = read(request, buffer, offset); + if (result) { + stats.incrementCacheHit(); + } + else { + stats.incrementCacheMiss(); + } + + return result; + } + + @Override + public void put(FileReadRequest key, Slice data) + { + if (stats.getInMemoryRetainedBytes() + data.length() >= maxInflightBytes) { + // cannot accept more requests + return; + } + + // make a copy given the input data could be a reusable buffer + stats.addInMemoryRetainedBytes(data.length()); + byte[] copy = data.getBytes(); + + cacheFlushExecutor.submit(() -> { + Path newFilePath = new Path(baseDirectory.toUri() + "/" + randomUUID() + EXTENSION); + if (!write(key, copy, newFilePath)) { + log.warn("%s Fail to persist cache %s with length %s ", Thread.currentThread().getName(), newFilePath, key.getLength()); + } + stats.addInMemoryRetainedBytes(-copy.length); + }); + } + + private boolean read(FileReadRequest request, byte[] buffer, int offset) + { + if (request.getLength() <= 0) { + // no-op + return true; + } + + try { + // hint the cache no matter what + cache.get(request.getPath(), () -> true); + } + catch (ExecutionException e) { + // ignore + } + + // check if the file is cached on local disk + CacheRange cacheRange = persistedRanges.get(request.getPath()); + if (cacheRange == null) { + return false; + } + + LocalCacheFile cacheFile; + Lock readLock = cacheRange.getLock().readLock(); + readLock.lock(); + try { + Map, LocalCacheFile> diskRanges = cacheRange.getRange().subRangeMap(Range.closedOpen(request.getOffset(), request.getLength() + request.getOffset())).asMapOfRanges(); + if (diskRanges.size() != 1) { + // no range or there is a hole in between + return false; + } + + cacheFile = getOnlyElement(diskRanges.entrySet().iterator()).getValue(); + } + finally { + readLock.unlock(); + } + + try (RandomAccessFile file = new RandomAccessFile(new File(cacheFile.getPath().toUri()), "r")) { + file.seek(request.getOffset() - cacheFile.getOffset()); + file.readFully(buffer, offset, request.getLength()); + return true; + } + catch (IOException e) { + // there might be a chance the file has been deleted + return false; + } + } + + private boolean write(FileReadRequest key, byte[] data, Path newFilePath) + { + Path targetFile = key.getPath(); + persistedRanges.putIfAbsent(targetFile, new CacheRange()); + + LocalCacheFile previousCacheFile; + LocalCacheFile followingCacheFile; + + CacheRange cacheRange = persistedRanges.get(targetFile); + if (cacheRange == null) { + // there is a chance the cache has just expired. + return false; + } + + Lock readLock = cacheRange.getLock().readLock(); + readLock.lock(); + try { + RangeMap cache = cacheRange.getRange(); + + // check if it can be merged with the previous or following range + previousCacheFile = cache.get(key.getOffset() - 1); + followingCacheFile = cache.get(key.getOffset() + key.getLength()); + } + finally { + readLock.unlock(); + } + + if (previousCacheFile != null && cacheFileEquals(previousCacheFile, followingCacheFile)) { + log.debug("%s found covered range %s", Thread.currentThread().getName(), previousCacheFile.getPath()); + // this range has already been covered by someone else + return true; + } + + long newFileOffset; + int newFileLength; + File newFile = new File(newFilePath.toUri()); + try { + if (previousCacheFile == null) { + // a new file + Files.write(newFile.toPath(), data, CREATE_NEW); + + // update new range info + newFileLength = data.length; + newFileOffset = key.getOffset(); + } + else { + // copy previous file's data to the new file + int previousFileLength = appendToFile(previousCacheFile, 0, newFile); + long previousFileOffset = previousCacheFile.getOffset(); + + // remove the overlapping part and append the remaining cache data + int remainingCacheFileOffset = toIntExact(previousFileLength + previousFileOffset - key.getOffset()); + int remainingCacheFileLength = toIntExact((key.getLength() + key.getOffset()) - (previousFileLength + previousFileOffset)); + + try (RandomAccessFile randomAccessFile = new RandomAccessFile(newFile, "rw")) { + randomAccessFile.seek(randomAccessFile.length()); + randomAccessFile.write(data, remainingCacheFileOffset, remainingCacheFileLength); + } + + // update new range info + newFileLength = previousFileLength + remainingCacheFileLength; + newFileOffset = previousFileOffset; + } + + if (followingCacheFile != null) { + // remove the overlapping part and append the remaining following file data + newFileLength += appendToFile(followingCacheFile, key.getOffset() + key.getLength() - followingCacheFile.getOffset(), newFile); + } + } + catch (IOException e) { + log.warn(e, "%s encountered an error while flushing file %s", Thread.currentThread().getName(), newFilePath); + tryDeleteFile(newFilePath); + return false; + } + + // use a flag so that file deletion can be done outside the lock + boolean updated; + Set cacheFilesToDelete = new HashSet<>(); + + Lock writeLock = persistedRanges.get(targetFile).getLock().writeLock(); + writeLock.lock(); + try { + RangeMap cache = persistedRanges.get(targetFile).getRange(); + // check again if the previous or following range has been updated by someone else + LocalCacheFile newPreviousCacheFile = cache.get(key.getOffset() - 1); + LocalCacheFile newFollowingCacheFile = cache.get(key.getOffset() + key.getLength()); + + if (!cacheFileEquals(previousCacheFile, newPreviousCacheFile) || !cacheFileEquals(followingCacheFile, newFollowingCacheFile)) { + // someone else has updated the cache; delete the newly created file + updated = false; + } + else { + updated = true; + + // remove all the files that can be covered by the current range + cacheFilesToDelete = cache.subRangeMap(Range.closedOpen(key.getOffset(), key.getOffset() + key.getLength())).asMapOfRanges().values().stream() + .map(LocalCacheFile::getPath).collect(Collectors.toSet()); + + // update the range + Range newRange = Range.closedOpen(newFileOffset, newFileOffset + newFileLength); + cache.remove(newRange); + cache.put(newRange, new LocalCacheFile(newFileOffset, newFilePath)); + } + } + finally { + writeLock.unlock(); + } + + // no lock is needed for the following operation + if (updated) { + // remove the the previous or following file as well + if (previousCacheFile != null) { + cacheFilesToDelete.add(previousCacheFile.getPath()); + } + if (followingCacheFile != null) { + cacheFilesToDelete.add(followingCacheFile.getPath()); + } + } + else { + cacheFilesToDelete = ImmutableSet.of(newFilePath); + } + + cacheFilesToDelete.forEach(LocalRangeCacheManager::tryDeleteFile); + return true; + } + + private int appendToFile(LocalCacheFile source, long offset, File destination) + throws IOException + { + int totalBytesRead = 0; + try (FileInputStream fileInputStream = new FileInputStream(new File(source.getPath().toUri()))) { + fileInputStream.getChannel().position(offset); + int readBytes; + byte[] buffer = buffers.get(); + + while ((readBytes = fileInputStream.read(buffer)) > 0) { + if (!Files.exists(destination.toPath())) { + Files.createFile(destination.toPath()); + } + totalBytesRead += readBytes; + + if (readBytes != FILE_MERGE_BUFFER_SIZE) { + try (RandomAccessFile randomAccessFile = new RandomAccessFile(destination, "rw")) { + randomAccessFile.seek(destination.length()); + randomAccessFile.write(buffer, 0, readBytes); + } + } + else { + Files.write(destination.toPath(), buffer, APPEND); + } + } + } + return totalBytesRead; + } + + private static void tryDeleteFile(Path path) + { + try { + File file = new File(path.toUri()); + if (file.exists()) { + Files.delete(file.toPath()); + } + } + catch (IOException e) { + // ignore + } + } + + private static boolean cacheFileEquals(LocalCacheFile left, LocalCacheFile right) + { + if (left == null && right == null) { + return true; + } + + if (left == null || right == null) { + return false; + } + + return left.equals(right); + } + + private static class LocalCacheFile + { + private final long offset; // the original file offset + private final Path path; // the cache location on disk + + public LocalCacheFile(long offset, Path path) + { + this.offset = offset; + this.path = path; + } + + public long getOffset() + { + return offset; + } + + public Path getPath() + { + return path; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LocalCacheFile that = (LocalCacheFile) o; + return Objects.equals(offset, that.offset) && Objects.equals(path, that.path); + } + + @Override + public int hashCode() + { + return Objects.hash(offset, path); + } + } + + private static class CacheRange + { + private final RangeMap range = TreeRangeMap.create(); + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + + public RangeMap getRange() + { + return range; + } + + public ReadWriteLock getLock() + { + return lock; + } + } + + private class CacheRemovalListener + implements RemovalListener + { + @Override + public void onRemoval(RemovalNotification notification) + { + Path path = notification.getKey(); + CacheRange cacheRange = persistedRanges.remove(path); + if (cacheRange == null) { + return; + } + + cacheRemovalExecutor.submit(() -> { + Collection files; + cacheRange.lock.readLock().lock(); + try { + files = cacheRange.getRange().asMapOfRanges().values(); + } + finally { + cacheRange.lock.readLock().unlock(); + } + + // There is a chance of the files to be deleted are being read. + // We may just fail the cache hit and do it in a simple way given the chance is low. + for (LocalCacheFile file : files) { + try { + Files.delete(new File(file.getPath().toUri()).toPath()); + } + catch (IOException e) { + // ignore + } + } + }); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceFactory.java b/presto-cache/src/main/java/com/facebook/presto/cache/NoOpCacheManager.java similarity index 64% rename from presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceFactory.java rename to presto-cache/src/main/java/com/facebook/presto/cache/NoOpCacheManager.java index 14e4fed325e99..e5ae4a7dadf70 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceFactory.java +++ b/presto-cache/src/main/java/com/facebook/presto/cache/NoOpCacheManager.java @@ -11,20 +11,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.metadata; +package com.facebook.presto.cache; -public class StaticFunctionNamespaceFactory - implements FunctionNamespaceFactory +import io.airlift.slice.Slice; + +public class NoOpCacheManager + implements CacheManager { - @Override - public String getName() + public boolean get(FileReadRequest key, byte[] buffer, int offset) { - return "$static"; + return false; } - @Override - public FunctionHandleResolver getHandleResolver() + public void put(FileReadRequest key, Slice data) { - return new StaticFunctionNamespaceHandleResolver(); + // no op } } diff --git a/presto-cache/src/test/java/com/facebook/presto/cache/TestCacheConfig.java b/presto-cache/src/test/java/com/facebook/presto/cache/TestCacheConfig.java new file mode 100644 index 0000000000000..e9ccf6a9e9285 --- /dev/null +++ b/presto-cache/src/test/java/com/facebook/presto/cache/TestCacheConfig.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.cache; + +import com.google.common.collect.ImmutableMap; +import io.airlift.units.DataSize; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.net.URI; +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.units.DataSize.Unit.GIGABYTE; +import static io.airlift.units.DataSize.Unit.MEGABYTE; +import static java.util.concurrent.TimeUnit.DAYS; + +public class TestCacheConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(CacheConfig.class) + .setBaseDirectory(null) + .setValidationEnabled(false) + .setMaxInMemoryCacheSize(new DataSize(2, GIGABYTE)) + .setMaxCachedEntries(1_000) + .setCacheTtl(new Duration(2, DAYS))); + } + + @Test + public void testExplicitPropertyMappings() + throws Exception + { + Map properties = new ImmutableMap.Builder() + .put("cache.base-directory", "tcp://abc") + .put("cache.validation-enabled", "true") + .put("cache.max-in-memory-cache-size", "42MB") + .put("cache.max-cached-entries", "10000") + .put("cache.ttl", "20d") + .build(); + + CacheConfig expected = new CacheConfig() + .setBaseDirectory(new URI("tcp://abc")) + .setValidationEnabled(true) + .setMaxInMemoryCacheSize(new DataSize(42, MEGABYTE)) + .setMaxCachedEntries(10_000) + .setCacheTtl(new Duration(20, DAYS)); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-cache/src/test/java/com/facebook/presto/cache/TestLocalRangeCacheManager.java b/presto-cache/src/test/java/com/facebook/presto/cache/TestLocalRangeCacheManager.java new file mode 100644 index 0000000000000..cc567317844a3 --- /dev/null +++ b/presto-cache/src/test/java/com/facebook/presto/cache/TestLocalRangeCacheManager.java @@ -0,0 +1,255 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.cache; + +import com.google.common.util.concurrent.SettableFuture; +import io.airlift.units.DataSize; +import io.airlift.units.Duration; +import org.apache.hadoop.fs.Path; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.net.URI; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; + +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.google.common.base.Preconditions.checkState; +import static io.airlift.slice.Slices.wrappedBuffer; +import static io.airlift.units.DataSize.Unit.KILOBYTE; +import static java.lang.Integer.max; +import static java.lang.String.format; +import static java.nio.file.Files.createTempDirectory; +import static java.nio.file.StandardOpenOption.CREATE_NEW; +import static java.util.concurrent.Executors.newScheduledThreadPool; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +public class TestLocalRangeCacheManager +{ + private static final int DATA_LENGTH = (int) new DataSize(20, KILOBYTE).toBytes(); + private final byte[] data = new byte[DATA_LENGTH]; + private final ExecutorService flushExecutor = newScheduledThreadPool(5, daemonThreadsNamed("test-cache-flusher-%s")); + private final ExecutorService removeExecutor = newScheduledThreadPool(5, daemonThreadsNamed("test-cache-remover-%s")); + + private URI cacheDirectory; + private URI fileDirectory; + private File dataFile; + + @BeforeClass + public void setup() + throws IOException + { + new Random().nextBytes(data); + + this.cacheDirectory = createTempDirectory("cache").toUri(); + this.fileDirectory = createTempDirectory("file").toUri(); + this.dataFile = new File(fileDirectory.getPath() + "/data"); + + Files.write((new File(dataFile.toString())).toPath(), data, CREATE_NEW); + } + + @AfterClass + public void close() + throws IOException + { + flushExecutor.shutdown(); + removeExecutor.shutdown(); + + checkState(cacheDirectory != null); + checkState(fileDirectory != null); + + Files.deleteIfExists(dataFile.toPath()); + File[] files = new File(cacheDirectory).listFiles(); + if (files != null) { + for (File file : files) { + Files.delete(file.toPath()); + } + } + + Files.deleteIfExists(new File(cacheDirectory).toPath()); + Files.deleteIfExists(new File(fileDirectory).toPath()); + } + + @Test(timeOut = 30_000) + public void testBasic() + throws InterruptedException, ExecutionException, IOException + { + TestingCacheStats stats = new TestingCacheStats(); + CacheManager cacheManager = localRangeCacheManager(stats); + byte[] buffer = new byte[1024]; + + // new read + assertFalse(readFully(cacheManager, 42, buffer, 0, 100)); + assertEquals(stats.getCacheMiss(), 1); + assertEquals(stats.getCacheHit(), 0); + stats.trigger(); + assertEquals(stats.getInMemoryRetainedBytes(), 0); + validateBuffer(42, buffer, 0, 100); + + // within the range of the cache + assertTrue(readFully(cacheManager, 47, buffer, 0, 90)); + assertEquals(stats.getCacheMiss(), 1); + assertEquals(stats.getCacheHit(), 1); + assertEquals(stats.getInMemoryRetainedBytes(), 0); + validateBuffer(47, buffer, 0, 90); + + // partially within the range of the cache + assertFalse(readFully(cacheManager, 52, buffer, 0, 100)); + assertEquals(stats.getCacheMiss(), 2); + assertEquals(stats.getCacheHit(), 1); + stats.trigger(); + assertEquals(stats.getInMemoryRetainedBytes(), 0); + validateBuffer(52, buffer, 0, 100); + + // partially within the range of the cache + assertFalse(readFully(cacheManager, 32, buffer, 10, 50)); + assertEquals(stats.getCacheMiss(), 3); + assertEquals(stats.getCacheHit(), 1); + stats.trigger(); + assertEquals(stats.getInMemoryRetainedBytes(), 0); + validateBuffer(32, buffer, 10, 50); + + // create a hole within two caches + assertFalse(readFully(cacheManager, 200, buffer, 40, 50)); + assertEquals(stats.getCacheMiss(), 4); + assertEquals(stats.getCacheHit(), 1); + stats.trigger(); + assertEquals(stats.getInMemoryRetainedBytes(), 0); + validateBuffer(200, buffer, 40, 50); + + // use a range to cover the hole + assertFalse(readFully(cacheManager, 40, buffer, 400, 200)); + assertEquals(stats.getCacheMiss(), 5); + assertEquals(stats.getCacheHit(), 1); + stats.trigger(); + assertEquals(stats.getInMemoryRetainedBytes(), 0); + validateBuffer(40, buffer, 400, 200); + } + + @Test(invocationCount = 10) + public void testStress() + throws ExecutionException, InterruptedException + { + CacheManager cacheManager = localRangeCacheManager( + new CacheConfig() + .setBaseDirectory(cacheDirectory) + .setCacheTtl(new Duration(10, MILLISECONDS))); + + ExecutorService executor = newScheduledThreadPool(5); + List> futures = new ArrayList<>(); + AtomicReference exception = new AtomicReference<>(); + + for (int i = 0; i < 5; i++) { + byte[] buffer = new byte[DATA_LENGTH]; + futures.add(executor.submit(() -> { + Random random = new Random(); + for (int j = 0; j < 200; j++) { + int position = random.nextInt(DATA_LENGTH - 1); + int length = random.nextInt(max((DATA_LENGTH - position) / 3, 1)); + int offset = random.nextInt(DATA_LENGTH - length); + + try { + readFully(cacheManager, position, buffer, offset, length); + } + catch (IOException e) { + exception.compareAndSet(null, e.getMessage()); + return; + } + validateBuffer(position, buffer, offset, length); + } + })); + } + + for (Future future : futures) { + future.get(); + } + + if (exception.get() != null) { + fail(exception.get()); + } + } + + private CacheManager localRangeCacheManager(CacheConfig cacheConfig) + { + return new LocalRangeCacheManager(cacheConfig, new CacheStats(), flushExecutor, removeExecutor); + } + + private CacheManager localRangeCacheManager(CacheStats cacheStats) + { + return new LocalRangeCacheManager(new CacheConfig().setBaseDirectory(cacheDirectory), cacheStats, flushExecutor, removeExecutor); + } + + private void validateBuffer(long position, byte[] buffer, int offset, int length) + { + for (int i = 0; i < length; i++) { + assertEquals(buffer[i + offset], data[i + (int) position], format("corrupted buffer at position %s offset %s", position, i)); + } + } + + private boolean readFully(CacheManager cacheManager, long position, byte[] buffer, int offset, int length) + throws IOException + { + FileReadRequest key = new FileReadRequest(new Path(dataFile.getAbsolutePath()), position, length); + if (!cacheManager.get(key, buffer, offset)) { + RandomAccessFile file = new RandomAccessFile(dataFile.getAbsolutePath(), "r"); + file.seek(position); + file.readFully(buffer, offset, length); + file.close(); + cacheManager.put(key, wrappedBuffer(buffer, offset, length)); + return false; + } + return true; + } + + private static class TestingCacheStats + extends CacheStats + { + private SettableFuture trigger; + + public TestingCacheStats() + { + this.trigger = SettableFuture.create(); + } + + @Override + public void addInMemoryRetainedBytes(long bytes) + { + super.addInMemoryRetainedBytes(bytes); + if (bytes < 0) { + trigger.set(null); + } + } + + public void trigger() + throws InterruptedException, ExecutionException + { + trigger.get(); + trigger = SettableFuture.create(); + } + } +} diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index beec0d84ecb5f..bbb3598694aa4 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-cassandra @@ -33,27 +33,27 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift configuration @@ -167,7 +167,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraClientConfig.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraClientConfig.java index 5293bd709d5ad..421ebe65f8ada 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraClientConfig.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraClientConfig.java @@ -16,12 +16,12 @@ import com.datastax.driver.core.ConsistencyLevel; import com.datastax.driver.core.ProtocolVersion; import com.datastax.driver.core.SocketOptions; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.ConfigSecuritySensitive; +import com.facebook.airlift.configuration.DefunctConfig; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.ConfigSecuritySensitive; -import io.airlift.configuration.DefunctConfig; import io.airlift.units.Duration; import io.airlift.units.MaxDuration; import io.airlift.units.MinDuration; diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraClientModule.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraClientModule.java index 159999719ac52..d2990a32bb800 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraClientModule.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraClientModule.java @@ -23,11 +23,11 @@ import com.datastax.driver.core.policies.RoundRobinPolicy; import com.datastax.driver.core.policies.TokenAwarePolicy; import com.datastax.driver.core.policies.WhiteListPolicy; +import com.facebook.airlift.json.JsonCodec; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.Scopes; -import io.airlift.json.JsonCodec; import javax.inject.Singleton; @@ -35,9 +35,9 @@ import java.util.ArrayList; import java.util.List; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnector.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnector.java index 5d8ebfa496090..f787974d4b371 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnector.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnector.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.cassandra; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.connector.ConnectorPageSinkProvider; @@ -21,8 +23,6 @@ import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spi.transaction.IsolationLevel; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnectorFactory.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnectorFactory.java index 428f335d7c6cf..a4ce305a9d7c9 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnectorFactory.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnectorFactory.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.cassandra; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorContext; @@ -20,8 +22,6 @@ import com.google.inject.Binder; import com.google.inject.Injector; import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import org.weakref.jmx.guice.MBeanModule; import javax.management.MBeanServer; diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraMetadata.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraMetadata.java index 8b76c08ca2438..2fa6c15a49e2f 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraMetadata.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraMetadata.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.cassandra; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.cassandra.util.CassandraCqlUtils; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; @@ -39,7 +40,6 @@ import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; import javax.inject.Inject; diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraPartitionManager.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraPartitionManager.java index b4be35dba012d..0fb8be0fe488f 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraPartitionManager.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraPartitionManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.cassandra; +import com.facebook.airlift.log.Logger; import com.facebook.presto.cassandra.util.CassandraCqlUtils; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorTableHandle; @@ -23,7 +24,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraRecordSetProvider.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraRecordSetProvider.java index 62d7c1b81d82e..f8d450bc76d70 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraRecordSetProvider.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraRecordSetProvider.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.cassandra; +import com.facebook.airlift.log.Logger; import com.facebook.presto.cassandra.util.CassandraCqlUtils; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorSession; @@ -20,7 +21,6 @@ import com.facebook.presto.spi.RecordSet; import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/NativeCassandraSession.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/NativeCassandraSession.java index cfa1f4eb27609..04455387a07be 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/NativeCassandraSession.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/NativeCassandraSession.java @@ -36,6 +36,8 @@ import com.datastax.driver.core.querybuilder.Clause; import com.datastax.driver.core.querybuilder.QueryBuilder; import com.datastax.driver.core.querybuilder.Select; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.facebook.presto.cassandra.util.CassandraCqlUtils; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.PrestoException; @@ -50,8 +52,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import io.airlift.units.Duration; import java.nio.ByteBuffer; diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/RebindSafeMBeanServer.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/RebindSafeMBeanServer.java index ecb68feede6fd..49dc3c6642368 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/RebindSafeMBeanServer.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/RebindSafeMBeanServer.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.cassandra; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import javax.annotation.concurrent.ThreadSafe; import javax.management.Attribute; diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/ReopeningCluster.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/ReopeningCluster.java index a8c988b460ed4..0ebdb248ddf70 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/ReopeningCluster.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/ReopeningCluster.java @@ -16,7 +16,7 @@ import com.datastax.driver.core.CloseFuture; import com.datastax.driver.core.Cluster; import com.datastax.driver.core.DelegatingCluster; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; diff --git a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/EmbeddedCassandra.java b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/EmbeddedCassandra.java index 9430fa5237187..9c8f90c271d1e 100644 --- a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/EmbeddedCassandra.java +++ b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/EmbeddedCassandra.java @@ -16,10 +16,10 @@ import com.datastax.driver.core.Cluster; import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Row; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.google.common.collect.ImmutableList; import com.google.common.io.Resources; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import io.airlift.units.Duration; import org.apache.cassandra.service.CassandraDaemon; diff --git a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraClientConfig.java b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraClientConfig.java index d2b3c09842c3f..331aba77a2504 100644 --- a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraClientConfig.java +++ b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraClientConfig.java @@ -15,8 +15,8 @@ import com.datastax.driver.core.ConsistencyLevel; import com.datastax.driver.core.SocketOptions; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.Duration; import org.testng.annotations.Test; diff --git a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraColumnHandle.java b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraColumnHandle.java index a084d901c8424..f0d3fc0a87081 100644 --- a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraColumnHandle.java +++ b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraColumnHandle.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.cassandra; +import com.facebook.airlift.json.JsonCodec; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; public class TestCassandraColumnHandle diff --git a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraConnector.java b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraConnector.java index c66f085d998c5..8d75dc9c9e633 100644 --- a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraConnector.java +++ b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraConnector.java @@ -48,6 +48,8 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.cassandra.CassandraTestingUtils.TABLE_ALL_TYPES; import static com.facebook.presto.cassandra.CassandraTestingUtils.createTestTables; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING; @@ -63,8 +65,6 @@ import static com.facebook.presto.spi.type.Varchars.isVarcharType; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.testing.Assertions.assertInstanceOf; import static java.util.Locale.ENGLISH; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; @@ -85,7 +85,8 @@ public class TestCassandraConnector System.currentTimeMillis(), new CassandraSessionProperties(new CassandraClientConfig()).getSessionProperties(), ImmutableMap.of(), - true); + true, + Optional.empty()); protected String database; protected SchemaTableName table; protected SchemaTableName tableUnpartitioned; diff --git a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraSplit.java b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraSplit.java index 8d4d4c1074e14..8336f783ec939 100644 --- a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraSplit.java +++ b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraSplit.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.cassandra; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.HostAddress; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; diff --git a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraTableHandle.java b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraTableHandle.java index 2da21d6822056..2e827de52f547 100644 --- a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraTableHandle.java +++ b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraTableHandle.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.cassandra; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; diff --git a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestJsonCassandraHandles.java b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestJsonCassandraHandles.java index 7b114437322c6..5891c80be6d8c 100644 --- a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestJsonCassandraHandles.java +++ b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestJsonCassandraHandles.java @@ -13,17 +13,17 @@ */ package com.facebook.presto.cassandra; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.SchemaTableName; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.ObjectMapperProvider; import org.testng.annotations.Test; import java.util.Map; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index aa41c038769ac..c13ecd6205ba7 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-cli @@ -38,17 +38,17 @@ - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift log-manager @@ -79,7 +79,7 @@ - io.airlift + com.facebook.airlift json diff --git a/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java b/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java index b9f859be30aaf..0f3d265dc8f78 100644 --- a/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java +++ b/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java @@ -162,7 +162,6 @@ public ClientSession toClientSession() clientInfo, catalog, schema, - null, TimeZone.getDefault().getID(), Locale.getDefault(), toResourceEstimates(resourceEstimates), diff --git a/presto-cli/src/main/java/com/facebook/presto/cli/Console.java b/presto-cli/src/main/java/com/facebook/presto/cli/Console.java index ce39b5e958c2f..022bc6b48dde2 100644 --- a/presto-cli/src/main/java/com/facebook/presto/cli/Console.java +++ b/presto-cli/src/main/java/com/facebook/presto/cli/Console.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.cli; +import com.facebook.airlift.log.Logging; +import com.facebook.airlift.log.LoggingConfiguration; import com.facebook.presto.cli.ClientOptions.OutputFormat; import com.facebook.presto.client.ClientSession; import com.facebook.presto.spi.security.SelectedRole; @@ -23,8 +25,6 @@ import com.google.common.io.Files; import io.airlift.airline.Command; import io.airlift.airline.HelpOption; -import io.airlift.log.Logging; -import io.airlift.log.LoggingConfiguration; import io.airlift.units.Duration; import jline.console.history.FileHistory; import jline.console.history.History; @@ -36,7 +36,6 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; -import java.io.UncheckedIOException; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -334,11 +333,6 @@ private static boolean process(QueryRunner queryRunner, String sql, OutputFormat builder = builder.withTransactionId(query.getStartedTransactionId()); } - // update path if present - if (query.getSetPath().isPresent()) { - builder = builder.withPath(query.getSetPath().get()); - } - // update session properties if present if (!query.getSetSessionProperties().isEmpty() || !query.getResetSessionProperties().isEmpty()) { Map sessionProperties = new HashMap<>(session.getProperties()); @@ -433,9 +427,6 @@ private static void initializeLogging(String logLevelsFile) Logging logging = Logging.initialize(); logging.configure(config); } - catch (IOException e) { - throw new UncheckedIOException(e); - } finally { System.setOut(out); System.setErr(err); diff --git a/presto-cli/src/main/java/com/facebook/presto/cli/Pager.java b/presto-cli/src/main/java/com/facebook/presto/cli/Pager.java index df793f9656913..26041d616504e 100644 --- a/presto-cli/src/main/java/com/facebook/presto/cli/Pager.java +++ b/presto-cli/src/main/java/com/facebook/presto/cli/Pager.java @@ -23,8 +23,8 @@ import java.util.List; import java.util.concurrent.CompletableFuture; +import static com.facebook.airlift.concurrent.MoreFutures.unmodifiableFuture; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.concurrent.MoreFutures.unmodifiableFuture; public class Pager extends FilterOutputStream diff --git a/presto-cli/src/main/java/com/facebook/presto/cli/Query.java b/presto-cli/src/main/java/com/facebook/presto/cli/Query.java index 104931f65422a..f1454eb248731 100644 --- a/presto-cli/src/main/java/com/facebook/presto/cli/Query.java +++ b/presto-cli/src/main/java/com/facebook/presto/cli/Query.java @@ -75,11 +75,6 @@ public Optional getSetSchema() return client.getSetSchema(); } - public Optional getSetPath() - { - return client.getSetPath(); - } - public Map getSetSessionProperties() { return client.getSetSessionProperties(); diff --git a/presto-cli/src/main/java/com/facebook/presto/cli/QueryPreprocessor.java b/presto-cli/src/main/java/com/facebook/presto/cli/QueryPreprocessor.java index 2ce2a99a16180..cf6adcd1028ee 100644 --- a/presto-cli/src/main/java/com/facebook/presto/cli/QueryPreprocessor.java +++ b/presto-cli/src/main/java/com/facebook/presto/cli/QueryPreprocessor.java @@ -33,11 +33,11 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; +import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; import static com.facebook.presto.cli.ConsolePrinter.REAL_TERMINAL; import static com.google.common.base.Strings.emptyToNull; import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.base.Throwables.propagateIfPossible; -import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-cli/src/main/java/com/facebook/presto/cli/StatusPrinter.java b/presto-cli/src/main/java/com/facebook/presto/cli/StatusPrinter.java index f371ef43fa2cd..cd5693ffd012d 100644 --- a/presto-cli/src/main/java/com/facebook/presto/cli/StatusPrinter.java +++ b/presto-cli/src/main/java/com/facebook/presto/cli/StatusPrinter.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.cli; +import com.facebook.airlift.log.Logger; import com.facebook.presto.client.QueryStatusInfo; import com.facebook.presto.client.StageStats; import com.facebook.presto.client.StatementClient; import com.facebook.presto.client.StatementStats; import com.google.common.base.Strings; import com.google.common.primitives.Ints; -import io.airlift.log.Logger; import io.airlift.units.DataSize; import io.airlift.units.Duration; diff --git a/presto-cli/src/main/java/com/facebook/presto/cli/TableNameCompleter.java b/presto-cli/src/main/java/com/facebook/presto/cli/TableNameCompleter.java index c1e18df3f66ed..6aa536516a7bb 100644 --- a/presto-cli/src/main/java/com/facebook/presto/cli/TableNameCompleter.java +++ b/presto-cli/src/main/java/com/facebook/presto/cli/TableNameCompleter.java @@ -28,8 +28,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.google.common.cache.CacheLoader.asyncReloading; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; diff --git a/presto-cli/src/test/java/com/facebook/presto/cli/TestQueryRunner.java b/presto-cli/src/test/java/com/facebook/presto/cli/TestQueryRunner.java index 5661ea8396fa9..b85f63ad118ff 100644 --- a/presto-cli/src/test/java/com/facebook/presto/cli/TestQueryRunner.java +++ b/presto-cli/src/test/java/com/facebook/presto/cli/TestQueryRunner.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.cli; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.client.ClientSession; import com.facebook.presto.client.Column; import com.facebook.presto.client.QueryResults; @@ -21,7 +22,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; import io.airlift.units.Duration; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -35,12 +35,12 @@ import java.util.Locale; import java.util.Optional; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.cli.ClientOptions.OutputFormat.CSV; import static com.google.common.io.ByteStreams.nullOutputStream; import static com.google.common.net.HttpHeaders.CONTENT_TYPE; import static com.google.common.net.HttpHeaders.LOCATION; import static com.google.common.net.HttpHeaders.SET_COOKIE; -import static io.airlift.json.JsonCodec.jsonCodec; import static java.util.concurrent.TimeUnit.MINUTES; import static org.testng.Assert.assertEquals; @@ -91,7 +91,6 @@ public void testCookie() "clientInfo", "catalog", "schema", - "path", "America/Los_Angeles", Locale.ENGLISH, ImmutableMap.of(), diff --git a/presto-client/pom.xml b/presto-client/pom.xml index b018c4e0e2cfe..b2337be66e201 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-client @@ -43,12 +43,12 @@ - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift security diff --git a/presto-client/src/main/java/com/facebook/presto/client/ClientSession.java b/presto-client/src/main/java/com/facebook/presto/client/ClientSession.java index 184ad567a909c..8bfb12eec3cba 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/ClientSession.java +++ b/presto-client/src/main/java/com/facebook/presto/client/ClientSession.java @@ -42,7 +42,6 @@ public class ClientSession private final String clientInfo; private final String catalog; private final String schema; - private final String path; private final TimeZoneKey timeZone; private final Locale locale; private final Map resourceEstimates; @@ -74,7 +73,6 @@ public ClientSession( String clientInfo, String catalog, String schema, - String path, String timeZoneId, Locale locale, Map resourceEstimates, @@ -93,7 +91,6 @@ public ClientSession( this.clientInfo = clientInfo; this.catalog = catalog; this.schema = schema; - this.path = path; this.locale = locale; this.timeZone = TimeZoneKey.getTimeZoneKey(timeZoneId); this.transactionId = transactionId; @@ -173,11 +170,6 @@ public String getSchema() return schema; } - public String getPath() - { - return path; - } - public TimeZoneKey getTimeZone() { return timeZone; @@ -241,7 +233,6 @@ public String toString() .add("clientInfo", clientInfo) .add("catalog", catalog) .add("schema", schema) - .add("path", path) .add("traceToken", traceToken.orElse(null)) .add("timeZone", timeZone) .add("locale", locale) @@ -261,7 +252,6 @@ public static final class Builder private String clientInfo; private String catalog; private String schema; - private String path; private TimeZoneKey timeZone; private Locale locale; private Map resourceEstimates; @@ -283,7 +273,6 @@ private Builder(ClientSession clientSession) clientInfo = clientSession.getClientInfo(); catalog = clientSession.getCatalog(); schema = clientSession.getSchema(); - path = clientSession.getPath(); timeZone = clientSession.getTimeZone(); locale = clientSession.getLocale(); resourceEstimates = clientSession.getResourceEstimates(); @@ -307,12 +296,6 @@ public Builder withSchema(String schema) return this; } - public Builder withPath(String path) - { - this.path = requireNonNull(path, "path is null"); - return this; - } - public Builder withProperties(Map properties) { this.properties = requireNonNull(properties, "properties is null"); @@ -360,7 +343,6 @@ public ClientSession build() clientInfo, catalog, schema, - path, timeZone.getId(), locale, resourceEstimates, diff --git a/presto-client/src/main/java/com/facebook/presto/client/ClientTypeSignatureParameter.java b/presto-client/src/main/java/com/facebook/presto/client/ClientTypeSignatureParameter.java index e23fb35e6edcf..daaf0dbdbb9f9 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/ClientTypeSignatureParameter.java +++ b/presto-client/src/main/java/com/facebook/presto/client/ClientTypeSignatureParameter.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.client; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.type.NamedTypeSignature; import com.facebook.presto.spi.type.ParameterKind; import com.facebook.presto.spi.type.TypeSignatureParameter; @@ -24,7 +25,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import io.airlift.json.ObjectMapperProvider; import javax.annotation.concurrent.Immutable; diff --git a/presto-client/src/main/java/com/facebook/presto/client/FixJsonDataUtils.java b/presto-client/src/main/java/com/facebook/presto/client/FixJsonDataUtils.java index c58c160fc65b3..5d90729ecf34f 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/FixJsonDataUtils.java +++ b/presto-client/src/main/java/com/facebook/presto/client/FixJsonDataUtils.java @@ -40,6 +40,7 @@ import static com.facebook.presto.spi.type.StandardTypes.INTERVAL_DAY_TO_SECOND; import static com.facebook.presto.spi.type.StandardTypes.INTERVAL_YEAR_TO_MONTH; import static com.facebook.presto.spi.type.StandardTypes.IPADDRESS; +import static com.facebook.presto.spi.type.StandardTypes.IPPREFIX; import static com.facebook.presto.spi.type.StandardTypes.JSON; import static com.facebook.presto.spi.type.StandardTypes.MAP; import static com.facebook.presto.spi.type.StandardTypes.REAL; @@ -169,6 +170,7 @@ private static Object fixValue(TypeSignature signature, Object value) case INTERVAL_YEAR_TO_MONTH: case INTERVAL_DAY_TO_SECOND: case IPADDRESS: + case IPPREFIX: case DECIMAL: case CHAR: case GEOMETRY: diff --git a/presto-client/src/main/java/com/facebook/presto/client/JsonResponse.java b/presto-client/src/main/java/com/facebook/presto/client/JsonResponse.java index f86d7d2096b29..7f906b83b999a 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/JsonResponse.java +++ b/presto-client/src/main/java/com/facebook/presto/client/JsonResponse.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.client; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import okhttp3.Headers; import okhttp3.MediaType; import okhttp3.OkHttpClient; diff --git a/presto-client/src/main/java/com/facebook/presto/client/OkHttpUtil.java b/presto-client/src/main/java/com/facebook/presto/client/OkHttpUtil.java index ead05170281d0..813d36db60fd7 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/OkHttpUtil.java +++ b/presto-client/src/main/java/com/facebook/presto/client/OkHttpUtil.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.client; +import com.facebook.airlift.security.pem.PemReader; import com.google.common.base.CharMatcher; import com.google.common.net.HostAndPort; -import io.airlift.security.pem.PemReader; import okhttp3.Call; import okhttp3.Callback; import okhttp3.Credentials; diff --git a/presto-client/src/main/java/com/facebook/presto/client/PrestoHeaders.java b/presto-client/src/main/java/com/facebook/presto/client/PrestoHeaders.java index 189e60277236e..711f5402d5b9b 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/PrestoHeaders.java +++ b/presto-client/src/main/java/com/facebook/presto/client/PrestoHeaders.java @@ -19,14 +19,12 @@ public final class PrestoHeaders public static final String PRESTO_SOURCE = "X-Presto-Source"; public static final String PRESTO_CATALOG = "X-Presto-Catalog"; public static final String PRESTO_SCHEMA = "X-Presto-Schema"; - public static final String PRESTO_PATH = "X-Presto-Path"; public static final String PRESTO_TIME_ZONE = "X-Presto-Time-Zone"; public static final String PRESTO_LANGUAGE = "X-Presto-Language"; public static final String PRESTO_TRACE_TOKEN = "X-Presto-Trace-Token"; public static final String PRESTO_SESSION = "X-Presto-Session"; public static final String PRESTO_SET_CATALOG = "X-Presto-Set-Catalog"; public static final String PRESTO_SET_SCHEMA = "X-Presto-Set-Schema"; - public static final String PRESTO_SET_PATH = "X-Presto-Set-Path"; public static final String PRESTO_SET_SESSION = "X-Presto-Set-Session"; public static final String PRESTO_CLEAR_SESSION = "X-Presto-Clear-Session"; public static final String PRESTO_SET_ROLE = "X-Presto-Set-Role"; @@ -39,7 +37,6 @@ public final class PrestoHeaders public static final String PRESTO_CLEAR_TRANSACTION_ID = "X-Presto-Clear-Transaction-Id"; public static final String PRESTO_CLIENT_INFO = "X-Presto-Client-Info"; public static final String PRESTO_CLIENT_TAGS = "X-Presto-Client-Tags"; - public static final String PRESTO_CLIENT_CAPABILITIES = "X-Presto-Client-Capabilities"; public static final String PRESTO_RESOURCE_ESTIMATE = "X-Presto-Resource-Estimate"; public static final String PRESTO_EXTRA_CREDENTIAL = "X-Presto-Extra-Credential"; diff --git a/presto-client/src/main/java/com/facebook/presto/client/StatementClient.java b/presto-client/src/main/java/com/facebook/presto/client/StatementClient.java index 19528fca2f95d..33b06218be0f1 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/StatementClient.java +++ b/presto-client/src/main/java/com/facebook/presto/client/StatementClient.java @@ -50,8 +50,6 @@ public interface StatementClient Optional getSetSchema(); - Optional getSetPath(); - Map getSetSessionProperties(); Set getResetSessionProperties(); diff --git a/presto-client/src/main/java/com/facebook/presto/client/StatementClientV1.java b/presto-client/src/main/java/com/facebook/presto/client/StatementClientV1.java index 04594e5699503..e18c461bfc882 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/StatementClientV1.java +++ b/presto-client/src/main/java/com/facebook/presto/client/StatementClientV1.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.client; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.client.OkHttpUtil.NullCallback; import com.facebook.presto.spi.security.SelectedRole; import com.facebook.presto.spi.type.TimeZoneKey; @@ -21,7 +22,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import io.airlift.json.JsonCodec; import io.airlift.units.Duration; import okhttp3.Headers; import okhttp3.HttpUrl; @@ -46,23 +46,21 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.client.PrestoHeaders.PRESTO_ADDED_PREPARE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CATALOG; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLEAR_SESSION; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLEAR_TRANSACTION_ID; -import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_CAPABILITIES; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_INFO; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_TAGS; import static com.facebook.presto.client.PrestoHeaders.PRESTO_DEALLOCATED_PREPARE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_EXTRA_CREDENTIAL; import static com.facebook.presto.client.PrestoHeaders.PRESTO_LANGUAGE; -import static com.facebook.presto.client.PrestoHeaders.PRESTO_PATH; import static com.facebook.presto.client.PrestoHeaders.PRESTO_PREPARED_STATEMENT; import static com.facebook.presto.client.PrestoHeaders.PRESTO_RESOURCE_ESTIMATE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SCHEMA; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SESSION; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SET_CATALOG; -import static com.facebook.presto.client.PrestoHeaders.PRESTO_SET_PATH; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SET_ROLE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SET_SCHEMA; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SET_SESSION; @@ -75,7 +73,6 @@ import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkState; import static com.google.common.net.HttpHeaders.USER_AGENT; -import static io.airlift.json.JsonCodec.jsonCodec; import static java.lang.String.format; import static java.net.HttpURLConnection.HTTP_OK; import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; @@ -100,7 +97,6 @@ class StatementClientV1 private final AtomicReference currentResults = new AtomicReference<>(); private final AtomicReference setCatalog = new AtomicReference<>(); private final AtomicReference setSchema = new AtomicReference<>(); - private final AtomicReference setPath = new AtomicReference<>(); private final Map setSessionProperties = new ConcurrentHashMap<>(); private final Set resetSessionProperties = Sets.newConcurrentHashSet(); private final Map setRoles = new ConcurrentHashMap<>(); @@ -111,7 +107,6 @@ class StatementClientV1 private final TimeZoneKey timeZone; private final Duration requestTimeoutNanos; private final String user; - private final String clientCapabilities; private final AtomicReference state = new AtomicReference<>(State.RUNNING); @@ -126,7 +121,6 @@ public StatementClientV1(OkHttpClient httpClient, ClientSession session, String this.query = query; this.requestTimeoutNanos = session.getClientRequestTimeout(); this.user = session.getUser(); - this.clientCapabilities = Joiner.on(",").join(ClientCapabilities.values()); Request request = buildQueryRequest(session, query); @@ -168,9 +162,6 @@ private Request buildQueryRequest(ClientSession session, String query) if (session.getSchema() != null) { builder.addHeader(PRESTO_SCHEMA, session.getSchema()); } - if (session.getPath() != null) { - builder.addHeader(PRESTO_PATH, session.getPath()); - } builder.addHeader(PRESTO_TIME_ZONE, session.getTimeZone().getId()); if (session.getLocale() != null) { builder.addHeader(PRESTO_LANGUAGE, session.getLocale().toLanguageTag()); @@ -203,8 +194,6 @@ private Request buildQueryRequest(ClientSession session, String query) builder.addHeader(PRESTO_TRANSACTION_ID, session.getTransactionId() == null ? "NONE" : session.getTransactionId()); - builder.addHeader(PRESTO_CLIENT_CAPABILITIES, clientCapabilities); - return builder.build(); } @@ -279,12 +268,6 @@ public Optional getSetSchema() return Optional.ofNullable(setSchema.get()); } - @Override - public Optional getSetPath() - { - return Optional.ofNullable(setPath.get()); - } - @Override public Map getSetSessionProperties() { @@ -409,7 +392,6 @@ private void processResponse(Headers headers, QueryResults results) { setCatalog.set(headers.get(PRESTO_SET_CATALOG)); setSchema.set(headers.get(PRESTO_SET_SCHEMA)); - setPath.set(headers.get(PRESTO_SET_PATH)); for (String setSession : headers.values(PRESTO_SET_SESSION)) { List keyValue = SESSION_HEADER_SPLITTER.splitToList(setSession); diff --git a/presto-client/src/test/java/com/facebook/presto/client/TestClientTypeSignature.java b/presto-client/src/test/java/com/facebook/presto/client/TestClientTypeSignature.java index 508a61cca63dd..d6ca3b257d0d5 100644 --- a/presto-client/src/test/java/com/facebook/presto/client/TestClientTypeSignature.java +++ b/presto-client/src/test/java/com/facebook/presto/client/TestClientTypeSignature.java @@ -13,15 +13,15 @@ */ package com.facebook.presto.client; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.type.NamedTypeSignature; import com.facebook.presto.spi.type.RowFieldName; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.spi.type.TypeSignatureParameter; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; import org.testng.annotations.Test; import java.util.Optional; diff --git a/presto-client/src/test/java/com/facebook/presto/client/TestFixJsonDataUtils.java b/presto-client/src/test/java/com/facebook/presto/client/TestFixJsonDataUtils.java index 451dd918db0e0..a83efba59d97a 100644 --- a/presto-client/src/test/java/com/facebook/presto/client/TestFixJsonDataUtils.java +++ b/presto-client/src/test/java/com/facebook/presto/client/TestFixJsonDataUtils.java @@ -53,6 +53,7 @@ public void testFixData() assertQueryResult("map(bigint,bigint)", ImmutableMap.of(1, 3, 2, 4), ImmutableMap.of(1L, 3L, 2L, 4L)); assertQueryResult("json", "{\"json\": {\"a\": 1}}", "{\"json\": {\"a\": 1}}"); assertQueryResult("ipaddress", "1.2.3.4", "1.2.3.4"); + assertQueryResult("ipprefix", "1.2.3.4/32", "1.2.3.4/32"); assertQueryResult("Geometry", "POINT (1.2 3.4)", "POINT (1.2 3.4)"); assertQueryResult("map(BingTile,bigint)", ImmutableMap.of("BingTile{x=1, y=2, zoom_level=10}", 1), ImmutableMap.of("BingTile{x=1, y=2, zoom_level=10}", 1L)); } diff --git a/presto-client/src/test/java/com/facebook/presto/client/TestQueryResults.java b/presto-client/src/test/java/com/facebook/presto/client/TestQueryResults.java index 39a7a8e17aa88..c18db489e3cb4 100644 --- a/presto-client/src/test/java/com/facebook/presto/client/TestQueryResults.java +++ b/presto-client/src/test/java/com/facebook/presto/client/TestQueryResults.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.client; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import org.testng.annotations.Test; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; public class TestQueryResults diff --git a/presto-client/src/test/java/com/facebook/presto/client/TestServerInfo.java b/presto-client/src/test/java/com/facebook/presto/client/TestServerInfo.java index 1a28512735dd4..34d80bf134720 100644 --- a/presto-client/src/test/java/com/facebook/presto/client/TestServerInfo.java +++ b/presto-client/src/test/java/com/facebook/presto/client/TestServerInfo.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.client; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import io.airlift.units.Duration; import org.testng.annotations.Test; import java.util.Optional; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.client.NodeVersion.UNKNOWN; -import static io.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; public class TestServerInfo diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 9923f24456b78..2bb59e732737b 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-docs diff --git a/presto-docs/src/main/sphinx/admin/properties.rst b/presto-docs/src/main/sphinx/admin/properties.rst index 9a7e865c39809..d8c16fd3693fe 100644 --- a/presto-docs/src/main/sphinx/admin/properties.rst +++ b/presto-docs/src/main/sphinx/admin/properties.rst @@ -116,6 +116,19 @@ Memory Management Properties This is the amount of memory set aside as headroom/buffer in the JVM heap for allocations that are not tracked by Presto. +``query.low-memory-killer.policy`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + * **Type:** ``string`` + * **Default value:** ``none`` + + The policy used for selecting the query to kill when the cluster is out of memory (OOM). + This property can have one of the following values: ``none``, ``total-reservation``, + or ``total-reservation-on-blocked-nodes``. ``none`` disables the cluster OOM killer. + The value of ``total-reservation`` configures a policy that kills the query with the largest + memory reservation across the cluster. The value of ``total-reservation-on-blocked-nodes`` + configures a policy that kills the query using the most memory on the workers that are out of memory (blocked). + .. _tuning-spilling: Spilling Properties diff --git a/presto-docs/src/main/sphinx/admin/spill.rst b/presto-docs/src/main/sphinx/admin/spill.rst index c0d1146b52741..5c80504fab615 100644 --- a/presto-docs/src/main/sphinx/admin/spill.rst +++ b/presto-docs/src/main/sphinx/admin/spill.rst @@ -48,6 +48,24 @@ memory intensive queries. It is still possible that the query runner will fail to divide intermediate data into chunks small enough that every chunk fits into memory, leading to ``Out of memory`` errors while loading the data from disk. +Revocable memory and reserved pool +---------------------------------- + +Both reserved memory pool and revocable memory are designed to cope with low memory conditions. +When user memory pool is exhausted then a single query will be promoted to a reserved pool. +In such case only that query is allowed to progress thus reducing cluster +concurrency. Revocable memory will try to prevent that by triggering spill. +Reserved pool is of ``query.max-total-memory-per-node`` size. If +``query.max-total-memory-per-node`` is large compared to the total memory +available on the node, then the general memory pool may not have enough +memory to run larger queries. If spill is enabled, then this will cause +excessive spilling for queries that consume large amounts of memory per node. +These queries would finish much quicker if spill were disabled because they +would execute in the reserved pool. However, doing so could also significantly +reduce cluster concurrency. In such a situation we recommend disabling the +reserved memory pool via the ``experimental.reserved-pool-enabled`` config +property. + Spill Disk Space ---------------- diff --git a/presto-docs/src/main/sphinx/conf.py b/presto-docs/src/main/sphinx/conf.py index 6e6c4dd3738f2..f0b2196d49958 100644 --- a/presto-docs/src/main/sphinx/conf.py +++ b/presto-docs/src/main/sphinx/conf.py @@ -60,9 +60,9 @@ def get_version(): # -- General configuration ----------------------------------------------------- -needs_sphinx = '1.1' +needs_sphinx = '1.6.5' -extensions = ['download', 'issue', 'pr'] +extensions = ['download', 'issue', 'pr', 'sphinx.ext.mathjax'] templates_path = ['_templates'] diff --git a/presto-docs/src/main/sphinx/connector.rst b/presto-docs/src/main/sphinx/connector.rst index 677437f35e26a..1479993ceafd9 100644 --- a/presto-docs/src/main/sphinx/connector.rst +++ b/presto-docs/src/main/sphinx/connector.rst @@ -22,6 +22,7 @@ from different data sources. connector/memory connector/mongodb connector/mysql + connector/pinot connector/postgresql connector/redis connector/redshift diff --git a/presto-docs/src/main/sphinx/connector/elasticsearch.rst b/presto-docs/src/main/sphinx/connector/elasticsearch.rst index a528891c96bba..1790bb3770c2b 100644 --- a/presto-docs/src/main/sphinx/connector/elasticsearch.rst +++ b/presto-docs/src/main/sphinx/connector/elasticsearch.rst @@ -30,7 +30,7 @@ replacing the properties as appropriate: elasticsearch.default-schema=default elasticsearch.table-description-directory=etc/elasticsearch/ elasticsearch.scroll-size=1000 - elasticsearch.scroll-timeout=60000 + elasticsearch.scroll-timeout=2s elasticsearch.request-timeout=2s elasticsearch.max-request-retries=5 elasticsearch.max-request-retry-time=10s @@ -46,7 +46,7 @@ Property Name Description ``elasticsearch.default-schema`` Default schema name for tables. ``elasticsearch.table-description-directory`` Directory containing JSON table description files. ``elasticsearch.scroll-size`` Maximum number of hits to be returned with each Elasticsearch scroll request. -``elasticsearch.scroll-timeout`` Amount of time (ms) Elasticsearch will keep the search context alive for scroll requests. +``elasticsearch.scroll-timeout`` Amount of time Elasticsearch will keep the search context alive for scroll requests. ``elasticsearch.max-hits`` Maximum number of hits a single Elasticsearch request can fetch. ``elasticsearch.request-timeout`` Timeout for Elasticsearch requests. ``elasticsearch.max-request-retries`` Maximum number of Elasticsearch request retries. @@ -80,9 +80,9 @@ This property is optional; the default is ``1000``. ``elasticsearch.scroll-timeout`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -This property defines the amount of time (ms) Elasticsearch will keep the `search context alive`_ for scroll requests +This property defines the amount of time Elasticsearch will keep the `search context alive`_ for scroll requests -This property is optional; the default is ``20s``. +This property is optional; the default is ``1s``. .. _search context alive: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html#scroll-search-context diff --git a/presto-docs/src/main/sphinx/connector/hive.rst b/presto-docs/src/main/sphinx/connector/hive.rst index 27f0c22f89b17..1862bdd090cb4 100644 --- a/presto-docs/src/main/sphinx/connector/hive.rst +++ b/presto-docs/src/main/sphinx/connector/hive.rst @@ -423,6 +423,48 @@ If your workload experiences the error *Timeout waiting for connection from pool*, increase the value of both ``hive.s3select-pushdown.max-connections`` and the maximum connections configuration for the file system you are using. +Alluxio Configuration +--------------------- + +Presto can read and write tables stored in the Alluxio Data Orchestration System +`Alluxio `_, +leveraging Alluxio's distributed block-level read/write caching functionality. +The tables must be created in the Hive metastore with the ``alluxio://`` location prefix +(see `Running Apache Hive with Alluxio `_ +for details and examples). +Presto queries will then transparently retrieve and cache files +or objects from a variety of disparate storage systems including HDFS and S3. + +Alluxio Client-Side Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To configure Alluxio client-side properties on Presto, append the Alluxio +configuration directory (``${ALLUXIO_HOME}/conf``) to the Presto JVM classpath, +so that the Alluxio properties file ``alluxio-site.properties`` can be loaded as a resource. +Update the Presto :ref:`presto_jvm_config` file ``etc/jvm.config`` to include the following: + +.. code-block:: none + + -Xbootclasspath/a: + +The advantage of this approach is that all the Alluxio properties are set in +the single ``alluxio-site.properties`` file. For details, see `Customize Alluxio User Properties +`_. + +Alternatively, add Alluxio configuration properties to the Hadoop configuration +files (``core-site.xml``, ``hdfs-site.xml``) and configure the Hive connector +to use the `Hadoop configuration files <#hdfs-configuration>`__ via the +``hive.config.resources`` connector property. + +Deploy Alluxio with Presto +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To achieve the best performance running Presto on Alluxio, it is recommended +to collocate Presto workers with Alluxio workers. This allows reads and writes +to bypass the network. See `Performance Tuning Tips for Presto with Alluxio +`_ +for more details. + Table Statistics ---------------- diff --git a/presto-docs/src/main/sphinx/connector/kafka-tutorial.rst b/presto-docs/src/main/sphinx/connector/kafka-tutorial.rst index f9dcea003b8d1..5d02c23e3f14d 100644 --- a/presto-docs/src/main/sphinx/connector/kafka-tutorial.rst +++ b/presto-docs/src/main/sphinx/connector/kafka-tutorial.rst @@ -63,7 +63,7 @@ Now run the ``kafka-tpch`` program to preload a number of topics with tpch data: .. code-block:: none $ ./kafka-tpch load --brokers localhost:9092 --prefix tpch. --tpch-type tiny - 2014-07-28T17:17:07.594-0700 INFO main io.airlift.log.Logging Logging to stderr + 2014-07-28T17:17:07.594-0700 INFO main com.facebook.airlift.log.Logging Logging to stderr 2014-07-28T17:17:07.623-0700 INFO main de.softwareforge.kafka.LoadCommand Processing tables: [customer, orders, lineitem, part, partsupp, supplier, nation, region] 2014-07-28T17:17:07.981-0700 INFO pool-1-thread-1 de.softwareforge.kafka.LoadCommand Loading table 'customer' into topic 'tpch.customer'... 2014-07-28T17:17:07.981-0700 INFO pool-1-thread-2 de.softwareforge.kafka.LoadCommand Loading table 'orders' into topic 'tpch.orders'... diff --git a/presto-docs/src/main/sphinx/connector/pinot.rst b/presto-docs/src/main/sphinx/connector/pinot.rst new file mode 100644 index 0000000000000..15c513c967755 --- /dev/null +++ b/presto-docs/src/main/sphinx/connector/pinot.rst @@ -0,0 +1,100 @@ +=============== +Pinot Connector +=============== + +The Pinot connector allows querying and creating tables in an external Pinot +database. This can be used to query pinot data or join pinot data with +something else. + +Configuration +------------- + +To configure the Pinot connector, create a catalog properties file +in ``etc/catalog`` named, for example, ``pinot.properties``, to +mount the Pinot connector as the ``pinot`` catalog. +Create the file with the following contents, replacing the +connection properties as appropriate for your setup: + +.. code-block:: none + + connector.name=pinot + pinot.controller-urls=controller_host1:9000,controller_host2:9000 + +Where the ``pinot.controller-urls`` property allows you to specify a +comma separated list of the pinot controller host/port pairs. + +Multiple Pinot Servers +^^^^^^^^^^^^^^^^^^^^^^ + +You can have as many catalogs as you need, so if you have additional +Pinot clusters, simply add another properties file to ``etc/catalog`` +with a different name (making sure it ends in ``.properties``). For +example, if you name the property file ``sales.properties``, Presto +will create a catalog named ``sales`` using the configured connector. + +Querying Pinot +-------------- + +The Pinot catalog exposes all pinot tables inside a flat schema. The +schema name is immaterial when querying but running ``SHOW SCHEMAS``, +will show just one schema entry of ``default``. + +The name of the pinot catalog is the catalog file you created above +without the ``.properties`` extension. + +For example, if you created a +file called ``mypinotcluster.properties``, you can see all the tables +in it using the command:: + + SHOW TABLES from mypinotcluster.default + +OR:: + + SHOW TABLES from mypinotcluster.foo + +Both of these commands will list all the tables in your pinot cluster. +This is because Pinot does not have a notion of schemas. + +Consider you have a table called ``clicks`` in the ``mypinotcluster``. +You can see a list of the columns in the ``clicks`` table using either +of the following:: + + DESCRIBE mypinotcluster.dontcare.clicks; + SHOW COLUMNS FROM mypinotcluster.dontcare.clicks; + +Finally, you can access the ``clicks`` table:: + + SELECT count(*) FROM mypinotcluster.default.clicks; + + +How the Pinot connector works +----------------------------- + +The connector tries to push the maximal subquery inferred from the +presto query into pinot. It can push down everything Pinot supports +including aggregations, group by, all UDFs etc. It generates the +correct Pinot PQL keeping Pinot's quirks in mind. + +By default, it sends aggregation and limit queries to the Pinot broker +and does a parallel scan for non-aggregation/non-limit queries. The +pinot broker queries create a single split that lets the Pinot broker +do the scatter gather. Whereas, in the parallel scan mode, there is +one split created for one-or-more Pinot segments and the Pinot servers +are directly contacted by the Presto servers (ie., the Pinot broker is +not involved in the parallel scan mode) + +There are a few configurations that control this behavior: + +* ``pinot.prefer-broker-queries``: This config is true by default. + Setting it to false will also create parallel plans for + aggregation and limit queries. +* ``pinot.forbid-segment-queries``: This config is false by default. + Setting it to true will forbid parallel querying and force all + querying to happen via the broker. +* ``pinot.non-aggregate-limit-for-broker-queries``: To prevent + overwhelming the broker, the connector only allows querying the + pinot broker for ``short`` queries. We define a ``short`` query to + be either an aggregation (or group-by) query or a query with a limit + less than the value configured for + ``pinot.non-aggregate-limit-for-broker-queries``. The default value + for this limit is 25K rows. diff --git a/presto-docs/src/main/sphinx/functions.rst b/presto-docs/src/main/sphinx/functions.rst index d2f7a524822b0..8eb3dc0af02bc 100644 --- a/presto-docs/src/main/sphinx/functions.rst +++ b/presto-docs/src/main/sphinx/functions.rst @@ -23,6 +23,7 @@ Functions and Operators functions/array functions/map functions/url + functions/ip functions/geospatial functions/hyperloglog functions/qdigest diff --git a/presto-docs/src/main/sphinx/functions/aggregate.rst b/presto-docs/src/main/sphinx/functions/aggregate.rst index 67d638278b44a..06dd6dc83cd03 100644 --- a/presto-docs/src/main/sphinx/functions/aggregate.rst +++ b/presto-docs/src/main/sphinx/functions/aggregate.rst @@ -105,7 +105,7 @@ General Aggregate Functions Returns ``n`` smallest values of all input values of ``x``. -.. function:: reduce_agg(inputValue T, initialState S, inputFunction(S, T, S), combineFunction(S, S, S)) -> S +.. function:: reduce_agg(inputValue T, initialState S, inputFunction(S,T,S), combineFunction(S,S,S)) -> S Reduces all input values into a single value. ```inputFunction`` will be invoked for each input value. In addition to taking the input value, ``inputFunction`` @@ -116,7 +116,7 @@ General Aggregate Functions SELECT id, reduce_agg(value, 0, (a, b) -> a + b, (a, b) -> a + b) FROM ( VALUES - (1, 2) + (1, 2), (1, 3), (1, 4), (2, 20), @@ -279,12 +279,7 @@ Approximate Aggregate Functions Computes an approximate histogram with up to ``buckets`` number of buckets for all ``value``\ s with a per-item weight of ``weight``. The keys of the returned map are roughly the center of the bin, and the entry is the total - weight of the bin. The algorithm is based loosely on: - - .. code-block:: none - - Yael Ben-Haim and Elad Tom-Tov, "A streaming parallel decision tree algorithm", - J. Machine Learning Research 11 (2010), pp. 849--872. + weight of the bin. The algorithm is based loosely on [BenHaimTomTov2010]_. ``buckets`` must be a ``bigint``. ``value`` and ``weight`` must be numeric. @@ -295,6 +290,7 @@ Approximate Aggregate Functions :func:`numeric_histogram` that takes a ``weight``, with a per-item weight of ``1``. In this case, the total weight in the returned map is the count of items in the bin. + Statistical Aggregate Functions ------------------------------- @@ -314,9 +310,9 @@ Statistical Aggregate Functions Returns the log-2 entropy of count input-values. - .. code-block:: none + .. math:: - entropy(c) = \sum_i [ c_i / \sum_j [c_j] \log_2(\sum_j [c_j] / c_i) ] + \mathrm{entropy}(c) = \sum_i \left[ {c_i \over \sum_j [c_j]} \log_2\left({\sum_j [c_j] \over c_i}\right) \right]. ``c`` must be a ``bigint`` column of non-negative values. @@ -328,44 +324,201 @@ Statistical Aggregate Functions Returns the excess kurtosis of all input values. Unbiased estimate using the following expression: + .. math:: + + \mathrm{kurtosis}(x) = {n(n+1) \over (n-1)(n-2)(n-3)} { \sum[(x_i-\mu)^4] \over \sigma^4} -3{ (n-1)^2 \over (n-2)(n-3) }, + + where :math:`\mu` is the mean, and :math:`\sigma` is the standard deviation. + +.. function:: regr_intercept(y, x) -> double + + Returns linear regression intercept of input values. ``y`` is the dependent + value. ``x`` is the independent value. + +.. function:: regr_slope(y, x) -> double + + Returns linear regression slope of input values. ``y`` is the dependent + value. ``x`` is the independent value. + +.. function:: skewness(x) -> double + + Returns the skewness of all input values. + +.. function:: stddev(x) -> double + + This is an alias for :func:`stddev_samp`. + +.. function:: stddev_pop(x) -> double + + Returns the population standard deviation of all input values. + +.. function:: stddev_samp(x) -> double + + Returns the sample standard deviation of all input values. + +.. function:: variance(x) -> double + + This is an alias for :func:`var_samp`. + +.. function:: var_pop(x) -> double + + Returns the population variance of all input values. + +.. function:: var_samp(x) -> double + + Returns the sample variance of all input values. + + +Classification Metrics Aggregate Functions +------------------------------------------ + +The following functions each measure how some metric of a binary +`confusion matrix `_ changes as a function of +classification thresholds. They are meant to be used in conjunction. + +For example, to find the `precision-recall curve `_, use + .. code-block:: none - kurtosis(x) = n(n+1)/((n-1)(n-2)(n-3))sum[(x_i-mean)^4]/stddev(x)^4-3(n-1)^2/((n-2)(n-3)) + WITH + recall_precision AS ( + SELECT + CLASSIFICATION_RECALL(10000, correct, pred) AS recalls, + CLASSIFICATION_PRECISION(10000, correct, pred) AS precisions + FROM + classification_dataset + ) + SELECT + recall, + precision + FROM + recall_precision + CROSS JOIN UNNEST(recalls, precisions) AS t(recall, precision) + +To get the corresponding thresholds for these values, use + + .. code-block:: none + + WITH + recall_precision AS ( + SELECT + CLASSIFICATION_THRESHOLDS(10000, correct, pred) AS thresholds, + CLASSIFICATION_RECALL(10000, correct, pred) AS recalls, + CLASSIFICATION_PRECISION(10000, correct, pred) AS precisions + FROM + classification_dataset + ) + SELECT + threshold, + recall, + precision + FROM + recall_precision + CROSS JOIN UNNEST(thresholds, recalls, precisions) AS t(threshold, recall, precision) + +To find the `ROC curve `_, use + + .. code-block:: none + + WITH + fallout_recall AS ( + SELECT + CLASSIFICATION_FALLOUT(10000, correct, pred) AS fallouts, + CLASSIFICATION_RECALL(10000, correct, pred) AS recalls + FROM + classification_dataset + ) + SELECT + fallout + recall, + FROM + recall_fallout + CROSS JOIN UNNEST(fallouts, recalls) AS t(fallout, recall) + .. function:: classification_miss_rate(buckets, y, x, weight) -> array - Computes the miss-rate part of the receiver operator curve with up to ``buckets`` number of buckets. Returns - an array of miss-rate values. ``y`` should be a boolean outcome value; ``x`` should be predictions, each + Computes the miss-rate with up to ``buckets`` number of buckets. Returns + an array of miss-rate values. + + ``y`` should be a boolean outcome value; ``x`` should be predictions, each between 0 and 1; ``weight`` should be non-negative values, indicating the weight of the instance. + The + `miss-rate `_ + is defined as a sequence whose :math:`j`-th entry is + + .. math :: + + { + \sum_{i \;|\; x_i \leq t_j \bigwedge y_i = 1} \left[ w_i \right] + \over + \sum_{i \;|\; x_i \leq t_j \bigwedge y_i = 1} \left[ w_i \right] + + + \sum_{i \;|\; x_i > t_j \bigwedge y_i = 1} \left[ w_i \right] + }, + + where :math:`t_j` is the :math:`j`-th smallest threshold, + and :math:`y_i`, :math:`x_i`, and :math:`w_i` are the :math:`i`-th + entries of ``y``, ``x``, and ``weight``, respectively. + +.. function:: classification_miss_rate(buckets, y, x) -> array + + This function is equivalent to the variant of + :func:`classification_miss_rate` that takes a ``weight``, with a per-item weight of ``1``. + .. function:: classification_fall_out(buckets, y, x, weight) -> array - Computes the fall-out part of the receiver operator curve with up to ``buckets`` number of buckets. Returns - an array of miss-rate values. ``y`` should be a boolean outcome value; ``x`` should be predictions, each + Computes the fall-out with up to ``buckets`` number of buckets. Returns + an array of fall-out values. + + ``y`` should be a boolean outcome value; ``x`` should be predictions, each between 0 and 1; ``weight`` should be non-negative values, indicating the weight of the instance. - To get an ROC map, use this in conjunction with :func:`classification_recall`: + The + `fall-out `_ + is defined as a sequence whose :math:`j`-th entry is - .. code-block:: none + .. math :: - MAP(classification_fall_out(1000, outcome, prediction), classification_recall(1000, outcome, prediction)) + { + \sum_{i \;|\; x_i \leq t_j \bigwedge y_i = 0} \left[ w_i \right] + \over + \sum_{i \;|\; y_i = 0} \left[ w_i \right] + }, -.. function:: classification_precision(buckets, y, x) -> array + where :math:`t_j` is the :math:`j`-th smallest threshold, + and :math:`y_i`, :math:`x_i`, and :math:`w_i` are the :math:`i`-th + entries of ``y``, ``x``, and ``weight``, respectively. + +.. function:: classification_fall_out(buckets, y, x) -> array This function is equivalent to the variant of - :func:`classification_precision` that takes a ``weight``, with a per-item weight of ``1``. + :func:`classification_fall_out` that takes a ``weight``, with a per-item weight of ``1``. .. function:: classification_precision(buckets, y, x, weight) -> array - Computes the precision part of the precision-recall curve with up to ``buckets`` number of buckets. Returns - an array of precision values. ``y`` should be a boolean outcome value; ``x`` should be predictions, each + Computes the precision with up to ``buckets`` number of buckets. Returns + an array of precision values. + + ``y`` should be a boolean outcome value; ``x`` should be predictions, each between 0 and 1; ``weight`` should be non-negative values, indicating the weight of the instance. - To get a map of recall to precision, use this in conjunction with :func:`classification_recall`: + The + `precision `_ + is defined as a sequence whose :math:`j`-th entry is - .. code-block:: none + .. math :: - MAP(classification_recall(1000, outcome, prediction), classification_precision(1000, outcome, prediction)) + { + \sum_{i \;|\; x_i > t_j \bigwedge y_i = 1} \left[ w_i \right] + \over + \sum_{i \;|\; x_i > t_j} \left[ w_i \right] + }, + + where :math:`t_j` is the :math:`j`-th smallest threshold, + and :math:`y_i`, :math:`x_i`, and :math:`w_i` are the :math:`i`-th + entries of ``y``, ``x``, and ``weight``, respectively. .. function:: classification_precision(buckets, y, x) -> array @@ -374,16 +527,27 @@ Statistical Aggregate Functions .. function:: classification_recall(buckets, y, x, weight) -> array - Computes the recall part of the precision-recall curve or the receiver operator charateristic curve - with up to ``buckets`` number of buckets. Returns an array of recall values. + Computes the recall with up to ``buckets`` number of buckets. Returns + an array of recall values. + ``y`` should be a boolean outcome value; ``x`` should be predictions, each between 0 and 1; ``weight`` should be non-negative values, indicating the weight of the instance. - To get a map of recall to precision, use this in conjunction with :func:`classification_recall`: + The + `recall `_ + is defined as a sequence whose :math:`j`-th entry is - .. code-block:: none + .. math :: + + { + \sum_{i \;|\; x_i > t_j \bigwedge y_i = 1} \left[ w_i \right] + \over + \sum_{i \;|\; y_i = 1} \left[ w_i \right] + }, - MAP(classification_recall(1000, outcome, prediction), classification_precision(1000, outcome, prediction)) + where :math:`t_j` is the :math:`j`-th smallest threshold, + and :math:`y_i`, :math:`x_i`, and :math:`w_i` are the :math:`i`-th + entries of ``y``, ``x``, and ``weight``, respectively. .. function:: classification_recall(buckets, y, x) -> array @@ -392,56 +556,156 @@ Statistical Aggregate Functions .. function:: classification_thresholds(buckets, y, x) -> array - Computes the thresholds part of the precision-recall curve with up to ``buckets`` number of buckets. Returns - an array of thresholds. ``y`` should be a boolean outcome value; ``x`` should be predictions, each + Computes the thresholds with up to ``buckets`` number of buckets. Returns + an array of threshold values. + + ``y`` should be a boolean outcome value; ``x`` should be predictions, each between 0 and 1. - To get a map of thresholds to precision, use this in conjunction with :func:`classification_precision`: + The thresholds are defined as a sequence whose :math:`j`-th entry is the :math:`j`-th smallest threshold. + + +Differential Entropy Functions +------------------------------- + +The following functions approximate the binary `differential entropy `_. +That is, for a random variable :math:`x`, they approximate + +.. math :: + + h(x) = - \int x \log_2\left(f(x)\right) dx, + +where :math:`f(x)` is the partial density function of :math:`x`. + +.. function:: differential_entropy(sample_size, x) + + Returns the approximate log-2 differential entropy from a random variable's sample outcomes. The function internally + creates a reservoir (see [Black2015]_), then calculates the + entropy from the sample results by approximating the derivative of the cumulative distribution + (see [Alizadeh2010]_). + + ``sample_size`` (``long``) is the maximal number of reservoir samples. + + ``x`` (``double``) is the samples. + + For example, to find the differential entropy of ``x`` of ``data`` using 1000000 reservoir samples, use .. code-block:: none - MAP(classification_thresholds(1000, outcome, prediction), classification_precision(1000, outcome, prediction)) + SELECT + differential_entropy(1000000, x) + FROM + data + + .. note:: + + If :math:`x` has a known lower and upper bound, + prefer the versions taking ``(bucket_count, x, 1.0, "fixed_histogram_mle", min, max)``, + or ``(bucket_count, x, 1.0, "fixed_histogram_jacknife", min, max)``, + as they have better convergence. + +.. function:: differential_entropy(sample_size, x, weight) + + Returns the approximate log-2 differential entropy from a random variable's sample outcomes. The function + internally creates a weighted reservoir (see [Efraimidis2006]_), then calculates the + entropy from the sample results by approximating the derivative of the cumulative distribution + (see [Alizadeh2010]_). + + ``sample_size`` is the maximal number of reservoir samples. + + ``x`` (``double``) is the samples. - To get a map of thresholds to recall, use this in conjunction with :func:`classification_recall`: + ``weight`` (``double``) is a non-negative double value indicating the weight of the sample. + + For example, to find the differential entropy of ``x`` with weights ``weight`` of ``data`` + using 1000000 reservoir samples, use .. code-block:: none - MAP(classification_thresholds(1000, outcome, prediction), classification_recall(1000, outcome, prediction)) + SELECT + differential_entropy(1000000, x, weight) + FROM + data -.. function:: regr_intercept(y, x) -> double + .. note:: - Returns linear regression intercept of input values. ``y`` is the dependent - value. ``x`` is the independent value. + If :math:`x` has a known lower and upper bound, + prefer the versions taking ``(bucket_count, x, weight, "fixed_histogram_mle", min, max)``, + or ``(bucket_count, x, weight, "fixed_histogram_jacknife", min, max)``, + as they have better convergence. -.. function:: regr_slope(y, x) -> double +.. function:: differential_entropy(bucket_count, x, weight, method, min, max) -> double - Returns linear regression slope of input values. ``y`` is the dependent - value. ``x`` is the independent value. + Returns the approximate log-2 differential entropy from a random variable's sample outcomes. The function + internally creates a conceptual histogram of the sample values, calculates the counts, and + then approximates the entropy using maximum likelihood with or without Jacknife + correction, based on the ``method`` parameter. If Jacknife correction (see [Beirlant2001]_) is used, the + estimate is -.. function:: skewness(x) -> double + .. math :: - Returns the skewness of all input values. + n H(x) - (n - 1) \sum_{i = 1}^n H\left(x_{(i)}\right) -.. function:: stddev(x) -> double + where :math:`n` is the length of the sequence, and :math:`x_{(i)}` is the sequence with the :math:`i`-th element + removed. - This is an alias for :func:`stddev_samp`. + ``bucket_count`` (``long``) determines the number of histogram buckets. -.. function:: stddev_pop(x) -> double + ``x`` (``double``) is the samples. - Returns the population standard deviation of all input values. + ``method`` (``varchar``) is either ``'fixed_histogram_mle'`` (for the maximum likelihood estimate) + or ``'fixed_histogram_jacknife'`` (for the jacknife-corrected maximum likelihood estimate). -.. function:: stddev_samp(x) -> double + ``min`` and ``max`` (both ``double``) are the minimal and maximal values, respectively; + the function will throw if there is an input outside this range. - Returns the sample standard deviation of all input values. + ``weight`` (``double``) is the weight of the sample, and must be non-negative. -.. function:: variance(x) -> double + For example, to find the differential entropy of ``x``, each between ``0.0`` and ``1.0``, + with weights 1.0 of ``data`` using 1000000 bins and jacknife estimates, use - This is an alias for :func:`var_samp`. + .. code-block:: none -.. function:: var_pop(x) -> double + SELECT + differential_entropy(1000000, x, 1.0, 'fixed_histogram_jacknife', 0.0, 1.0) + FROM + data - Returns the population variance of all input values. + To find the differential entropy of ``x``, each between ``-2.0`` and ``2.0``, + with weights ``weight`` of ``data`` using 1000000 buckets and maximum-likelihood estimates, use -.. function:: var_samp(x) -> double + .. code-block:: none - Returns the sample variance of all input values. + SELECT + differential_entropy(1000000, x, weight, 'fixed_histogram_mle', -2.0, 2.0) + FROM + data + + .. note:: + + If :math:`x` doesn't have known lower and upper bounds, prefer the versions taking ``(sample_size, x)`` + (unweighted case) or ``(sample_size, x, weight)`` (weighted case), as they use reservoir + sampling which doesn't require a known range for samples. + + Otherwise, if the number of distinct weights is low, + especially if the number of samples is low, consider using the version taking + ``(bucket_count, x, weight, "fixed_histogram_jacknife", min, max)``, as jacknife bias correction, + is better than maximum likelihood estimation. However, if the number of distinct weights is high, + consider using the version taking ``(bucket_count, x, weight, "fixed_histogram_mle", min, max)``, + as this will reduce memory and running time. + + +--------------------------- + +.. [Alizadeh2010] Alizadeh Noughabi, Hadi & Arghami, N. (2010). "A New Estimator of Entropy". + +.. [Beirlant2001] Beirlant, Dudewicz, Gyorfi, and van der Meulen, + "Nonparametric entropy estimation: an overview", (2001) + +.. [BenHaimTomTov2010] Yael Ben-Haim and Elad Tom-Tov, "A streaming parallel decision tree algorithm", + J. Machine Learning Research 11 (2010), pp. 849--872. + +.. [Black2015] Black, Paul E. (26 January 2015). "Reservoir sampling". Dictionary of Algorithms and Data Structures. + +.. [Efraimidis2006] Efraimidis, Pavlos S.; Spirakis, Paul G. (2006-03-16). "Weighted random sampling with a reservoir". + Information Processing Letters. 97 (5): 181–185. diff --git a/presto-docs/src/main/sphinx/functions/array.rst b/presto-docs/src/main/sphinx/functions/array.rst index 37a1ed8f1809e..da8bf665a4246 100644 --- a/presto-docs/src/main/sphinx/functions/array.rst +++ b/presto-docs/src/main/sphinx/functions/array.rst @@ -21,22 +21,32 @@ The ``||`` operator is used to concatenate an array with an array or an element Array Functions --------------- -.. function:: array_distinct(x) -> array +.. function:: all_match(array(T), function(T,boolean)) -> boolean - Remove duplicate values from the array ``x``. + Returns whether all elements of an array match the given predicate. Returns ``true`` if all the elements + match the predicate (a special case is when the array is empty); ``false`` if one or more elements don't + match; ``NULL`` if the predicate function returns ``NULL`` for one or more elements and ``true`` for all + other elements. -.. function:: array_intersect(x, y) -> array +.. function:: any_match(array(T), function(T,boolean)) -> boolean - Returns an array of the elements in the intersection of ``x`` and ``y``, without duplicates. + Returns whether any elements of an array match the given predicate. Returns ``true`` if one or more + elements match the predicate; ``false`` if none of the elements matches (a special case is when the + array is empty); ``NULL`` if the predicate function returns ``NULL`` for one or more elements and ``false`` + for all other elements. -.. function:: array_union(x, y) -> array +.. function:: array_distinct(x) -> array - Returns an array of the elements in the union of ``x`` and ``y``, without duplicates. + Remove duplicate values from the array ``x``. .. function:: array_except(x, y) -> array Returns an array of elements in ``x`` but not in ``y``, without duplicates. +.. function:: array_intersect(x, y) -> array + + Returns an array of the elements in the intersection of ``x`` and ``y``, without duplicates. + .. function:: array_join(x, delimiter, null_replacement) -> varchar Concatenates the elements of the given array using the delimiter and an optional string to replace nulls. @@ -94,9 +104,13 @@ Array Functions .. function:: arrays_overlap(x, y) -> boolean - Tests if arrays ``x`` and ``y`` have any any non-null elements in common. + Tests if arrays ``x`` and ``y`` have any non-null elements in common. Returns null if there are no non-null elements in common but either array contains null. +.. function:: array_union(x, y) -> array + + Returns an array of the elements in the union of ``x`` and ``y``, without duplicates. + .. function:: cardinality(x) -> bigint Returns the cardinality (size) of the array ``x``. @@ -107,6 +121,18 @@ Array Functions Concatenates the arrays ``array1``, ``array2``, ``...``, ``arrayN``. This function provides the same functionality as the SQL-standard concatenation operator (``||``). +.. function:: combinations(array(T), n) -> array(array(T)) + + Returns n-element combinations of the input array. + If the input array has no duplicates, ``combinations`` returns n-element subsets. + Order of subgroup is deterministic but unspecified. Order of elements within + a subgroup are deterministic but unspecified. ``n`` must not be greater than 5, + and the total size of subgroups generated must be smaller than 100000:: + + SELECT combinations(ARRAY['foo', 'bar', 'boo'],2); --[['foo', 'bar'], ['foo', 'boo']['bar', 'boo']] + SELECT combinations(ARRAY[1,2,3,4,5],3); --[[1,2,3], [1,2,4], [1,3,4], [2,3,4]] + SELECT combinations(ARRAY[1,2,2],2); --[[1,2],[1,2],[2,2]] + .. function:: contains(x, element) -> boolean Returns true if the array ``x`` contains the ``element``. @@ -139,6 +165,12 @@ Array Functions SELECT ngrams(ARRAY['foo', 'bar', 'baz', 'foo'], 5); -- [['foo', 'bar', 'baz', 'foo']] SELECT ngrams(ARRAY[1, 2, 3, 4], 2); -- [[1, 2], [2, 3], [3, 4]] +.. function:: none_match(array(T), function(T,boolean)) -> boolean + + Returns whether no elements of an array match the given predicate. Returns ``true`` if none of the elements + matches the predicate (a special case is when the array is empty); ``false`` if one or more elements match; + ``NULL`` if the predicate function returns ``NULL`` for one or more elements and ``false`` for all other elements. + .. function:: reduce(array(T), initialState S, inputFunction(S,T,S), outputFunction(S,R)) -> R Returns a single value reduced from ``array``. ``inputFunction`` will diff --git a/presto-docs/src/main/sphinx/functions/geospatial.rst b/presto-docs/src/main/sphinx/functions/geospatial.rst index b278ee5bcdedc..2ffaedd735d7a 100644 --- a/presto-docs/src/main/sphinx/functions/geospatial.rst +++ b/presto-docs/src/main/sphinx/functions/geospatial.rst @@ -21,35 +21,34 @@ Binary (WKB) form of spatial objects: * ``MULTIPOLYGON (((0 0, 4 0, 4 4, 0 4, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1)), ((-1 -1, -1 -2, -2 -2, -2 -1, -1 -1)))`` * ``GEOMETRYCOLLECTION (POINT(2 3), LINESTRING (2 3, 3 4))`` -Use ST_GeometryFromText and ST_GeomFromBinary functions to create geometry -objects from WKT or WKB. In WKT/WKB, the coordinate order is ``(x, y)``. -For spherical/geospatial uses, this implies ``(longitude, latitude)`` instead -of ``(latitude, longitude)``. +Use ``ST_GeometryFromText`` and ``ST_GeomFromBinary`` functions to create +geometry objects from WKT or WKB. In WKT/WKB, the coordinate order is +``(x, y)``. For spherical/geospatial uses, this implies +``(longitude, latitude)`` instead of ``(latitude, longitude)``. -The SphericalGeography type provides native support for spatial features -represented on "geographic" coordinates (sometimes called "geodetic" -coordinates, or "lat/lon", or "lon/lat"). Geographic coordinates are spherical -coordinates expressed in angular units (degrees). - -The basis for the Geometry type is a plane. The shortest path between two +The basis for the ``Geometry`` type is a plane. The shortest path between two points on the plane is a straight line. That means calculations on geometries (areas, distances, lengths, intersections, etc) can be calculated using cartesian mathematics and straight line vectors. -The basis for the SphericalGeography type is a sphere. The shortest path +The ``SphericalGeography`` type provides native support for spatial features +represented on "geographic" coordinates (sometimes called "geodetic" +coordinates, or "lat/lon", or "lon/lat"). Geographic coordinates are spherical +coordinates expressed in angular units (degrees). + +The basis for the ``SphericalGeography`` type is a sphere. The shortest path between two points on the sphere is a great circle arc. That means that calculations on geographies (areas, distances, lengths, intersections, etc) must be calculated on the sphere, using more complicated mathematics. More accurate measurements that take the actual spheroidal shape of the world into account are not supported. -Values returned by the measurement functions ST_Distance and ST_Length are in -the unit of meters; values returned by ST_Area are in square meters. +For ``SphericalGeography`` objects, values returned by the measurement functions +``ST_Distance`` and ``ST_Length`` are in the unit of meters; values returned by +``ST_Area`` are in square meters. Use ``to_spherical_geography()`` function to convert a geometry object to -geography object. - -For example, +geography object. For example, ``ST_Distance(ST_Point(-71.0882, 42.3607), ST_Point(-74.1197, 40.6976))`` returns 3.4577 in the unit of the passed-in values on the euclidean plane, while @@ -116,7 +115,7 @@ Constructors of such geometries. For each point of the input geometry, it verifies that point.x is within [-180.0, 180.0] and point.y is within [-90.0, 90.0], and uses them as (longitude, latitude) degrees to construct the shape of the - SphericalGeography result. + ``SphericalGeography`` result. .. function:: to_geometry(SphericalGeography) -> Geometry @@ -187,7 +186,9 @@ Operations .. function:: ST_Buffer(Geometry, distance) -> Geometry Returns the geometry that represents all points whose distance from the - specified geometry is less than or equal to the specified distance. + specified geometry is less than or equal to the specified distance. If the + points of the geometry are extremely close together (``delta < 1e-8``), this + might return an empty geometry. .. function:: ST_Difference(Geometry, Geometry) -> Geometry @@ -204,6 +205,13 @@ Operations the bounding rectangular polygon of a geometry. Returns ``null`` if input geometry is empty. +.. function:: expand_envelope(Geometry, double) -> Geometry + + Returns the bounding rectangular polygon of a geometry, expanded by a + distance. Empty geometries will return an empty polygon. Negative or NaN + distances will return an error. Positive infinity distances may lead to + undefined results. + .. function:: ST_ExteriorRing(Geometry) -> Geometry Returns a line string representing the exterior ring of the input polygon. @@ -241,7 +249,7 @@ Accessors Returns the area of a polygon or multi-polygon in square meters using a spherical model for Earth. -.. function:: ST_Centroid(Geometry) -> Geometry +.. function:: ST_Centroid(Geometry) -> Point Returns the point value that is the mathematical centroid of a geometry. @@ -334,34 +342,29 @@ Accessors .. function:: ST_XMax(Geometry) -> double - Returns X maxima of a bounding box of a geometry. + Returns the X maximum of the geometry's bounding box. .. function:: ST_YMax(Geometry) -> double - Returns Y maxima of a bounding box of a geometry. + Returns the Y maximum of the geometry's bounding box. .. function:: ST_XMin(Geometry) -> double - Returns X minima of a bounding box of a geometry. + Returns the X minimum of the geometry's bounding box. .. function:: ST_YMin(Geometry) -> double - Returns Y minima of a bounding box of a geometry. + Returns the Y minimum of the geometry's bounding box. .. function:: ST_StartPoint(Geometry) -> point Returns the first point of a LineString geometry as a Point. - This is a shortcut for ST_PointN(geometry, 1). - -.. function:: simplify_geometry(Geometry, double) -> Geometry - - Returns a "simplified" version of the input geometry using the Douglas-Peucker algorithm. - Will avoid creating derived geometries (polygons in particular) that are invalid. + This is a shortcut for ``ST_PointN(geometry, 1)``. .. function:: ST_EndPoint(Geometry) -> point Returns the last point of a LineString geometry as a Point. - This is a shortcut for ST_PointN(geometry, ST_NumPoints(geometry)). + This is a shortcut for ``ST_PointN(geometry, ST_NumPoints(geometry))``. .. function:: ST_X(Point) -> double @@ -371,7 +374,7 @@ Accessors Return the Y coordinate of the point. -.. function:: ST_InteriorRings(Geometry) -> Geometry +.. function:: ST_InteriorRings(Geometry) -> array(Geometry) Returns an array of all interior rings found in the input geometry, or an empty array if the polygon has no interior rings. Returns ``null`` if the input geometry @@ -380,16 +383,23 @@ Accessors .. function:: ST_NumGeometries(Geometry) -> bigint Returns the number of geometries in the collection. - If the geometry is a collection of geometries (e.g., GEOMETRYCOLLECTION or MULTI*), - returns the number of geometries, - for single geometries returns 1, - for empty geometries returns 0. + If the geometry is a collection of geometries (e.g., GEOMETRYCOLLECTION or + MULTI*), returns the number of geometries, for single geometries returns 1, + for empty geometries returns 0. Note that empty geometries inside of a + GEOMETRYCOLLECTION will count as a geometry; eg + ``ST_NumGeometries(ST_GeometryFromText('GEOMETRYCOLLECTION(MULTIPOINT EMPTY)'))`` + will evaluate to 1. -.. function:: ST_Geometries(Geometry) -> Geometry +.. function:: ST_Geometries(Geometry) -> array(Geometry) Returns an array of geometries in the specified collection. Returns a one-element array if the input geometry is not a multi-geometry. Returns ``null`` if input geometry is empty. + For example, a MultiLineString will create an array of LineStrings. A GeometryCollection + will produce an un-flattened array of its constituents: + ``GEOMETRYCOLLECTION(MULTIPOINT(0 0, 1 1), GEOMETRYCOLLECTION(MULTILINESTRING((2 2, 3 3))))`` + would produce ``array[MULTIPOINT(0 0, 1 1), GEOMETRYCOLLECTION(MULTILINESTRING((2 2, 3 3)))]``. + .. function:: ST_NumPoints(Geometry) -> bigint Returns the number of points in a geometry. This is an extension to the SQL/MM @@ -399,6 +409,11 @@ Accessors Returns the cardinality of the collection of interior rings of a polygon. +.. function:: simplify_geometry(Geometry, double) -> Geometry + + Returns a "simplified" version of the input geometry using the Douglas-Peucker algorithm. + Will avoid creating derived geometries (polygons in particular) that are invalid. + .. function:: line_locate_point(LineString, Point) -> double Returns a float between 0 and 1 representing the location of the closest point on @@ -406,6 +421,14 @@ Accessors Returns ``null`` if a LineString or a Point is empty or ``null``. +.. function:: line_interpolate_point(LineString, double) -> Geometry + + Returns the Point on the LineString at a fractional distance given by the + double argument. Throws an exception if the distance is not between 0 and 1. + + Returns an empty Point if the LineString is empty. Returns ``null`` if + either the LineString or double is null. + .. function:: geometry_invalid_reason(Geometry) -> varchar Returns the reason for why the input geometry is not valid. diff --git a/presto-docs/src/main/sphinx/functions/ip.rst b/presto-docs/src/main/sphinx/functions/ip.rst new file mode 100644 index 0000000000000..6b9140dec8834 --- /dev/null +++ b/presto-docs/src/main/sphinx/functions/ip.rst @@ -0,0 +1,12 @@ +=================== +IP Functions +=================== + +.. function:: ip_prefix(ip_address, prefix_bits) -> ipprefix + + Returns the IP prefix of a given ``ip_address`` with subnet size of ``prefix_bits``. + ``ip_address`` can be either of type ``VARCHAR`` or type ``IPADDRESS``. :: + + SELECT ip_prefix(CAST('192.168.255.255' AS IPADDRESS), 9); -- {192.128.0.0/9} + SELECT ip_prefix('2001:0db8:85a3:0001:0001:8a2e:0370:7334', 48); -- {2001:db8:85a3::/48} + diff --git a/presto-docs/src/main/sphinx/functions/string.rst b/presto-docs/src/main/sphinx/functions/string.rst index e3703f9a216e2..d27576484f6a3 100644 --- a/presto-docs/src/main/sphinx/functions/string.rst +++ b/presto-docs/src/main/sphinx/functions/string.rst @@ -119,12 +119,12 @@ String Functions each pair into key and value. Note that ``entryDelimiter`` and ``keyValueDelimiter`` are interpreted literally, i.e., as full string matches. -.. function:: split_to_map(string, entryDelimiter, keyValueDelimiter, function(k, v1, v2, res)) -> map +.. function:: split_to_map(string, entryDelimiter, keyValueDelimiter, function(K,V1,V2,R)) -> map Splits ``string`` by ``entryDelimiter`` and ``keyValueDelimiter`` and returns a map. ``entryDelimiter`` splits ``string`` into key-value pairs. ``keyValueDelimiter`` splits each pair into key and value. Note that ``entryDelimiter`` and ``keyValueDelimiter`` are - interpreted literally, i.e., as full string matches. ``function(k, v1, v2, res)`` + interpreted literally, i.e., as full string matches. ``function(K,V1,V2,R)`` is invoked in cases of duplicate keys to resolve the value that should be in the map. SELECT(split_to_map('a:1;b:2;a:3', ';', ':', (k, v1, v2) -> v1)); -- {"a": "1", "b": "2"} diff --git a/presto-docs/src/main/sphinx/functions/window.rst b/presto-docs/src/main/sphinx/functions/window.rst index 0807672a7ca32..76ef29d48b62d 100644 --- a/presto-docs/src/main/sphinx/functions/window.rst +++ b/presto-docs/src/main/sphinx/functions/window.rst @@ -90,6 +90,12 @@ Ranking Functions Value Functions --------------- +Value functions provide an option to specify how null values should be treated when evaluating the +function. Nulls can either be ignored (``IGNORE NULLS``) or respected (``RESPECT NULLS``). By default, +null values are respected. If ``IGNORE NULLS`` is specified, all rows where the value expresssion is +null are excluded from the calculation. If ``IGNORE NULLS`` is specified and the value expression is +null for all rows, the ``default_value`` is returned, or if it is not specified, ``null`` is returned. + .. function:: first_value(x) -> [same as input] Returns the first value of the window. @@ -110,14 +116,14 @@ Value Functions Returns the value at ``offset`` rows after the current row in the window. Offsets start at ``0``, which is the current row. The - offset can be any scalar expression. The default ``offset`` is ``1``. If the + offset can be any scalar expression. The default ``offset`` is ``1``. If the offset is null or larger than the window, the ``default_value`` is returned, or if it is not specified ``null`` is returned. .. function:: lag(x[, offset [, default_value]]) -> [same as input] Returns the value at ``offset`` rows before the current row in the window - Offsets start at ``0``, which is the current row. The - offset can be any scalar expression. The default ``offset`` is ``1``. If the + Offsets start at ``0``, which is the current row. The + offset can be any scalar expression. The default ``offset`` is ``1``. If the offset is null or larger than the window, the ``default_value`` is returned, or if it is not specified ``null`` is returned. diff --git a/presto-docs/src/main/sphinx/installation/verifier.rst b/presto-docs/src/main/sphinx/installation/verifier.rst index ddc444c7bd9b8..037324a4e55cd 100644 --- a/presto-docs/src/main/sphinx/installation/verifier.rst +++ b/presto-docs/src/main/sphinx/installation/verifier.rst @@ -20,14 +20,15 @@ Create a MySQL database with the following table and load it with the queries yo control_catalog varchar(256) NOT NULL, control_schema varchar(256) NOT NULL, control_query text NOT NULL, + control_username varchar(256) DEFAULT NULL, + control_password varchar(256) DEFAULT NULL, + control_session_properties text DEFAULT NULL, test_catalog varchar(256) NOT NULL, test_schema varchar(256) NOT NULL, test_query text NOT NULL, - control_username varchar(256) NOT NULL DEFAULT 'verifier-test', - control_password varchar(256) DEFAULT NULL, - test_username varchar(256) NOT NULL DEFAULT 'verifier-test', + test_username varchar(256) DEFAULT NULL, test_password varchar(256) DEFAULT NULL, - session_properties_json varchar(2048) DEFAULT NULL) + test_session_properties text DEFAULT NULL) Next, create a properties file to configure the verifier: @@ -50,32 +51,36 @@ make it executable with ``chmod +x``, then run it: Configuration Reference ----------------------- -================================= ======================================================================= -Name Description -================================= ======================================================================= -``control.timeout`` The maximum execution time of the control queries. -``test.timeout`` The maximum execution time of the test queries. -``metadata.timeout`` The maximum execution time of the queries that are required for - obtaining table metadata or rewriting queries. -``checksum.timeout`` The maximum execution time of the queries that computes checksum for - the control and the test results. -``whitelist`` A comma-separated list that specifies names of the queries within the - suite to verify. -``blacklist`` A comma-separated list that specifies names of the queries to be - excluded from suite. ``blacklist`` is applied after ``whitelist``. -``source-query.table-name`` Specifies the MySQL table from which to read the source queries for - verification. -``event-clients`` A comma-separated list that specifies where the output events should be - emitted. Valid individual values are ``json`` and ``human-readable``. -``json.log-file`` Specifies the output files for JSON events. If ``json`` is specified in - ``event-clients`` but this property is not set, JSON events are emitted - to ``stdout``. -``human-readable.log-file`` Specifies the output files for human readable events. If - ``human-readable`` is specified in ``event-clients`` but this property - is not set, human readable events are emitted to ``stdout``. -``max-concurrency`` Specifies the maximum concurrent verification. Alternatively speaking, - the maximum concurrent queries that will be submitted to control and - test clusters combined. -``relative-error-margin`` Specified the maximum tolerable relative error between control and test - queries for floating point columns. -================================= ======================================================================= +=========================================== ======================================================================= +Name Description +=========================================== ======================================================================= +``control.timeout`` The maximum execution time of the control queries. +``test.timeout`` The maximum execution time of the test queries. +``metadata.timeout`` The maximum execution time of the queries that are required for + obtaining table metadata or rewriting queries. +``checksum.timeout`` The maximum execution time of the queries that computes checksum for + the control and the test results. +``whitelist`` A comma-separated list that specifies names of the queries within the + suite to verify. +``blacklist`` A comma-separated list that specifies names of the queries to be + excluded from suite. ``blacklist`` is applied after ``whitelist``. +``source-query.table-name`` Specifies the MySQL table from which to read the source queries for + verification. +``event-clients`` A comma-separated list that specifies where the output events should be + emitted. Valid individual values are ``json`` and ``human-readable``. +``json.log-file`` Specifies the output files for JSON events. If ``json`` is specified in + ``event-clients`` but this property is not set, JSON events are emitted + to ``stdout``. +``human-readable.log-file`` Specifies the output files for human readable events. If + ``human-readable`` is specified in ``event-clients`` but this property + is not set, human readable events are emitted to ``stdout``. +``max-concurrency`` Specifies the maximum concurrent verification. Alternatively speaking, + the maximum concurrent queries that will be submitted to control and + test clusters combined. +``relative-error-margin`` Specified the maximum tolerable relative error between control and test + queries for floating point columns. +``max-determinism-analysis-runs`` Maximum number of reruns of the control queries in case of a result + mismatch to determine whether the query is deterministic. +``run-teardown-for-determinism-analysis`` Whether temporary tables created in determinism analysis runs are + teared down. +=========================================== ======================================================================= diff --git a/presto-docs/src/main/sphinx/language/reserved.rst b/presto-docs/src/main/sphinx/language/reserved.rst index bc39537fdc5e2..14b769a996f6f 100644 --- a/presto-docs/src/main/sphinx/language/reserved.rst +++ b/presto-docs/src/main/sphinx/language/reserved.rst @@ -21,7 +21,6 @@ Keyword SQL:2016 SQL-92 ``CROSS`` reserved reserved ``CUBE`` reserved ``CURRENT_DATE`` reserved reserved -``CURRENT_PATH`` reserved ``CURRENT_TIME`` reserved reserved ``CURRENT_TIMESTAMP`` reserved reserved ``CURRENT_USER`` reserved diff --git a/presto-docs/src/main/sphinx/language/types.rst b/presto-docs/src/main/sphinx/language/types.rst index 039f60dcec024..90e6e70e2537c 100644 --- a/presto-docs/src/main/sphinx/language/types.rst +++ b/presto-docs/src/main/sphinx/language/types.rst @@ -88,8 +88,8 @@ Fixed-Precision .. note:: For compatibility reasons decimal literals without explicit type specifier (e.g. ``1.2``) - are treated as the values of the ``DOUBLE`` type by default, but this is subject to change - in future releases. This behavior is controlled by: + are treated as values of the ``DOUBLE`` type by default up to version 0.198. + After 0.198 they are parsed as DECIMAL. - System wide property: ``parse-decimal-literals-as-double`` - Session wide property: ``parse_decimal_literals_as_double`` @@ -225,16 +225,38 @@ Network Address ``IPADDRESS`` ^^^^^^^^^^^^^ - An IP address that can represent either an IPv4 or IPv6 address. Internally, - the type is a pure IPv6 address. Support for IPv4 is handled using the - *IPv4-mapped IPv6 address* range (:rfc:`4291#section-2.5.5.2`). + An IP address that can represent either an IPv4 or IPv6 address. + + Internally, the type is a pure IPv6 address. Support for IPv4 is handled + using the *IPv4-mapped IPv6 address* range (:rfc:`4291#section-2.5.5.2`). When creating an ``IPADDRESS``, IPv4 addresses will be mapped into that range. + When formatting an ``IPADDRESS``, any address within the mapped range will be formatted as an IPv4 address. Other addresses will be formatted as IPv6 using the canonical format defined in :rfc:`5952`. Examples: ``IPADDRESS '10.0.0.1'``, ``IPADDRESS '2001:db8::1'`` +.. _ipprefix_type: + +``IPPREFIX`` +^^^^^^^^^^^^ + + An IP routing prefix that can represent either an IPv4 or IPv6 address. + + Internally, an address is a pure IPv6 address. Support for IPv4 is handled + using the *IPv4-mapped IPv6 address* range (:rfc:`4291#section-2.5.5.2`). + When creating an ``IPPREFIX``, IPv4 addresses will be mapped into that range. + Additionally, addresses will be reduced to the first address of a network. + + ``IPPREFIX`` values will be formatted in CIDR notation, written as an IP + address, a slash ('/') character, and the bit-length of the prefix. Any + address within the IPv4-mapped IPv6 address range will be formatted as an + IPv4 address. Other addresses will be formatted as IPv6 using the canonical + format defined in :rfc:`5952`. + + Examples: ``IPPREFIX '10.0.1.0/24'``, ``IPPREFIX '2001:db8::/48'`` + HyperLogLog ----------- diff --git a/presto-docs/src/main/sphinx/release.rst b/presto-docs/src/main/sphinx/release.rst index c25051f2df72b..54e6d90fd4319 100644 --- a/presto-docs/src/main/sphinx/release.rst +++ b/presto-docs/src/main/sphinx/release.rst @@ -5,6 +5,12 @@ Release Notes .. toctree:: :maxdepth: 1 + release/release-0.229 + release/release-0.228 + release/release-0.227 + release/release-0.226 + release/release-0.225 + release/release-0.224 release/release-0.223 release/release-0.222 release/release-0.221 diff --git a/presto-docs/src/main/sphinx/release/release-0.224.rst b/presto-docs/src/main/sphinx/release/release-0.224.rst new file mode 100644 index 0000000000000..33b986d501914 --- /dev/null +++ b/presto-docs/src/main/sphinx/release/release-0.224.rst @@ -0,0 +1,45 @@ +============= +Release 0.224 +============= + +General Changes +--------------- + +* Fix an issue where CPU time and scheduled time are swapped in query plans. +* Fix a correctness issue for queries containing nested coercible type casts. +* Add support for recoverable grouped execution when writing unpartitioned table. +* Add support for index-based access to fields of type ``ROW`` with subscript. + +Security Changes +---------------- + +* Fix a security issue in the client protocol. + +Web UI Changes +-------------- + +* Add ``Planning Time`` in ``Query Details`` page. + +Hive Connector Changes +---------------------- + +* Improve performance for bucketed table insertion when some buckets are empty. +* Add config property ``hive.max-buckets-for-grouped-execution`` and session + property ``max_buckets_for_grouped_execution`` to limit the number of buckets + a query can access while still taking the advantage of grouped execution. + If more than the configured number of buckets are queried, the query will + run without grouped execution. +* Add session property ``hive.virtual_bucket_count`` to support grouped + execution for queries that read unbucketed tables and have no joins or + aggregations. This session property controls the number of virtual buckets + that the connector will create for unbucketed tables. + +Raptor Connector Changes +------------------------ + +* Fix query failures on colocated joins when one table has empty data. + +Verifier Changes +---------------- + +* Add the reason that a test case is skipped to the output event. diff --git a/presto-docs/src/main/sphinx/release/release-0.225.rst b/presto-docs/src/main/sphinx/release/release-0.225.rst new file mode 100644 index 0000000000000..71302397b21e6 --- /dev/null +++ b/presto-docs/src/main/sphinx/release/release-0.225.rst @@ -0,0 +1,50 @@ +============= +Release 0.225 +============= + +General Changes +_______________ +* Fix under-reporting of output data size from a fragment. (:issue:`11770`) +* Fix a bug where spatial joins would cause workers to run out of memory and crash. (:pr:`13251`) +* Fix leak in operator peak memory computations. (:issue:`13210`) +* Allow overriding session zone sent by clients. (:issue:`13140`) +* Throw an exception when a spatial partition is made with 0 rows. +* Add ``TableStatistics`` field containing statistics estimates for input tables to the ``QueryInputMetadata`` in ``QueryCompletedEvent`` for join queries. (:pr:`12808`) +* Add configuration property ``experimental.max-total-running-task-count`` and ``experimental.max-query-running-task-count`` + to limit number of running tasks for all queries and a single query, respectively. Query will be killed only if both conditions are violated. (:pr:`13228`) + +JDBC Changes +____________ +* Match schema and table names case insensitively. This behavior can be enabled by setting the ``case-insensitive-name-matching`` catalog configuration option to true. + +Web UI Changes +______________ +* Display tasks per stage on the query page. + +Elasticsearch Connector Changes +_______________________________ +* Fix and support nested fields in Elasticsearch Connector. (:issue:`12642`) + +Hive Changes +____________ +* Improve performance of file reads on S3. (:pr:`13222`) +* Add ability to use JSON key file to access Google Cloud Storage. The file path can be specified using the configuration property ``hive.gcs.json-key-file-path=/path/to/gcs_keyfile.json``. +* Add ability to use client-provided OAuth token to access Google Cloud Storage. This can be configured using the configuration property ``hive.gcs.use-access-token=false``. + +Raptor Changes +______________ +* Return an error when creating tables with unsupported types like ROW type. (:pr:`13209`) +* Remove legacy ORC writer. Configuration property ``storage.orc.optimized-writer-stage`` is enabled by default. Option ``DISABLED`` is removed. +* Add a new procedure ``trigger_bucket_balancer`` to trigger bucket balancing job. + +SPI Changes +___________ +* Add ``TableHandle`` parameter to ``ConnectorPageSourceProvider#createPageSource`` method. This gives connector access to ``ConnectorTableLayoutHandle`` during execution. +* Add ``columnHandles`` parameter to ``ConnectorMetadata.getTableStatistics`` method. The new parameter allows connectors to prune statistics to the + desired list of columns and subfields and fixes compatibility issue between subfield pruning and CBO. (:issue:`13082`) + +Verifier Changes +________________ +* Add support for overriding session properties for all queries in a suite. +* Add cluster type and retryable information for ``QueryFailure``. +* Add final query failure information to Verifier output event. diff --git a/presto-docs/src/main/sphinx/release/release-0.226.rst b/presto-docs/src/main/sphinx/release/release-0.226.rst new file mode 100644 index 0000000000000..f4b8d955be110 --- /dev/null +++ b/presto-docs/src/main/sphinx/release/release-0.226.rst @@ -0,0 +1,20 @@ +============= +Release 0.226 +============= + +General Changes +_______________ +* Fix ST_Buffer for tiny geometries (:issue:`13194`) +* Fix ST_Centroid for tiny geometries (:issue:`10629`) +* Fix failure when reading unpartitioned table from some connectors with session property ``grouped_execution_for_eligible_table_scans`` turned on. +* Improve memory tracking for geospatial query indexing and prevent worker node from crashing due to out of memory. +* Improve memory usage of spatial joins. +* Add peak running task count to statistics field of QueryCompletedEvent. + +Hive Changes +____________ +* Fix computation for number of buckets accessed for ``max_buckets_for_grouped_execution``. +* Fix a bug where the bucket column was not available if ``max_buckets_for_grouped_execution`` was exceeded. +* Fix high CPU usage when writing ORC files with too many row groups. +* Improve parallelism of writes to hive bucketed tables by respecting the ``task.writer-count`` configuration property. +* Add debug mode that allows reading from offline table and partition. This is controlled by session property ``offline_data_debug_mode_enabled``. diff --git a/presto-docs/src/main/sphinx/release/release-0.227.rst b/presto-docs/src/main/sphinx/release/release-0.227.rst new file mode 100644 index 0000000000000..910f95382373c --- /dev/null +++ b/presto-docs/src/main/sphinx/release/release-0.227.rst @@ -0,0 +1,57 @@ +============= +Release 0.227 +============= + +General Changes +_______________ +* Fix a bug where index joins would fail with the error ``driver should never block``. + Queries will now fail if the index is unable to load within the time specified by + the ``index-loader-timeout`` configuration property and the ``index_loader_timeout`` + session property. +* Fix a bug that could cause ``NullPointerException`` in spatial join with geometry + collections on the build side. +* Improve performance for queries with ``FULL OUTER JOIN`` where join keys have the + ``COALESCE`` function applied. +* Improve performance for ``UNNEST`` queries. +* Improve performance of repartitioning data between stages. The optimization can be + enabled by the ``optimized_repartitioning`` session property or the + ``experimental.optimized-repartitioning`` configuration property. +* Add spatial join (broadcast and partitioned) support for :func:`ST_Equals`, + :func:`ST_Overlaps`, :func:`ST_Crosses`, and :func:`ST_Touches`. +* Add ``task_partitioned_writer_count`` session property to allow setting the number + of concurrent writers for partitioned (bucketed) writes. +* Add ``IPPREFIX`` type and :func:`ip_prefix` function. +* Add :func:`differential_entropy` functions to compute differential entropy. +* Remove syntax support for ``SET PATH`` and ``CURRENT_PATH``. The path information was + never used by Presto. + +Hive Changes +____________ +* Fix a bug that might lead to corruption when writing sorted table in the recoverable + grouped execution mode. +* Fix ORC stripe skipping when using bloom filter. +* Improve the CPU load on coordinator by reducing the cost of serializing ``HiveSplit``. +* Improve GC pressure from Parquet reader by constraining the maximum column read size. + This can be configured by the ``parquet_max_read_block_size`` session property or the + ``hive.parquet.max-read-block-size`` configuration property. +* Add support for sub-field pruning when reading Parquet files, so that only necessary + sub-fields are extracted from struct columns. +* Add configuration property ``hive.s3-file-system-type=HADOOP_DEFAULT`` to allow + users to switch different Hadoop file system implementations for ``s3://`` addresses. + The corresponding Hadoop File system implementation should be specified in ``core-site.xml`` + +Raptor Changes +______________ +* Fix memory leak in file descriptor during shard compaction. The regression was introduced in 0.219. + +Verifier Changes +________________ +* Add support for auto-resolving query failures with ``HIVE_TOO_MANY_OPEN_PARTITIONS`` error. +* Add support to perform additional determinism analysis for queries with ``LIMIT`` clause. +* Add detailed determinism analysis result to ``VerifierOutputEvent``. + +SPI Changes +________________ +* Move ``AggregationNode`` to SPI. Connectors can now push down aggregation to table scan. +* Move ``ProjectNode`` to SPI. Connectors can now push down projection to table scan. +* Rename ``Block#getObject`` to ``Block#getBlock`` and remove unnecessary ``clazz`` parameter. \ No newline at end of file diff --git a/presto-docs/src/main/sphinx/release/release-0.228.rst b/presto-docs/src/main/sphinx/release/release-0.228.rst new file mode 100644 index 0000000000000..3f4ee1a77fa9c --- /dev/null +++ b/presto-docs/src/main/sphinx/release/release-0.228.rst @@ -0,0 +1,21 @@ +============= +Release 0.228 +============= + +General Changes +_______________ +* Reduce excessive memory usage by ExchangeClient. +* Improve coordinator stability on parsing functions that may create large constant + arrays/maps/rows (e.g., ``SEQUENCE``, ``REPEAT``, ``ARRAY[...]``, etc) by delaying the + evaluation on these functions on workers. +* Optimize queries with ``LIMIT 0``. +* Allow Bing Tiles at zoom level 0. + +Hive Changes +____________ +* Fix ORC writer rollback failure due to exception thrown during rollback. +* Improve ORC read performance for variants and fixed width numbers. + +Spi Changes +___________ +* Move most ``RowExpression`` utilities to the ``presto-expressions`` module. diff --git a/presto-docs/src/main/sphinx/release/release-0.229.rst b/presto-docs/src/main/sphinx/release/release-0.229.rst new file mode 100644 index 0000000000000..28da5f8e37e2d --- /dev/null +++ b/presto-docs/src/main/sphinx/release/release-0.229.rst @@ -0,0 +1,36 @@ +============= +Release 0.229 +============= + +General Changes +_______________ +* Fix an issue that would cause query failure when calling :func:`geometry_to_bing_tiles` on certain degenerate geometries. +* Add geospatial function :func:`line_interpolate_point`. +* Add support for ``CREATE FUNCTION`` +* Add support for passing ``X_Forwarded_For`` header from Proxy to coordinator. +* Add support to respect configuration property ``stage.max-tasks-per-stage`` for limiting the number of tasks per scan. +* Add configuration property ``experimental.internal-communication.max-task-update-size`` to limit the size of the ``TaskUpdate``. +* Add configuration properties ``internal-communication.https.trust-store-path``, ``internal-communication.https.included-cipher``, + and ``internal-communication.https.excluded-cipher`` to easily set common https configurations for all internal communications at one place. +* Add peak task memory distribution of each stage to ``QueryStatistics``. + +Pinot Connector Changes +_______________________ +* Add Pinot connector. + +Hive Connector Changes +______________________ +* Fix parquet predicate pushdown on dictionaries to consider more than just the first predicate column. +* Improve parquet predicate pushdown on dictionaries to avoid reading additional data after successfully eliminating a block. + +Raptor Connector Changes +________________________ +* Add support for using remote HDFS as the storage in Raptor. Configuration property ``storage.data-directory`` is changed from a ``File`` to a ``URI``. + For deployment on local flash, scheme ``file:/`` must be prepended. +* Rename error code ``RAPTOR_LOCAL_FILE_SYSTEM_ERROR`` to ``RAPTOR_FILE_SYSTEM_ERROR``. + +SPI Changes +___________ +* Add support for connectors to alter query plans involving ``UNION``, ``INTERSECT``, and ``EXCEPT``, by moving ``SetOperationNode`` to SPI. +* Improve interface ``ConnectorPlanOptimizerProvider`` to allow connectors to participate in query optimization in two phases, ``LOGICAL`` and ``PHYSICAL``. + The two phases correspond to post-shuffle and post-shuffle optimization, respectively. diff --git a/presto-elasticsearch/pom.xml b/presto-elasticsearch/pom.xml index 9f435344ed95b..858c2660528fd 100644 --- a/presto-elasticsearch/pom.xml +++ b/presto-elasticsearch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-elasticsearch Presto - Elasticsearch Connector @@ -39,27 +39,27 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift configuration - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift log @@ -175,7 +175,7 @@ - io.airlift + com.facebook.airlift log-manager runtime @@ -231,19 +231,19 @@ - io.airlift + com.facebook.airlift http-server test - io.airlift + com.facebook.airlift node test - io.airlift + com.facebook.airlift testing test diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchClient.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchClient.java index edb385d42aee6..4d64c959c1e82 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchClient.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchClient.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.elasticsearch; +import com.facebook.airlift.json.ObjectMapperProvider; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; @@ -27,8 +29,6 @@ import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; import com.google.common.io.Closer; -import io.airlift.json.ObjectMapperProvider; -import io.airlift.log.Logger; import io.airlift.units.Duration; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; @@ -59,6 +59,7 @@ import java.util.TreeMap; import java.util.concurrent.ExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.elasticsearch.ElasticsearchErrorCode.ELASTICSEARCH_CORRUPTED_MAPPING_METADATA; import static com.facebook.presto.elasticsearch.RetryDriver.retry; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -81,7 +82,6 @@ import static com.google.common.base.Verify.verify; import static com.google.common.cache.CacheLoader.asyncReloading; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Map.Entry; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newFixedThreadPool; @@ -275,7 +275,7 @@ private List buildColumns(ElasticsearchTableDescription tab Iterator indexIterator = mappings.keysIt(); while (indexIterator.hasNext()) { - // TODO use io.airlift.json.JsonCodec + // TODO use com.facebook.airlift.json.JsonCodec MappingMetaData mappingMetaData = mappings.get(indexIterator.next()).get(tableDescription.getType()); JsonNode rootNode; try { diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnector.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnector.java index bef9a286f4c1f..1a5b0ed9ef42d 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnector.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnector.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.elasticsearch; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.transaction.IsolationLevel; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorConfig.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorConfig.java index 34392842bf2be..ab9db4e0b1179 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorConfig.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.elasticsearch; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.ConfigSecuritySensitive; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.ConfigSecuritySensitive; import io.airlift.units.Duration; import javax.validation.constraints.Min; diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorFactory.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorFactory.java index b50844e74ebde..35e5bb5abfab6 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorFactory.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorFactory.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.elasticsearch; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.connector.Connector; @@ -21,8 +23,6 @@ import com.facebook.presto.spi.type.TypeManager; import com.google.inject.Injector; import com.google.inject.Scopes; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorModule.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorModule.java index fecbc59d108ee..24f37376b9bbd 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorModule.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchConnectorModule.java @@ -28,11 +28,11 @@ import java.io.IOException; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.util.Objects.requireNonNull; public class ElasticsearchConnectorModule diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchQueryBuilder.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchQueryBuilder.java index 82bfd5e4d8107..b370b2c2fe951 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchQueryBuilder.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchQueryBuilder.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.elasticsearch; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.Range; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.Type; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import io.airlift.units.Duration; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -42,8 +42,6 @@ import static com.facebook.presto.elasticsearch.ElasticsearchClient.createTransportClient; import static com.facebook.presto.elasticsearch.ElasticsearchErrorCode.ELASTICSEARCH_CONNECTION_ERROR; -import static com.facebook.presto.spi.predicate.Marker.Bound.ABOVE; -import static com.facebook.presto.spi.predicate.Marker.Bound.BELOW; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchRecordCursor.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchRecordCursor.java index 803d4501b6044..14963e7c489c3 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchRecordCursor.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchRecordCursor.java @@ -13,26 +13,29 @@ */ package com.facebook.presto.elasticsearch; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.RecordCursor; import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; import io.airlift.units.Duration; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.SearchHit; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.elasticsearch.ElasticsearchErrorCode.ELASTICSEARCH_MAX_HITS_EXCEEDED; import static com.facebook.presto.elasticsearch.ElasticsearchUtils.serializeObject; import static com.facebook.presto.elasticsearch.RetryDriver.retry; @@ -43,7 +46,6 @@ import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.slice.Slices.EMPTY_SLICE; import static io.airlift.slice.Slices.utf8Slice; import static java.lang.String.format; @@ -250,11 +252,86 @@ private Object getFieldValue(int field) private void extractFromSource(SearchHit hit) { - Map map = hit.getSourceAsMap(); - for (Map.Entry entry : map.entrySet()) { - String jsonPath = entry.getKey(); - Object entryValue = entry.getValue(); - setFieldIfExists(jsonPath, entryValue); + List fields = new ArrayList<>(); + for (Map.Entry entry : hit.getSourceAsMap().entrySet()) { + fields.add(new Field(entry.getKey(), entry.getValue())); + } + Collections.sort(fields, Comparator.comparing(Field::getName)); + + for (Map.Entry entry : unflatten(fields).entrySet()) { + setFieldIfExists(entry.getKey(), entry.getValue()); + } + } + + private static Map unflatten(List fields) + { + return unflatten(fields, 0, 0, fields.size()); + } + + private static Map unflatten(List fields, int level, int start, int length) + { + checkArgument(length > 0, "length must be > 0"); + + int limit = start + length; + + Map result = new HashMap<>(); + int anchor = start; + int current = start; + + do { + Field field = fields.get(anchor); + String name = field.getPathElement(level); + + current++; + if (current == limit || !name.equals(fields.get(current).getPathElement(level))) { + // We assume that fields can't be both leaves and intermediate nodes + Object value; + if (level < field.getDepth() - 1) { + value = unflatten(fields, level + 1, anchor, current - anchor); + } + else { + value = field.getValue(); + } + result.put(name, value); + anchor = current; + } + } + while (current < limit); + + return result; + } + + private static final class Field + { + private final String name; + private final List path; + private final Object value; + + public Field(String name, Object value) + { + this.name = name; + this.path = Arrays.asList(name.split("\\.")); + this.value = value; + } + + public String getName() + { + return name; + } + + public int getDepth() + { + return path.size(); + } + + public String getPathElement(int level) + { + return path.get(level); + } + + public Object getValue() + { + return value; } } } diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchTableDescriptionProvider.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchTableDescriptionProvider.java index f1a2c56bb0f8c..f68b94beb2730 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchTableDescriptionProvider.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchTableDescriptionProvider.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.elasticsearch; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.SchemaTableName; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; import javax.inject.Inject; diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchUtils.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchUtils.java index 53c670b6eadee..7c6502a2548cf 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchUtils.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/ElasticsearchUtils.java @@ -46,7 +46,8 @@ public static Block serializeObject(Type type, BlockBuilder builder, Object obje if (MAP.equals(type.getTypeSignature().getBase()) || ARRAY.equals(type.getTypeSignature().getBase())) { throw new IllegalArgumentException("Type not supported: " + type.getDisplayName()); } - return serializePrimitive(type, builder, object); + serializePrimitive(type, builder, object); + return null; } private static Block serializeStruct(Type type, BlockBuilder builder, Object object) @@ -54,15 +55,20 @@ private static Block serializeStruct(Type type, BlockBuilder builder, Object obj if (object == null) { requireNonNull(builder, "builder is null"); builder.appendNull(); - return builder.build(); + return null; } + List typeParameters = type.getTypeParameters(); + + BlockBuilder currentBuilder; + + boolean builderSynthesized = false; if (builder == null) { + builderSynthesized = true; builder = type.createBlockBuilder(null, 1); } - BlockBuilder currentBuilder = builder.beginBlockEntry(); - List typeParameters = type.getTypeParameters(); + currentBuilder = builder.beginBlockEntry(); for (int i = 0; i < typeParameters.size(); i++) { Optional fieldName = type.getTypeSignature().getParameters().get(i).getNamedTypeSignature().getName(); @@ -75,16 +81,19 @@ private static Block serializeStruct(Type type, BlockBuilder builder, Object obj } builder.closeEntry(); - return (Block) type.getObject(builder, 0); + if (builderSynthesized) { + return (Block) type.getObject(builder, 0); + } + return null; } - private static Block serializePrimitive(Type type, BlockBuilder builder, Object object) + private static void serializePrimitive(Type type, BlockBuilder builder, Object object) { requireNonNull(builder, "builder is null"); if (object == null) { builder.appendNull(); - return builder.build(); + return; } if (type.equals(BOOLEAN)) { @@ -105,6 +114,5 @@ else if (type.equals(VARCHAR) || type.equals(VARBINARY)) { else { throw new IllegalArgumentException("Unknown primitive type: " + type.getDisplayName()); } - return builder.build(); } } diff --git a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/RetryDriver.java b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/RetryDriver.java index e4512e5a384e1..623e7730f5656 100644 --- a/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/RetryDriver.java +++ b/presto-elasticsearch/src/main/java/com/facebook/presto/elasticsearch/RetryDriver.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.elasticsearch; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import io.airlift.units.Duration; import java.util.ArrayList; diff --git a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/CodecSupplier.java b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/CodecSupplier.java index a67cfccaf9916..aee5d99a92d43 100644 --- a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/CodecSupplier.java +++ b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/CodecSupplier.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.elasticsearch; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; import java.util.function.Supplier; diff --git a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchQueryRunner.java b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchQueryRunner.java index 060d1da310f81..a6a5f5d628a49 100644 --- a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchQueryRunner.java +++ b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/ElasticsearchQueryRunner.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.elasticsearch; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.log.Logging; import com.facebook.presto.Session; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.QualifiedObjectName; @@ -21,19 +24,16 @@ import com.facebook.presto.tests.TestingPrestoClient; import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; -import io.airlift.log.Logging; import io.airlift.tpch.TpchTable; import java.io.File; import java.net.URL; import java.util.Map; +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static com.google.common.io.Resources.getResource; -import static io.airlift.testing.Closeables.closeAllSuppress; import static io.airlift.units.Duration.nanosSince; import static java.lang.String.format; import static java.util.Locale.ENGLISH; diff --git a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestElasticsearchConnectorConfig.java b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestElasticsearchConnectorConfig.java index 5f4d8ae9ea1de..afa2fe235bdde 100644 --- a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestElasticsearchConnectorConfig.java +++ b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestElasticsearchConnectorConfig.java @@ -20,11 +20,11 @@ import java.io.File; import java.util.Map; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static com.facebook.presto.elasticsearch.SearchGuardCertificateFormat.NONE; import static com.facebook.presto.elasticsearch.SearchGuardCertificateFormat.PEM; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestElasticsearchIntegrationSmokeTest.java b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestElasticsearchIntegrationSmokeTest.java index d3368607cd396..68f6468c1198e 100644 --- a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestElasticsearchIntegrationSmokeTest.java +++ b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestElasticsearchIntegrationSmokeTest.java @@ -17,6 +17,7 @@ import com.facebook.presto.testing.MaterializedRow; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestIntegrationSmokeTest; +import com.google.common.collect.ImmutableMap; import com.google.common.io.Closer; import io.airlift.tpch.TpchTable; import org.testng.annotations.AfterClass; @@ -31,6 +32,7 @@ import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.assertions.Assert.assertEquals; import static java.lang.String.format; +import static org.elasticsearch.client.Requests.refreshRequest; public class TestElasticsearchIntegrationSmokeTest extends AbstractTestIntegrationSmokeTest @@ -90,4 +92,28 @@ public void testDescribeTable() .row("comment", "varchar", "", "").build(); assertEquals(actualResult, expectedColumns, format("%s != %s", actualResult, expectedColumns)); } + + @Test + public void testNestedFields() + { + String indexName = "data"; + embeddedElasticsearchNode.getClient() + .prepareIndex(indexName, "doc") + .setSource(ImmutableMap.builder() + .put("name", "nestfield") + .put("fields.fielda", 32) + .put("fields.fieldb", "valueb") + .build()) + .get(); + + embeddedElasticsearchNode.getClient() + .admin() + .indices() + .refresh(refreshRequest(indexName)) + .actionGet(); + + assertQuery( + "SELECT name, fields.fielda, fields.fieldb FROM nested.data", + "VALUES ('nestfield', 32, 'valueb')"); + } } diff --git a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestingElasticsearchConnectorFactory.java b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestingElasticsearchConnectorFactory.java index 64926c81149f6..bba402034f3d9 100644 --- a/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestingElasticsearchConnectorFactory.java +++ b/presto-elasticsearch/src/test/java/com/facebook/presto/elasticsearch/TestingElasticsearchConnectorFactory.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.elasticsearch; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.type.TypeManager; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; diff --git a/presto-elasticsearch/src/test/resources/queryrunner/nested.data.json b/presto-elasticsearch/src/test/resources/queryrunner/nested.data.json new file mode 100644 index 0000000000000..7cee724418005 --- /dev/null +++ b/presto-elasticsearch/src/test/resources/queryrunner/nested.data.json @@ -0,0 +1,25 @@ +{ + "tableName": "data", + "schemaName": "nested", + "host": "localhost", + "port": "9300", + "clusterName": "test", + "index": "data", + "type": "doc", + "columns": [ + { + "name": "name", + "type": "varchar", + "jsonPath": "name", + "jsonType": "varchar", + "ordinalPosition": "0" + }, + { + "name": "fields", + "type": "row(fielda integer, fieldb varchar)", + "jsonPath": "fields", + "jsonType": "varchar", + "ordinalPosition": "1" + } + ] +} diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index cbe321cdfc667..890f7b84f1a48 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-example-http @@ -17,22 +17,22 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift configuration @@ -100,19 +100,19 @@ - io.airlift + com.facebook.airlift testing test - io.airlift + com.facebook.airlift http-server test - io.airlift + com.facebook.airlift node test diff --git a/presto-example-http/src/main/java/com/facebook/presto/example/ExampleClient.java b/presto-example-http/src/main/java/com/facebook/presto/example/ExampleClient.java index 2bf4070d659e8..07c7b1bc1367d 100644 --- a/presto-example-http/src/main/java/com/facebook/presto/example/ExampleClient.java +++ b/presto-example-http/src/main/java/com/facebook/presto/example/ExampleClient.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.example; +import com.facebook.airlift.json.JsonCodec; import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; @@ -20,7 +21,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.io.Resources; -import io.airlift.json.JsonCodec; import javax.inject.Inject; diff --git a/presto-example-http/src/main/java/com/facebook/presto/example/ExampleConfig.java b/presto-example-http/src/main/java/com/facebook/presto/example/ExampleConfig.java index e875966f998eb..7ef582d3195a0 100644 --- a/presto-example-http/src/main/java/com/facebook/presto/example/ExampleConfig.java +++ b/presto-example-http/src/main/java/com/facebook/presto/example/ExampleConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.example; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import javax.validation.constraints.NotNull; diff --git a/presto-example-http/src/main/java/com/facebook/presto/example/ExampleConnector.java b/presto-example-http/src/main/java/com/facebook/presto/example/ExampleConnector.java index ecb9e40218964..64bf524813eb4 100644 --- a/presto-example-http/src/main/java/com/facebook/presto/example/ExampleConnector.java +++ b/presto-example-http/src/main/java/com/facebook/presto/example/ExampleConnector.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.example; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.transaction.IsolationLevel; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-example-http/src/main/java/com/facebook/presto/example/ExampleConnectorFactory.java b/presto-example-http/src/main/java/com/facebook/presto/example/ExampleConnectorFactory.java index 4b185ee70dbef..8002998d0cd2b 100644 --- a/presto-example-http/src/main/java/com/facebook/presto/example/ExampleConnectorFactory.java +++ b/presto-example-http/src/main/java/com/facebook/presto/example/ExampleConnectorFactory.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.example; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.connector.ConnectorFactory; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; diff --git a/presto-example-http/src/main/java/com/facebook/presto/example/ExampleModule.java b/presto-example-http/src/main/java/com/facebook/presto/example/ExampleModule.java index 1723c034997a0..e8395e7272a20 100644 --- a/presto-example-http/src/main/java/com/facebook/presto/example/ExampleModule.java +++ b/presto-example-http/src/main/java/com/facebook/presto/example/ExampleModule.java @@ -23,11 +23,11 @@ import javax.inject.Inject; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static io.airlift.json.JsonCodec.listJsonCodec; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.util.Objects.requireNonNull; public class ExampleModule diff --git a/presto-example-http/src/test/java/com/facebook/presto/example/ExampleHttpServer.java b/presto-example-http/src/test/java/com/facebook/presto/example/ExampleHttpServer.java index 1d78d2d600d39..02bec0eef6f26 100644 --- a/presto-example-http/src/test/java/com/facebook/presto/example/ExampleHttpServer.java +++ b/presto-example-http/src/test/java/com/facebook/presto/example/ExampleHttpServer.java @@ -13,18 +13,18 @@ */ package com.facebook.presto.example; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.http.server.TheServlet; +import com.facebook.airlift.http.server.testing.TestingHttpServer; +import com.facebook.airlift.http.server.testing.TestingHttpServerModule; +import com.facebook.airlift.node.testing.TestingNodeModule; import com.google.common.collect.ImmutableMap; import com.google.common.io.Resources; import com.google.inject.Binder; import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.TypeLiteral; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.http.server.TheServlet; -import io.airlift.http.server.testing.TestingHttpServer; -import io.airlift.http.server.testing.TestingHttpServerModule; -import io.airlift.node.testing.TestingNodeModule; import javax.servlet.Servlet; import javax.servlet.http.HttpServlet; diff --git a/presto-example-http/src/test/java/com/facebook/presto/example/MetadataUtil.java b/presto-example-http/src/test/java/com/facebook/presto/example/MetadataUtil.java index 7d817ef67a88b..3813a209bcefa 100644 --- a/presto-example-http/src/test/java/com/facebook/presto/example/MetadataUtil.java +++ b/presto-example-http/src/test/java/com/facebook/presto/example/MetadataUtil.java @@ -13,24 +13,24 @@ */ package com.facebook.presto.example; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; import java.util.List; import java.util.Map; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType; -import static io.airlift.json.JsonCodec.listJsonCodec; import static java.util.Locale.ENGLISH; public final class MetadataUtil diff --git a/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleColumnHandle.java b/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleColumnHandle.java index 44973efb1c053..13dc5354da6fe 100644 --- a/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleColumnHandle.java +++ b/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleColumnHandle.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.example; -import io.airlift.testing.EquivalenceTester; +import com.facebook.airlift.testing.EquivalenceTester; import org.testng.annotations.Test; import static com.facebook.presto.example.MetadataUtil.COLUMN_CODEC; diff --git a/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleConfig.java b/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleConfig.java index aceee8b103d2b..4a41b5b27f10a 100644 --- a/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleConfig.java +++ b/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.example; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import org.testng.annotations.Test; import java.net.URI; diff --git a/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleSplit.java b/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleSplit.java index d8c693ae0d161..10f6e2ba67422 100644 --- a/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleSplit.java +++ b/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleSplit.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.example; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.HostAddress; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; import java.net.URI; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; public class TestExampleSplit diff --git a/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleTableHandle.java b/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleTableHandle.java index 759bdd9747c55..a6e65bc9bb032 100644 --- a/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleTableHandle.java +++ b/presto-example-http/src/test/java/com/facebook/presto/example/TestExampleTableHandle.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.example; -import io.airlift.json.JsonCodec; -import io.airlift.testing.EquivalenceTester; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.testing.EquivalenceTester; import org.testng.annotations.Test; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; public class TestExampleTableHandle diff --git a/presto-expressions/pom.xml b/presto-expressions/pom.xml new file mode 100644 index 0000000000000..c4c84e2810905 --- /dev/null +++ b/presto-expressions/pom.xml @@ -0,0 +1,48 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.231-SNAPSHOT + + + presto-expressions + presto-expressions + + + ${project.parent.basedir} + + + + + com.google.guava + guava + + + + com.facebook.presto + presto-spi + + + + + org.testng + testng + test + + + + org.openjdk.jmh + jmh-core + test + + + + org.openjdk.jmh + jmh-generator-annprocess + test + + + diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/DefaultRowExpressionTraversalVisitor.java b/presto-expressions/src/main/java/com/facebook/presto/expressions/DefaultRowExpressionTraversalVisitor.java similarity index 78% rename from presto-spi/src/main/java/com/facebook/presto/spi/relation/DefaultRowExpressionTraversalVisitor.java rename to presto-expressions/src/main/java/com/facebook/presto/expressions/DefaultRowExpressionTraversalVisitor.java index 5c4325498e621..cb1ed947d9789 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/relation/DefaultRowExpressionTraversalVisitor.java +++ b/presto-expressions/src/main/java/com/facebook/presto/expressions/DefaultRowExpressionTraversalVisitor.java @@ -11,7 +11,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.spi.relation; +package com.facebook.presto.expressions; + +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.RowExpressionVisitor; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; /** * The default visitor serves as a template for "consumer-like" tree traversal. diff --git a/presto-expressions/src/main/java/com/facebook/presto/expressions/LogicalRowExpressions.java b/presto-expressions/src/main/java/com/facebook/presto/expressions/LogicalRowExpressions.java new file mode 100644 index 0000000000000..6592b3ff3e760 --- /dev/null +++ b/presto-expressions/src/main/java/com/facebook/presto/expressions/LogicalRowExpressions.java @@ -0,0 +1,812 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.expressions; + +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.DeterminismEvaluator; +import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.RowExpressionVisitor; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression.Form; +import com.facebook.presto.spi.relation.VariableReferenceExpression; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Queue; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static com.facebook.presto.spi.function.OperatorType.EQUAL; +import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; +import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN_OR_EQUAL; +import static com.facebook.presto.spi.function.OperatorType.LESS_THAN; +import static com.facebook.presto.spi.function.OperatorType.LESS_THAN_OR_EQUAL; +import static com.facebook.presto.spi.function.OperatorType.NOT_EQUAL; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.OR; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static java.lang.Math.min; +import static java.util.Arrays.asList; +import static java.util.Arrays.stream; +import static java.util.Collections.singletonList; +import static java.util.Collections.unmodifiableList; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toList; + +public final class LogicalRowExpressions +{ + public static final ConstantExpression TRUE_CONSTANT = new ConstantExpression(true, BOOLEAN); + public static final ConstantExpression FALSE_CONSTANT = new ConstantExpression(false, BOOLEAN); + // 10000 is very conservative estimation + private static final int ELIMINATE_COMMON_SIZE_LIMIT = 10000; + + private final DeterminismEvaluator determinismEvaluator; + private final StandardFunctionResolution functionResolution; + private final FunctionMetadataManager functionMetadataManager; + + public LogicalRowExpressions(DeterminismEvaluator determinismEvaluator, StandardFunctionResolution functionResolution, FunctionMetadataManager functionMetadataManager) + { + this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + } + + public static List extractConjuncts(RowExpression expression) + { + return extractPredicates(AND, expression); + } + + public static List extractDisjuncts(RowExpression expression) + { + return extractPredicates(OR, expression); + } + + public static List extractPredicates(RowExpression expression) + { + if (expression instanceof SpecialFormExpression) { + Form form = ((SpecialFormExpression) expression).getForm(); + if (form == AND || form == OR) { + return extractPredicates(form, expression); + } + } + return singletonList(expression); + } + + public static List extractPredicates(Form form, RowExpression expression) + { + if (expression instanceof SpecialFormExpression && ((SpecialFormExpression) expression).getForm() == form) { + SpecialFormExpression specialFormExpression = (SpecialFormExpression) expression; + if (specialFormExpression.getArguments().size() != 2) { + throw new IllegalStateException("logical binary expression requires exactly 2 operands"); + } + + List predicates = new ArrayList<>(); + predicates.addAll(extractPredicates(form, specialFormExpression.getArguments().get(0))); + predicates.addAll(extractPredicates(form, specialFormExpression.getArguments().get(1))); + return unmodifiableList(predicates); + } + + return singletonList(expression); + } + + public static RowExpression and(RowExpression... expressions) + { + return and(asList(expressions)); + } + + public static RowExpression and(Collection expressions) + { + return binaryExpression(AND, expressions); + } + + public static RowExpression or(RowExpression... expressions) + { + return or(asList(expressions)); + } + + public static RowExpression or(Collection expressions) + { + return binaryExpression(OR, expressions); + } + + public static RowExpression binaryExpression(Form form, Collection expressions) + { + requireNonNull(form, "operator is null"); + requireNonNull(expressions, "expressions is null"); + + if (expressions.isEmpty()) { + switch (form) { + case AND: + return TRUE_CONSTANT; + case OR: + return FALSE_CONSTANT; + default: + throw new IllegalArgumentException("Unsupported binary expression operator"); + } + } + + // Build balanced tree for efficient recursive processing that + // preserves the evaluation order of the input expressions. + // + // The tree is built bottom up by combining pairs of elements into + // binary AND expressions. + // + // Example: + // + // Initial state: + // a b c d e + // + // First iteration: + // + // /\ /\ e + // a b c d + // + // Second iteration: + // + // / \ e + // /\ /\ + // a b c d + // + // + // Last iteration: + // + // / \ + // / \ e + // /\ /\ + // a b c d + + Queue queue = new ArrayDeque<>(expressions); + while (queue.size() > 1) { + Queue buffer = new ArrayDeque<>(); + + // combine pairs of elements + while (queue.size() >= 2) { + List arguments = asList(queue.remove(), queue.remove()); + buffer.add(new SpecialFormExpression(form, BOOLEAN, arguments)); + } + + // if there's and odd number of elements, just append the last one + if (!queue.isEmpty()) { + buffer.add(queue.remove()); + } + + // continue processing the pairs that were just built + queue = buffer; + } + + return queue.remove(); + } + + public RowExpression combinePredicates(Form form, RowExpression... expressions) + { + return combinePredicates(form, asList(expressions)); + } + + public RowExpression combinePredicates(Form form, Collection expressions) + { + if (form == AND) { + return combineConjuncts(expressions); + } + return combineDisjuncts(expressions); + } + + public RowExpression combineConjuncts(RowExpression... expressions) + { + return combineConjuncts(asList(expressions)); + } + + public RowExpression combineConjuncts(Collection expressions) + { + requireNonNull(expressions, "expressions is null"); + + List conjuncts = expressions.stream() + .flatMap(e -> extractConjuncts(e).stream()) + .filter(e -> !e.equals(TRUE_CONSTANT)) + .collect(toList()); + + conjuncts = removeDuplicates(conjuncts); + + if (conjuncts.contains(FALSE_CONSTANT)) { + return FALSE_CONSTANT; + } + + return and(conjuncts); + } + + public RowExpression combineDisjuncts(RowExpression... expressions) + { + return combineDisjuncts(asList(expressions)); + } + + public RowExpression combineDisjuncts(Collection expressions) + { + return combineDisjunctsWithDefault(expressions, FALSE_CONSTANT); + } + + public RowExpression combineDisjunctsWithDefault(Collection expressions, RowExpression emptyDefault) + { + requireNonNull(expressions, "expressions is null"); + + List disjuncts = expressions.stream() + .flatMap(e -> extractDisjuncts(e).stream()) + .filter(e -> !e.equals(FALSE_CONSTANT)) + .collect(toList()); + + disjuncts = removeDuplicates(disjuncts); + + if (disjuncts.contains(TRUE_CONSTANT)) { + return TRUE_CONSTANT; + } + + return disjuncts.isEmpty() ? emptyDefault : or(disjuncts); + } + + /** + * Given a logical expression, the goal is to push negation to the leaf nodes. + * This only applies to propositional logic and comparison. this utility cannot be applied to high-order logic. + * Examples of non-applicable cases could be f(a AND b) > 5 + * + * An applicable example: + * + * NOT + * | + * ___OR_ AND + * / \ / \ + * NOT OR ==> AND AND + * | / \ / \ / \ + * AND c NOT a b NOT d + * / \ | | + * a b d c + */ + public RowExpression pushNegationToLeaves(RowExpression expression) + { + return expression.accept(new PushNegationVisitor(), null); + } + + /** + * Given a logical expression, the goal is to convert to conjuctive normal form (CNF). + * This requires making a call to `pushNegationToLeaves`. There is no guarantee as to + * the balance of the resulting expression tree. + * + * This only applies to propositional logic. this utility cannot be applied to high-order logic. + * Examples of non-applicable cases could be f(a AND b) > 5 + * + * NOTE: This may exponentially increase the number of RowExpressions in the expression. + * + * An applicable example: + * + * NOT + * | + * ___OR_ AND + * / \ / \ + * NOT OR ==> OR AND + * | / \ / \ / \ + * OR c NOT a b NOT d + * / \ | | + * a b d c + */ + public RowExpression convertToConjunctiveNormalForm(RowExpression expression) + { + return convertToNormalForm(expression, AND); + } + + /** + * Given a logical expression, the goal is to convert to disjunctive normal form (DNF). + * The same limitations, format, and risks apply as for converting to conjunctive normal form (CNF). + * + * An applicable example: + * + * NOT OR + * | / \ + * ___OR_ AND AND + * / \ / \ / \ + * NOT OR ==> a AND b AND + * | / \ / \ / \ + * OR c NOT NOT d NOT d + * / \ | | | + * a b d c c + */ + public RowExpression convertToDisjunctiveNormalForm(RowExpression expression) + { + return convertToNormalForm(expression, OR); + } + + public RowExpression minimalNormalForm(RowExpression expression) + { + RowExpression conjunctiveNormalForm = convertToConjunctiveNormalForm(expression); + RowExpression disjunctiveNormalForm = convertToDisjunctiveNormalForm(expression); + return numOfClauses(conjunctiveNormalForm) > numOfClauses(disjunctiveNormalForm) ? disjunctiveNormalForm : conjunctiveNormalForm; + } + + public RowExpression convertToNormalForm(RowExpression expression, Form clauseJoiner) + { + return pushNegationToLeaves(expression).accept(new ConvertNormalFormVisitor(), rootContext(clauseJoiner)); + } + + public RowExpression filterDeterministicConjuncts(RowExpression expression) + { + return filterConjuncts(expression, this.determinismEvaluator::isDeterministic); + } + + public RowExpression filterNonDeterministicConjuncts(RowExpression expression) + { + return filterConjuncts(expression, predicate -> !this.determinismEvaluator.isDeterministic(predicate)); + } + + public RowExpression filterConjuncts(RowExpression expression, Predicate predicate) + { + List conjuncts = extractConjuncts(expression).stream() + .filter(predicate) + .collect(toList()); + + return combineConjuncts(conjuncts); + } + + /** + * Removes duplicate deterministic expressions. Preserves the relative order + * of the expressions in the list. + */ + private List removeDuplicates(List expressions) + { + Set seen = new HashSet<>(); + + List result = new ArrayList<>(); + for (RowExpression expression : expressions) { + if (determinismEvaluator.isDeterministic(expression)) { + if (!seen.contains(expression)) { + result.add(expression); + seen.add(expression); + } + } + else { + result.add(expression); + } + } + + return unmodifiableList(result); + } + + private boolean isConjunctionOrDisjunction(RowExpression expression) + { + if (expression instanceof SpecialFormExpression) { + Form form = ((SpecialFormExpression) expression).getForm(); + return form == AND || form == OR; + } + return false; + } + + private final class PushNegationVisitor + implements RowExpressionVisitor + { + @Override + public RowExpression visitCall(CallExpression call, Void context) + { + if (!isNegationExpression(call)) { + return call; + } + + checkArgument(call.getArguments().size() == 1, "Not expression should have exactly one argument"); + RowExpression argument = call.getArguments().get(0); + + // eliminate two consecutive negations + if (isNegationExpression(argument)) { + return ((CallExpression) argument).getArguments().get(0).accept(new PushNegationVisitor(), null); + } + + if (isComparisonExpression(argument)) { + return negateComparison((CallExpression) argument); + } + + if (!isConjunctionOrDisjunction(argument)) { + return call; + } + + // push negation through conjunction or disjunction + SpecialFormExpression specialForm = ((SpecialFormExpression) argument); + RowExpression left = specialForm.getArguments().get(0); + RowExpression right = specialForm.getArguments().get(1); + if (specialForm.getForm() == AND) { + // !(a AND b) ==> !a OR !b + return or(notCallExpression(left).accept(new PushNegationVisitor(), null), notCallExpression(right).accept(this, null)); + } + // !(a OR b) ==> !a AND !b + return and(notCallExpression(left).accept(new PushNegationVisitor(), null), notCallExpression(right).accept(this, null)); + } + + private RowExpression negateComparison(CallExpression expression) + { + OperatorType newOperator = negate(getOperator(expression).orElse(null)); + if (newOperator == null) { + return new CallExpression("NOT", functionResolution.notFunction(), BOOLEAN, singletonList(expression)); + } + checkArgument(expression.getArguments().size() == 2, "Comparison expression must have exactly two arguments"); + RowExpression left = expression.getArguments().get(0).accept(this, null); + RowExpression right = expression.getArguments().get(1).accept(this, null); + return new CallExpression( + newOperator.getOperator(), + functionResolution.comparisonFunction(newOperator, left.getType(), right.getType()), + BOOLEAN, + asList(left, right)); + } + + @Override + public RowExpression visitSpecialForm(SpecialFormExpression specialForm, Void context) + { + if (!isConjunctionOrDisjunction(specialForm)) { + return specialForm; + } + + RowExpression left = specialForm.getArguments().get(0); + RowExpression right = specialForm.getArguments().get(1); + + if (specialForm.getForm() == AND) { + return and(left.accept(this, null), right.accept(this, null)); + } + return or(left.accept(this, null), right.accept(this, null)); + } + + @Override + public RowExpression visitInputReference(InputReferenceExpression reference, Void context) + { + return reference; + } + + @Override + public RowExpression visitConstant(ConstantExpression literal, Void context) + { + return literal; + } + + @Override + public RowExpression visitLambda(LambdaDefinitionExpression lambda, Void context) + { + return lambda; + } + + @Override + public RowExpression visitVariableReference(VariableReferenceExpression reference, Void context) + { + return reference; + } + } + + private static ConvertNormalFormVisitorContext rootContext(Form clauseJoiner) + { + return new ConvertNormalFormVisitorContext(clauseJoiner, 0); + } + + private static class ConvertNormalFormVisitorContext + { + private final Form expectedClauseJoiner; + private final int depth; + + public ConvertNormalFormVisitorContext(Form expectedClauseJoiner, int depth) + { + this.expectedClauseJoiner = expectedClauseJoiner; + this.depth = depth; + } + + public ConvertNormalFormVisitorContext childContext() + { + return new ConvertNormalFormVisitorContext(expectedClauseJoiner, depth + 1); + } + } + + private class ConvertNormalFormVisitor + implements RowExpressionVisitor + { + @Override + public RowExpression visitSpecialForm(SpecialFormExpression specialFormExpression, ConvertNormalFormVisitorContext context) + { + if (!isConjunctionOrDisjunction(specialFormExpression)) { + return specialFormExpression; + } + // Attempt to convert sub expression to expected normal form, deduplicate and fold constants. + RowExpression rewritten = combinePredicates( + specialFormExpression.getForm(), + extractPredicates(specialFormExpression.getForm(), specialFormExpression).stream() + .map(subPredicate -> subPredicate.accept(this, context.childContext())) + .collect(toList())); + + if (!isConjunctionOrDisjunction(rewritten)) { + return rewritten; + } + + SpecialFormExpression rewrittenSpecialForm = (SpecialFormExpression) rewritten; + Form expressionClauseJoiner = rewrittenSpecialForm.getForm(); + List> groupedClauses = getGroupedClauses(rewrittenSpecialForm); + + if (groupedClauses.stream().mapToInt(List::size).sum() > ELIMINATE_COMMON_SIZE_LIMIT) { + return rewritten; + } + groupedClauses = eliminateCommonPredicates(groupedClauses); + + // extractCommonPredicates can produce opposite expectedClauseJoiner + List> groupedClausesWithFlippedJoiner = extractCommonPredicates(expressionClauseJoiner, groupedClauses); + if (groupedClausesWithFlippedJoiner != null) { + groupedClauses = groupedClausesWithFlippedJoiner; + expressionClauseJoiner = flip(expressionClauseJoiner); + } + + int numClauses = groupedClauses.stream().mapToInt(List::size).sum(); + + int numClausesProducedByDistributiveLaw = groupedClauses.size(); + for (List group : groupedClauses) { + numClausesProducedByDistributiveLaw *= group.size(); + // If distributive rule will produce too many sub expressions, return what we have instead. + if (context.depth > 0 || numClausesProducedByDistributiveLaw > numClauses * 2) { + return combineGroupedClauses(expressionClauseJoiner, groupedClauses); + } + } + // size unchanged means distributive law will not apply, we can save an unnecessary crossProduct call. + // For example, distributive law cannot apply to (a || b || c). + if (numClausesProducedByDistributiveLaw == numClauses) { + return combineGroupedClauses(expressionClauseJoiner, groupedClauses); + } + + // TODO if the non-deterministic operation only appears in the only sub-predicates that has size >1, we can still expand it. + // For example: a && b && c && (d || e) can still be expanded if d or e is non-deterministic. + boolean deterministic = groupedClauses.stream() + .flatMap(List::stream) + .allMatch(determinismEvaluator::isDeterministic); + + // Do not apply distributive law if there is non-deterministic element or we have already got expected expectedClauseJoiner. + if (expressionClauseJoiner == context.expectedClauseJoiner || !deterministic) { + return combineGroupedClauses(expressionClauseJoiner, groupedClauses); + } + + // else, we apply distributive law and rewrite based on distributive property of Boolean algebra, for example + // (l1 OR l2) AND (r1 OR r2) <=> (l1 AND r1) OR (l1 AND r2) OR (l2 AND r1) OR (l2 AND r2) + groupedClauses = crossProduct(groupedClauses); + return combineGroupedClauses(context.expectedClauseJoiner, groupedClauses); + } + + @Override + public RowExpression visitCall(CallExpression call, ConvertNormalFormVisitorContext context) + { + return call; + } + + @Override + public RowExpression visitInputReference(InputReferenceExpression reference, ConvertNormalFormVisitorContext context) + { + return reference; + } + + @Override + public RowExpression visitConstant(ConstantExpression literal, ConvertNormalFormVisitorContext context) + { + return literal; + } + + @Override + public RowExpression visitLambda(LambdaDefinitionExpression lambda, ConvertNormalFormVisitorContext context) + { + return lambda; + } + + @Override + public RowExpression visitVariableReference(VariableReferenceExpression reference, ConvertNormalFormVisitorContext context) + { + return reference; + } + } + + private boolean isNegationExpression(RowExpression expression) + { + return expression instanceof CallExpression && ((CallExpression) expression).getFunctionHandle().equals(functionResolution.notFunction()); + } + + private boolean isComparisonExpression(RowExpression expression) + { + return expression instanceof CallExpression && functionResolution.isComparisonFunction(((CallExpression) expression).getFunctionHandle()); + } + + /** + * Extract the component predicates as a list of list in which is grouped so that the outer level has same conjunctive/disjunctive joiner as original predicate and + * inner level has opposite joiner. + * For example, (a or b) and (a or c) or ( a or c) returns [[a,b], [a,c], [a,c]] + */ + private List> getGroupedClauses(SpecialFormExpression expression) + { + return extractPredicates(expression.getForm(), expression).stream() + .map(LogicalRowExpressions::extractPredicates) + .collect(toList()); + } + + private int numOfClauses(RowExpression expression) + { + if (expression instanceof SpecialFormExpression) { + return getGroupedClauses((SpecialFormExpression) expression).stream().mapToInt(List::size).sum(); + } + return 1; + } + + /** + * Eliminate a sub predicate if its sub predicates contain its peer. + * For example: (a || b) && a = a, (a && b) || b = b + */ + private List> eliminateCommonPredicates(List> groupedClauses) + { + if (groupedClauses.size() < 2) { + return groupedClauses; + } + // initialize to self + int[] reduceTo = IntStream.range(0, groupedClauses.size()).toArray(); + for (int i = 0; i < groupedClauses.size(); i++) { + // Do not eliminate predicates contain non-deterministic value + // (a || b) && a should be kept same if a is non-deterministic. + // TODO We can eliminate (a || b) && a if a is deterministic even b is not. + if (groupedClauses.get(i).stream().allMatch(determinismEvaluator::isDeterministic)) { + for (int j = 0; j < groupedClauses.size(); j++) { + if (isSuperSet(groupedClauses.get(reduceTo[i]), groupedClauses.get(j))) { + reduceTo[i] = j; //prefer smaller set + } + else if (isSameSet(groupedClauses.get(reduceTo[i]), groupedClauses.get(j))) { + reduceTo[i] = min(reduceTo[i], j); //prefer predicates that appears earlier. + } + } + } + } + + return unmodifiableList(stream(reduceTo) + .distinct() + .boxed() + .map(groupedClauses::get) + .collect(toList())); + } + + /** + * Eliminate a sub predicate if its component predicates contain its peer. Will return null if cannot extract common predicates otherwise return a nested list with flipped form + * For example: + * (a || b || c || d) && (a || b || e || f) -> a || b || ((c || d) && (e || f)) + * (a || b) && (c || d) -> null + */ + private List> extractCommonPredicates(Form rootClauseJoiner, List> groupedPredicates) + { + if (groupedPredicates.isEmpty()) { + return null; + } + Set commonPredicates = new LinkedHashSet<>(groupedPredicates.get(0)); + for (int i = 1; i < groupedPredicates.size(); i++) { + // remove all non-common predicates + commonPredicates.retainAll(groupedPredicates.get(i)); + } + + if (commonPredicates.isEmpty()) { + return null; + } + // extract the component predicates that are not in common predicates: [(c || d), (e || f)] + List remainingPredicates = new ArrayList<>(); + for (List group : groupedPredicates) { + List remaining = group.stream() + .filter(predicate -> !commonPredicates.contains(predicate)) + .collect(toList()); + remainingPredicates.add(combinePredicates(flip(rootClauseJoiner), remaining)); + } + // combine common predicates and remaining predicates to flipped nested form. For example: [[a], [b], [ (c || d), (e || f)] + return Stream.concat(commonPredicates.stream().map(predicate -> singletonList(predicate)), Stream.of(remainingPredicates)) + .collect(toList()); + } + + private RowExpression combineGroupedClauses(Form clauseJoiner, List> nestedPredicates) + { + return combinePredicates(clauseJoiner, nestedPredicates.stream() + .map(predicate -> combinePredicates(flip(clauseJoiner), predicate)) + .collect(toList())); + } + + /** + * Cartesian cross product of List of List. + * For example, [[a], [b, c], [d]] becomes [[a,b,d], [a,c,d]] + */ + private static List> crossProduct(List> groupedPredicates) + { + checkArgument(groupedPredicates.size() > 0, "Must contains more than one child"); + List> result = groupedPredicates.get(0).stream().map(Collections::singletonList).collect(toList()); + for (int i = 1; i < groupedPredicates.size(); i++) { + result = crossProduct(result, groupedPredicates.get(i)); + } + return result; + } + + private static List> crossProduct(List> previousCrossProduct, List clauses) + { + List> result = new ArrayList<>(); + for (List previousClauses : previousCrossProduct) { + for (RowExpression newClause : clauses) { + List newClauses = new ArrayList<>(previousClauses); + newClauses.add(newClause); + result.add(newClauses); + } + } + return result; + } + + private static Form flip(Form binaryLogicalOperation) + { + switch (binaryLogicalOperation) { + case AND: + return OR; + case OR: + return AND; + } + throw new UnsupportedOperationException("Invalid binary logical operation: " + binaryLogicalOperation); + } + + private Optional getOperator(RowExpression expression) + { + if (expression instanceof CallExpression) { + return functionMetadataManager.getFunctionMetadata(((CallExpression) expression).getFunctionHandle()).getOperatorType(); + } + return Optional.empty(); + } + + private RowExpression notCallExpression(RowExpression argument) + { + return new CallExpression("not", functionResolution.notFunction(), BOOLEAN, singletonList(argument)); + } + + private static OperatorType negate(OperatorType operator) + { + switch (operator) { + case EQUAL: + return NOT_EQUAL; + case NOT_EQUAL: + return EQUAL; + case GREATER_THAN: + return LESS_THAN_OR_EQUAL; + case LESS_THAN: + return GREATER_THAN_OR_EQUAL; + case LESS_THAN_OR_EQUAL: + return GREATER_THAN; + case GREATER_THAN_OR_EQUAL: + return LESS_THAN; + } + return null; + } + + private static void checkArgument(boolean condition, String message, Object... arguments) + { + if (!condition) { + throw new IllegalArgumentException(String.format(message, arguments)); + } + } + + private static boolean isSuperSet(Collection a, Collection b) + { + // We assumes a, b both are de-duplicated collections. + return a.size() > b.size() && a.containsAll(b); + } + + private static boolean isSameSet(Collection a, Collection b) + { + // We assumes a, b both are de-duplicated collections. + return a.size() == b.size() && a.containsAll(b) && b.containsAll(a); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionNodeInliner.java b/presto-expressions/src/main/java/com/facebook/presto/expressions/RowExpressionNodeInliner.java similarity index 93% rename from presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionNodeInliner.java rename to presto-expressions/src/main/java/com/facebook/presto/expressions/RowExpressionNodeInliner.java index b8bd6ed71f4b4..195628d4cbee1 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionNodeInliner.java +++ b/presto-expressions/src/main/java/com/facebook/presto/expressions/RowExpressionNodeInliner.java @@ -11,7 +11,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.spi.relation; +package com.facebook.presto.expressions; + +import com.facebook.presto.spi.relation.RowExpression; import java.util.Map; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionRewriter.java b/presto-expressions/src/main/java/com/facebook/presto/expressions/RowExpressionRewriter.java similarity index 80% rename from presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionRewriter.java rename to presto-expressions/src/main/java/com/facebook/presto/expressions/RowExpressionRewriter.java index cf906fd92acc4..d0c65a22a516a 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionRewriter.java +++ b/presto-expressions/src/main/java/com/facebook/presto/expressions/RowExpressionRewriter.java @@ -11,7 +11,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.spi.relation; +package com.facebook.presto.expressions; + +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; public class RowExpressionRewriter { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionTreeRewriter.java b/presto-expressions/src/main/java/com/facebook/presto/expressions/RowExpressionTreeRewriter.java similarity index 92% rename from presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionTreeRewriter.java rename to presto-expressions/src/main/java/com/facebook/presto/expressions/RowExpressionTreeRewriter.java index b81f51c5bfd41..fe8141440ba05 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionTreeRewriter.java +++ b/presto-expressions/src/main/java/com/facebook/presto/expressions/RowExpressionTreeRewriter.java @@ -11,7 +11,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.spi.relation; +package com.facebook.presto.expressions; + +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.RowExpressionVisitor; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; import java.util.ArrayList; import java.util.Collection; diff --git a/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/FunctionTranslator.java b/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/FunctionTranslator.java new file mode 100644 index 0000000000000..f8c717ef770bb --- /dev/null +++ b/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/FunctionTranslator.java @@ -0,0 +1,78 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.expressions.translator; + +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.relation.RowExpression; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.lang.invoke.MethodHandle; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.expressions.translator.TranslatedExpression.untranslated; +import static com.facebook.presto.expressions.translator.TranslatorAnnotationParser.removeTypeParameters; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +public class FunctionTranslator +{ + private final Map functionMapping; + + public static FunctionTranslator buildFunctionTranslator(Set> translatorContainers) + { + ImmutableMap.Builder functionMappingBuilder = new ImmutableMap.Builder<>(); + translatorContainers.stream() + .map(TranslatorAnnotationParser::parseFunctionDefinitions) + .forEach(functionMappingBuilder::putAll); + return new FunctionTranslator<>(functionMappingBuilder.build()); + } + + public TranslatedExpression translate(FunctionMetadata functionMetadata, RowExpression original, List> translatedExpressions) + throws Throwable + { + functionMetadata = removeTypeParameters(functionMetadata); + if (!functionMapping.containsKey(functionMetadata) + || !translatedExpressions.stream().map(TranslatedExpression::getTranslated).allMatch(Optional::isPresent)) { + return untranslated(original, translatedExpressions); + } + + List translatedArguments = translatedExpressions.stream() + .map(TranslatedExpression::getTranslated) + .map(Optional::get) + .collect(toImmutableList()); + + return new TranslatedExpression<>(Optional.of((T) functionMapping.get(functionMetadata).invokeWithArguments(translatedArguments)), original, translatedExpressions); + } + + public TranslatedExpression translate(FunctionMetadata functionMetadata, RowExpression original, TranslatedExpression... translatedArguments) + throws Throwable + { + return translate(functionMetadata, original, ImmutableList.copyOf(translatedArguments)); + } + + public TranslatedExpression translate(FunctionMetadata functionMetadata, RowExpression original) + throws Throwable + { + return translate(functionMetadata, original, ImmutableList.of()); + } + + private FunctionTranslator(Map functionMapping) + { + this.functionMapping = requireNonNull(functionMapping, "functionMapping is null"); + } +} diff --git a/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/RowExpressionTranslator.java b/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/RowExpressionTranslator.java new file mode 100644 index 0000000000000..2a2aa0d0725d9 --- /dev/null +++ b/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/RowExpressionTranslator.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.expressions.translator; + +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; + +import static com.facebook.presto.expressions.translator.TranslatedExpression.untranslated; + +public class RowExpressionTranslator +{ + public TranslatedExpression translateConstant(ConstantExpression literal, C context, RowExpressionTreeTranslator rowExpressionTreeTranslator) + { + return untranslated(literal); + } + + public TranslatedExpression translateVariable(VariableReferenceExpression variable, C context, RowExpressionTreeTranslator rowExpressionTreeTranslator) + { + return untranslated(variable); + } + + public TranslatedExpression translateLambda(LambdaDefinitionExpression lambda, C context, RowExpressionTreeTranslator rowExpressionTreeTranslator) + { + return untranslated(lambda); + } + + public TranslatedExpression translateCall(CallExpression call, C context, RowExpressionTreeTranslator rowExpressionTreeTranslator) + { + return untranslated(call); + } + + public TranslatedExpression translateSpecialForm(SpecialFormExpression specialForm, C context, RowExpressionTreeTranslator rowExpressionTreeTranslator) + { + return untranslated(specialForm); + } +} diff --git a/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/RowExpressionTreeTranslator.java b/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/RowExpressionTreeTranslator.java new file mode 100644 index 0000000000000..f5b457b9fdfb6 --- /dev/null +++ b/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/RowExpressionTreeTranslator.java @@ -0,0 +1,91 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.expressions.translator; + +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.RowExpressionVisitor; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; + +import static java.util.Objects.requireNonNull; + +public class RowExpressionTreeTranslator +{ + private final RowExpressionTranslator rowExpressionTranslator; + private final RowExpressionVisitor, C> visitor; + + private RowExpressionTreeTranslator(RowExpressionTranslator rowExpressionTranslator) + { + this.rowExpressionTranslator = requireNonNull(rowExpressionTranslator, "rowExpressionTranslator is null"); + this.visitor = new TranslatingVisitor(); + } + + public TranslatedExpression rewrite(RowExpression node, C context) + { + return node.accept(this.visitor, context); + } + + public static TranslatedExpression translateWith( + RowExpression expression, + RowExpressionTranslator translator, + C context) + { + return expression.accept(new RowExpressionTreeTranslator<>(translator).visitor, context); + } + + private class TranslatingVisitor + implements RowExpressionVisitor, C> + { + @Override + public TranslatedExpression visitCall(CallExpression call, C context) + { + return rowExpressionTranslator.translateCall(call, context, RowExpressionTreeTranslator.this); + } + + @Override + public TranslatedExpression visitInputReference(InputReferenceExpression reference, C context) + { + // InputReferenceExpression should only be used by Presto engine rather than connectors + throw new UnsupportedOperationException("Cannot translate RowExpression that contains inputReferenceExpression"); + } + + @Override + public TranslatedExpression visitConstant(ConstantExpression literal, C context) + { + return rowExpressionTranslator.translateConstant(literal, context, RowExpressionTreeTranslator.this); + } + + @Override + public TranslatedExpression visitLambda(LambdaDefinitionExpression lambda, C context) + { + return rowExpressionTranslator.translateLambda(lambda, context, RowExpressionTreeTranslator.this); + } + + @Override + public TranslatedExpression visitVariableReference(VariableReferenceExpression reference, C context) + { + return rowExpressionTranslator.translateVariable(reference, context, RowExpressionTreeTranslator.this); + } + + @Override + public TranslatedExpression visitSpecialForm(SpecialFormExpression specialForm, C context) + { + return rowExpressionTranslator.translateSpecialForm(specialForm, context, RowExpressionTreeTranslator.this); + } + } +} diff --git a/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/TranslatedExpression.java b/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/TranslatedExpression.java new file mode 100644 index 0000000000000..63322275bc169 --- /dev/null +++ b/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/TranslatedExpression.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.expressions.translator; + +import com.facebook.presto.spi.relation.RowExpression; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class TranslatedExpression +{ + private final Optional translated; + private final RowExpression originalExpression; + private final List> translatedArguments; + + public TranslatedExpression(Optional translated, RowExpression originalExpression, List> translatedArguments) + { + this.translated = requireNonNull(translated); + this.originalExpression = requireNonNull(originalExpression); + this.translatedArguments = requireNonNull(translatedArguments); + } + + public Optional getTranslated() + { + return translated; + } + + public RowExpression getOriginalExpression() + { + return originalExpression; + } + + public List> getTranslatedArguments() + { + return translatedArguments; + } + + public static TranslatedExpression untranslated(RowExpression originalExpression) + { + return new TranslatedExpression<>(Optional.empty(), originalExpression, ImmutableList.of()); + } + + public static TranslatedExpression untranslated(RowExpression originalExpression, List> translatedArguments) + { + return new TranslatedExpression<>(Optional.empty(), originalExpression, translatedArguments); + } +} diff --git a/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/TranslatorAnnotationParser.java b/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/TranslatorAnnotationParser.java new file mode 100644 index 0000000000000..05a0ce064de97 --- /dev/null +++ b/presto-expressions/src/main/java/com/facebook/presto/expressions/translator/TranslatorAnnotationParser.java @@ -0,0 +1,292 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.expressions.translator; + +import com.facebook.presto.spi.CatalogSchemaName; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.ScalarOperator; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.type.TypeSignature; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.lang.annotation.Annotation; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.AnnotatedElement; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Parameter; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; + +import static com.facebook.presto.spi.function.FunctionImplementationType.BUILTIN; +import static com.facebook.presto.spi.function.FunctionKind.SCALAR; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.google.common.base.CaseFormat.LOWER_CAMEL; +import static com.google.common.base.CaseFormat.LOWER_UNDERSCORE; +import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +class TranslatorAnnotationParser +{ + private TranslatorAnnotationParser() + { + } + + public static Map parseFunctionDefinitions(Class clazz) + { + ImmutableMap.Builder translatorBuilder = ImmutableMap.builder(); + for (ScalarHeaderAndMethods methods : findScalarsFromSetClass(clazz)) { + translatorBuilder.putAll(scalarToFunctionMetadata(methods)); + } + + return translatorBuilder.build(); + } + + private static TypeSignature removeTypeParameters(TypeSignature typeSignature) + { + return new TypeSignature(typeSignature.getBase()); + } + + public static FunctionMetadata removeTypeParameters(FunctionMetadata metadata) + { + ImmutableList.Builder argumentsBuilder = ImmutableList.builder(); + for (TypeSignature typeSignature : metadata.getArgumentTypes()) { + argumentsBuilder.add(removeTypeParameters(typeSignature)); + } + + if (metadata.getOperatorType().isPresent()) { + return new FunctionMetadata( + metadata.getOperatorType().get(), + argumentsBuilder.build(), + metadata.getReturnType(), + metadata.getFunctionKind(), + metadata.getImplementationType(), + metadata.isDeterministic(), + metadata.isCalledOnNullInput()); + } + return new FunctionMetadata( + metadata.getName(), + argumentsBuilder.build(), + metadata.getReturnType(), + metadata.getFunctionKind(), + metadata.getImplementationType(), + metadata.isDeterministic(), + metadata.isCalledOnNullInput()); + } + + private static MethodHandle getMethodHandle(Method method) + { + try { + return MethodHandles.lookup().unreflect(method); + } + catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + private static List findScalarsFromSetClass(Class clazz) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (Method method : findPublicMethodsWithAnnotation(clazz, SqlType.class, ScalarFunction.class, ScalarOperator.class)) { + boolean annotationCondition = (method.getAnnotation(ScalarFunction.class) != null) || (method.getAnnotation(ScalarOperator.class) != null); + checkArgument(annotationCondition, format("Method [%s] annotated with @SqlType is missing @ScalarFunction or @ScalarOperator", method)); + + for (ScalarTranslationHeader header : ScalarTranslationHeader.fromAnnotatedElement(method)) { + builder.add(new ScalarHeaderAndMethods(header, ImmutableSet.of(method))); + } + } + List methods = builder.build(); + checkArgument(!methods.isEmpty(), "Class [%s] does not have any methods annotated with @ScalarFunction or @ScalarOperator", clazz.getName()); + + return methods; + } + + private static Map scalarToFunctionMetadata(ScalarHeaderAndMethods scalar) + { + ScalarTranslationHeader header = scalar.getHeader(); + + ImmutableMap.Builder metadataBuilder = ImmutableMap.builder(); + for (Method method : scalar.getMethods()) { + FunctionMetadata metadata = methodToFunctionMetadata(header, method); + metadataBuilder.put(removeTypeParameters(metadata), getMethodHandle(method)); + } + + return metadataBuilder.build(); + } + + private static FunctionMetadata methodToFunctionMetadata(ScalarTranslationHeader header, Method method) + { + requireNonNull(header, "header is null"); + + // return type + SqlType annotatedReturnType = method.getAnnotation(SqlType.class); + checkArgument(annotatedReturnType != null, format("Method [%s] is missing @SqlType annotation", method)); + TypeSignature returnType = parseTypeSignature(annotatedReturnType.value(), ImmutableSet.of()); + + // argument type + ImmutableList.Builder argumentTypes = new ImmutableList.Builder<>(); + for (Parameter parameter : method.getParameters()) { + Annotation[] annotations = parameter.getAnnotations(); + + SqlType type = Stream.of(annotations) + .filter(SqlType.class::isInstance) + .map(SqlType.class::cast) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(format("Method [%s] is missing @SqlType annotation for parameter", method))); + TypeSignature typeSignature = parseTypeSignature(type.value(), ImmutableSet.of()); + argumentTypes.add(typeSignature); + } + + if (header.getOperatorType().isPresent()) { + return new FunctionMetadata(header.getOperatorType().get(), argumentTypes.build(), returnType, SCALAR, BUILTIN, header.isDeterministic(), header.isCalledOnNullInput()); + } + return new FunctionMetadata(header.getName(), argumentTypes.build(), returnType, SCALAR, BUILTIN, header.isDeterministic(), header.isCalledOnNullInput()); + } + + @SafeVarargs + private static Set findPublicMethodsWithAnnotation(Class clazz, Class... annotationClasses) + { + ImmutableSet.Builder methods = ImmutableSet.builder(); + for (Method method : clazz.getDeclaredMethods()) { + for (Annotation annotation : method.getAnnotations()) { + for (Class annotationClass : annotationClasses) { + if (annotationClass.isInstance(annotation)) { + checkArgument(Modifier.isPublic(method.getModifiers()), "Method [%s] annotated with @%s must be public", method, annotationClass.getSimpleName()); + methods.add(method); + } + } + } + } + return methods.build(); + } + + private static class ScalarHeaderAndMethods + { + private final ScalarTranslationHeader header; + private final Set methods; + + public ScalarHeaderAndMethods(ScalarTranslationHeader header, Set methods) + { + this.header = requireNonNull(header, "header is null"); + this.methods = requireNonNull(methods, "methods are null"); + } + + public ScalarTranslationHeader getHeader() + { + return header; + } + + public Set getMethods() + { + return methods; + } + } + + private static class ScalarTranslationHeader + { + private final QualifiedFunctionName name; + private final Optional operatorType; + private final boolean deterministic; + private final boolean calledOnNullInput; + + public static List fromAnnotatedElement(AnnotatedElement annotated) + { + ScalarFunction scalarFunction = annotated.getAnnotation(ScalarFunction.class); + ScalarOperator scalarOperator = annotated.getAnnotation(ScalarOperator.class); + + ImmutableList.Builder builder = ImmutableList.builder(); + + if (scalarFunction != null) { + String baseName = scalarFunction.value().isEmpty() ? camelToSnake(annotatedName(annotated)) : scalarFunction.value(); + builder.add(new ScalarTranslationHeader(baseName, scalarFunction.deterministic(), scalarFunction.calledOnNullInput())); + + for (String alias : scalarFunction.alias()) { + builder.add(new ScalarTranslationHeader(alias, scalarFunction.deterministic(), scalarFunction.calledOnNullInput())); + } + } + + if (scalarOperator != null) { + builder.add(new ScalarTranslationHeader(scalarOperator.value(), true, scalarOperator.value().isCalledOnNullInput())); + } + + List result = builder.build(); + checkArgument(!result.isEmpty()); + return result; + } + + private ScalarTranslationHeader(String name, boolean deterministic, boolean calledOnNullInput) + { + // TODO This is a hack. Engine should provide an API for connectors to overwrite functions. Connector should not hard code the builtin function namespace. + this.name = requireNonNull(QualifiedFunctionName.of(new CatalogSchemaName("presto", "default"), name)); + this.operatorType = Optional.empty(); + this.deterministic = deterministic; + this.calledOnNullInput = calledOnNullInput; + } + + private ScalarTranslationHeader(OperatorType operatorType, boolean deterministic, boolean calledOnNullInput) + { + this.name = operatorType.getFunctionName(); + this.operatorType = Optional.of(operatorType); + this.deterministic = deterministic; + this.calledOnNullInput = calledOnNullInput; + } + + private static String annotatedName(AnnotatedElement annotatedElement) + { + if (annotatedElement instanceof Class) { + return ((Class) annotatedElement).getSimpleName(); + } + else if (annotatedElement instanceof Method) { + return ((Method) annotatedElement).getName(); + } + + throw new UnsupportedOperationException("Only Classes and Methods are supported as annotated elements."); + } + + private static String camelToSnake(String name) + { + return LOWER_CAMEL.to(LOWER_UNDERSCORE, name); + } + + QualifiedFunctionName getName() + { + return name; + } + + Optional getOperatorType() + { + return operatorType; + } + + boolean isDeterministic() + { + return deterministic; + } + + boolean isCalledOnNullInput() + { + return calledOnNullInput; + } + } +} diff --git a/presto-geospatial-toolkit/pom.xml b/presto-geospatial-toolkit/pom.xml index 3f2abd54ae267..48c46a4ebc010 100644 --- a/presto-geospatial-toolkit/pom.xml +++ b/presto-geospatial-toolkit/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-geospatial-toolkit @@ -46,7 +46,7 @@ - io.airlift + com.facebook.airlift json @@ -74,6 +74,21 @@ test + + com.facebook.presto + presto-array + + + + com.facebook.presto + presto-spi + + + + it.unimi.dsi + fastutil + + diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/GeometryUtils.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/GeometryUtils.java index 066e1dbbcef7b..eb4dae5abc9a9 100644 --- a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/GeometryUtils.java +++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/GeometryUtils.java @@ -23,12 +23,26 @@ import com.esri.core.geometry.ogc.OGCGeometry; import com.esri.core.geometry.ogc.OGCPoint; import com.esri.core.geometry.ogc.OGCPolygon; +import com.facebook.presto.spi.PrestoException; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateSequence; +import org.locationtech.jts.geom.CoordinateSequenceFactory; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.impl.PackedCoordinateSequenceFactory; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKTReader; +import org.locationtech.jts.io.WKTWriter; import java.util.HashSet; import java.util.Set; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; + public final class GeometryUtils { + private static final CoordinateSequenceFactory COORDINATE_SEQUENCE_FACTORY = new PackedCoordinateSequenceFactory(); + private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory(COORDINATE_SEQUENCE_FACTORY); + private GeometryUtils() {} /** @@ -96,6 +110,58 @@ public static Envelope getEnvelope(OGCGeometry ogcGeometry) } } + /** + * Get the bounding box for an OGCGeometry. + *

+ * If the geometry is empty, return a Retangle with NaN coordinates. + * + * @param ogcGeometry + * @return Rectangle bounding box + */ + public static Rectangle getExtent(OGCGeometry ogcGeometry) + { + return getExtent(ogcGeometry, 0.0); + } + + /** + * Get the bounding box for an OGCGeometry, inflated by radius. + *

+ * If the geometry is empty, return a Retangle with NaN coordinates. + * + * @param ogcGeometry + * @return Rectangle bounding box + */ + public static Rectangle getExtent(OGCGeometry ogcGeometry, double radius) + { + com.esri.core.geometry.Envelope envelope = getEnvelope(ogcGeometry); + + return new Rectangle( + envelope.getXMin() - radius, + envelope.getYMin() - radius, + envelope.getXMax() + radius, + envelope.getYMax() + radius); + } + + public static org.locationtech.jts.geom.Envelope getJtsEnvelope(OGCGeometry ogcGeometry, double radius) + { + Envelope esriEnvelope = getEnvelope(ogcGeometry); + + if (esriEnvelope.isEmpty()) { + return new org.locationtech.jts.geom.Envelope(); + } + + return new org.locationtech.jts.geom.Envelope( + esriEnvelope.getXMin() - radius, + esriEnvelope.getXMax() + radius, + esriEnvelope.getYMin() - radius, + esriEnvelope.getYMax() + radius); + } + + public static org.locationtech.jts.geom.Envelope getJtsEnvelope(OGCGeometry ogcGeometry) + { + return getJtsEnvelope(ogcGeometry, 0.0); + } + public static boolean disjoint(Envelope envelope, OGCGeometry ogcGeometry) { GeometryCursor cursor = ogcGeometry.getEsriGeometryCursor(); @@ -160,4 +226,54 @@ public static boolean isPointOrRectangle(OGCGeometry ogcGeometry, Envelope envel return true; } + + public static org.locationtech.jts.geom.Geometry jtsGeometryFromWkt(String wkt) + { + try { + return new WKTReader(GEOMETRY_FACTORY).read(wkt); + } + catch (ParseException | IllegalArgumentException e) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Invalid WKT: " + e.getMessage(), e); + } + } + + public static String wktFromJtsGeometry(org.locationtech.jts.geom.Geometry geometry) + { + return new WKTWriter().write(geometry); + } + + public static org.locationtech.jts.geom.Point createJtsEmptyPoint() + { + return GEOMETRY_FACTORY.createPoint(); + } + + public static org.locationtech.jts.geom.Point createJtsPoint(Coordinate coordinate) + { + return GEOMETRY_FACTORY.createPoint(coordinate); + } + + public static org.locationtech.jts.geom.Point createJtsPoint(double x, double y) + { + return createJtsPoint(new Coordinate(x, y)); + } + + public static org.locationtech.jts.geom.MultiPoint createJtsMultiPoint(CoordinateSequence coordinates) + { + return GEOMETRY_FACTORY.createMultiPoint(coordinates); + } + + public static org.locationtech.jts.geom.Geometry createJtsEmptyLineString() + { + return GEOMETRY_FACTORY.createLineString(); + } + + public static org.locationtech.jts.geom.Geometry createJtsLineString(CoordinateSequence coordinates) + { + return GEOMETRY_FACTORY.createLineString(coordinates); + } + + public static org.locationtech.jts.geom.Geometry createJtsEmptyPolygon() + { + return GEOMETRY_FACTORY.createPolygon(); + } } diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/KdbTreeUtils.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/KdbTreeUtils.java index a2899bf4571eb..25aa68bdd2538 100644 --- a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/KdbTreeUtils.java +++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/KdbTreeUtils.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.geospatial; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; import static java.util.Objects.requireNonNull; diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/Rectangle.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/Rectangle.java index 8f7794f2c53cd..73f94594dac2d 100644 --- a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/Rectangle.java +++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/Rectangle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.geospatial; +import com.facebook.presto.geospatial.rtree.HasExtent; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.openjdk.jol.info.ClassLayout; @@ -26,6 +27,7 @@ import static java.util.Objects.requireNonNull; public final class Rectangle + implements HasExtent { private static final int INSTANCE_SIZE = ClassLayout.parseClass(Rectangle.class).instanceSize(); @@ -103,7 +105,20 @@ public Rectangle merge(Rectangle other) return new Rectangle(min(this.xMin, other.xMin), min(this.yMin, other.yMin), max(this.xMax, other.xMax), max(this.yMax, other.yMax)); } - public int estimateMemorySize() + public boolean contains(double x, double y) + { + return xMin <= x && x <= xMax + && yMin <= y && y <= yMax; + } + + @Override + public Rectangle getExtent() + { + return this; + } + + @Override + public long getEstimatedSizeInBytes() { return INSTANCE_SIZE; } diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/rtree/Flatbush.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/rtree/Flatbush.java new file mode 100644 index 0000000000000..99f518a5e14ad --- /dev/null +++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/rtree/Flatbush.java @@ -0,0 +1,306 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.geospatial.rtree; + +import com.facebook.presto.array.DoubleBigArray; +import com.facebook.presto.geospatial.Rectangle; +import com.google.common.annotations.VisibleForTesting; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import org.openjdk.jol.info.ClassLayout; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.function.Consumer; + +import static com.google.common.base.Preconditions.checkArgument; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.util.Objects.requireNonNull; + +/** + * A fast, low memory footprint static RTree. + *

+ * Packed Hilbert RTrees -- aka Flatbushes -- create very few objects, instead + * storing the tree in a flat array. They support the standard RTree queries, + * but cannot be modified once built. It is quite possible to semi-efficiently + * remove objects. + *

+ * Consider an RTree with branching factor `b`. Each non-leaf node has two + * pieces of information: + * 1. The minimum bounding box of all descendants, and + * 2. Pointers to its children. + *

+ * The former is simply four doubles. The latter can be derived from the + * node's level and which sibling it is. This means we can actually flatten + * the tree into a single array of doubles, four per node. We can + * programmatically find the indices of a node's children (it will be faster if + * we pre-compute level offsets), and do envelope checks with just float + * operations. "padded" empty nodes will have NaN entries, which will naturally + * return false for all comparison operators, thus being automatically not + * selected. + *

+ * A critical choice in RTree implementation is how to group leaf nodes as + * children (and recursively, their parents). One method that is very efficient + * to construct and comparable to best lookup performance is sorting by an + * object's Hilbert curve index. Hilbert curves are naturally hierarchical, so + * successively grouping children and their parents will give a naturally nested + * structure. This means we only need to sort the items once. + *

+ * If sort time is a problem, we actually just need to "partition" into groups + * of `degree`, since the order of the children of a single parent doesn't + * matter. This could be done with quicksort, stopping once all items at index + * `n * degree` are correctly placed. + *

+ * Original implementation in Javascript: https://github.com/mourner/flatbush + */ +public class Flatbush +{ + // Number of coordinates to define an envelope + @VisibleForTesting + static final int ENVELOPE_SIZE = 4; + private static final int INSTANCE_SIZE = ClassLayout.parseClass(Flatbush.class).instanceSize(); + private static final int DEFAULT_DEGREE = 16; + // Number of children per node + private final int degree; + // Offsets in tree for each level + private final int[] levelOffsets; + // Each node has four doubles: xMin, yMin, xMax, yMax + private final DoubleBigArray tree; + private final T[] items; + + /** + * Build Flatbush RTree for `items`. + * + * @param items Items to index. + * @param degree Number of children for each intermediate node. + */ + public Flatbush(T[] items, int degree) + { + checkArgument(degree > 0, "degree must be positive"); + this.degree = degree; + this.items = requireNonNull(items, "items is null"); + this.levelOffsets = calculateLevelOffsets(items.length, degree); + this.tree = buildTree(); + } + + /** + * Build Flatbush RTree for `items` with default number of children per node. + *

+ * This will use the default degree. + * + * @param items Items to index. + */ + public Flatbush(T[] items) + { + this(items, DEFAULT_DEGREE); + } + + /** + * Calculate the indices for each level. + * + * A given level contains a certain number of items. We give it a capacity + * equal to the next multiple of `degree`, so that each parent will have + * an equal number (`degree`) of children. This means the next level will + * have `capacity/degree` nodes, which yields a capacity equal to the next + * multiple, and so on, until the number of items is 1. + * + * Since we are storing each node as 4 doubles, we will actually multiply + * every index by 4. + */ + private static int[] calculateLevelOffsets(int numItems, int degree) + { + List offsets = new ArrayList<>(); + // Leaf nodes start at 0, root is the last element. + offsets.add(0); + int level = 0; + while (numItems > 1) { + // The number of children will be the smallest multiple of degree >= numItems + int numChildren = (int) Math.ceil(1.0 * numItems / degree) * degree; + offsets.add(offsets.get(level) + ENVELOPE_SIZE * numChildren); + numItems = numChildren / degree; + level += 1; + } + return offsets.stream().mapToInt(Integer::intValue).toArray(); + } + + private DoubleBigArray buildTree() + { + // We initialize it to NaN, because all comparisons with NaN are false. + // Thus the normal intersection logic will not select uninitialized + // nodes. + DoubleBigArray tree = new DoubleBigArray(Double.NaN); + tree.ensureCapacity(levelOffsets[levelOffsets.length - 1] + ENVELOPE_SIZE); + + if (items.length > degree) { + sortByHilbertIndex(items); + } + + int writeOffset = 0; + for (T item : items) { + tree.set(writeOffset++, item.getExtent().getXMin()); + tree.set(writeOffset++, item.getExtent().getYMin()); + tree.set(writeOffset++, item.getExtent().getXMax()); + tree.set(writeOffset++, item.getExtent().getYMax()); + } + + int numChildren = items.length; + for (int level = 0; level < levelOffsets.length - 1; level++) { + int readOffset = levelOffsets[level]; + writeOffset = levelOffsets[level + 1]; + int numParents = 0; + double xMin = Double.POSITIVE_INFINITY; + double yMin = Double.POSITIVE_INFINITY; + double xMax = Double.NEGATIVE_INFINITY; + double yMax = Double.NEGATIVE_INFINITY; + int child = 0; + for (; child < numChildren; child++) { + xMin = min(xMin, tree.get(readOffset++)); + yMin = min(yMin, tree.get(readOffset++)); + xMax = max(xMax, tree.get(readOffset++)); + yMax = max(yMax, tree.get(readOffset++)); + + if ((child + 1) % degree == 0) { + numParents++; + tree.set(writeOffset++, xMin); + tree.set(writeOffset++, yMin); + tree.set(writeOffset++, xMax); + tree.set(writeOffset++, yMax); + xMin = Double.POSITIVE_INFINITY; + yMin = Double.POSITIVE_INFINITY; + xMax = Double.NEGATIVE_INFINITY; + yMax = Double.NEGATIVE_INFINITY; + } + } + + if (child % degree != 0) { + numParents++; + tree.set(writeOffset++, xMin); + tree.set(writeOffset++, yMin); + tree.set(writeOffset++, xMax); + tree.set(writeOffset++, yMax); + } + numChildren = numParents; + } + + return tree; + } + + /** + * Find intersection candidates for `query` rectangle. + *

+ * This will feed to `consumer` each object in the rtree whose bounding + * rectangle intersects the query rectangle. The actual intersection + * check will need to be performed by the caller. + * + * @param query Rectangle for which to search for intersection. + * @param consumer Function to call for each intersection candidate. + */ + public void findIntersections(Rectangle query, Consumer consumer) + { + IntArrayList todoNodes = new IntArrayList(levelOffsets.length * degree); + IntArrayList todoLevels = new IntArrayList(levelOffsets.length * degree); + + int rootLevel = levelOffsets.length - 1; + int rootIndex = levelOffsets[rootLevel]; + if (doesIntersect(query, rootIndex)) { + todoNodes.push(rootIndex); + todoLevels.push(rootLevel); + } + + while (!todoNodes.isEmpty()) { + int nodeIndex = todoNodes.popInt(); + int level = todoLevels.popInt(); + + if (level == 0) { + // This is a leaf node + consumer.accept(items[nodeIndex / ENVELOPE_SIZE]); + } + else { + int childrenOffset = getChildrenOffset(nodeIndex, level); + for (int i = 0; i < degree; i++) { + int childIndex = childrenOffset + ENVELOPE_SIZE * i; + if (doesIntersect(query, childIndex)) { + todoNodes.push(childIndex); + todoLevels.push(level - 1); + } + } + } + } + } + + private boolean doesIntersect(Rectangle query, int nodeIndex) + { + return query.getXMax() >= tree.get(nodeIndex) // xMin + && query.getYMax() >= tree.get(nodeIndex + 1) // yMin + && query.getXMin() <= tree.get(nodeIndex + 2) // xMax + && query.getYMin() <= tree.get(nodeIndex + 3); // yMax + } + + /** + * Get the offset of the first child for the node. + * + * @param nodeIndex Index in tree of first entry for node + * @param level Level of node + */ + @VisibleForTesting + int getChildrenOffset(int nodeIndex, int level) + { + int indexInLevel = nodeIndex - levelOffsets[level]; + return levelOffsets[level - 1] + degree * indexInLevel; + } + + @VisibleForTesting + int getHeight() + { + return levelOffsets.length; + } + + /* + * Sorts items in-place by the Hilbert index of the envelope center. + */ + private void sortByHilbertIndex(T[] items) + { + if (items == null || items.length < 2) { + return; + } + + Rectangle totalExtent = items[0].getExtent(); + for (int i = 1; i < items.length; i++) { + totalExtent = totalExtent.merge(items[i].getExtent()); + } + + HilbertIndex hilbert = new HilbertIndex(totalExtent); + Arrays.parallelSort(items, Comparator.comparing(item -> + hilbert.indexOf( + (item.getExtent().getXMin() + item.getExtent().getXMax()) / 2, + (item.getExtent().getYMin() + item.getExtent().getYMax()) / 2))); + } + + public boolean isEmpty() + { + return items.length == 0; + } + + public long getEstimatedSizeInBytes() + { + long result = INSTANCE_SIZE + sizeOf(levelOffsets) + tree.sizeOf(); + for (T item : items) { + result += item.getEstimatedSizeInBytes(); + } + return result; + } +} diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/rtree/HasExtent.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/rtree/HasExtent.java new file mode 100644 index 0000000000000..71c93bf6e309f --- /dev/null +++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/rtree/HasExtent.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.geospatial.rtree; + +import com.facebook.presto.geospatial.Rectangle; + +public interface HasExtent +{ + Rectangle getExtent(); + long getEstimatedSizeInBytes(); +} diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/rtree/HilbertIndex.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/rtree/HilbertIndex.java new file mode 100644 index 0000000000000..95aff025819ab --- /dev/null +++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/rtree/HilbertIndex.java @@ -0,0 +1,148 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.geospatial.rtree; + +import com.facebook.presto.geospatial.Rectangle; + +import static java.util.Objects.requireNonNull; + +/** + * A class that can calculate Hilbert indices. + *

+ * A Hilbert index is the index on a Hilbert curve; a Hilbert curve is a + * space-filling curve in a rectangle. This class is instantiated with the + * rectangle within which it can calculate the index. + *

+ * The (fast) index algorithm is adapted from the C++ from + * https://github.com/rawrunprotected/hilbert_curves , + * original algorithm by http://threadlocalmutex.com/?p=126 . + */ +public class HilbertIndex +{ + private static final int HILBERT_BITS = 16; + private static final double HILBERT_MAX = (1 << HILBERT_BITS) - 1; + + private final Rectangle rectangle; + private final double xScale; + private final double yScale; + + /** + * @param rectangle Rectangle defining bounds of Hilbert curve + */ + public HilbertIndex(Rectangle rectangle) + { + this.rectangle = requireNonNull(rectangle, "rectangle is null"); + if (rectangle.getXMax() == rectangle.getXMin()) { + this.xScale = 0; + } + else { + this.xScale = HILBERT_MAX / (rectangle.getXMax() - rectangle.getXMin()); + } + if (rectangle.getYMax() == rectangle.getYMin()) { + this.yScale = 0; + } + else { + this.yScale = HILBERT_MAX / (rectangle.getYMax() - rectangle.getYMin()); + } + } + + /** + * Calculate Hilbert index of coordinates in rectangle. + *

+ * This gives a reasonable index for coordinates contained in the bounding + * rectangle; coordinates not in the box will return `Long.MAX_VALUE`. + * + * @param x + * @param y + * @return Hilbert curve index, relative to rectangle + */ + public long indexOf(double x, double y) + { + if (!rectangle.contains(x, y)) { + // Put things outside the box at the end + // This will also handle infinities and NaNs + return Long.MAX_VALUE; + } + + int xInt = (int) (xScale * (x - rectangle.getXMin())); + int yInt = (int) (yScale * (y - rectangle.getYMin())); + return discreteIndexOf(xInt, yInt); + } + + /** + * Calculate the Hilbert index of a discrete coordinate. + *

+ * Since Java doesn't have unsigned ints, we put incoming ints into the + * lower 32 bits of a long and do the calculations there. + * + * @param x discrete positive x coordinate + * @param y discrete positive y coordinate + * @return Hilbert curve index + */ + private long discreteIndexOf(int x, int y) + { + int a = x ^ y; + int b = 0x0000FFFF ^ a; + int c = 0x0000FFFF ^ (x | y); + int d = x & (y ^ 0x0000FFFF); + + int e = a | (b >>> 1); + int f = (a >>> 1) ^ a; + int g = ((c >>> 1) ^ (b & (d >>> 1))) ^ c; + int h = ((a & (c >>> 1)) ^ (d >>> 1)) ^ d; + + a = e; + b = f; + c = g; + d = h; + e = (a & (a >>> 2)) ^ (b & (b >>> 2)); + f = (a & (b >>> 2)) ^ (b & ((a ^ b) >>> 2)); + g ^= (a & (c >>> 2)) ^ (b & (d >>> 2)); + h ^= (b & (c >>> 2)) ^ ((a ^ b) & (d >>> 2)); + + a = e; + b = f; + c = g; + d = h; + e = (a & (a >>> 4)) ^ (b & (b >>> 4)); + f = (a & (b >>> 4)) ^ (b & ((a ^ b) >>> 4)); + g ^= (a & (c >>> 4)) ^ (b & (d >>> 4)); + h ^= (b & (c >>> 4)) ^ ((a ^ b) & (d >>> 4)); + + a = e; + b = f; + c = g; + d = h; + g ^= (a & (c >>> 8)) ^ (b & (d >>> 8)); + h ^= (b & (c >>> 8)) ^ ((a ^ b) & (d >>> 8)); + + a = (g ^ (g >>> 1)); + b = (h ^ (h >>> 1)); + + int i0 = (x ^ y); + int i1 = (b | (0x0000FFFF ^ (i0 | a))); + + i0 = (i0 | (i0 << 8)) & 0x00FF00FF; + i0 = (i0 | (i0 << 4)) & 0x0F0F0F0F; + i0 = (i0 | (i0 << 2)) & 0x33333333; + i0 = (i0 | (i0 << 1)) & 0x55555555; + + i1 = (i1 | (i1 << 8)) & 0x00FF00FF; + i1 = (i1 | (i1 << 4)) & 0x0F0F0F0F; + i1 = (i1 | (i1 << 2)) & 0x33333333; + i1 = (i1 | (i1 << 1)) & 0x55555555; + + return (((long) ((i1 << 1) | i0)) << 32) >>> 32; + } +} diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/GeometrySerde.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/EsriGeometrySerde.java similarity index 92% rename from presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/GeometrySerde.java rename to presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/EsriGeometrySerde.java index 1f51a909acbd2..536c81089b0e2 100644 --- a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/GeometrySerde.java +++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/EsriGeometrySerde.java @@ -31,6 +31,7 @@ import com.esri.core.geometry.ogc.OGCPoint; import com.esri.core.geometry.ogc.OGCPolygon; import com.facebook.presto.geospatial.GeometryType; +import com.google.common.annotations.VisibleForTesting; import io.airlift.slice.BasicSliceInput; import io.airlift.slice.DynamicSliceOutput; import io.airlift.slice.Slice; @@ -51,9 +52,9 @@ import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; -public class GeometrySerde +public class EsriGeometrySerde { - private GeometrySerde() {} + private EsriGeometrySerde() {} public static Slice serialize(OGCGeometry input) { @@ -66,13 +67,9 @@ public static Slice serialize(OGCGeometry input) public static Slice serialize(Envelope envelope) { requireNonNull(envelope, "envelope is null"); - verify(!envelope.isEmpty()); DynamicSliceOutput output = new DynamicSliceOutput(100); output.appendByte(GeometrySerializationType.ENVELOPE.code()); - output.appendDouble(envelope.getXMin()); - output.appendDouble(envelope.getYMin()); - output.appendDouble(envelope.getXMax()); - output.appendDouble(envelope.getYMax()); + writeEnvelopeCoordinates(output, envelope); return output.slice(); } @@ -218,7 +215,8 @@ private static OGCGeometry readSimpleGeometry(BasicSliceInput input, Slice input return createFromEsriGeometry(esriGeometry, type.geometryType().isMultitype()); } - private static OGCGeometry createFromEsriGeometry(Geometry geometry, boolean multiType) + @VisibleForTesting + static OGCGeometry createFromEsriGeometry(Geometry geometry, boolean multiType) { Geometry.Type type = geometry.getType(); switch (type) { @@ -307,7 +305,7 @@ private static Envelope getEnvelope(BasicSliceInput input, GeometrySerialization private static Envelope getGeometryCollectionOverallEnvelope(BasicSliceInput input) { - Envelope overallEnvelope = null; + Envelope overallEnvelope = new Envelope(); while (input.available() > 0) { int length = input.readInt() - 1; GeometrySerializationType type = GeometrySerializationType.getForCode(input.readByte()); @@ -321,20 +319,12 @@ private static Envelope getSimpleGeometryEnvelope(BasicSliceInput input, int len { // skip type injected by esri input.readInt(); - - double xMin = input.readDouble(); - double yMin = input.readDouble(); - double xMax = input.readDouble(); - double yMax = input.readDouble(); + Envelope envelope = readEnvelope(input); int skipLength = length - (4 * Double.BYTES) - Integer.BYTES; verify(input.skip(skipLength) == skipLength); - if (isEsriNaN(xMin) || isEsriNaN(yMin) || isEsriNaN(xMax) || isEsriNaN(yMax)) { - // TODO: isn't it better to return empty envelope instead? - return null; - } - return new Envelope(xMin, yMin, xMax, yMax); + return envelope; } private static Envelope getPointEnvelope(BasicSliceInput input) @@ -342,8 +332,7 @@ private static Envelope getPointEnvelope(BasicSliceInput input) double x = input.readDouble(); double y = input.readDouble(); if (isNaN(x) || isNaN(y)) { - // TODO: isn't it better to return empty envelope instead? - return null; + return new Envelope(); } return new Envelope(x, y, x, y); } @@ -355,9 +344,28 @@ private static Envelope readEnvelope(SliceInput input) double yMin = input.readDouble(); double xMax = input.readDouble(); double yMax = input.readDouble(); + if (isEsriNaN(xMin) || isEsriNaN(yMin) || isEsriNaN(xMin) || isEsriNaN(yMin)) { + return new Envelope(); + } return new Envelope(xMin, yMin, xMax, yMax); } + private static void writeEnvelopeCoordinates(DynamicSliceOutput output, Envelope envelope) + { + if (envelope.isEmpty()) { + output.appendDouble(NaN); + output.appendDouble(NaN); + output.appendDouble(NaN); + output.appendDouble(NaN); + } + else { + output.appendDouble(envelope.getXMin()); + output.appendDouble(envelope.getYMin()); + output.appendDouble(envelope.getXMax()); + output.appendDouble(envelope.getYMax()); + } + } + @Nullable private static Envelope merge(@Nullable Envelope left, @Nullable Envelope right) { diff --git a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/JtsGeometrySerde.java b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/JtsGeometrySerde.java index e8a3e05100481..420748c48d267 100644 --- a/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/JtsGeometrySerde.java +++ b/presto-geospatial-toolkit/src/main/java/com/facebook/presto/geospatial/serde/JtsGeometrySerde.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.List; +import static com.facebook.presto.geospatial.GeometryUtils.isEsriNaN; import static com.facebook.presto.geospatial.GeometryUtils.translateToAVNaN; import static com.google.common.base.Verify.verify; import static com.google.common.collect.Iterables.getOnlyElement; @@ -216,6 +217,10 @@ private static Geometry readEnvelope(SliceInput input) double xMax = input.readDouble(); double yMax = input.readDouble(); + if (isEsriNaN(xMin) || isEsriNaN(yMin) || isEsriNaN(xMax) || isEsriNaN(yMax)) { + return GEOMETRY_FACTORY.createPolygon(); + } + Coordinate[] coordinates = new Coordinate[5]; coordinates[0] = new Coordinate(xMin, yMin); coordinates[1] = new Coordinate(xMin, yMax); @@ -275,6 +280,8 @@ private static void writeGeometry(Geometry geometry, DynamicSliceOutput output) writeMultiPoint((MultiPoint) geometry, output); break; case "LineString": + case "LinearRing": + // LinearRings are a subclass of LineString writePolyline(geometry, output, false); break; case "MultiLineString": diff --git a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/TestGeometryUtils.java b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/TestGeometryUtils.java new file mode 100644 index 0000000000000..3b282be7b9b92 --- /dev/null +++ b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/TestGeometryUtils.java @@ -0,0 +1,72 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.geospatial; + +import com.esri.core.geometry.ogc.OGCGeometry; +import org.locationtech.jts.geom.Envelope; +import org.testng.annotations.Test; + +import static com.facebook.presto.geospatial.GeometryUtils.getExtent; +import static org.testng.Assert.assertEquals; + +public class TestGeometryUtils +{ + @Test + public void testGetJtsEnvelope() + { + assertJtsEnvelope( + "MULTIPOLYGON EMPTY", + new Envelope()); + assertJtsEnvelope( + "POINT (-23.4 12.2)", + new Envelope(-23.4, -23.4, 12.2, 12.2)); + assertJtsEnvelope( + "LINESTRING (-75.9375 23.6359, -75.9375 23.6364)", + new Envelope(-75.9375, -75.9375, 23.6359, 23.6364)); + assertJtsEnvelope( + "GEOMETRYCOLLECTION (" + + " LINESTRING (-75.9375 23.6359, -75.9375 23.6364)," + + " MULTIPOLYGON (((-75.9375 23.45520, -75.9371 23.4554, -75.9375 23.46023325, -75.9375 23.45520)))" + + ")", + new Envelope(-75.9375, -75.9371, 23.4552, 23.6364)); + } + + private void assertJtsEnvelope(String wkt, Envelope expected) + { + Envelope calculated = GeometryUtils.getJtsEnvelope(OGCGeometry.fromText(wkt)); + assertEquals(calculated, expected); + } + + @Test + public void testGetExtent() + { + assertGetExtent( + "POINT (-23.4 12.2)", + new Rectangle(-23.4, 12.2, -23.4, 12.2)); + assertGetExtent( + "LINESTRING (-75.9375 23.6359, -75.9375 23.6364)", + new Rectangle(-75.9375, 23.6359, -75.9375, 23.6364)); + assertGetExtent( + "GEOMETRYCOLLECTION (" + + " LINESTRING (-75.9375 23.6359, -75.9375 23.6364)," + + " MULTIPOLYGON (((-75.9375 23.45520, -75.9371 23.4554, -75.9375 23.46023325, -75.9375 23.45520)))" + + ")", + new Rectangle(-75.9375, 23.4552, -75.9371, 23.6364)); + } + + private void assertGetExtent(String wkt, Rectangle expected) + { + assertEquals(getExtent(OGCGeometry.fromText(wkt)), expected); + } +} diff --git a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkFlatbushBuild.java b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkFlatbushBuild.java new file mode 100644 index 0000000000000..efd4945eaeef3 --- /dev/null +++ b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkFlatbushBuild.java @@ -0,0 +1,97 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.geospatial.rtree; + +import com.facebook.presto.geospatial.Rectangle; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.VerboseMode; + +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import static com.facebook.presto.geospatial.rtree.RtreeTestUtils.makeRectangles; + +@State(Scope.Thread) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Fork(2) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 2, timeUnit = TimeUnit.SECONDS) +@BenchmarkMode(Mode.AverageTime) +public class BenchmarkFlatbushBuild +{ + private static final int SEED = 613; + + @Benchmark + @OperationsPerInvocation(1) + public void buildRtree(BenchmarkData data, Blackhole blackhole) + { + blackhole.consume(new Flatbush<>( + data.getBuildRectangles().toArray(new Rectangle[] {}), + data.getRtreeDegree())); + } + + @State(Scope.Thread) + public static class BenchmarkData + { + @Param({"8", "16", "32"}) + private int rtreeDegree; + @Param({"1000", "3000", "10000", "30000", "100000", "300000", "1000000"}) + private int numBuildRectangles; + + private List buildRectangles; + + @Setup + public void setup() + { + Random random = new Random(SEED); + buildRectangles = makeRectangles(random, numBuildRectangles); + } + + public int getRtreeDegree() + { + return rtreeDegree; + } + + public List getBuildRectangles() + { + return buildRectangles; + } + } + + public static void main(String[] args) + throws Throwable + { + Options options = new OptionsBuilder() + .verbosity(VerboseMode.NORMAL) + .include(".*" + BenchmarkFlatbushBuild.class.getSimpleName() + ".*") + .build(); + new Runner(options).run(); + } +} diff --git a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkFlatbushQuery.java b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkFlatbushQuery.java new file mode 100644 index 0000000000000..9b60033c830d2 --- /dev/null +++ b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkFlatbushQuery.java @@ -0,0 +1,105 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.geospatial.rtree; + +import com.facebook.presto.geospatial.Rectangle; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.VerboseMode; + +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import static com.facebook.presto.geospatial.rtree.RtreeTestUtils.makeRectangles; + +@State(Scope.Thread) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(2) +@Warmup(iterations = 3, time = 2, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +@BenchmarkMode(Mode.AverageTime) +public class BenchmarkFlatbushQuery +{ + private static final int SEED = 613; + private static final int NUM_PROBE_RECTANGLES = 1000; + + @Benchmark + @OperationsPerInvocation(NUM_PROBE_RECTANGLES) + public void rtreeQuery(BenchmarkData data, Blackhole blackhole) + { + for (Rectangle query : data.getProbeRectangles()) { + data.getRtree().findIntersections(query, blackhole::consume); + } + } + + @State(Scope.Thread) + public static class BenchmarkData + { + @Param({"8", "16", "32"}) + private int rtreeDegree; + @Param({"1000", "3000", "10000", "30000", "100000", "300000", "1000000"}) + private int numBuildRectangles; + + private List probeRectangles; + private Flatbush rtree; + + @Setup + public void setup() + { + Random random = new Random(SEED); + probeRectangles = makeRectangles(random, NUM_PROBE_RECTANGLES); + rtree = buildRtree(makeRectangles(random, numBuildRectangles)); + } + + public List getProbeRectangles() + { + return probeRectangles; + } + + public Flatbush getRtree() + { + return rtree; + } + + private Flatbush buildRtree(List rectangles) + { + return new Flatbush<>(rectangles.toArray(new Rectangle[] {}), rtreeDegree); + } + } + + public static void main(String[] args) + throws Throwable + { + Options options = new OptionsBuilder() + .verbosity(VerboseMode.NORMAL) + .include(".*" + BenchmarkFlatbushQuery.class.getSimpleName() + ".*") + .build(); + new Runner(options).run(); + } +} diff --git a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkJtsStrTreeBuild.java b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkJtsStrTreeBuild.java new file mode 100644 index 0000000000000..3ec768fb2f9af --- /dev/null +++ b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkJtsStrTreeBuild.java @@ -0,0 +1,96 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.geospatial.rtree; + +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.index.strtree.STRtree; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.VerboseMode; + +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static com.facebook.presto.geospatial.rtree.RtreeTestUtils.makeRectangles; + +@State(Scope.Thread) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Fork(2) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 2, timeUnit = TimeUnit.SECONDS) +@BenchmarkMode(Mode.AverageTime) +public class BenchmarkJtsStrTreeBuild +{ + private static final int SEED = 613; + + @Benchmark + @OperationsPerInvocation(1) + public void buildRtree(BenchmarkData data, Blackhole blackhole) + { + STRtree rtree = new STRtree(); + for (Envelope envelope : data.getBuildEnvelopes()) { + rtree.insert(envelope, envelope); + } + rtree.build(); + blackhole.consume(rtree); + } + + @State(Scope.Thread) + public static class BenchmarkData + { + @Param({"1000", "3000", "10000", "30000", "100000", "300000", "1000000"}) + private int numBuildEnvelopes; + + private List buildEnvelopes; + + @Setup + public void setup() + { + buildEnvelopes = makeRectangles(new Random(SEED), numBuildEnvelopes).stream() + .map(rectangle -> new Envelope(rectangle.getXMin(), rectangle.getXMax(), rectangle.getYMin(), rectangle.getYMax())) + .collect(Collectors.toList()); + } + + public List getBuildEnvelopes() + { + return buildEnvelopes; + } + } + + public static void main(String[] args) + throws Throwable + { + Options options = new OptionsBuilder() + .verbosity(VerboseMode.NORMAL) + .include(".*" + BenchmarkJtsStrTreeBuild.class.getSimpleName() + ".*") + .build(); + new Runner(options).run(); + } +} diff --git a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkJtsStrTreeQuery.java b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkJtsStrTreeQuery.java new file mode 100644 index 0000000000000..1f9dd08a64055 --- /dev/null +++ b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/BenchmarkJtsStrTreeQuery.java @@ -0,0 +1,115 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.geospatial.rtree; + +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.index.strtree.STRtree; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.VerboseMode; + +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static com.facebook.presto.geospatial.rtree.RtreeTestUtils.makeRectangles; + +@State(Scope.Thread) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(2) +@Warmup(iterations = 3, time = 2, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +@BenchmarkMode(Mode.AverageTime) +public class BenchmarkJtsStrTreeQuery +{ + private static final int SEED = 613; + private static final int NUM_PROBE_RECTANGLES = 1000; + + @Benchmark + @OperationsPerInvocation(NUM_PROBE_RECTANGLES) + public void rtreeQuery(BenchmarkData data, Blackhole blackhole) + { + for (Envelope query : data.getProbeEnvs()) { + data.getRtree().query(query, blackhole::consume); + } + } + + @State(Scope.Thread) + public static class BenchmarkData + { + @Param({"1000", "3000", "10000", "30000", "100000", "300000", "1000000"}) + private int numBuildRectangles; + + private List probeEnvs; + private STRtree rtree; + + @Setup + public void setup() + { + Random random = new Random(SEED); + probeEnvs = makeRectangles(random, NUM_PROBE_RECTANGLES).stream() + .map(rect -> new Envelope(rect.getXMin(), rect.getXMax(), rect.getYMin(), rect.getYMax())) + .collect(Collectors.toList()); + List buildEnvs = makeRectangles(random, numBuildRectangles).stream() + .map(rect -> new Envelope(rect.getXMin(), rect.getXMax(), rect.getYMin(), rect.getYMax())) + .collect(Collectors.toList()); + rtree = buildRtree(buildEnvs); + } + + public List getProbeEnvs() + { + return probeEnvs; + } + + public STRtree getRtree() + { + return rtree; + } + + private STRtree buildRtree(List envs) + { + STRtree rtree = new STRtree(); + for (Envelope env : envs) { + rtree.insert(env, env); + } + rtree.build(); + return rtree; + } + } + + public static void main(String[] args) + throws Throwable + { + Options options = new OptionsBuilder() + .verbosity(VerboseMode.NORMAL) + .include(".*" + BenchmarkJtsStrTreeQuery.class.getSimpleName() + ".*") + .build(); + new Runner(options).run(); + } +} diff --git a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/RtreeTestUtils.java b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/RtreeTestUtils.java new file mode 100644 index 0000000000000..e527511602d34 --- /dev/null +++ b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/RtreeTestUtils.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.geospatial.rtree; + +import com.facebook.presto.geospatial.Rectangle; + +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public final class RtreeTestUtils +{ + private RtreeTestUtils() {} + + public static List makeRectangles(Random random, int numRectangles) + { + return IntStream.range(0, numRectangles) + .mapToObj(i -> makeRectangle(random)) + .collect(Collectors.toList()); + } + + /* + * Make a random rectangle at a random origin of size < 10. + */ + private static Rectangle makeRectangle(Random random) + { + double minX = randomDouble(random, -100, 100); + double minY = randomDouble(random, -100, 100); + double sizeX = randomDouble(random, 0.0, 10); + double sizeY = randomDouble(random, 0.0, 10); + return new Rectangle(minX, minY, minX + sizeX, minY + sizeY); + } + + private static double randomDouble(Random random, double min, double max) + { + return min + random.nextDouble() * (max - min); + } +} diff --git a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/TestFlatbush.java b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/TestFlatbush.java new file mode 100644 index 0000000000000..8ae8687057dcd --- /dev/null +++ b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/TestFlatbush.java @@ -0,0 +1,279 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.geospatial.rtree; + +import com.esri.core.geometry.Point; +import com.esri.core.geometry.ogc.OGCGeometry; +import com.esri.core.geometry.ogc.OGCPoint; +import com.facebook.presto.geospatial.GeometryUtils; +import com.facebook.presto.geospatial.Rectangle; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Random; + +import static com.facebook.presto.geospatial.rtree.Flatbush.ENVELOPE_SIZE; +import static com.facebook.presto.geospatial.rtree.RtreeTestUtils.makeRectangles; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.Double.NEGATIVE_INFINITY; +import static java.lang.Double.POSITIVE_INFINITY; +import static java.util.stream.Collectors.toList; +import static org.testng.Assert.assertEquals; + +public class TestFlatbush +{ + private static final Rectangle EVERYTHING = new Rectangle(NEGATIVE_INFINITY, NEGATIVE_INFINITY, POSITIVE_INFINITY, POSITIVE_INFINITY); + + private static final Comparator RECTANGLE_COMPARATOR = Comparator + .comparing(Rectangle::getXMin) + .thenComparing(Rectangle::getYMin) + .thenComparing(Rectangle::getXMax) + .thenComparing(Rectangle::getYMax); + + // 2 intersecting polygons: A and B + private static final OGCGeometry POLYGON_A = OGCGeometry.fromText("POLYGON ((0 0, -0.5 2.5, 0 5, 2.5 5.5, 5 5, 5.5 2.5, 5 0, 2.5 -0.5, 0 0))"); + private static final OGCGeometry POLYGON_B = OGCGeometry.fromText("POLYGON ((4 4, 3.5 7, 4 10, 7 10.5, 10 10, 10.5 7, 10 4, 7 3.5, 4 4))"); + + // A set of points: X in A, Y in A and B, Z in B, W outside of A and B + private static final OGCGeometry POINT_X = new OGCPoint(new Point(1.0, 1.0), null); + private static final OGCGeometry POINT_Y = new OGCPoint(new Point(4.5, 4.5), null); + private static final OGCGeometry POINT_Z = new OGCPoint(new Point(6.0, 6.0), null); + private static final OGCGeometry POINT_W = new OGCPoint(new Point(20.0, 20.0), null); + + @Test + public void testEmptyFlatbush() + { + Flatbush rtree = new Flatbush<>(new Rectangle[] {}); + assertEquals(findIntersections(rtree, EVERYTHING), ImmutableList.of()); + } + + @Test + public void testSingletonFlatbush() + { + List items = ImmutableList.of(new Rectangle(0, 0, 1, 1)); + Flatbush rtree = new Flatbush<>(items.toArray(new Rectangle[] {})); + + assertEquals(findIntersections(rtree, EVERYTHING), items); + // hit + assertEquals(findIntersections(rtree, new Rectangle(1, 1, 2, 2)), items); + // miss + assertEquals(findIntersections(rtree, new Rectangle(-1, -1, -0.1, -0.1)), ImmutableList.of()); + } + + @Test + public void testSingletonFlatbushXY() + { + // Because mixing up x and y is easy to do... + List items = ImmutableList.of(new Rectangle(0, 10, 1, 11)); + Flatbush rtree = new Flatbush<>(items.toArray(new Rectangle[] {})); + + // hit + assertEquals(findIntersections(rtree, new Rectangle(1, 11, 2, 12)), items); + // miss + assertEquals(findIntersections(rtree, new Rectangle(11, 1, 12, 2)), ImmutableList.of()); + } + + @Test + public void testDoubletonFlatbush() + { + // This is the smallest Rtree with height > 1 + // Also test for some degeneracies + Rectangle rect0 = new Rectangle(1, 1, 1, 1); + Rectangle rect1 = new Rectangle(-1, -2, -1, -1); + List items = ImmutableList.of(rect0, rect1); + + Flatbush rtree = new Flatbush<>(items.toArray(new Rectangle[] {})); + + List allResults = findIntersections(rtree, EVERYTHING); + assertEqualsSorted(allResults, items, RECTANGLE_COMPARATOR); + + assertEquals(findIntersections(rtree, new Rectangle(1, 1, 2, 2)), ImmutableList.of(rect0)); + assertEquals(findIntersections(rtree, new Rectangle(-2, -2, -1, -2)), ImmutableList.of(rect1)); + // This should test missing at the root level + assertEquals(findIntersections(rtree, new Rectangle(10, 10, 12, 12)), ImmutableList.of()); + // This should test missing at the leaf level + assertEquals(findIntersections(rtree, new Rectangle(0, 0, 0, 0)), ImmutableList.of()); + } + + @Test + public void testTwoLevelFlatbush() + { + // This is the smallest Rtree with height > 2 + // Also test for NaN behavior + Rectangle rect0 = new Rectangle(1, 1, 1, 1); + Rectangle rect1 = new Rectangle(-1, -1, -1, -1); + Rectangle rect2 = new Rectangle(1, -1, 1, -1); + List items = ImmutableList.of(rect0, rect1, rect2); + + Flatbush rtree = new Flatbush<>(items.toArray(new Rectangle[] {}), 2); + + List allResults = findIntersections(rtree, EVERYTHING); + assertEqualsSorted(allResults, items, RECTANGLE_COMPARATOR); + + assertEquals(findIntersections(rtree, new Rectangle(1, 1, 1, 1)), ImmutableList.of(rect0)); + assertEquals(findIntersections(rtree, new Rectangle(-1, -1, -1, -1)), ImmutableList.of(rect1)); + assertEquals(findIntersections(rtree, new Rectangle(1, -1, 1, -1)), ImmutableList.of(rect2)); + // Test hitting across parent nodes + List results12 = findIntersections(rtree, new Rectangle(-1, -1, 1, -1)); + assertEqualsSorted(results12, ImmutableList.of(rect1, rect2), RECTANGLE_COMPARATOR); + + // This should test missing at the root level + assertEquals(findIntersections(rtree, new Rectangle(10, 10, 12, 12)), ImmutableList.of()); + // This should test missing at the leaf level + assertEquals(findIntersections(rtree, new Rectangle(0, 0, 0, 0)), ImmutableList.of()); + } + + @Test + public void testOctagonQuery() + { + OGCGeometryWrapper octagonA = new OGCGeometryWrapper(POLYGON_A); + OGCGeometryWrapper octagonB = new OGCGeometryWrapper(POLYGON_B); + + OGCGeometryWrapper pointX = new OGCGeometryWrapper(POINT_X); + OGCGeometryWrapper pointY = new OGCGeometryWrapper(POINT_Y); + OGCGeometryWrapper pointZ = new OGCGeometryWrapper(POINT_Z); + OGCGeometryWrapper pointW = new OGCGeometryWrapper(POINT_W); + + Flatbush rtree = new Flatbush<>(new OGCGeometryWrapper[] {pointX, pointY, pointZ, pointW}); + + List resultsA = findIntersections(rtree, octagonA.getExtent()); + assertEqualsSorted(resultsA, ImmutableList.of(pointX, pointY), Comparator.naturalOrder()); + + List resultsB = findIntersections(rtree, octagonB.getExtent()); + assertEqualsSorted(resultsB, ImmutableList.of(pointY, pointZ), Comparator.naturalOrder()); + } + + @Test + public void testOctagonTree() + { + OGCGeometryWrapper octagonA = new OGCGeometryWrapper(POLYGON_A); + OGCGeometryWrapper octagonB = new OGCGeometryWrapper(POLYGON_B); + + OGCGeometryWrapper pointX = new OGCGeometryWrapper(POINT_X); + OGCGeometryWrapper pointY = new OGCGeometryWrapper(POINT_Y); + OGCGeometryWrapper pointZ = new OGCGeometryWrapper(POINT_Z); + OGCGeometryWrapper pointW = new OGCGeometryWrapper(POINT_W); + + Flatbush rtree = new Flatbush<>(new OGCGeometryWrapper[] {octagonA, octagonB}); + + assertEquals(findIntersections(rtree, pointX.getExtent()), ImmutableList.of(octagonA)); + + List results = findIntersections(rtree, pointY.getExtent()); + assertEqualsSorted(results, ImmutableList.of(octagonA, octagonB), Comparator.naturalOrder()); + + assertEquals(findIntersections(rtree, pointZ.getExtent()), ImmutableList.of(octagonB)); + + assertEquals(findIntersections(rtree, pointW.getExtent()), ImmutableList.of()); + } + + @DataProvider(name = "rectangle-counts") + private Object[][] rectangleCounts() + { + return new Object[][] {{100, 1000, 42}, {1000, 10_000, 123}, {5000, 50_000, 321}}; + } + + @Test(dataProvider = "rectangle-counts") + public void testRectangleCollection(int numBuildRectangles, int numProbeRectangles, int seed) + { + Random random = new Random(seed); + List buildRectangles = makeRectangles(random, numBuildRectangles); + List probeRectangles = makeRectangles(random, numProbeRectangles); + + Flatbush rtree = new Flatbush<>(buildRectangles.toArray(new Rectangle[] {})); + for (Rectangle query : probeRectangles) { + List actual = findIntersections(rtree, query); + List expected = buildRectangles.stream() + .filter(rect -> rect.intersects(query)) + .collect(toList()); + assertEqualsSorted(actual, expected, RECTANGLE_COMPARATOR); + } + } + + @Test + public void testChildrenOffsets() + { + int numRectangles = 10; + int degree = 8; + Random random = new Random(122); + + int firstParentIndex = 2 * degree * ENVELOPE_SIZE; + int secondParentIndex = firstParentIndex + ENVELOPE_SIZE; + int grandparentIndex = 3 * degree * ENVELOPE_SIZE; + + List rectangles = makeRectangles(random, numRectangles); + Flatbush rtree = new Flatbush<>(rectangles.toArray(new Rectangle[] {}), degree); + assertEquals(rtree.getHeight(), 3); + assertEquals(rtree.getChildrenOffset(firstParentIndex, 1), 0); + assertEquals(rtree.getChildrenOffset(secondParentIndex, 1), degree * ENVELOPE_SIZE); + assertEquals(rtree.getChildrenOffset(grandparentIndex, 2), 2 * degree * ENVELOPE_SIZE); + } + + private static List findIntersections(Flatbush rtree, Rectangle rectangle) + { + List results = new ArrayList<>(); + rtree.findIntersections(rectangle, results::add); + return results; + } + + /* + * Asserts the two lists of Rectangles are equal after sorting. + */ + private static void assertEqualsSorted(List actual, List expected, Comparator comparator) + { + List actualSorted = actual.stream().sorted(comparator).collect(toImmutableList()); + List expectedSorted = expected.stream().sorted(comparator).collect(toImmutableList()); + + assertEquals(actualSorted, expectedSorted); + } + + private static final class OGCGeometryWrapper + implements HasExtent, Comparable + { + private final OGCGeometry geometry; + private final Rectangle extent; + + public OGCGeometryWrapper(OGCGeometry geometry) + { + this.geometry = geometry; + this.extent = GeometryUtils.getExtent(geometry); + } + + public OGCGeometry getGeometry() + { + return geometry; + } + + @Override + public Rectangle getExtent() + { + return extent; + } + + @Override + public long getEstimatedSizeInBytes() + { + return geometry.estimateMemorySize(); + } + + @Override + public int compareTo(OGCGeometryWrapper other) + { + return RECTANGLE_COMPARATOR.compare(this.getExtent(), other.getExtent()); + } + } +} diff --git a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/TestHilbertIndex.java b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/TestHilbertIndex.java new file mode 100644 index 0000000000000..7ef288b403c52 --- /dev/null +++ b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/rtree/TestHilbertIndex.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.geospatial.rtree; + +import com.facebook.presto.geospatial.Rectangle; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestHilbertIndex +{ + @Test + public void testOrder() + { + HilbertIndex hilbert = new HilbertIndex(new Rectangle(0, 0, 4, 4)); + long h0 = hilbert.indexOf(0., 0.); + long h1 = hilbert.indexOf(1., 1.); + long h2 = hilbert.indexOf(1., 3.); + long h3 = hilbert.indexOf(3., 3.); + long h4 = hilbert.indexOf(3., 1.); + assertTrue(h0 < h1); + assertTrue(h1 < h2); + assertTrue(h2 < h3); + assertTrue(h3 < h4); + } + + @Test + public void testOutOfBounds() + { + HilbertIndex hilbert = new HilbertIndex(new Rectangle(0, 0, 1, 1)); + assertEquals(hilbert.indexOf(2., 2.), Long.MAX_VALUE); + } + + @Test + public void testDegenerateRectangle() + { + HilbertIndex hilbert = new HilbertIndex(new Rectangle(0, 0, 0, 0)); + assertEquals(hilbert.indexOf(0., 0.), 0); + assertEquals(hilbert.indexOf(2., 2.), Long.MAX_VALUE); + } + + @Test + public void testDegenerateHorizontalRectangle() + { + HilbertIndex hilbert = new HilbertIndex(new Rectangle(0, 0, 4, 0)); + assertEquals(hilbert.indexOf(0., 0.), 0); + assertTrue(hilbert.indexOf(1., 0.) < hilbert.indexOf(2., 0.)); + assertEquals(hilbert.indexOf(0., 2.), Long.MAX_VALUE); + assertEquals(hilbert.indexOf(2., 2.), Long.MAX_VALUE); + } + + @Test + public void testDegenerateVerticalRectangle() + { + HilbertIndex hilbert = new HilbertIndex(new Rectangle(0, 0, 0, 4)); + assertEquals(hilbert.indexOf(0., 0.), 0); + assertTrue(hilbert.indexOf(0., 1.) < hilbert.indexOf(0., 2.)); + assertEquals(hilbert.indexOf(2., 0.), Long.MAX_VALUE); + assertEquals(hilbert.indexOf(2., 2.), Long.MAX_VALUE); + } +} diff --git a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/serde/BenchmarkGeometrySerde.java b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/serde/BenchmarkGeometrySerde.java index a8f8e1432ffb8..0dd0ecb987ba6 100644 --- a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/serde/BenchmarkGeometrySerde.java +++ b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/serde/BenchmarkGeometrySerde.java @@ -40,9 +40,9 @@ import static com.facebook.presto.geospatial.serde.BenchmarkGeometrySerializationData.POINT; import static com.facebook.presto.geospatial.serde.BenchmarkGeometrySerializationData.POLYGON; import static com.facebook.presto.geospatial.serde.BenchmarkGeometrySerializationData.readResource; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserialize; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserializeEnvelope; -import static com.facebook.presto.geospatial.serde.GeometrySerde.serialize; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserialize; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserializeEnvelope; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.serialize; import static java.util.concurrent.TimeUnit.SECONDS; import static org.openjdk.jmh.annotations.Mode.Throughput; diff --git a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/serde/TestGeometrySerialization.java b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/serde/TestGeometrySerialization.java index 5da02a7bd1a87..fe90756fe84dd 100644 --- a/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/serde/TestGeometrySerialization.java +++ b/presto-geospatial-toolkit/src/test/java/com/facebook/presto/geospatial/serde/TestGeometrySerialization.java @@ -21,11 +21,11 @@ import org.locationtech.jts.io.WKTReader; import org.testng.annotations.Test; -import static com.esri.core.geometry.ogc.OGCGeometry.createFromEsriGeometry; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserialize; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserializeEnvelope; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserializeType; -import static com.facebook.presto.geospatial.serde.GeometrySerde.serialize; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.createFromEsriGeometry; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserialize; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserializeEnvelope; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserializeType; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.serialize; import static com.facebook.presto.geospatial.serde.GeometrySerializationType.ENVELOPE; import static com.facebook.presto.geospatial.serde.GeometrySerializationType.GEOMETRY_COLLECTION; import static com.facebook.presto.geospatial.serde.GeometrySerializationType.LINE_STRING; @@ -83,7 +83,6 @@ public void testPolygon() testSerialization("POLYGON ((30 10, 40 40, 20 40, 30 10))"); testSerialization("POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))"); testSerialization("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))"); - testSerialization("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))"); testSerialization("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))"); testSerialization("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0), (0.75 0.25, 0.75 0.75, 0.25 0.75, 0.25 0.25, 0.75 0.25))"); testSerialization("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0), (0.25 0.25, 0.25 0.75, 0.75 0.75, 0.75 0.25, 0.25 0.25))"); @@ -134,6 +133,7 @@ public void testGeometryCollection() @Test public void testEnvelope() { + testEnvelopeSerialization(new Envelope()); testEnvelopeSerialization(new Envelope(0, 0, 1, 1)); testEnvelopeSerialization(new Envelope(1, 2, 3, 4)); testEnvelopeSerialization(new Envelope(10101, -2.05, -3e5, 0)); @@ -141,9 +141,9 @@ public void testEnvelope() private void testEnvelopeSerialization(Envelope envelope) { - assertEquals(deserialize(serialize(envelope)), createFromEsriGeometry(envelope, null)); + assertEquals(deserialize(serialize(envelope)), createFromEsriGeometry(envelope, false)); assertEquals(deserializeEnvelope(serialize(envelope)), envelope); - assertEquals(JtsGeometrySerde.serialize(JtsGeometrySerde.deserialize(serialize(envelope))), serialize(createFromEsriGeometry(envelope, null))); + assertEquals(JtsGeometrySerde.serialize(JtsGeometrySerde.deserialize(serialize(envelope))), serialize(createFromEsriGeometry(envelope, false))); } @Test @@ -154,9 +154,9 @@ public void testDeserializeEnvelope() assertDeserializeEnvelope("POLYGON ((0 0, 0 4, 4 0))", new Envelope(0, 0, 4, 4)); assertDeserializeEnvelope("MULTIPOLYGON (((0 0 , 0 2, 2 2, 2 0)), ((2 2, 2 4, 4 4, 4 2)))", new Envelope(0, 0, 4, 4)); assertDeserializeEnvelope("GEOMETRYCOLLECTION (POINT (3 7), LINESTRING (4 6, 7 10))", new Envelope(3, 6, 7, 10)); - assertDeserializeEnvelope("POLYGON EMPTY", null); + assertDeserializeEnvelope("POLYGON EMPTY", new Envelope()); assertDeserializeEnvelope("POINT (1 2)", new Envelope(1, 2, 1, 2)); - assertDeserializeEnvelope("POINT EMPTY", null); + assertDeserializeEnvelope("POINT EMPTY", new Envelope()); assertDeserializeEnvelope("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (2 7), LINESTRING (4 6, 7 10)), POINT (3 7), LINESTRING (4 6, 7 10))", new Envelope(2, 6, 7, 10)); } @@ -185,6 +185,7 @@ private static void testSerialization(String wkt) { testEsriSerialization(wkt); testJtsSerialization(wkt); + testCrossSerialization(wkt); } private static void testEsriSerialization(String wkt) @@ -195,19 +196,24 @@ private static void testEsriSerialization(String wkt) } private static void testJtsSerialization(String wkt) + { + Geometry expected = createJtsGeometry(wkt); + Geometry actual = JtsGeometrySerde.deserialize(JtsGeometrySerde.serialize(expected)); + assertGeometryEquals(actual, expected); + } + + private static void testCrossSerialization(String wkt) { Geometry jtsGeometry = createJtsGeometry(wkt); OGCGeometry esriGeometry = OGCGeometry.fromText(wkt); Slice jtsSerialized = JtsGeometrySerde.serialize(jtsGeometry); - Slice esriSerialized = GeometrySerde.serialize(esriGeometry); - assertEquals(jtsSerialized, esriSerialized); - - Geometry jtsDeserialized = JtsGeometrySerde.deserialize(jtsSerialized); - assertGeometryEquals(jtsDeserialized, jtsGeometry); + Slice esriSerialized = EsriGeometrySerde.serialize(esriGeometry); - OGCGeometry esriDeserialized = GeometrySerde.deserialize(esriSerialized); - assertGeometryEquals(esriDeserialized, esriGeometry); + OGCGeometry esriFromJts = EsriGeometrySerde.deserialize(jtsSerialized); + Geometry jtsFromEsri = JtsGeometrySerde.deserialize(esriSerialized); + assertGeometryEquals(esriFromJts, esriGeometry); + assertGeometryEquals(jtsFromEsri, jtsGeometry); } private static Slice geometryFromText(String wkt) diff --git a/presto-geospatial/pom.xml b/presto-geospatial/pom.xml index 41955a8d39b04..be8dc72d3104d 100644 --- a/presto-geospatial/pom.xml +++ b/presto-geospatial/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-geospatial @@ -94,6 +94,12 @@ test + + com.facebook.presto + presto-hive-metastore + test + + com.facebook.presto presto-tpch @@ -119,19 +125,19 @@ - io.airlift + com.facebook.airlift concurrent test - io.airlift + com.facebook.airlift log-manager test - io.airlift + com.facebook.airlift log test diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTile.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTile.java index 2d098271f6337..5f0ad94430063 100644 --- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTile.java +++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTile.java @@ -33,7 +33,9 @@ public final class BingTile private BingTile(int x, int y, int zoomLevel) { - checkArgument(zoomLevel <= MAX_ZOOM_LEVEL); + checkArgument(0 <= zoomLevel && zoomLevel <= MAX_ZOOM_LEVEL); + checkArgument(0 <= x && x < (1 << zoomLevel)); + checkArgument(0 <= y && y < (1 << zoomLevel)); this.x = x; this.y = y; this.zoomLevel = zoomLevel; diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTileFunctions.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTileFunctions.java index 485a5a3ea1cd9..ddbe94e9e7585 100644 --- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTileFunctions.java +++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/BingTileFunctions.java @@ -25,6 +25,7 @@ import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.StandardTypes; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; @@ -33,8 +34,8 @@ import static com.facebook.presto.geospatial.GeometryUtils.getEnvelope; import static com.facebook.presto.geospatial.GeometryUtils.getPointCount; import static com.facebook.presto.geospatial.GeometryUtils.isPointOrRectangle; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserialize; -import static com.facebook.presto.geospatial.serde.GeometrySerde.serialize; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserialize; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.serialize; import static com.facebook.presto.plugin.geospatial.BingTile.MAX_ZOOM_LEVEL; import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY_TYPE_NAME; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; @@ -63,9 +64,12 @@ public class BingTileFunctions { private static final int TILE_PIXELS = 256; - private static final double MAX_LATITUDE = 85.05112878; + + @VisibleForTesting + static final double MAX_LATITUDE = 85.05112878; private static final double MIN_LATITUDE = -85.05112878; - private static final double MIN_LONGITUDE = -180; + @VisibleForTesting + static final double MIN_LONGITUDE = -180; private static final double MAX_LONGITUDE = 180; private static final double EARTH_RADIUS_KM = 6371.01; private static final int OPTIMIZED_TILING_MIN_ZOOM_LEVEL = 10; @@ -75,9 +79,8 @@ public class BingTileFunctions private static final String LATITUDE_SPAN_OUT_OF_RANGE = String.format("Latitude span for the geometry must be in [%.2f, %.2f] range", MIN_LATITUDE, MAX_LATITUDE); private static final String LONGITUDE_OUT_OF_RANGE = "Longitude must be between " + MIN_LONGITUDE + " and " + MAX_LONGITUDE; private static final String LONGITUDE_SPAN_OUT_OF_RANGE = String.format("Longitude span for the geometry must be in [%.2f, %.2f] range", MIN_LONGITUDE, MAX_LONGITUDE); - private static final String QUAD_KEY_EMPTY = "QuadKey must not be empty string"; private static final String QUAD_KEY_TOO_LONG = "QuadKey must be " + MAX_ZOOM_LEVEL + " characters or less"; - private static final String ZOOM_LEVEL_TOO_SMALL = "Zoom level must be > 0"; + private static final String ZOOM_LEVEL_TOO_SMALL = "Zoom level must be >= 0"; private static final String ZOOM_LEVEL_TOO_LARGE = "Zoom level must be <= " + MAX_ZOOM_LEVEL; private BingTileFunctions() {} @@ -367,7 +370,22 @@ public static Block geometryToBingTiles(@SqlType(GEOMETRY_TYPE_NAME) Slice input boolean pointOrRectangle = isPointOrRectangle(ogcGeometry, envelope); BingTile leftUpperTile = latitudeLongitudeToTile(envelope.getYMax(), envelope.getXMin(), zoomLevel); - BingTile rightLowerTile = getTileCoveringLowerRightCorner(envelope, zoomLevel); + BingTile rightLowerTile = latitudeLongitudeToTile(envelope.getYMin(), envelope.getXMax(), zoomLevel); + + // If the tile covering the lower right corner of the envelope overlaps the envelope only + // at the border then return a tile shifted to the left and/or top + int deltaX = 0; + int deltaY = 0; + Point upperLeftCorner = tileXYToLatitudeLongitude(rightLowerTile.getX(), rightLowerTile.getY(), rightLowerTile.getZoomLevel()); + if (rightLowerTile.getX() > leftUpperTile.getX() && upperLeftCorner.getX() == envelope.getXMax()) { + deltaX = -1; + } + if (rightLowerTile.getY() > leftUpperTile.getY() && upperLeftCorner.getY() == envelope.getYMin()) { + deltaY = -1; + } + if (deltaX != 0 || deltaY != 0) { + rightLowerTile = BingTile.fromCoordinates(rightLowerTile.getX() + deltaX, rightLowerTile.getY() + deltaY, rightLowerTile.getZoomLevel()); + } // XY coordinates start at (0,0) in the left upper corner and increase left to right and top to bottom long tileCount = (long) (rightLowerTile.getX() - leftUpperTile.getX() + 1) * (rightLowerTile.getY() - leftUpperTile.getY() + 1); @@ -405,29 +423,6 @@ public static Block geometryToBingTiles(@SqlType(GEOMETRY_TYPE_NAME) Slice input return blockBuilder.build(); } - private static BingTile getTileCoveringLowerRightCorner(Envelope envelope, int zoomLevel) - { - BingTile tile = latitudeLongitudeToTile(envelope.getYMin(), envelope.getXMax(), zoomLevel); - - // If the tile covering the lower right corner of the envelope overlaps the envelope only - // at the border then return a tile shifted to the left and/or top - int deltaX = 0; - int deltaY = 0; - Point upperLeftCorner = tileXYToLatitudeLongitude(tile.getX(), tile.getY(), tile.getZoomLevel()); - if (upperLeftCorner.getX() == envelope.getXMax()) { - deltaX = -1; - } - if (upperLeftCorner.getY() == envelope.getYMin()) { - deltaY = -1; - } - - if (deltaX != 0 || deltaY != 0) { - return BingTile.fromCoordinates(tile.getX() + deltaX, tile.getY() + deltaY, tile.getZoomLevel()); - } - - return tile; - } - private static void checkGeometryToBingTilesLimits(OGCGeometry ogcGeometry, Envelope envelope, boolean pointOrRectangle, long tileCount, int zoomLevel) { if (pointOrRectangle) { @@ -644,7 +639,7 @@ private static Envelope tileToEnvelope(BingTile tile) private static void checkZoomLevel(long zoomLevel) { - checkCondition(zoomLevel > 0, ZOOM_LEVEL_TOO_SMALL); + checkCondition(zoomLevel >= 0, ZOOM_LEVEL_TOO_SMALL); checkCondition(zoomLevel <= MAX_ZOOM_LEVEL, ZOOM_LEVEL_TOO_LARGE); } @@ -655,7 +650,6 @@ private static void checkCoordinate(long coordinate, long zoomLevel) private static void checkQuadKey(@SqlType(StandardTypes.VARCHAR) Slice quadkey) { - checkCondition(quadkey.length() > 0, QUAD_KEY_EMPTY); checkCondition(quadkey.length() <= MAX_ZOOM_LEVEL, QUAD_KEY_TOO_LONG); } diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeoFunctions.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeoFunctions.java index df4b74a224018..0ee404f93c780 100644 --- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeoFunctions.java +++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeoFunctions.java @@ -17,7 +17,6 @@ import com.esri.core.geometry.GeometryCursor; import com.esri.core.geometry.ListeningGeometryCursor; import com.esri.core.geometry.MultiPath; -import com.esri.core.geometry.MultiPoint; import com.esri.core.geometry.MultiVertexGeometry; import com.esri.core.geometry.NonSimpleResult; import com.esri.core.geometry.NonSimpleResult.Reason; @@ -30,15 +29,11 @@ import com.esri.core.geometry.ogc.OGCGeometry; import com.esri.core.geometry.ogc.OGCGeometryCollection; import com.esri.core.geometry.ogc.OGCLineString; -import com.esri.core.geometry.ogc.OGCMultiPolygon; -import com.esri.core.geometry.ogc.OGCPoint; -import com.esri.core.geometry.ogc.OGCPolygon; import com.facebook.presto.geospatial.GeometryType; import com.facebook.presto.geospatial.KdbTree; import com.facebook.presto.geospatial.Rectangle; -import com.facebook.presto.geospatial.serde.GeometrySerde; +import com.facebook.presto.geospatial.serde.EsriGeometrySerde; import com.facebook.presto.geospatial.serde.GeometrySerializationType; -import com.facebook.presto.geospatial.serde.JtsGeometrySerde; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; @@ -51,8 +46,15 @@ import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import io.airlift.slice.BasicSliceInput; import io.airlift.slice.Slice; +import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryCollection; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.MultiLineString; +import org.locationtech.jts.geom.impl.PackedCoordinateSequenceFactory; import org.locationtech.jts.linearref.LengthIndexedLine; import java.util.ArrayDeque; @@ -81,11 +83,19 @@ import static com.facebook.presto.geospatial.GeometryType.MULTI_POLYGON; import static com.facebook.presto.geospatial.GeometryType.POINT; import static com.facebook.presto.geospatial.GeometryType.POLYGON; +import static com.facebook.presto.geospatial.GeometryUtils.createJtsEmptyLineString; +import static com.facebook.presto.geospatial.GeometryUtils.createJtsEmptyPoint; +import static com.facebook.presto.geospatial.GeometryUtils.createJtsEmptyPolygon; +import static com.facebook.presto.geospatial.GeometryUtils.createJtsLineString; +import static com.facebook.presto.geospatial.GeometryUtils.createJtsMultiPoint; +import static com.facebook.presto.geospatial.GeometryUtils.createJtsPoint; import static com.facebook.presto.geospatial.GeometryUtils.getPointCount; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserialize; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserializeEnvelope; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserializeType; -import static com.facebook.presto.geospatial.serde.GeometrySerde.serialize; +import static com.facebook.presto.geospatial.GeometryUtils.jtsGeometryFromWkt; +import static com.facebook.presto.geospatial.GeometryUtils.wktFromJtsGeometry; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserializeEnvelope; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserializeType; +import static com.facebook.presto.geospatial.serde.JtsGeometrySerde.deserialize; +import static com.facebook.presto.geospatial.serde.JtsGeometrySerde.serialize; import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY; import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY_TYPE_NAME; import static com.facebook.presto.plugin.geospatial.SphericalGeographyType.SPHERICAL_GEOGRAPHY_TYPE_NAME; @@ -118,8 +128,7 @@ public final class GeoFunctions { private static final Joiner OR_JOINER = Joiner.on(" or "); - private static final Slice EMPTY_POLYGON = serialize(new OGCPolygon(new Polygon(), null)); - private static final Slice EMPTY_MULTIPOINT = serialize(createFromEsriGeometry(new MultiPoint(), null, true)); + private static final Slice EMPTY_POLYGON = serialize(createJtsEmptyPolygon()); private static final double EARTH_RADIUS_KM = 6371.01; private static final double EARTH_RADIUS_M = EARTH_RADIUS_KM * 1000.0; private static final Map NON_SIMPLE_REASONS = ImmutableMap.builder() @@ -149,7 +158,7 @@ private GeoFunctions() {} @SqlType(GEOMETRY_TYPE_NAME) public static Slice parseLine(@SqlType(VARCHAR) Slice input) { - OGCGeometry geometry = geometryFromText(input); + Geometry geometry = jtsGeometryFromWkt(input.toStringUtf8()); validateType("ST_LineFromText", geometry, EnumSet.of(LINE_STRING)); return serialize(geometry); } @@ -159,39 +168,12 @@ public static Slice parseLine(@SqlType(VARCHAR) Slice input) @SqlType(GEOMETRY_TYPE_NAME) public static Slice stLineString(@SqlType("array(" + GEOMETRY_TYPE_NAME + ")") Block input) { - MultiPath multipath = new Polyline(); - OGCPoint previousPoint = null; - for (int i = 0; i < input.getPositionCount(); i++) { - Slice slice = GEOMETRY.getSlice(input, i); - - if (slice.getInput().available() == 0) { - throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to ST_LineString: null point at index %s", i + 1)); - } - - OGCGeometry geometry = deserialize(slice); - if (!(geometry instanceof OGCPoint)) { - throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("ST_LineString takes only an array of valid points, %s was passed", geometry.geometryType())); - } - OGCPoint point = (OGCPoint) geometry; - - if (point.isEmpty()) { - throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to ST_LineString: empty point at index %s", i + 1)); - } - - if (previousPoint == null) { - multipath.startPath(point.X(), point.Y()); - } - else { - if (point.Equals(previousPoint)) { - throw new PrestoException(INVALID_FUNCTION_ARGUMENT, - format("Invalid input to ST_LineString: consecutive duplicate points at index %s", i + 1)); - } - multipath.lineTo(point.X(), point.Y()); - } - previousPoint = point; + CoordinateSequence coordinates = readPointCoordinates(input, "ST_LineString", true); + if (coordinates.size() < 2) { + return serialize(createJtsEmptyLineString()); } - OGCLineString linestring = new OGCLineString(multipath, 0, null); - return serialize(linestring); + + return serialize(createJtsLineString(coordinates)); } @Description("Returns a Geometry type Point object with the given coordinate values") @@ -199,8 +181,7 @@ public static Slice stLineString(@SqlType("array(" + GEOMETRY_TYPE_NAME + ")") B @SqlType(GEOMETRY_TYPE_NAME) public static Slice stPoint(@SqlType(DOUBLE) double x, @SqlType(DOUBLE) double y) { - OGCGeometry geometry = createFromEsriGeometry(new Point(x, y), null); - return serialize(geometry); + return serialize(createJtsPoint(x, y)); } @SqlNullable @@ -209,28 +190,49 @@ public static Slice stPoint(@SqlType(DOUBLE) double x, @SqlType(DOUBLE) double y @SqlType(GEOMETRY_TYPE_NAME) public static Slice stMultiPoint(@SqlType("array(" + GEOMETRY_TYPE_NAME + ")") Block input) { - MultiPoint multipoint = new MultiPoint(); + CoordinateSequence coordinates = readPointCoordinates(input, "ST_MultiPoint", false); + if (coordinates.size() == 0) { + return null; + } + + return serialize(createJtsMultiPoint(coordinates)); + } + + private static CoordinateSequence readPointCoordinates(Block input, String functionName, boolean forbidDuplicates) + { + PackedCoordinateSequenceFactory coordinateSequenceFactory = new PackedCoordinateSequenceFactory(); + double[] coordinates = new double[2 * input.getPositionCount()]; + double lastX = Double.NaN; + double lastY = Double.NaN; for (int i = 0; i < input.getPositionCount(); i++) { if (input.isNull(i)) { - throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to ST_MultiPoint: null at index %s", i + 1)); + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to %s: null at index %s", functionName, i + 1)); + } + + BasicSliceInput slice = new BasicSliceInput(GEOMETRY.getSlice(input, i)); + GeometrySerializationType type = GeometrySerializationType.getForCode(slice.readByte()); + if (type != GeometrySerializationType.POINT) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to %s: geometry is not a point: %s at index %s", functionName, type.toString(), i + 1)); } - Slice slice = GEOMETRY.getSlice(input, i); - OGCGeometry geometry = deserialize(slice); - if (!(geometry instanceof OGCPoint)) { - throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to ST_MultiPoint: geometry is not a point: %s at index %s", geometry.geometryType(), i + 1)); + double x = slice.readDouble(); + double y = slice.readDouble(); + + if (Double.isNaN(x) || Double.isNaN(x)) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to %s: empty point at index %s", functionName, i + 1)); } - OGCPoint point = (OGCPoint) geometry; - if (point.isEmpty()) { - throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid input to ST_MultiPoint: empty point at index %s", i + 1)); + if (forbidDuplicates && x == lastX && y == lastY) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, + format("Invalid input to %s: consecutive duplicate points at index %s", functionName, i + 1)); } - multipoint.add(point.X(), point.Y()); - } - if (multipoint.getPointCount() == 0) { - return null; + lastX = x; + lastY = y; + coordinates[2 * i] = x; + coordinates[2 * i + 1] = y; } - return serialize(createFromEsriGeometry(multipoint, null, true)); + + return coordinateSequenceFactory.create(coordinates, 2); } @Description("Returns a Geometry type Polygon object from Well-Known Text representation (WKT)") @@ -238,7 +240,7 @@ public static Slice stMultiPoint(@SqlType("array(" + GEOMETRY_TYPE_NAME + ")") B @SqlType(GEOMETRY_TYPE_NAME) public static Slice stPolygon(@SqlType(VARCHAR) Slice input) { - OGCGeometry geometry = geometryFromText(input); + Geometry geometry = jtsGeometryFromWkt(input.toStringUtf8()); validateType("ST_Polygon", geometry, EnumSet.of(POLYGON)); return serialize(geometry); } @@ -248,24 +250,7 @@ public static Slice stPolygon(@SqlType(VARCHAR) Slice input) @SqlType(DOUBLE) public static double stArea(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); - - // The Esri geometry library does not support area for geometry collections. We compute the area - // of collections by summing the area of the individual components. - GeometryType type = GeometryType.getForEsriGeometryType(geometry.geometryType()); - if (type == GeometryType.GEOMETRY_COLLECTION) { - double area = 0.0; - GeometryCursor cursor = geometry.getEsriGeometryCursor(); - while (true) { - com.esri.core.geometry.Geometry esriGeometry = cursor.next(); - if (esriGeometry == null) { - return area; - } - - area += esriGeometry.calculateArea2D(); - } - } - return geometry.getEsriGeometry().calculateArea2D(); + return deserialize(input).getArea(); } @Description("Returns a Geometry type object from Well-Known Text representation (WKT)") @@ -273,7 +258,7 @@ public static double stArea(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(GEOMETRY_TYPE_NAME) public static Slice stGeometryFromText(@SqlType(VARCHAR) Slice input) { - return serialize(geometryFromText(input)); + return serialize(jtsGeometryFromWkt(input.toStringUtf8())); } @Description("Returns a Geometry type object from Well-Known Binary representation (WKB)") @@ -281,7 +266,7 @@ public static Slice stGeometryFromText(@SqlType(VARCHAR) Slice input) @SqlType(GEOMETRY_TYPE_NAME) public static Slice stGeomFromBinary(@SqlType(VARBINARY) Slice input) { - return serialize(geomFromBinary(input)); + return EsriGeometrySerde.serialize(geomFromBinary(input)); } @Description("Converts a Geometry object to a SphericalGeography object") @@ -291,13 +276,13 @@ public static Slice toSphericalGeography(@SqlType(GEOMETRY_TYPE_NAME) Slice inpu { // "every point in input is in range" <=> "the envelope of input is in range" Envelope envelope = deserializeEnvelope(input); - if (envelope != null) { + if (!envelope.isEmpty()) { checkLatitude(envelope.getYMin()); checkLatitude(envelope.getYMax()); checkLongitude(envelope.getXMin()); checkLongitude(envelope.getXMax()); } - OGCGeometry geometry = deserialize(input); + OGCGeometry geometry = EsriGeometrySerde.deserialize(input); if (geometry.is3D()) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Cannot convert 3D geometry to a spherical geography"); } @@ -331,7 +316,7 @@ public static Slice toGeometry(@SqlType(SPHERICAL_GEOGRAPHY_TYPE_NAME) Slice inp @SqlType(VARCHAR) public static Slice stAsText(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - return utf8Slice(deserialize(input).asText()); + return utf8Slice(wktFromJtsGeometry(deserialize(input))); } @Description("Returns the Well-Known Binary (WKB) representation of the geometry") @@ -339,7 +324,7 @@ public static Slice stAsText(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(VARBINARY) public static Slice stAsBinary(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - return wrappedBuffer(deserialize(input).asBinary()); + return wrappedBuffer(EsriGeometrySerde.deserialize(input).asBinary()); } @SqlNullable @@ -360,11 +345,11 @@ public static Slice stBuffer(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @SqlType( return input; } - OGCGeometry geometry = deserialize(input); + OGCGeometry geometry = EsriGeometrySerde.deserialize(input); if (geometry.isEmpty()) { return null; } - return serialize(geometry.buffer(distance)); + return EsriGeometrySerde.serialize(geometry.buffer(distance)); } @Description("Returns the Point value that is the mathematical centroid of a Geometry") @@ -372,45 +357,17 @@ public static Slice stBuffer(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @SqlType( @SqlType(GEOMETRY_TYPE_NAME) public static Slice stCentroid(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_Centroid", geometry, EnumSet.of(POINT, MULTI_POINT, LINE_STRING, MULTI_LINE_STRING, POLYGON, MULTI_POLYGON)); - GeometryType geometryType = GeometryType.getForEsriGeometryType(geometry.geometryType()); + GeometryType geometryType = GeometryType.getForJtsGeometryType(geometry.getGeometryType()); if (geometryType == GeometryType.POINT) { return input; } - int pointCount = ((MultiVertexGeometry) geometry.getEsriGeometry()).getPointCount(); - if (pointCount == 0) { - return serialize(createFromEsriGeometry(new Point(), geometry.getEsriSpatialReference())); - } - - Point centroid; - try { - switch (geometryType) { - case MULTI_POINT: - centroid = computePointsCentroid((MultiVertexGeometry) geometry.getEsriGeometry()); - break; - case LINE_STRING: - case MULTI_LINE_STRING: - centroid = computeLineCentroid((Polyline) geometry.getEsriGeometry()); - break; - case POLYGON: - centroid = computePolygonCentroid((Polygon) geometry.getEsriGeometry()); - break; - case MULTI_POLYGON: - centroid = computeMultiPolygonCentroid((OGCMultiPolygon) geometry); - break; - default: - throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Unexpected geometry type: " + geometryType); - } + if (geometry.getNumPoints() == 0) { + return serialize(createJtsEmptyPoint()); } - catch (RuntimeException e) { - if (e instanceof PrestoException && ((PrestoException) e).getErrorCode() == INVALID_FUNCTION_ARGUMENT.toErrorCode()) { - throw e; - } - throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Cannot compute centroid: %s Use ST_IsValid to confirm that input geometry is valid or compute centroid for a bounding box using ST_Envelope.", e.getMessage()), e); - } - return serialize(createFromEsriGeometry(centroid, geometry.getEsriSpatialReference())); + return serialize(geometry.getCentroid()); } @Description("Returns the minimum convex geometry that encloses all input geometries") @@ -418,14 +375,14 @@ public static Slice stCentroid(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(GEOMETRY_TYPE_NAME) public static Slice stConvexHull(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + OGCGeometry geometry = EsriGeometrySerde.deserialize(input); if (geometry.isEmpty()) { return input; } if (GeometryType.getForEsriGeometryType(geometry.geometryType()) == POINT) { return input; } - return serialize(geometry.convexHull()); + return EsriGeometrySerde.serialize(geometry.convexHull()); } @Description("Return the coordinate dimension of the Geometry") @@ -433,7 +390,7 @@ public static Slice stConvexHull(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(TINYINT) public static long stCoordinateDimension(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - return deserialize(input).coordinateDimension(); + return EsriGeometrySerde.deserialize(input).coordinateDimension(); } @Description("Returns the inherent dimension of this Geometry object, which must be less than or equal to the coordinate dimension") @@ -441,7 +398,7 @@ public static long stCoordinateDimension(@SqlType(GEOMETRY_TYPE_NAME) Slice inpu @SqlType(TINYINT) public static long stDimension(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - return deserialize(input).dimension(); + return deserialize(input).getDimension(); } @SqlNullable @@ -450,18 +407,17 @@ public static long stDimension(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(BOOLEAN) public static Boolean stIsClosed(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_IsClosed", geometry, EnumSet.of(LINE_STRING, MULTI_LINE_STRING)); - MultiPath lines = (MultiPath) geometry.getEsriGeometry(); - int pathCount = lines.getPathCount(); - for (int i = 0; i < pathCount; i++) { - Point start = lines.getPoint(lines.getPathStart(i)); - Point end = lines.getPoint(lines.getPathEnd(i) - 1); - if (!end.equals(start)) { - return false; - } + if (geometry instanceof LineString) { + return ((LineString) geometry).isClosed(); } - return true; + else if (geometry instanceof MultiLineString) { + return ((MultiLineString) geometry).isClosed(); + } + + // This would be handled in validateType, but for completeness. + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Invalid type for isClosed: %s", geometry.getGeometryType())); } @SqlNullable @@ -470,8 +426,7 @@ public static Boolean stIsClosed(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(BOOLEAN) public static Boolean stIsEmpty(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - Envelope envelope = deserializeEnvelope(input); - return envelope == null || envelope.isEmpty(); + return deserializeEnvelope(input).isEmpty(); } @Description("Returns TRUE if this Geometry has no anomalous geometric points, such as self intersection or self tangency") @@ -479,7 +434,7 @@ public static Boolean stIsEmpty(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(BOOLEAN) public static boolean stIsSimple(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + OGCGeometry geometry = EsriGeometrySerde.deserialize(input); return geometry.isEmpty() || geometry.isSimple(); } @@ -488,7 +443,7 @@ public static boolean stIsSimple(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(BOOLEAN) public static boolean stIsValid(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - GeometryCursor cursor = deserialize(input).getEsriGeometryCursor(); + GeometryCursor cursor = EsriGeometrySerde.deserialize(input).getEsriGeometryCursor(); while (true) { com.esri.core.geometry.Geometry geometry = cursor.next(); if (geometry == null) { @@ -507,7 +462,7 @@ public static boolean stIsValid(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlNullable public static Slice invalidReason(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - GeometryCursor cursor = deserialize(input).getEsriGeometryCursor(); + GeometryCursor cursor = EsriGeometrySerde.deserialize(input).getEsriGeometryCursor(); NonSimpleResult result = new NonSimpleResult(); while (true) { com.esri.core.geometry.Geometry geometry = cursor.next(); @@ -544,9 +499,9 @@ public static Slice invalidReason(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(DOUBLE) public static double stLength(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_Length", geometry, EnumSet.of(LINE_STRING, MULTI_LINE_STRING)); - return geometry.getEsriGeometry().calculateLength2D(); + return geometry.getLength(); } @SqlNullable @@ -555,8 +510,8 @@ public static double stLength(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(DOUBLE) public static Double lineLocatePoint(@SqlType(GEOMETRY_TYPE_NAME) Slice lineSlice, @SqlType(GEOMETRY_TYPE_NAME) Slice pointSlice) { - Geometry line = JtsGeometrySerde.deserialize(lineSlice); - Geometry point = JtsGeometrySerde.deserialize(pointSlice); + Geometry line = deserialize(lineSlice); + Geometry point = deserialize(pointSlice); if (line.isEmpty() || point.isEmpty()) { return null; @@ -575,6 +530,27 @@ public static Double lineLocatePoint(@SqlType(GEOMETRY_TYPE_NAME) Slice lineSlic return new LengthIndexedLine(line).indexOf(point.getCoordinate()) / line.getLength(); } + @Description("Returns the point in the line at the fractional length.") + @ScalarFunction("line_interpolate_point") + @SqlType(GEOMETRY_TYPE_NAME) + public static Slice lineInterpolatePoint(@SqlType(GEOMETRY_TYPE_NAME) Slice lineSlice, @SqlType(DOUBLE) double fraction) + { + if (!(0.0 <= fraction && fraction <= 1.0)) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("line_interpolate_point: Fraction must be between 0 and 1, but is %s", fraction)); + } + + Geometry geometry = deserialize(lineSlice); + validateType("line_interpolate_point", geometry, ImmutableSet.of(LINE_STRING)); + LineString line = (LineString) geometry; + + if (line.isEmpty()) { + return serialize(createJtsEmptyPoint()); + } + + org.locationtech.jts.geom.Coordinate coordinate = new LengthIndexedLine(line).extractPoint(fraction * line.getLength()); + return serialize(createJtsPoint(coordinate)); + } + @SqlNullable @Description("Returns X maxima of a bounding box of a Geometry") @ScalarFunction("ST_XMax") @@ -582,7 +558,7 @@ public static Double lineLocatePoint(@SqlType(GEOMETRY_TYPE_NAME) Slice lineSlic public static Double stXMax(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { Envelope envelope = deserializeEnvelope(input); - if (envelope == null) { + if (envelope.isEmpty()) { return null; } return envelope.getXMax(); @@ -595,7 +571,7 @@ public static Double stXMax(@SqlType(GEOMETRY_TYPE_NAME) Slice input) public static Double stYMax(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { Envelope envelope = deserializeEnvelope(input); - if (envelope == null) { + if (envelope.isEmpty()) { return null; } return envelope.getYMax(); @@ -608,7 +584,7 @@ public static Double stYMax(@SqlType(GEOMETRY_TYPE_NAME) Slice input) public static Double stXMin(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { Envelope envelope = deserializeEnvelope(input); - if (envelope == null) { + if (envelope.isEmpty()) { return null; } return envelope.getXMin(); @@ -621,7 +597,7 @@ public static Double stXMin(@SqlType(GEOMETRY_TYPE_NAME) Slice input) public static Double stYMin(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { Envelope envelope = deserializeEnvelope(input); - if (envelope == null) { + if (envelope.isEmpty()) { return null; } return envelope.getYMin(); @@ -633,12 +609,12 @@ public static Double stYMin(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(BIGINT) public static Long stNumInteriorRings(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_NumInteriorRing", geometry, EnumSet.of(POLYGON)); if (geometry.isEmpty()) { return null; } - return Long.valueOf(((OGCPolygon) geometry).numInteriorRing()); + return Long.valueOf(((org.locationtech.jts.geom.Polygon) geometry).getNumInteriorRing()); } @SqlNullable @@ -647,16 +623,16 @@ public static Long stNumInteriorRings(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType("array(" + GEOMETRY_TYPE_NAME + ")") public static Block stInteriorRings(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_InteriorRings", geometry, EnumSet.of(POLYGON)); if (geometry.isEmpty()) { return null; } - OGCPolygon polygon = (OGCPolygon) geometry; - BlockBuilder blockBuilder = GEOMETRY.createBlockBuilder(null, polygon.numInteriorRing()); - for (int i = 0; i < polygon.numInteriorRing(); i++) { - GEOMETRY.writeSlice(blockBuilder, serialize(polygon.interiorRingN(i))); + org.locationtech.jts.geom.Polygon polygon = (org.locationtech.jts.geom.Polygon) geometry; + BlockBuilder blockBuilder = GEOMETRY.createBlockBuilder(null, polygon.getNumInteriorRing()); + for (int i = 0; i < polygon.getNumInteriorRing(); i++) { + GEOMETRY.writeSlice(blockBuilder, serialize((LineString) polygon.getInteriorRingN(i))); } return blockBuilder.build(); } @@ -666,15 +642,11 @@ public static Block stInteriorRings(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(INTEGER) public static long stNumGeometries(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); if (geometry.isEmpty()) { return 0; } - GeometryType type = GeometryType.getForEsriGeometryType(geometry.geometryType()); - if (!type.isMultitype()) { - return 1; - } - return ((OGCGeometryCollection) geometry).numGeometries(); + return geometry.getNumGeometries(); } @Description("Returns a geometry that represents the point set union of the input geometries.") @@ -718,7 +690,7 @@ private static Slice stUnion(Iterable slices) continue; } - for (OGCGeometry geometry : flattenCollection(deserialize(slice))) { + for (OGCGeometry geometry : flattenCollection(EsriGeometrySerde.deserialize(slice))) { int dimension = geometry.dimension(); cursorsByDimension[dimension].tick(geometry.getEsriGeometry()); operatorsByDimension[dimension].tock(); @@ -734,9 +706,9 @@ private static Slice stUnion(Iterable slices) } if (outputs.size() == 1) { - return serialize(outputs.get(0)); + return EsriGeometrySerde.serialize(outputs.get(0)); } - return serialize(new OGCConcreteGeometryCollection(outputs, null).flattenAndRemoveOverlaps().reduceFromMulti()); + return EsriGeometrySerde.serialize(new OGCConcreteGeometryCollection(outputs, null).flattenAndRemoveOverlaps().reduceFromMulti()); } @SqlNullable @@ -745,23 +717,22 @@ private static Slice stUnion(Iterable slices) @SqlType(GEOMETRY_TYPE_NAME) public static Slice stGeometryN(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @SqlType(INTEGER) long index) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); if (geometry.isEmpty()) { return null; } - GeometryType type = GeometryType.getForEsriGeometryType(geometry.geometryType()); + GeometryType type = GeometryType.getForJtsGeometryType(geometry.getGeometryType()); if (!type.isMultitype()) { if (index == 1) { return input; } return null; } - OGCGeometryCollection geometryCollection = ((OGCGeometryCollection) geometry); - if (index < 1 || index > geometryCollection.numGeometries()) { + GeometryCollection geometryCollection = ((GeometryCollection) geometry); + if (index < 1 || index > geometryCollection.getNumGeometries()) { return null; } - OGCGeometry ogcGeometry = geometryCollection.geometryN((int) index - 1); - return serialize(ogcGeometry); + return serialize(geometryCollection.getGeometryN((int) index - 1)); } @SqlNullable @@ -770,14 +741,14 @@ public static Slice stGeometryN(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @SqlTy @SqlType(GEOMETRY_TYPE_NAME) public static Slice stPointN(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @SqlType(INTEGER) long index) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_PointN", geometry, EnumSet.of(LINE_STRING)); - OGCLineString linestring = (OGCLineString) geometry; - if (index < 1 || index > linestring.numPoints()) { + LineString linestring = (LineString) geometry; + if (index < 1 || index > linestring.getNumPoints()) { return null; } - return serialize(linestring.pointN(toIntExact(index) - 1)); + return serialize(linestring.getPointN(toIntExact(index) - 1)); } @SqlNullable @@ -786,22 +757,22 @@ public static Slice stPointN(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @SqlType( @SqlType("array(" + GEOMETRY_TYPE_NAME + ")") public static Block stGeometries(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); if (geometry.isEmpty()) { return null; } - GeometryType type = GeometryType.getForEsriGeometryType(geometry.geometryType()); + GeometryType type = GeometryType.getForJtsGeometryType(geometry.getGeometryType()); if (!type.isMultitype()) { BlockBuilder blockBuilder = GEOMETRY.createBlockBuilder(null, 1); GEOMETRY.writeSlice(blockBuilder, serialize(geometry)); return blockBuilder.build(); } - OGCGeometryCollection collection = (OGCGeometryCollection) geometry; - BlockBuilder blockBuilder = GEOMETRY.createBlockBuilder(null, collection.numGeometries()); - for (int i = 0; i < collection.numGeometries(); i++) { - GEOMETRY.writeSlice(blockBuilder, serialize(collection.geometryN(i))); + GeometryCollection collection = (GeometryCollection) geometry; + BlockBuilder blockBuilder = GEOMETRY.createBlockBuilder(null, collection.getNumGeometries()); + for (int i = 0; i < collection.getNumGeometries(); i++) { + GEOMETRY.writeSlice(blockBuilder, serialize(collection.getGeometryN(i))); } return blockBuilder.build(); } @@ -812,14 +783,13 @@ public static Block stGeometries(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(GEOMETRY_TYPE_NAME) public static Slice stInteriorRingN(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @SqlType(INTEGER) long index) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_InteriorRingN", geometry, EnumSet.of(POLYGON)); - OGCPolygon polygon = (OGCPolygon) geometry; - if (index < 1 || index > polygon.numInteriorRing()) { + org.locationtech.jts.geom.Polygon polygon = (org.locationtech.jts.geom.Polygon) geometry; + if (index < 1 || index > polygon.getNumInteriorRing()) { return null; } - OGCGeometry interiorRing = polygon.interiorRingN(toIntExact(index) - 1); - return serialize(interiorRing); + return serialize(polygon.getInteriorRingN(toIntExact(index) - 1)); } @Description("Returns the number of points in a Geometry") @@ -827,7 +797,7 @@ public static Slice stInteriorRingN(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @S @SqlType(BIGINT) public static long stNumPoints(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - return getPointCount(deserialize(input)); + return getPointCount(EsriGeometrySerde.deserialize(input)); } @SqlNullable @@ -836,7 +806,7 @@ public static long stNumPoints(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(BOOLEAN) public static Boolean stIsRing(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + OGCGeometry geometry = EsriGeometrySerde.deserialize(input); validateType("ST_IsRing", geometry, EnumSet.of(LINE_STRING)); OGCLineString line = (OGCLineString) geometry; return line.isClosed() && line.isSimple(); @@ -848,13 +818,12 @@ public static Boolean stIsRing(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(GEOMETRY_TYPE_NAME) public static Slice stStartPoint(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_StartPoint", geometry, EnumSet.of(LINE_STRING)); if (geometry.isEmpty()) { return null; } - MultiPath lines = (MultiPath) geometry.getEsriGeometry(); - return serialize(createFromEsriGeometry(lines.getPoint(0), null)); + return serialize(((LineString) geometry).getStartPoint()); } @Description("Returns a \"simplified\" version of the given geometry") @@ -874,7 +843,7 @@ public static Slice simplifyGeometry(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @ return input; } - return JtsGeometrySerde.serialize(simplify(JtsGeometrySerde.deserialize(input), distanceTolerance)); + return serialize(simplify(deserialize(input), distanceTolerance)); } @SqlNullable @@ -883,13 +852,12 @@ public static Slice simplifyGeometry(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @ @SqlType(GEOMETRY_TYPE_NAME) public static Slice stEndPoint(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_EndPoint", geometry, EnumSet.of(LINE_STRING)); if (geometry.isEmpty()) { return null; } - MultiPath lines = (MultiPath) geometry.getEsriGeometry(); - return serialize(createFromEsriGeometry(lines.getPoint(lines.getPointCount() - 1), null)); + return serialize(((LineString) geometry).getEndPoint()); } @SqlNullable @@ -898,15 +866,15 @@ public static Slice stEndPoint(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType("array(" + GEOMETRY_TYPE_NAME + ")") public static Block stPoints(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_Points", geometry, EnumSet.of(LINE_STRING)); if (geometry.isEmpty()) { return null; } - MultiPath lines = (MultiPath) geometry.getEsriGeometry(); - BlockBuilder blockBuilder = GEOMETRY.createBlockBuilder(null, lines.getPointCount()); - for (int i = 0; i < lines.getPointCount(); i++) { - GEOMETRY.writeSlice(blockBuilder, serialize(createFromEsriGeometry(lines.getPoint(i), null))); + LineString lineString = (LineString) geometry; + BlockBuilder blockBuilder = GEOMETRY.createBlockBuilder(null, lineString.getNumPoints()); + for (int i = 0; i < lineString.getNumPoints(); i++) { + GEOMETRY.writeSlice(blockBuilder, serialize(lineString.getPointN(i))); } return blockBuilder.build(); } @@ -917,12 +885,12 @@ public static Block stPoints(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(DOUBLE) public static Double stX(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_X", geometry, EnumSet.of(POINT)); if (geometry.isEmpty()) { return null; } - return ((OGCPoint) geometry).X(); + return ((org.locationtech.jts.geom.Point) geometry).getX(); } @SqlNullable @@ -931,12 +899,12 @@ public static Double stX(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(DOUBLE) public static Double stY(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_Y", geometry, EnumSet.of(POINT)); if (geometry.isEmpty()) { return null; } - return ((OGCPoint) geometry).Y(); + return ((org.locationtech.jts.geom.Point) geometry).getY(); } @Description("Returns the closure of the combinatorial boundary of this Geometry") @@ -944,12 +912,7 @@ public static Double stY(@SqlType(GEOMETRY_TYPE_NAME) Slice input) @SqlType(GEOMETRY_TYPE_NAME) public static Slice stBoundary(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); - if (geometry.isEmpty() && GeometryType.getForEsriGeometryType(geometry.geometryType()) == LINE_STRING) { - // OCGGeometry#boundary crashes with NPE for LINESTRING EMPTY - return EMPTY_MULTIPOINT; - } - return serialize(geometry.boundary()); + return serialize(deserialize(input).getBoundary()); } @Description("Returns the bounding rectangular polygon of a Geometry") @@ -958,10 +921,10 @@ public static Slice stBoundary(@SqlType(GEOMETRY_TYPE_NAME) Slice input) public static Slice stEnvelope(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { Envelope envelope = deserializeEnvelope(input); - if (envelope == null) { + if (envelope.isEmpty()) { return EMPTY_POLYGON; } - return serialize(envelope); + return EsriGeometrySerde.serialize(envelope); } @SqlNullable @@ -971,26 +934,50 @@ public static Slice stEnvelope(@SqlType(GEOMETRY_TYPE_NAME) Slice input) public static Block stEnvelopeAsPts(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { Envelope envelope = deserializeEnvelope(input); - if (envelope == null) { + if (envelope.isEmpty()) { return null; } BlockBuilder blockBuilder = GEOMETRY.createBlockBuilder(null, 2); - Point lowerLeftCorner = new Point(envelope.getXMin(), envelope.getYMin()); - Point upperRightCorner = new Point(envelope.getXMax(), envelope.getYMax()); - GEOMETRY.writeSlice(blockBuilder, serialize(createFromEsriGeometry(lowerLeftCorner, null, false))); - GEOMETRY.writeSlice(blockBuilder, serialize(createFromEsriGeometry(upperRightCorner, null, false))); + org.locationtech.jts.geom.Point lowerLeftCorner = createJtsPoint(envelope.getXMin(), envelope.getYMin()); + org.locationtech.jts.geom.Point upperRightCorner = createJtsPoint(envelope.getXMax(), envelope.getYMax()); + GEOMETRY.writeSlice(blockBuilder, serialize(lowerLeftCorner)); + GEOMETRY.writeSlice(blockBuilder, serialize(upperRightCorner)); return blockBuilder.build(); } + @Description("Returns the bounding rectangle of a Geometry expanded by distance.") + @ScalarFunction("expand_envelope") + @SqlType(GEOMETRY_TYPE_NAME) + public static Slice expandEnvelope(@SqlType(GEOMETRY_TYPE_NAME) Slice input, @SqlType(DOUBLE) double distance) + { + if (isNaN(distance)) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "expand_envelope: distance is NaN"); + } + + if (distance < 0) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("expand_envelope: distance %s is negative", distance)); + } + + Envelope envelope = deserializeEnvelope(input); + if (envelope.isEmpty()) { + return EMPTY_POLYGON; + } + return EsriGeometrySerde.serialize(new Envelope( + envelope.getXMin() - distance, + envelope.getYMin() - distance, + envelope.getXMax() + distance, + envelope.getYMax() + distance)); + } + @Description("Returns the Geometry value that represents the point set difference of two geometries") @ScalarFunction("ST_Difference") @SqlType(GEOMETRY_TYPE_NAME) public static Slice stDifference(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right) { - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); - return serialize(leftGeometry.difference(rightGeometry)); + return EsriGeometrySerde.serialize(leftGeometry.difference(rightGeometry)); } @SqlNullable @@ -999,8 +986,8 @@ public static Slice stDifference(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTy @SqlType(DOUBLE) public static Double stDistance(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right) { - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); return leftGeometry.isEmpty() || rightGeometry.isEmpty() ? null : leftGeometry.distance(rightGeometry); } @@ -1011,12 +998,12 @@ public static Double stDistance(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTyp @SqlType(GEOMETRY_TYPE_NAME) public static Slice stExteriorRing(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + Geometry geometry = deserialize(input); validateType("ST_ExteriorRing", geometry, EnumSet.of(POLYGON)); if (geometry.isEmpty()) { return null; } - return serialize(((OGCPolygon) geometry).exteriorRing()); + return serialize(((org.locationtech.jts.geom.Polygon) geometry).getExteriorRing()); } @Description("Returns the Geometry value that represents the point set intersection of two Geometries") @@ -1036,22 +1023,22 @@ public static Slice stIntersection(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @Sql Envelope intersection = leftEnvelope; if (intersection.getXMin() == intersection.getXMax()) { if (intersection.getYMin() == intersection.getYMax()) { - return serialize(createFromEsriGeometry(new Point(intersection.getXMin(), intersection.getXMax()), null)); + return EsriGeometrySerde.serialize(createFromEsriGeometry(new Point(intersection.getXMin(), intersection.getXMax()), null)); } - return serialize(createFromEsriGeometry(new Polyline(new Point(intersection.getXMin(), intersection.getYMin()), new Point(intersection.getXMin(), intersection.getYMax())), null)); + return EsriGeometrySerde.serialize(createFromEsriGeometry(new Polyline(new Point(intersection.getXMin(), intersection.getYMin()), new Point(intersection.getXMin(), intersection.getYMax())), null)); } if (intersection.getYMin() == intersection.getYMax()) { - return serialize(createFromEsriGeometry(new Polyline(new Point(intersection.getXMin(), intersection.getYMin()), new Point(intersection.getXMax(), intersection.getYMin())), null)); + return EsriGeometrySerde.serialize(createFromEsriGeometry(new Polyline(new Point(intersection.getXMin(), intersection.getYMin()), new Point(intersection.getXMax(), intersection.getYMin())), null)); } - return serialize(intersection); + return EsriGeometrySerde.serialize(intersection); } - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); - return serialize(leftGeometry.intersection(rightGeometry)); + return EsriGeometrySerde.serialize(leftGeometry.intersection(rightGeometry)); } @Description("Returns the Geometry value that represents the point set symmetric difference of two Geometries") @@ -1059,10 +1046,10 @@ public static Slice stIntersection(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @Sql @SqlType(GEOMETRY_TYPE_NAME) public static Slice stSymmetricDifference(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right) { - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); - return serialize(leftGeometry.symDifference(rightGeometry)); + return EsriGeometrySerde.serialize(leftGeometry.symDifference(rightGeometry)); } @SqlNullable @@ -1074,8 +1061,8 @@ public static Boolean stContains(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTy if (!envelopes(left, right, Envelope::contains)) { return false; } - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); return leftGeometry.contains(rightGeometry); } @@ -1089,8 +1076,8 @@ public static Boolean stCrosses(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTyp if (!envelopes(left, right, Envelope::intersect)) { return false; } - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); return leftGeometry.crosses(rightGeometry); } @@ -1104,8 +1091,8 @@ public static Boolean stDisjoint(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTy if (!envelopes(left, right, Envelope::intersect)) { return true; } - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); return leftGeometry.disjoint(rightGeometry); } @@ -1116,10 +1103,10 @@ public static Boolean stDisjoint(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTy @SqlType(BOOLEAN) public static Boolean stEquals(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right) { - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); - return leftGeometry.equals(rightGeometry); + return leftGeometry.Equals(rightGeometry); } @SqlNullable @@ -1131,8 +1118,8 @@ public static Boolean stIntersects(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @Sql if (!envelopes(left, right, Envelope::intersect)) { return false; } - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); return leftGeometry.intersects(rightGeometry); } @@ -1146,8 +1133,8 @@ public static Boolean stOverlaps(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTy if (!envelopes(left, right, Envelope::intersect)) { return false; } - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); return leftGeometry.overlaps(rightGeometry); } @@ -1158,8 +1145,8 @@ public static Boolean stOverlaps(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTy @SqlType(BOOLEAN) public static Boolean stRelate(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType(GEOMETRY_TYPE_NAME) Slice right, @SqlType(VARCHAR) Slice relation) { - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); return leftGeometry.relate(rightGeometry, relation.toStringUtf8()); } @@ -1173,8 +1160,8 @@ public static Boolean stTouches(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlTyp if (!envelopes(left, right, Envelope::intersect)) { return false; } - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); return leftGeometry.touches(rightGeometry); } @@ -1188,8 +1175,8 @@ public static Boolean stWithin(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType if (!envelopes(right, left, Envelope::contains)) { return false; } - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); verifySameSpatialReference(leftGeometry, rightGeometry); return leftGeometry.within(rightGeometry); } @@ -1199,7 +1186,7 @@ public static Boolean stWithin(@SqlType(GEOMETRY_TYPE_NAME) Slice left, @SqlType @SqlType(VARCHAR) public static Slice stGeometryType(@SqlType(GEOMETRY_TYPE_NAME) Slice input) { - return GeometrySerde.getGeometryType(input).standardName(); + return EsriGeometrySerde.getGeometryType(input).standardName(); } @ScalarFunction @@ -1209,7 +1196,7 @@ public static Slice stGeometryType(@SqlType(GEOMETRY_TYPE_NAME) Slice input) public static Block spatialPartitions(@SqlType(KdbTreeType.NAME) Object kdbTree, @SqlType(GEOMETRY_TYPE_NAME) Slice geometry) { Envelope envelope = deserializeEnvelope(geometry); - if (envelope == null) { + if (envelope.isEmpty()) { // Empty geometry return null; } @@ -1236,7 +1223,7 @@ public static Block spatialPartitions(@SqlType(KdbTreeType.NAME) Object kdbTree, } Envelope envelope = deserializeEnvelope(geometry); - if (envelope == null) { + if (envelope.isEmpty()) { return null; } @@ -1320,19 +1307,6 @@ private static void checkLongitude(double longitude) } } - private static OGCGeometry geometryFromText(Slice input) - { - OGCGeometry geometry; - try { - geometry = OGCGeometry.fromText(input.toStringUtf8()); - } - catch (IllegalArgumentException e) { - throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Invalid WKT: " + input.toStringUtf8(), e); - } - geometry.setSpatialReference(null); - return geometry; - } - private static OGCGeometry geomFromBinary(Slice input) { requireNonNull(input, "input is null"); @@ -1355,129 +1329,24 @@ private static void validateType(String function, OGCGeometry geometry, Set validTypes) { - int pathCount = polygon.getPathCount(); - - if (pathCount == 1) { - return getPolygonSansHolesCentroid(polygon); - } - - double xSum = 0; - double ySum = 0; - double areaSum = 0; - - for (int i = 0; i < pathCount; i++) { - int startIndex = polygon.getPathStart(i); - int endIndex = polygon.getPathEnd(i); - - Polygon sansHoles = getSubPolygon(polygon, startIndex, endIndex); - - Point centroid = getPolygonSansHolesCentroid(sansHoles); - double area = sansHoles.calculateArea2D(); - - xSum += centroid.getX() * area; - ySum += centroid.getY() * area; - areaSum += area; - } - - return new Point(xSum / areaSum, ySum / areaSum); - } - - private static Polygon getSubPolygon(Polygon polygon, int startIndex, int endIndex) - { - Polyline boundary = new Polyline(); - boundary.startPath(polygon.getPoint(startIndex)); - for (int i = startIndex + 1; i < endIndex; i++) { - Point current = polygon.getPoint(i); - boundary.lineTo(current); - } - - final Polygon newPolygon = new Polygon(); - newPolygon.add(boundary, false); - return newPolygon; - } - - // Polygon sans holes centroid: - // c[x] = (Sigma(x[i] + x[i + 1]) * (x[i] * y[i + 1] - x[i + 1] * y[i]), for i = 0 to N - 1) / (6 * signedArea) - // c[y] = (Sigma(y[i] + y[i + 1]) * (x[i] * y[i + 1] - x[i + 1] * y[i]), for i = 0 to N - 1) / (6 * signedArea) - private static Point getPolygonSansHolesCentroid(Polygon polygon) - { - int pointCount = polygon.getPointCount(); - double xSum = 0; - double ySum = 0; - double signedArea = 0; - for (int i = 0; i < pointCount; i++) { - Point current = polygon.getPoint(i); - Point next = polygon.getPoint((i + 1) % polygon.getPointCount()); - double ladder = current.getX() * next.getY() - next.getX() * current.getY(); - xSum += (current.getX() + next.getX()) * ladder; - ySum += (current.getY() + next.getY()) * ladder; - signedArea += ladder / 2; + GeometryType type = GeometryType.getForJtsGeometryType(geometry.getGeometryType()); + if (!validTypes.contains(type)) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("%s only applies to %s. Input type is: %s", function, OR_JOINER.join(validTypes), type)); } - return new Point(xSum / (signedArea * 6), ySum / (signedArea * 6)); } - // MultiPolygon centroid is weighted mean of each polygon, weight in terms of polygon area - private static Point computeMultiPolygonCentroid(OGCMultiPolygon multiPolygon) + private static void verifySameSpatialReference(OGCGeometry leftGeometry, OGCGeometry rightGeometry) { - double xSum = 0; - double ySum = 0; - double weightSum = 0; - for (int i = 0; i < multiPolygon.numGeometries(); i++) { - Point centroid = computePolygonCentroid((Polygon) multiPolygon.geometryN(i).getEsriGeometry()); - Polygon polygon = (Polygon) multiPolygon.geometryN(i).getEsriGeometry(); - double weight = polygon.calculateArea2D(); - weightSum += weight; - xSum += centroid.getX() * weight; - ySum += centroid.getY() * weight; - } - return new Point(xSum / weightSum, ySum / weightSum); + checkArgument(Objects.equals(leftGeometry.getEsriSpatialReference(), rightGeometry.getEsriSpatialReference()), "Input geometries must have the same spatial reference"); } private static boolean envelopes(Slice left, Slice right, EnvelopesPredicate predicate) { Envelope leftEnvelope = deserializeEnvelope(left); Envelope rightEnvelope = deserializeEnvelope(right); - if (leftEnvelope == null || rightEnvelope == null) { + if (leftEnvelope.isEmpty() || rightEnvelope.isEmpty()) { return false; } return predicate.apply(leftEnvelope, rightEnvelope); @@ -1494,8 +1363,8 @@ private interface EnvelopesPredicate @SqlType(DOUBLE) public static Double stSphericalDistance(@SqlType(SPHERICAL_GEOGRAPHY_TYPE_NAME) Slice left, @SqlType(SPHERICAL_GEOGRAPHY_TYPE_NAME) Slice right) { - OGCGeometry leftGeometry = deserialize(left); - OGCGeometry rightGeometry = deserialize(right); + OGCGeometry leftGeometry = EsriGeometrySerde.deserialize(left); + OGCGeometry rightGeometry = EsriGeometrySerde.deserialize(right); if (leftGeometry.isEmpty() || rightGeometry.isEmpty()) { return null; } @@ -1524,7 +1393,7 @@ private static void validateSphericalType(String function, OGCGeometry geometry, @SqlType(DOUBLE) public static Double stSphericalArea(@SqlType(SPHERICAL_GEOGRAPHY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + OGCGeometry geometry = EsriGeometrySerde.deserialize(input); if (geometry.isEmpty()) { return null; } @@ -1557,7 +1426,7 @@ public static Double stSphericalArea(@SqlType(SPHERICAL_GEOGRAPHY_TYPE_NAME) Sli @SqlType(DOUBLE) public static Double stSphericalLength(@SqlType(SPHERICAL_GEOGRAPHY_TYPE_NAME) Slice input) { - OGCGeometry geometry = deserialize(input); + OGCGeometry geometry = EsriGeometrySerde.deserialize(input); if (geometry.isEmpty()) { return null; } diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeometryType.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeometryType.java index 2b007eb380739..724f855382a02 100644 --- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeometryType.java +++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/GeometryType.java @@ -20,7 +20,7 @@ import com.facebook.presto.spi.type.TypeSignature; import io.airlift.slice.Slice; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserialize; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserialize; public class GeometryType extends AbstractVariableWidthType diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningInternalAggregateFunction.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningInternalAggregateFunction.java index 9d4ad04b87f5c..58ab1e44b99a8 100644 --- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningInternalAggregateFunction.java +++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningInternalAggregateFunction.java @@ -16,6 +16,7 @@ import com.esri.core.geometry.Envelope; import com.facebook.presto.geospatial.KdbTreeUtils; import com.facebook.presto.geospatial.Rectangle; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AggregationFunction; import com.facebook.presto.spi.function.CombineFunction; @@ -30,9 +31,10 @@ import java.util.concurrent.ThreadLocalRandom; import static com.facebook.presto.geospatial.KdbTree.buildKdbTree; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserializeEnvelope; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserializeEnvelope; import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY_TYPE_NAME; import static com.facebook.presto.plugin.geospatial.SpatialPartitioningAggregateFunction.NAME; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.type.StandardTypes.INTEGER; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static java.lang.Math.toIntExact; @@ -48,7 +50,7 @@ private SpatialPartitioningInternalAggregateFunction() {} public static void input(SpatialPartitioningState state, @SqlType(GEOMETRY_TYPE_NAME) Slice slice, @SqlType(INTEGER) long partitionCount) { Envelope envelope = deserializeEnvelope(slice); - if (envelope == null) { + if (envelope.isEmpty()) { return; } @@ -88,8 +90,7 @@ public static void combine(SpatialPartitioningState state, SpatialPartitioningSt public static void output(SpatialPartitioningState state, BlockBuilder out) { if (state.getCount() == 0) { - out.appendNull(); - return; + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "No rows supplied to spatial partition."); } List samples = state.getSamples(); diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningStateFactory.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningStateFactory.java index bc37ab6c3c747..9d480dd8a20a8 100644 --- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningStateFactory.java +++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningStateFactory.java @@ -207,7 +207,7 @@ public void setSamples(List samples) @Override public long getEstimatedSize() { - return INSTANCE_SIZE + (envelope != null ? envelope.estimateMemorySize() * (1 + samples.size()) : 0); + return INSTANCE_SIZE + (envelope != null ? envelope.getEstimatedSizeInBytes() * (1 + samples.size()) : 0); } } } diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SphericalGeographyType.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SphericalGeographyType.java index 1b6461f1ae52b..bf277b8b55acc 100644 --- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SphericalGeographyType.java +++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SphericalGeographyType.java @@ -20,7 +20,7 @@ import com.facebook.presto.spi.type.TypeSignature; import io.airlift.slice.Slice; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserialize; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserialize; public class SphericalGeographyType extends AbstractVariableWidthType diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/ConvexHullAggregation.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/ConvexHullAggregation.java index 276f66b74dbe0..da3b23352d22c 100644 --- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/ConvexHullAggregation.java +++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/ConvexHullAggregation.java @@ -15,7 +15,7 @@ import com.esri.core.geometry.ogc.OGCGeometry; import com.facebook.presto.geospatial.GeometryType; -import com.facebook.presto.geospatial.serde.GeometrySerde; +import com.facebook.presto.geospatial.serde.EsriGeometrySerde; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AggregationFunction; @@ -51,7 +51,7 @@ private ConvexHullAggregation() {} public static void input(@AggregationState GeometryState state, @SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = GeometrySerde.deserialize(input); + OGCGeometry geometry = EsriGeometrySerde.deserialize(input); if (state.getGeometry() == null) { state.setGeometry(geometry.convexHull(), 0); } @@ -81,7 +81,7 @@ public static void output(@AggregationState GeometryState state, BlockBuilder ou out.appendNull(); } else { - GEOMETRY.writeSlice(out, GeometrySerde.serialize(state.getGeometry())); + GEOMETRY.writeSlice(out, EsriGeometrySerde.serialize(state.getGeometry())); } } diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/GeometryStateSerializer.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/GeometryStateSerializer.java index 2654d9299fe7d..83649fda1fd31 100644 --- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/GeometryStateSerializer.java +++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/GeometryStateSerializer.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.plugin.geospatial.aggregation; -import com.facebook.presto.geospatial.serde.GeometrySerde; +import com.facebook.presto.geospatial.serde.EsriGeometrySerde; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AccumulatorStateSerializer; @@ -37,7 +37,7 @@ public void serialize(GeometryState state, BlockBuilder out) out.appendNull(); } else { - GEOMETRY.writeSlice(out, GeometrySerde.serialize(state.getGeometry())); + GEOMETRY.writeSlice(out, EsriGeometrySerde.serialize(state.getGeometry())); } } @@ -45,6 +45,6 @@ public void serialize(GeometryState state, BlockBuilder out) public void deserialize(Block block, int index, GeometryState state) { long previousMemorySize = state.getGeometry() != null ? state.getGeometry().estimateMemorySize() : 0; - state.setGeometry(GeometrySerde.deserialize(GEOMETRY.getSlice(block, index)), previousMemorySize); + state.setGeometry(EsriGeometrySerde.deserialize(GEOMETRY.getSlice(block, index)), previousMemorySize); } } diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/GeometryUnionAgg.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/GeometryUnionAgg.java index 14c0f34fa2801..6991e416a6171 100644 --- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/GeometryUnionAgg.java +++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/aggregation/GeometryUnionAgg.java @@ -14,7 +14,7 @@ package com.facebook.presto.plugin.geospatial.aggregation; import com.esri.core.geometry.ogc.OGCGeometry; -import com.facebook.presto.geospatial.serde.GeometrySerde; +import com.facebook.presto.geospatial.serde.EsriGeometrySerde; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AggregationFunction; import com.facebook.presto.spi.function.AggregationState; @@ -41,7 +41,7 @@ private GeometryUnionAgg() {} @InputFunction public static void input(@AggregationState GeometryState state, @SqlType(GEOMETRY_TYPE_NAME) Slice input) { - OGCGeometry geometry = GeometrySerde.deserialize(input); + OGCGeometry geometry = EsriGeometrySerde.deserialize(input); if (state.getGeometry() == null) { state.setGeometry(geometry, 0); } @@ -70,7 +70,7 @@ public static void output(@AggregationState GeometryState state, BlockBuilder ou out.appendNull(); } else { - GEOMETRY.writeSlice(out, GeometrySerde.serialize(state.getGeometry())); + GEOMETRY.writeSlice(out, EsriGeometrySerde.serialize(state.getGeometry())); } } } diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkEnvelopeIntersection.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkEnvelopeIntersection.java index 99900b376ab6e..808e2130182a0 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkEnvelopeIntersection.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkEnvelopeIntersection.java @@ -33,7 +33,7 @@ import java.util.concurrent.TimeUnit; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserialize; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserialize; import static com.facebook.presto.plugin.geospatial.GeoFunctions.stEnvelope; import static com.facebook.presto.plugin.geospatial.GeoFunctions.stGeometryFromText; import static com.facebook.presto.plugin.geospatial.GeoFunctions.stIntersection; diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkSTArea.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkSTArea.java index 2c3f91cb59c53..a095c3935ce35 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkSTArea.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkSTArea.java @@ -111,10 +111,10 @@ public static void verify() throws IOException data.setup(); BenchmarkSTArea benchmark = new BenchmarkSTArea(); - assertEquals(Math.round(1000 * (Double) benchmark.stSphericalArea(data) / 3.659E8), 1000); - assertEquals(Math.round(1000 * (Double) benchmark.stSphericalArea500k(data) / 38842273735.0), 1000); - assertEquals(benchmark.stArea(data), 0.05033099592771004); - assertEquals(Math.round(1000 * (Double) benchmark.stArea500k(data) / Math.PI), 1000); + assertEquals((Double) benchmark.stSphericalArea(data) / 3.659E8, 1.0, 1E-3); + assertEquals((Double) benchmark.stSphericalArea500k(data) / 38842273735.0, 1.0, 1E-3); + assertEquals((Double) benchmark.stArea(data) / 0.0503309959277, 1.0, 1E-3); + assertEquals((Double) benchmark.stArea500k(data) / Math.PI, 1.0, 1E-3); } private static String createPolygon(double numVertices) @@ -130,6 +130,8 @@ private static String createPolygon(double numVertices) sb.append(Math.sin(angle)); separator = ","; } + sb.append(separator); + sb.append("1 0"); sb.append("))"); return sb.toString(); } diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkSTContains.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkSTContains.java index 6eb1bda1f6668..89909dd232ab2 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkSTContains.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkSTContains.java @@ -36,8 +36,8 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserialize; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserializeEnvelope; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserialize; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserializeEnvelope; import static com.facebook.presto.plugin.geospatial.GeometryBenchmarkUtils.loadPolygon; @State(Scope.Thread) diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkSTIntersects.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkSTIntersects.java index 89d4fe00967cf..990d96ebaa419 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkSTIntersects.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/BenchmarkSTIntersects.java @@ -121,7 +121,7 @@ public static class BenchmarkData public void setup() throws IOException { - simpleGeometry = stGeometryFromText(utf8Slice("POLYGON ((16.5 54, 16.5 54.1, 16.51 54.1, 16.8 54))")); + simpleGeometry = stGeometryFromText(utf8Slice("POLYGON ((16.5 54, 16.5 54.1, 16.51 54.1, 16.8 54, 16.5 54))")); geometry = stGeometryFromText(utf8Slice(loadPolygon("large_polygon.txt"))); innerLine = stGeometryFromText(utf8Slice("LINESTRING (16.6 54.0167, 16.6 54.017)")); crossingLine = stGeometryFromText(utf8Slice("LINESTRING (16.6 53, 16.6 56)")); diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/GeoQueryRunner.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/GeoQueryRunner.java index f1a03e620d5f4..e1f22e2b7f07b 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/GeoQueryRunner.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/GeoQueryRunner.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.plugin.geospatial; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.log.Logging; import com.facebook.presto.tests.DistributedQueryRunner; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import io.airlift.log.Logging; import java.util.Map; diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestBingTileFunctions.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestBingTileFunctions.java index dbb8df50f94bc..e572b54f5942f 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestBingTileFunctions.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestBingTileFunctions.java @@ -36,6 +36,8 @@ import static com.facebook.presto.operator.aggregation.AggregationTestUtils.assertAggregation; import static com.facebook.presto.operator.scalar.ApplyFunction.APPLY_FUNCTION; import static com.facebook.presto.plugin.geospatial.BingTile.fromCoordinates; +import static com.facebook.presto.plugin.geospatial.BingTileFunctions.MAX_LATITUDE; +import static com.facebook.presto.plugin.geospatial.BingTileFunctions.MIN_LONGITUDE; import static com.facebook.presto.plugin.geospatial.BingTileType.BING_TILE; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -60,8 +62,8 @@ protected void registerFunctions() for (Type type : plugin.getTypes()) { functionAssertions.getTypeRegistry().addType(type); } - functionAssertions.getMetadata().addFunctions(extractFunctions(plugin.getFunctions())); - functionAssertions.getMetadata().addFunctions(ImmutableList.of(APPLY_FUNCTION)); + functionAssertions.getMetadata().registerBuiltInFunctions(extractFunctions(plugin.getFunctions())); + functionAssertions.getMetadata().registerBuiltInFunctions(ImmutableList.of(APPLY_FUNCTION)); FunctionManager functionManager = functionAssertions.getMetadata().getFunctionManager(); approxDistinct = functionManager.getAggregateFunctionImplementation( functionManager.lookupFunction("approx_distinct", fromTypes(BING_TILE))); @@ -90,14 +92,15 @@ public void testArrayOfBingTiles() @Test public void testBingTile() { + assertFunction("bing_tile_quadkey(bing_tile(''))", VARCHAR, ""); assertFunction("bing_tile_quadkey(bing_tile('213'))", VARCHAR, "213"); assertFunction("bing_tile_quadkey(bing_tile('123030123010121'))", VARCHAR, "123030123010121"); + assertFunction("bing_tile_quadkey(bing_tile(0, 0, 0))", VARCHAR, ""); assertFunction("bing_tile_quadkey(bing_tile(3, 5, 3))", VARCHAR, "213"); assertFunction("bing_tile_quadkey(bing_tile(21845, 13506, 15))", VARCHAR, "123030123010121"); // Invalid calls: corrupt quadkeys - assertInvalidFunction("bing_tile('')", "QuadKey must not be empty string"); assertInvalidFunction("bing_tile('test')", "Invalid QuadKey digit sequence: test"); assertInvalidFunction("bing_tile('12345')", "Invalid QuadKey digit sequence: 12345"); assertInvalidFunction("bing_tile('101010101010101010101010101010100101010101001010')", "QuadKey must be 23 characters or less"); @@ -124,7 +127,7 @@ public void testPointToBingTile() // Latitude out of range assertInvalidFunction("bing_tile_at(300.12, 60, 15)", "Latitude must be between -85.05112878 and 85.05112878"); // Invalid zoom levels - assertInvalidFunction("bing_tile_at(30.12, 60, 0)", "Zoom level must be > 0"); + assertInvalidFunction("bing_tile_at(30.12, 60, -1)", "Zoom level must be >= 0"); assertInvalidFunction("bing_tile_at(30.12, 60, 40)", "Zoom level must be <= 23"); } @@ -381,8 +384,8 @@ public void testBingTileZoomLevel() @Test public void testBingTilePolygon() { - assertFunction("ST_AsText(bing_tile_polygon(bing_tile('123030123010121')))", VARCHAR, "POLYGON ((59.996337890625 30.11662158281937, 60.00732421875 30.11662158281937, 60.00732421875 30.12612436422458, 59.996337890625 30.12612436422458, 59.996337890625 30.11662158281937))"); - assertFunction("ST_AsText(ST_Centroid(bing_tile_polygon(bing_tile('123030123010121'))))", VARCHAR, "POINT (60.0018310442288 30.121372968273892)"); + assertFunction("ST_AsText(bing_tile_polygon(bing_tile('123030123010121')))", VARCHAR, "POLYGON ((59.996337890625 30.11662158281937, 59.996337890625 30.12612436422458, 60.00732421875 30.12612436422458, 60.00732421875 30.11662158281937, 59.996337890625 30.11662158281937))"); + assertFunction("ST_AsText(ST_Centroid(bing_tile_polygon(bing_tile('123030123010121'))))", VARCHAR, "POINT (60.0018310546875 30.12137297352197)"); // Check bottom right corner of a stack of tiles at different zoom levels assertFunction("ST_AsText(apply(bing_tile_polygon(bing_tile(1, 1, 1)), g -> ST_Point(ST_XMax(g), ST_YMin(g))))", VARCHAR, "POINT (180 -85.05112877980659)"); @@ -430,12 +433,13 @@ public void testLargeGeometryToBingTiles() public void testGeometryToBingTiles() throws Exception { + assertGeometryToBingTiles("POINT (60 30.12)", 0, ImmutableList.of("")); assertGeometryToBingTiles("POINT (60 30.12)", 10, ImmutableList.of("1230301230")); assertGeometryToBingTiles("POINT (60 30.12)", 15, ImmutableList.of("123030123010121")); assertGeometryToBingTiles("POINT (60 30.12)", 16, ImmutableList.of("1230301230101212")); - assertGeometryToBingTiles("POLYGON ((0 0, 0 10, 10 10, 10 0))", 6, ImmutableList.of("122220", "122222", "122221", "122223")); - assertGeometryToBingTiles("POLYGON ((0 0, 0 10, 10 10))", 6, ImmutableList.of("122220", "122222", "122221")); + assertGeometryToBingTiles("POLYGON ((0 0, 0 10, 10 10, 10 0, 0 0))", 6, ImmutableList.of("122220", "122222", "122221", "122223")); + assertGeometryToBingTiles("POLYGON ((0 0, 0 10, 10 10, 0 0))", 6, ImmutableList.of("122220", "122222", "122221")); assertGeometryToBingTiles("POLYGON ((10 10, -10 10, -20 -15, 10 10))", 3, ImmutableList.of("033", "211", "122")); assertGeometryToBingTiles("POLYGON ((10 10, -10 10, -20 -15, 10 10))", 6, ImmutableList.of("211102", "211120", "033321", "033323", "211101", "211103", "211121", "033330", "033332", "211110", "211112", "033331", "033333", "211111", "122220", "122222", "122221")); @@ -456,15 +460,21 @@ public void testGeometryToBingTiles() assertGeometryToBingTiles("POLYGON EMPTY", 10, emptyList()); assertGeometryToBingTiles("GEOMETRYCOLLECTION EMPTY", 10, emptyList()); + // Geometries at MIN_LONGITUDE/MAX_LATITUDE + assertGeometryToBingTiles("LINESTRING (-180 -79.19245, -180 -79.17133464081945)", 8, ImmutableList.of("22200000")); + assertGeometryToBingTiles(format("POINT (%s 0)", MIN_LONGITUDE), 5, ImmutableList.of("20000")); + assertGeometryToBingTiles(format("POINT (0 %s)", MAX_LATITUDE), 5, ImmutableList.of("10000")); + assertGeometryToBingTiles(format("POINT (%s %s)", MIN_LONGITUDE, MAX_LATITUDE), 5, ImmutableList.of("00000")); + // Invalid input // Longitude out of range assertInvalidFunction("geometry_to_bing_tiles(ST_Point(600, 30.12), 10)", "Longitude span for the geometry must be in [-180.00, 180.00] range"); - assertInvalidFunction("geometry_to_bing_tiles(ST_GeometryFromText('POLYGON ((1000 10, -10 10, -20 -15))'), 10)", "Longitude span for the geometry must be in [-180.00, 180.00] range"); + assertInvalidFunction("geometry_to_bing_tiles(ST_GeometryFromText('POLYGON ((1000 10, -10 10, -20 -15, 1000 10))'), 10)", "Longitude span for the geometry must be in [-180.00, 180.00] range"); // Latitude out of range assertInvalidFunction("geometry_to_bing_tiles(ST_Point(60, 300.12), 10)", "Latitude span for the geometry must be in [-85.05, 85.05] range"); - assertInvalidFunction("geometry_to_bing_tiles(ST_GeometryFromText('POLYGON ((10 1000, -10 10, -20 -15))'), 10)", "Latitude span for the geometry must be in [-85.05, 85.05] range"); + assertInvalidFunction("geometry_to_bing_tiles(ST_GeometryFromText('POLYGON ((10 1000, -10 10, -20 -15, 10 1000))'), 10)", "Latitude span for the geometry must be in [-85.05, 85.05] range"); // Invalid zoom levels - assertInvalidFunction("geometry_to_bing_tiles(ST_Point(60, 30.12), 0)", "Zoom level must be > 0"); + assertInvalidFunction("geometry_to_bing_tiles(ST_Point(60, 30.12), -1)", "Zoom level must be >= 0"); assertInvalidFunction("geometry_to_bing_tiles(ST_Point(60, 30.12), 40)", "Zoom level must be <= 23"); // Input rectangle too large diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestGeoFunctions.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestGeoFunctions.java index 1901faba6c73e..0365132b11c2c 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestGeoFunctions.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestGeoFunctions.java @@ -14,9 +14,11 @@ package com.facebook.presto.plugin.geospatial; import com.esri.core.geometry.Point; +import com.esri.core.geometry.ogc.OGCGeometry; import com.esri.core.geometry.ogc.OGCPoint; import com.facebook.presto.geospatial.KdbTreeUtils; import com.facebook.presto.geospatial.Rectangle; +import com.facebook.presto.geospatial.serde.EsriGeometrySerde; import com.facebook.presto.operator.scalar.AbstractTestFunctions; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; @@ -30,6 +32,7 @@ import java.util.stream.Collectors; import static com.facebook.presto.geospatial.KdbTree.buildKdbTree; +import static com.facebook.presto.plugin.geospatial.GeoFunctions.stCentroid; import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -131,21 +134,24 @@ public void testSTLineFromText() assertFunction("ST_AsText(ST_LineFromText('LINESTRING EMPTY'))", VARCHAR, "LINESTRING EMPTY"); assertFunction("ST_AsText(ST_LineFromText('LINESTRING (1 1, 2 2, 1 3)'))", VARCHAR, "LINESTRING (1 1, 2 2, 1 3)"); assertInvalidFunction("ST_AsText(ST_LineFromText('MULTILINESTRING EMPTY'))", "ST_LineFromText only applies to LINE_STRING. Input type is: MULTI_LINE_STRING"); - assertInvalidFunction("ST_AsText(ST_LineFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'))", "ST_LineFromText only applies to LINE_STRING. Input type is: POLYGON"); + assertInvalidFunction("ST_AsText(ST_LineFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'))", "ST_LineFromText only applies to LINE_STRING. Input type is: POLYGON"); + assertInvalidFunction("ST_LineFromText('LINESTRING (0 0)')", INVALID_FUNCTION_ARGUMENT, "Invalid WKT: Invalid number of points in LineString (found 1 - must be 0 or >= 2)"); + assertInvalidFunction("ST_LineFromText('LINESTRING (0 0, 1)')", INVALID_FUNCTION_ARGUMENT, "Invalid WKT: Expected number but found ')' (line 1)"); } @Test public void testSTPolygon() { assertFunction("ST_AsText(ST_Polygon('POLYGON EMPTY'))", VARCHAR, "POLYGON EMPTY"); - assertFunction("ST_AsText(ST_Polygon('POLYGON ((1 1, 1 4, 4 4, 4 1))'))", VARCHAR, "POLYGON ((1 1, 4 1, 4 4, 1 4, 1 1))"); + assertFunction("ST_AsText(ST_Polygon('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'))", VARCHAR, "POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))"); assertInvalidFunction("ST_AsText(ST_Polygon('LINESTRING (1 1, 2 2, 1 3)'))", "ST_Polygon only applies to POLYGON. Input type is: LINE_STRING"); + assertInvalidFunction("ST_Polygon('POLYGON((-1 1, 1 -1))')", INVALID_FUNCTION_ARGUMENT, "Invalid WKT: Points of LinearRing do not form a closed linestring"); } @Test public void testSTArea() { - assertArea("POLYGON ((2 2, 2 6, 6 6, 6 2))", 16.0); + assertArea("POLYGON ((2 2, 2 6, 6 6, 6 2, 2 2))", 16.0); assertArea("POLYGON EMPTY", 0.0); assertArea("LINESTRING (1 4, 2 5)", 0.0); assertArea("LINESTRING EMPTY", 0.0); @@ -171,17 +177,17 @@ private void assertArea(String wkt, double expectedArea) @Test public void testSTBuffer() { - assertFunction("ST_AsText(ST_Buffer(ST_Point(0, 0), 0.5))", VARCHAR, "POLYGON ((0.5 0, 0.4989294616193014 0.03270156461507146, 0.49572243068690486 0.0652630961100257, 0.4903926402016149 0.09754516100806403, 0.4829629131445338 0.12940952255126026, 0.47346506474755257 0.16071973265158065, 0.46193976625564315 0.19134171618254472, 0.4484363707663439 0.22114434510950046, 0.43301270189221913 0.2499999999999998, 0.41573480615127245 0.2777851165098009, 0.39667667014561747 0.30438071450436016, 0.3759199037394886 0.32967290755003426, 0.3535533905932737 0.3535533905932736, 0.32967290755003437 0.3759199037394886, 0.3043807145043603 0.39667667014561747, 0.2777851165098011 0.4157348061512725, 0.24999999999999997 0.43301270189221924, 0.22114434510950062 0.4484363707663441, 0.19134171618254486 0.4619397662556433, 0.16071973265158077 0.4734650647475528, 0.12940952255126037 0.48296291314453416, 0.09754516100806412 0.4903926402016152, 0.06526309611002579 0.4957224306869052, 0.03270156461507153 0.49892946161930174, 0 0.5, -0.03270156461507146 0.4989294616193014, -0.0652630961100257 0.49572243068690486, -0.09754516100806403 0.4903926402016149, -0.12940952255126026 0.4829629131445338, -0.16071973265158065 0.47346506474755257, -0.19134171618254472 0.46193976625564315, -0.22114434510950046 0.4484363707663439, -0.2499999999999998 0.43301270189221913, -0.2777851165098009 0.41573480615127245, -0.30438071450436016 0.39667667014561747, -0.32967290755003426 0.3759199037394886, -0.3535533905932736 0.3535533905932737, -0.3759199037394886 0.32967290755003437, -0.39667667014561747 0.3043807145043603, -0.4157348061512725 0.2777851165098011, -0.43301270189221924 0.24999999999999997, -0.4484363707663441 0.22114434510950062, -0.4619397662556433 0.19134171618254486, -0.4734650647475528 0.16071973265158077, -0.48296291314453416 0.12940952255126037, -0.4903926402016152 0.09754516100806412, -0.4957224306869052 0.06526309611002579, -0.49892946161930174 0.03270156461507153, -0.5 0, -0.4989294616193014 -0.03270156461507146, -0.49572243068690486 -0.0652630961100257, -0.4903926402016149 -0.09754516100806403, -0.4829629131445338 -0.12940952255126026, -0.47346506474755257 -0.16071973265158065, -0.46193976625564315 -0.19134171618254472, -0.4484363707663439 -0.22114434510950046, -0.43301270189221913 -0.2499999999999998, -0.41573480615127245 -0.2777851165098009, -0.39667667014561747 -0.30438071450436016, -0.3759199037394886 -0.32967290755003426, -0.3535533905932737 -0.3535533905932736, -0.32967290755003437 -0.3759199037394886, -0.3043807145043603 -0.39667667014561747, -0.2777851165098011 -0.4157348061512725, -0.24999999999999997 -0.43301270189221924, -0.22114434510950062 -0.4484363707663441, -0.19134171618254486 -0.4619397662556433, -0.16071973265158077 -0.4734650647475528, -0.12940952255126037 -0.48296291314453416, -0.09754516100806412 -0.4903926402016152, -0.06526309611002579 -0.4957224306869052, -0.03270156461507153 -0.49892946161930174, 0 -0.5, 0.03270156461507146 -0.4989294616193014, 0.0652630961100257 -0.49572243068690486, 0.09754516100806403 -0.4903926402016149, 0.12940952255126026 -0.4829629131445338, 0.16071973265158065 -0.47346506474755257, 0.19134171618254472 -0.46193976625564315, 0.22114434510950046 -0.4484363707663439, 0.2499999999999998 -0.43301270189221913, 0.2777851165098009 -0.41573480615127245, 0.30438071450436016 -0.39667667014561747, 0.32967290755003426 -0.3759199037394886, 0.3535533905932736 -0.3535533905932737, 0.3759199037394886 -0.32967290755003437, 0.39667667014561747 -0.3043807145043603, 0.4157348061512725 -0.2777851165098011, 0.43301270189221924 -0.24999999999999997, 0.4484363707663441 -0.22114434510950062, 0.4619397662556433 -0.19134171618254486, 0.4734650647475528 -0.16071973265158077, 0.48296291314453416 -0.12940952255126037, 0.4903926402016152 -0.09754516100806412, 0.4957224306869052 -0.06526309611002579, 0.49892946161930174 -0.03270156461507153, 0.5 0))"); - assertFunction("ST_AsText(ST_Buffer(ST_LineFromText('LINESTRING (0 0, 1 1, 2 0.5)'), 0.2))", VARCHAR, "POLYGON ((0 -0.19999999999999996, 0.013080625846028537 -0.19957178464772052, 0.02610523844401036 -0.19828897227476194, 0.03901806440322564 -0.19615705608064593, 0.05176380902050415 -0.1931851652578136, 0.06428789306063232 -0.18938602589902098, 0.07653668647301792 -0.18477590650225728, 0.0884577380438003 -0.17937454830653754, 0.09999999999999987 -0.17320508075688767, 0.11111404660392044 -0.166293922460509, 0.12175228580174413 -0.15867066805824703, 0.13186916302001372 -0.15036796149579545, 0.14142135623730945 -0.14142135623730945, 1.0394906098164265 0.7566478973418078, 1.9105572809000084 0.32111456180001685, 1.9115422619561997 0.32062545169346235, 1.923463313526982 0.31522409349774266, 1.9357121069393677 0.3106139741009789, 1.9482361909794959 0.3068148347421863, 1.9609819355967744 0.3038429439193539, 1.9738947615559896 0.30171102772523795, 1.9869193741539715 0.30042821535227926, 2 0.3, 2.0130806258460288 0.3004282153522794, 2.02610523844401 0.30171102772523806, 2.0390180644032254 0.30384294391935407, 2.051763809020504 0.30681483474218646, 2.0642878930606323 0.31061397410097896, 2.076536686473018 0.3152240934977427, 2.0884577380438003 0.32062545169346246, 2.1 0.3267949192431123, 2.1111140466039204 0.333706077539491, 2.121752285801744 0.34132933194175297, 2.1318691630200135 0.34963203850420455, 2.1414213562373092 0.35857864376269055, 2.1503679614957956 0.3681308369799863, 2.158670668058247 0.37824771419825587, 2.166293922460509 0.38888595339607956, 2.1732050807568877 0.4, 2.1793745483065377 0.41154226195619975, 2.1847759065022574 0.4234633135269821, 2.189386025899021 0.4357121069393677, 2.193185165257814 0.44823619097949585, 2.1961570560806463 0.46098193559677436, 2.1982889722747623 0.4738947615559897, 2.1995717846477207 0.4869193741539714, 2.2 0.5, 2.1995717846477207 0.5130806258460285, 2.198288972274762 0.5261052384440102, 2.196157056080646 0.5390180644032256, 2.1931851652578134 0.5517638090205041, 2.189386025899021 0.5642878930606323, 2.1847759065022574 0.5765366864730179, 2.1793745483065377 0.5884577380438002, 2.1732050807568877 0.5999999999999999, 2.166293922460509 0.6111140466039204, 2.158670668058247 0.6217522858017441, 2.1503679614957956 0.6318691630200137, 2.1414213562373097 0.6414213562373094, 2.131869163020014 0.6503679614957955, 2.121752285801744 0.658670668058247, 2.1111140466039204 0.666293922460509, 2.1 0.6732050807568877, 2.0894427190999916 0.6788854381999831, 1.0894427190999916 1.1788854381999831, 1.0884577380438003 1.1793745483065377, 1.076536686473018 1.1847759065022574, 1.0642878930606323 1.189386025899021, 1.0517638090205041 1.1931851652578138, 1.0390180644032256 1.196157056080646, 1.0261052384440104 1.198288972274762, 1.0130806258460288 1.1995717846477207, 1 1.2, 0.9869193741539715 1.1995717846477205, 0.9738947615559896 1.1982889722747618, 0.9609819355967744 1.1961570560806458, 0.9482361909794959 1.1931851652578136, 0.9357121069393677 1.189386025899021, 0.9234633135269821 1.1847759065022574, 0.9115422619561997 1.1793745483065377, 0.9000000000000001 1.1732050807568877, 0.8888859533960796 1.166293922460509, 0.8782477141982559 1.158670668058247, 0.8681308369799863 1.1503679614957956, 0.8585786437626906 1.1414213562373094, -0.14142135623730967 0.1414213562373095, -0.15036796149579557 0.13186916302001372, -0.1586706680582468 0.12175228580174413, -0.1662939224605089 0.11111404660392044, -0.17320508075688767 0.09999999999999998, -0.17937454830653765 0.08845773804380025, -0.1847759065022574 0.07653668647301792, -0.18938602589902098 0.06428789306063232, -0.19318516525781382 0.05176380902050415, -0.19615705608064626 0.03901806440322564, -0.19828897227476228 0.026105238444010304, -0.19957178464772074 0.013080625846028593, -0.20000000000000018 0, -0.19957178464772074 -0.013080625846028537, -0.19828897227476183 -0.026105238444010248, -0.19615705608064582 -0.03901806440322564, -0.19318516525781337 -0.05176380902050415, -0.18938602589902098 -0.06428789306063232, -0.1847759065022574 -0.07653668647301792, -0.17937454830653765 -0.0884577380438002, -0.17320508075688767 -0.09999999999999987, -0.1662939224605089 -0.11111404660392044, -0.1586706680582468 -0.12175228580174413, -0.15036796149579557 -0.13186916302001372, -0.14142135623730967 -0.14142135623730945, -0.13186916302001395 -0.15036796149579545, -0.12175228580174391 -0.15867066805824703, -0.11111404660392044 -0.166293922460509, -0.10000000000000009 -0.17320508075688767, -0.0884577380438003 -0.17937454830653765, -0.07653668647301792 -0.1847759065022574, -0.06428789306063232 -0.1893860258990211, -0.05176380902050415 -0.1931851652578137, -0.03901806440322586 -0.19615705608064604, -0.026105238444010137 -0.19828897227476205, -0.01308062584602876 -0.19957178464772074, 0 -0.19999999999999996))"); - assertFunction("ST_AsText(ST_Buffer(ST_GeometryFromText('POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))'), 1.2))", VARCHAR, "POLYGON ((-1.2 0, -1.1974307078863233 -0.0784837550761715, -1.1897338336485717 -0.15663143066406168, -1.1769423364838756 -0.23410838641935366, -1.1591109915468811 -0.3105828541230246, -1.1363161553941261 -0.38572735836379357, -1.1086554390135435 -0.4592201188381073, -1.0762472898392252 -0.530746428262801, -1.0392304845413258 -0.5999999999999995, -0.9977635347630538 -0.6666842796235222, -0.9520240083494819 -0.7305137148104643, -0.9022077689747725 -0.7912149781200822, -0.8485281374238568 -0.8485281374238567, -0.7912149781200825 -0.9022077689747725, -0.7305137148104647 -0.9520240083494819, -0.6666842796235226 -0.997763534763054, -0.5999999999999999 -1.039230484541326, -0.5307464282628015 -1.0762472898392257, -0.45922011883810765 -1.108655439013544, -0.38572735836379385 -1.1363161553941266, -0.3105828541230249 -1.159110991546882, -0.2341083864193539 -1.1769423364838765, -0.15663143066406188 -1.1897338336485723, -0.07848375507617167 -1.1974307078863242, 0 -1.2, 5 -1.2, 5.078483755076172 -1.1974307078863233, 5.156631430664062 -1.1897338336485717, 5.234108386419353 -1.1769423364838756, 5.310582854123025 -1.1591109915468811, 5.385727358363794 -1.1363161553941261, 5.4592201188381075 -1.1086554390135435, 5.530746428262801 -1.0762472898392252, 5.6 -1.0392304845413258, 5.666684279623523 -0.9977635347630538, 5.730513714810464 -0.9520240083494819, 5.791214978120082 -0.9022077689747725, 5.848528137423857 -0.8485281374238568, 5.9022077689747725 -0.7912149781200825, 5.952024008349482 -0.7305137148104647, 5.997763534763054 -0.6666842796235226, 6.039230484541326 -0.5999999999999999, 6.076247289839226 -0.5307464282628015, 6.108655439013544 -0.45922011883810765, 6.136316155394127 -0.38572735836379385, 6.159110991546882 -0.3105828541230249, 6.176942336483877 -0.2341083864193539, 6.189733833648573 -0.15663143066406188, 6.197430707886324 -0.07848375507617167, 6.2 0, 6.2 5, 6.1974307078863236 5.078483755076172, 6.189733833648572 5.156631430664062, 6.176942336483876 5.234108386419353, 6.159110991546881 5.310582854123025, 6.136316155394126 5.385727358363794, 6.1086554390135435 5.4592201188381075, 6.076247289839225 5.530746428262801, 6.039230484541326 5.6, 5.997763534763054 5.666684279623523, 5.952024008349482 5.730513714810464, 5.9022077689747725 5.791214978120082, 5.848528137423857 5.848528137423857, 5.791214978120083 5.9022077689747725, 5.730513714810464 5.952024008349482, 5.666684279623523 5.997763534763054, 5.6 6.039230484541326, 5.530746428262802 6.076247289839226, 5.4592201188381075 6.108655439013544, 5.385727358363794 6.136316155394127, 5.310582854123025 6.159110991546882, 5.234108386419354 6.176942336483877, 5.156631430664062 6.189733833648573, 5.078483755076172 6.197430707886324, 5 6.2, 0 6.2, -0.0784837550761715 6.1974307078863236, -0.15663143066406168 6.189733833648572, -0.23410838641935366 6.176942336483876, -0.3105828541230246 6.159110991546881, -0.38572735836379357 6.136316155394126, -0.4592201188381073 6.1086554390135435, -0.530746428262801 6.076247289839225, -0.5999999999999995 6.039230484541326, -0.6666842796235222 5.997763534763054, -0.7305137148104643 5.952024008349482, -0.7912149781200822 5.9022077689747725, -0.8485281374238567 5.848528137423857, -0.9022077689747725 5.791214978120083, -0.9520240083494819 5.730513714810464, -0.997763534763054 5.666684279623523, -1.039230484541326 5.6, -1.0762472898392257 5.530746428262802, -1.108655439013544 5.4592201188381075, -1.1363161553941266 5.385727358363794, -1.159110991546882 5.310582854123025, -1.1769423364838765 5.234108386419354, -1.1897338336485723 5.156631430664062, -1.1974307078863242 5.078483755076172, -1.2 5, -1.2 0))"); + assertFunction("ST_AsText(ST_Buffer(ST_Point(0, 0), 0.5))", VARCHAR, "POLYGON ((0.5 0, 0.4989294616193017 -0.0327015646150715, 0.4957224306869052 -0.0652630961100258, 0.4903926402016152 -0.0975451610080641, 0.4829629131445342 -0.1294095225512604, 0.4734650647475528 -0.1607197326515808, 0.4619397662556433 -0.1913417161825449, 0.4484363707663441 -0.2211443451095006, 0.4330127018922192 -0.25, 0.4157348061512725 -0.2777851165098011, 0.3966766701456175 -0.3043807145043603, 0.3759199037394886 -0.3296729075500344, 0.3535533905932736 -0.3535533905932737, 0.3296729075500343 -0.3759199037394886, 0.3043807145043602 -0.3966766701456175, 0.2777851165098009 -0.4157348061512725, 0.2499999999999998 -0.4330127018922191, 0.2211443451095005 -0.4484363707663439, 0.1913417161825447 -0.4619397662556431, 0.1607197326515807 -0.4734650647475526, 0.1294095225512603 -0.4829629131445338, 0.097545161008064 -0.4903926402016149, 0.0652630961100257 -0.4957224306869049, 0.0327015646150715 -0.4989294616193014, 0 -0.5, -0.0327015646150715 -0.4989294616193017, -0.0652630961100258 -0.4957224306869052, -0.0975451610080641 -0.4903926402016152, -0.1294095225512604 -0.4829629131445342, -0.1607197326515808 -0.4734650647475528, -0.1913417161825449 -0.4619397662556433, -0.2211443451095006 -0.4484363707663441, -0.25 -0.4330127018922192, -0.2777851165098011 -0.4157348061512725, -0.3043807145043603 -0.3966766701456175, -0.3296729075500344 -0.3759199037394886, -0.3535533905932737 -0.3535533905932736, -0.3759199037394886 -0.3296729075500343, -0.3966766701456175 -0.3043807145043602, -0.4157348061512725 -0.2777851165098009, -0.4330127018922191 -0.2499999999999998, -0.4484363707663439 -0.2211443451095005, -0.4619397662556431 -0.1913417161825447, -0.4734650647475526 -0.1607197326515807, -0.4829629131445338 -0.1294095225512603, -0.4903926402016149 -0.097545161008064, -0.4957224306869049 -0.0652630961100257, -0.4989294616193014 -0.0327015646150715, -0.5 0, -0.4989294616193017 0.0327015646150715, -0.4957224306869052 0.0652630961100258, -0.4903926402016152 0.0975451610080641, -0.4829629131445342 0.1294095225512604, -0.4734650647475528 0.1607197326515808, -0.4619397662556433 0.1913417161825449, -0.4484363707663441 0.2211443451095006, -0.4330127018922192 0.25, -0.4157348061512725 0.2777851165098011, -0.3966766701456175 0.3043807145043603, -0.3759199037394886 0.3296729075500344, -0.3535533905932736 0.3535533905932737, -0.3296729075500343 0.3759199037394886, -0.3043807145043602 0.3966766701456175, -0.2777851165098009 0.4157348061512725, -0.2499999999999998 0.4330127018922191, -0.2211443451095005 0.4484363707663439, -0.1913417161825447 0.4619397662556431, -0.1607197326515807 0.4734650647475526, -0.1294095225512603 0.4829629131445338, -0.097545161008064 0.4903926402016149, -0.0652630961100257 0.4957224306869049, -0.0327015646150715 0.4989294616193014, 0 0.5, 0.0327015646150715 0.4989294616193017, 0.0652630961100258 0.4957224306869052, 0.0975451610080641 0.4903926402016152, 0.1294095225512604 0.4829629131445342, 0.1607197326515808 0.4734650647475528, 0.1913417161825449 0.4619397662556433, 0.2211443451095006 0.4484363707663441, 0.25 0.4330127018922192, 0.2777851165098011 0.4157348061512725, 0.3043807145043603 0.3966766701456175, 0.3296729075500344 0.3759199037394886, 0.3535533905932737 0.3535533905932736, 0.3759199037394886 0.3296729075500343, 0.3966766701456175 0.3043807145043602, 0.4157348061512725 0.2777851165098009, 0.4330127018922191 0.2499999999999998, 0.4484363707663439 0.2211443451095005, 0.4619397662556431 0.1913417161825447, 0.4734650647475526 0.1607197326515807, 0.4829629131445338 0.1294095225512603, 0.4903926402016149 0.097545161008064, 0.4957224306869049 0.0652630961100257, 0.4989294616193014 0.0327015646150715, 0.5 0))"); + assertFunction("ST_AsText(ST_Buffer(ST_LineFromText('LINESTRING (0 0, 1 1, 2 0.5)'), 0.2))", VARCHAR, "POLYGON ((0 -0.2, -0.0130806258460288 -0.1995717846477207, -0.0261052384440101 -0.1982889722747621, -0.0390180644032259 -0.196157056080646, -0.0517638090205041 -0.1931851652578137, -0.0642878930606323 -0.1893860258990211, -0.0765366864730179 -0.1847759065022574, -0.0884577380438003 -0.1793745483065377, -0.1000000000000001 -0.1732050807568877, -0.1111140466039204 -0.166293922460509, -0.1217522858017439 -0.158670668058247, -0.1318691630200139 -0.1503679614957955, -0.1414213562373097 -0.1414213562373094, -0.1503679614957956 -0.1318691630200137, -0.1586706680582468 -0.1217522858017441, -0.1662939224605089 -0.1111140466039204, -0.1732050807568877 -0.0999999999999999, -0.1793745483065377 -0.0884577380438002, -0.1847759065022574 -0.0765366864730179, -0.189386025899021 -0.0642878930606323, -0.1931851652578134 -0.0517638090205041, -0.1961570560806458 -0.0390180644032256, -0.1982889722747618 -0.0261052384440102, -0.1995717846477207 -0.0130806258460285, -0.2000000000000002 0, -0.1995717846477207 0.0130806258460286, -0.1982889722747623 0.0261052384440103, -0.1961570560806463 0.0390180644032256, -0.1931851652578138 0.0517638090205041, -0.189386025899021 0.0642878930606323, -0.1847759065022574 0.0765366864730179, -0.1793745483065377 0.0884577380438002, -0.1732050807568877 0.1, -0.1662939224605089 0.1111140466039204, -0.1586706680582468 0.1217522858017441, -0.1503679614957956 0.1318691630200137, -0.1414213562373097 0.1414213562373095, 0.8585786437626906 1.1414213562373094, 0.8681308369799863 1.1503679614957956, 0.8782477141982559 1.158670668058247, 0.8888859533960796 1.166293922460509, 0.9000000000000001 1.1732050807568877, 0.9115422619561997 1.1793745483065377, 0.9234633135269821 1.1847759065022574, 0.9357121069393677 1.189386025899021, 0.9482361909794959 1.1931851652578136, 0.9609819355967744 1.1961570560806458, 0.9738947615559896 1.1982889722747618, 0.9869193741539715 1.1995717846477205, 1 1.2, 1.0130806258460288 1.1995717846477207, 1.0261052384440104 1.198288972274762, 1.0390180644032256 1.196157056080646, 1.0517638090205041 1.1931851652578138, 1.0642878930606323 1.189386025899021, 1.076536686473018 1.1847759065022574, 1.0884577380438003 1.1793745483065377, 1.0894427190999916 1.1788854381999831, 2.0894427190999916 0.6788854381999831, 2.1 0.6732050807568877, 2.1111140466039204 0.666293922460509, 2.121752285801744 0.658670668058247, 2.131869163020014 0.6503679614957955, 2.1414213562373097 0.6414213562373094, 2.1503679614957956 0.6318691630200137, 2.158670668058247 0.6217522858017441, 2.166293922460509 0.6111140466039204, 2.1732050807568877 0.5999999999999999, 2.1793745483065377 0.5884577380438002, 2.1847759065022574 0.5765366864730179, 2.189386025899021 0.5642878930606323, 2.1931851652578134 0.5517638090205041, 2.196157056080646 0.5390180644032256, 2.198288972274762 0.5261052384440102, 2.1995717846477207 0.5130806258460285, 2.2 0.5, 2.1995717846477207 0.4869193741539714, 2.1982889722747623 0.4738947615559897, 2.1961570560806463 0.4609819355967744, 2.193185165257814 0.4482361909794959, 2.189386025899021 0.4357121069393677, 2.1847759065022574 0.4234633135269821, 2.1793745483065377 0.4115422619561998, 2.1732050807568877 0.4, 2.166293922460509 0.3888859533960796, 2.158670668058247 0.3782477141982559, 2.1503679614957956 0.3681308369799863, 2.1414213562373092 0.3585786437626906, 2.1318691630200135 0.3496320385042045, 2.121752285801744 0.341329331941753, 2.1111140466039204 0.333706077539491, 2.1 0.3267949192431123, 2.0884577380438003 0.3206254516934625, 2.076536686473018 0.3152240934977427, 2.0642878930606323 0.310613974100979, 2.051763809020504 0.3068148347421865, 2.0390180644032254 0.3038429439193541, 2.02610523844401 0.3017110277252381, 2.0130806258460288 0.3004282153522794, 2 0.3, 1.9869193741539715 0.3004282153522793, 1.9738947615559896 0.3017110277252379, 1.9609819355967744 0.3038429439193539, 1.9482361909794959 0.3068148347421863, 1.9357121069393677 0.3106139741009789, 1.923463313526982 0.3152240934977427, 1.9115422619561997 0.3206254516934623, 1.9105572809000084 0.3211145618000169, 1.0394906098164265 0.7566478973418078, 0.1414213562373094 -0.1414213562373094, 0.1318691630200137 -0.1503679614957955, 0.1217522858017441 -0.158670668058247, 0.1111140466039204 -0.166293922460509, 0.0999999999999999 -0.1732050807568877, 0.0884577380438003 -0.1793745483065375, 0.0765366864730179 -0.1847759065022573, 0.0642878930606323 -0.189386025899021, 0.0517638090205041 -0.1931851652578136, 0.0390180644032256 -0.1961570560806459, 0.0261052384440104 -0.1982889722747619, 0.0130806258460285 -0.1995717846477205, 0 -0.2))"); + assertFunction("ST_AsText(ST_Buffer(ST_GeometryFromText('POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))'), 1.2))", VARCHAR, "POLYGON ((-1.2 0, -1.2 5, -1.1974307078863242 5.078483755076172, -1.1897338336485723 5.156631430664062, -1.1769423364838765 5.234108386419354, -1.159110991546882 5.310582854123025, -1.1363161553941266 5.385727358363794, -1.108655439013544 5.4592201188381075, -1.0762472898392257 5.530746428262802, -1.039230484541326 5.6, -0.997763534763054 5.666684279623523, -0.9520240083494819 5.730513714810464, -0.9022077689747725 5.791214978120083, -0.8485281374238567 5.848528137423857, -0.7912149781200822 5.9022077689747725, -0.7305137148104643 5.952024008349482, -0.6666842796235222 5.997763534763054, -0.5999999999999995 6.039230484541326, -0.530746428262801 6.076247289839225, -0.4592201188381073 6.1086554390135435, -0.3857273583637936 6.136316155394126, -0.3105828541230246 6.159110991546881, -0.2341083864193537 6.176942336483876, -0.1566314306640617 6.189733833648572, -0.0784837550761715 6.1974307078863236, 0 6.2, 5 6.2, 5.078483755076172 6.197430707886324, 5.156631430664062 6.189733833648573, 5.234108386419354 6.176942336483877, 5.310582854123025 6.159110991546882, 5.385727358363794 6.136316155394127, 5.4592201188381075 6.108655439013544, 5.530746428262802 6.076247289839226, 5.6 6.039230484541326, 5.666684279623523 5.997763534763054, 5.730513714810464 5.952024008349482, 5.791214978120083 5.9022077689747725, 5.848528137423857 5.848528137423857, 5.9022077689747725 5.791214978120082, 5.952024008349482 5.730513714810464, 5.997763534763054 5.666684279623523, 6.039230484541326 5.6, 6.076247289839225 5.530746428262801, 6.1086554390135435 5.4592201188381075, 6.136316155394126 5.385727358363794, 6.159110991546881 5.310582854123025, 6.176942336483876 5.234108386419353, 6.189733833648572 5.156631430664062, 6.1974307078863236 5.078483755076172, 6.2 5, 6.2 0, 6.197430707886324 -0.0784837550761717, 6.189733833648573 -0.1566314306640619, 6.176942336483877 -0.2341083864193539, 6.159110991546882 -0.3105828541230249, 6.136316155394127 -0.3857273583637938, 6.108655439013544 -0.4592201188381076, 6.076247289839226 -0.5307464282628015, 6.039230484541326 -0.5999999999999999, 5.997763534763054 -0.6666842796235226, 5.952024008349482 -0.7305137148104647, 5.9022077689747725 -0.7912149781200825, 5.848528137423857 -0.8485281374238568, 5.791214978120082 -0.9022077689747725, 5.730513714810464 -0.9520240083494819, 5.666684279623523 -0.9977635347630538, 5.6 -1.0392304845413258, 5.530746428262801 -1.0762472898392252, 5.4592201188381075 -1.1086554390135435, 5.385727358363794 -1.1363161553941261, 5.310582854123025 -1.1591109915468811, 5.234108386419353 -1.1769423364838756, 5.156631430664062 -1.1897338336485717, 5.078483755076172 -1.1974307078863233, 5 -1.2, 0 -1.2, -0.0784837550761717 -1.1974307078863242, -0.1566314306640619 -1.1897338336485723, -0.2341083864193539 -1.1769423364838765, -0.3105828541230249 -1.159110991546882, -0.3857273583637938 -1.1363161553941266, -0.4592201188381076 -1.108655439013544, -0.5307464282628015 -1.0762472898392257, -0.5999999999999999 -1.039230484541326, -0.6666842796235226 -0.997763534763054, -0.7305137148104647 -0.9520240083494819, -0.7912149781200825 -0.9022077689747725, -0.8485281374238568 -0.8485281374238567, -0.9022077689747725 -0.7912149781200822, -0.9520240083494819 -0.7305137148104643, -0.9977635347630538 -0.6666842796235222, -1.0392304845413258 -0.5999999999999995, -1.0762472898392252 -0.530746428262801, -1.1086554390135435 -0.4592201188381073, -1.1363161553941261 -0.3857273583637936, -1.1591109915468811 -0.3105828541230246, -1.1769423364838756 -0.2341083864193537, -1.1897338336485717 -0.1566314306640617, -1.1974307078863233 -0.0784837550761715, -1.2 0))"); // zero distance assertFunction("ST_AsText(ST_Buffer(ST_Point(0, 0), 0))", VARCHAR, "POINT (0 0)"); assertFunction("ST_AsText(ST_Buffer(ST_LineFromText('LINESTRING (0 0, 1 1, 2 0.5)'), 0))", VARCHAR, "LINESTRING (0 0, 1 1, 2 0.5)"); - assertFunction("ST_AsText(ST_Buffer(ST_GeometryFromText('POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))'), 0))", VARCHAR, "POLYGON ((0 0, 5 0, 5 5, 0 5, 0 0))"); + assertFunction("ST_AsText(ST_Buffer(ST_GeometryFromText('POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))'), 0))", VARCHAR, "POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))"); // geometry collection - assertFunction("ST_AsText(ST_Buffer(ST_Intersection(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))')), 0.2))", VARCHAR, "MULTIPOLYGON (((5 0.8, 5.013080625846029 0.8004282153522794, 5.026105238444011 0.801711027725238, 5.039018064403225 0.803842943919354, 5.051763809020504 0.8068148347421864, 5.064287893060633 0.8106139741009789, 5.076536686473018 0.8152240934977427, 5.0884577380438 0.8206254516934623, 5.1 0.8267949192431123, 5.11111404660392 0.833706077539491, 5.121752285801744 0.841329331941753, 5.1318691630200135 0.8496320385042045, 5.141421356237309 0.8585786437626906, 5.150367961495795 0.8681308369799863, 5.158670668058247 0.8782477141982559, 5.166293922460509 0.8888859533960796, 5.173205080756888 0.9, 5.179374548306538 0.9115422619561997, 5.184775906502257 0.9234633135269821, 5.189386025899021 0.9357121069393677, 5.193185165257813 0.9482361909794959, 5.196157056080646 0.9609819355967744, 5.198288972274762 0.9738947615559896, 5.199571784647721 0.9869193741539714, 5.2 1, 5.199571784647721 1.0130806258460288, 5.198288972274762 1.0261052384440104, 5.196157056080646 1.0390180644032256, 5.193185165257813 1.0517638090205041, 5.189386025899021 1.0642878930606323, 5.184775906502257 1.076536686473018, 5.179374548306537 1.0884577380438003, 5.173205080756888 1.1, 5.166293922460509 1.1111140466039204, 5.158670668058247 1.1217522858017441, 5.150367961495795 1.1318691630200137, 5.141421356237309 1.1414213562373094, 5.1318691630200135 1.1503679614957956, 5.121752285801744 1.158670668058247, 5.11111404660392 1.1662939224605091, 5.1 1.1732050807568877, 5.0884577380438 1.1793745483065377, 5.076536686473018 1.1847759065022574, 5.064287893060632 1.1893860258990212, 5.051763809020504 1.1931851652578138, 5.039018064403225 1.196157056080646, 5.026105238444011 1.198288972274762, 5.013080625846029 1.1995717846477207, 5 1.2, 4.986919374153971 1.1995717846477207, 4.973894761555989 1.198288972274762, 4.960981935596775 1.196157056080646, 4.948236190979496 1.1931851652578136, 4.935712106939367 1.189386025899021, 4.923463313526982 1.1847759065022574, 4.9115422619562 1.1793745483065377, 4.9 1.1732050807568877, 4.88888595339608 1.166293922460509, 4.878247714198256 1.158670668058247, 4.8681308369799865 1.1503679614957956, 4.858578643762691 1.1414213562373094, 4.849632038504205 1.1318691630200137, 4.841329331941753 1.1217522858017441, 4.833706077539491 1.1111140466039204, 4.826794919243112 1.1, 4.820625451693462 1.0884577380438003, 4.815224093497743 1.076536686473018, 4.810613974100979 1.0642878930606323, 4.806814834742187 1.0517638090205041, 4.803842943919354 1.0390180644032256, 4.801711027725238 1.0261052384440104, 4.800428215352279 1.0130806258460285, 4.8 1, 4.800428215352279 0.9869193741539714, 4.801711027725238 0.9738947615559896, 4.803842943919354 0.9609819355967743, 4.806814834742187 0.9482361909794959, 4.810613974100979 0.9357121069393677, 4.815224093497743 0.923463313526982, 4.820625451693463 0.9115422619561997, 4.826794919243112 0.8999999999999999, 4.833706077539491 0.8888859533960796, 4.841329331941753 0.8782477141982559, 4.849632038504205 0.8681308369799862, 4.858578643762691 0.8585786437626904, 4.8681308369799865 0.8496320385042044, 4.878247714198256 0.841329331941753, 4.88888595339608 0.8337060775394909, 4.9 0.8267949192431122, 4.9115422619562 0.8206254516934623, 4.923463313526982 0.8152240934977426, 4.935712106939368 0.8106139741009788, 4.948236190979496 0.8068148347421863, 4.960981935596775 0.8038429439193538, 4.973894761555989 0.801711027725238, 4.986919374153971 0.8004282153522793, 5 0.8)), ((3 3.8, 4 3.8, 4.013080625846029 3.8004282153522793, 4.026105238444011 3.801711027725238, 4.039018064403225 3.803842943919354, 4.051763809020504 3.8068148347421866, 4.064287893060632 3.810613974100979, 4.076536686473018 3.8152240934977426, 4.0884577380438 3.8206254516934623, 4.1 3.8267949192431123, 4.11111404660392 3.833706077539491, 4.121752285801744 3.841329331941753, 4.1318691630200135 3.8496320385042044, 4.141421356237309 3.8585786437626903, 4.150367961495795 3.868130836979986, 4.158670668058247 3.878247714198256, 4.166293922460509 3.8888859533960796, 4.173205080756888 3.9, 4.179374548306537 3.9115422619561997, 4.184775906502257 3.923463313526982, 4.189386025899021 3.9357121069393677, 4.193185165257813 3.948236190979496, 4.196157056080646 3.960981935596774, 4.198288972274762 3.97389476155599, 4.199571784647721 3.9869193741539712, 4.2 4, 4.199571784647721 4.013080625846029, 4.198288972274762 4.026105238444011, 4.196157056080646 4.039018064403225, 4.193185165257813 4.051763809020504, 4.189386025899021 4.064287893060632, 4.184775906502257 4.076536686473018, 4.179374548306537 4.0884577380438, 4.173205080756888 4.1, 4.166293922460509 4.11111404660392, 4.158670668058247 4.121752285801744, 4.150367961495795 4.1318691630200135, 4.141421356237309 4.141421356237309, 4.1318691630200135 4.150367961495795, 4.121752285801744 4.158670668058247, 4.11111404660392 4.166293922460509, 4.1 4.173205080756888, 4.0884577380438 4.179374548306537, 4.076536686473018 4.184775906502257, 4.064287893060632 4.189386025899021, 4.051763809020504 4.193185165257813, 4.039018064403225 4.196157056080646, 4.026105238444011 4.198288972274762, 4.013080625846029 4.199571784647721, 4 4.2, 3 4.2, 2.9869193741539712 4.199571784647721, 2.9738947615559894 4.198288972274762, 2.9609819355967746 4.196157056080646, 2.948236190979496 4.193185165257813, 2.9357121069393677 4.189386025899021, 2.923463313526982 4.184775906502257, 2.9115422619561997 4.179374548306537, 2.9000000000000004 4.173205080756888, 2.8888859533960796 4.166293922460509, 2.878247714198256 4.158670668058247, 2.8681308369799865 4.150367961495795, 2.8585786437626908 4.141421356237309, 2.8496320385042044 4.1318691630200135, 2.841329331941753 4.121752285801744, 2.833706077539491 4.11111404660392, 2.8267949192431123 4.1, 2.8206254516934623 4.0884577380438, 2.8152240934977426 4.076536686473018, 2.8106139741009786 4.064287893060632, 2.8068148347421866 4.051763809020504, 2.8038429439193537 4.039018064403225, 2.801711027725238 4.026105238444011, 2.8004282153522793 4.013080625846029, 2.8 4, 2.8004282153522793 3.9869193741539712, 2.801711027725238 3.97389476155599, 2.8038429439193537 3.9609819355967746, 2.8068148347421866 3.948236190979496, 2.810613974100979 3.9357121069393677, 2.8152240934977426 3.923463313526982, 2.8206254516934623 3.9115422619561997, 2.8267949192431123 3.9, 2.833706077539491 3.8888859533960796, 2.841329331941753 3.878247714198256, 2.8496320385042044 3.8681308369799865, 2.8585786437626908 3.8585786437626908, 2.8681308369799865 3.8496320385042044, 2.878247714198256 3.841329331941753, 2.8888859533960796 3.833706077539491, 2.9 3.8267949192431123, 2.9115422619561997 3.8206254516934623, 2.923463313526982 3.8152240934977426, 2.9357121069393677 3.810613974100979, 2.948236190979496 3.806814834742186, 2.9609819355967746 3.8038429439193537, 2.9738947615559894 3.8017110277252377, 2.9869193741539712 3.8004282153522793, 3 3.8)))"); + assertFunction("ST_AsText(ST_Buffer(ST_Intersection(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))')), 0.2))", VARCHAR, "MULTIPOLYGON (((5 0.8, 4.986919374153971 0.8004282153522793, 4.973894761555989 0.801711027725238, 4.960981935596775 0.8038429439193538, 4.948236190979496 0.8068148347421863, 4.935712106939368 0.8106139741009788, 4.923463313526982 0.8152240934977426, 4.9115422619562 0.8206254516934623, 4.9 0.8267949192431122, 4.88888595339608 0.8337060775394909, 4.878247714198256 0.841329331941753, 4.8681308369799865 0.8496320385042044, 4.858578643762691 0.8585786437626904, 4.849632038504205 0.8681308369799862, 4.841329331941753 0.8782477141982559, 4.833706077539491 0.8888859533960796, 4.826794919243112 0.8999999999999999, 4.820625451693463 0.9115422619561997, 4.815224093497743 0.923463313526982, 4.810613974100979 0.9357121069393677, 4.806814834742187 0.9482361909794959, 4.803842943919354 0.9609819355967743, 4.801711027725238 0.9738947615559896, 4.800428215352279 0.9869193741539714, 4.8 1, 4.800428215352279 1.0130806258460285, 4.801711027725238 1.0261052384440104, 4.803842943919354 1.0390180644032256, 4.806814834742187 1.0517638090205041, 4.810613974100979 1.0642878930606323, 4.815224093497743 1.076536686473018, 4.820625451693462 1.0884577380438003, 4.826794919243112 1.1, 4.833706077539491 1.1111140466039204, 4.841329331941753 1.1217522858017441, 4.849632038504205 1.1318691630200137, 4.858578643762691 1.1414213562373094, 4.8681308369799865 1.1503679614957956, 4.878247714198256 1.158670668058247, 4.88888595339608 1.166293922460509, 4.9 1.1732050807568877, 4.9115422619562 1.1793745483065377, 4.923463313526982 1.1847759065022574, 4.935712106939367 1.189386025899021, 4.948236190979496 1.1931851652578136, 4.960981935596775 1.196157056080646, 4.973894761555989 1.198288972274762, 4.986919374153971 1.1995717846477207, 5 1.2, 5.013080625846029 1.1995717846477207, 5.026105238444011 1.198288972274762, 5.039018064403225 1.196157056080646, 5.051763809020504 1.1931851652578138, 5.064287893060632 1.1893860258990212, 5.076536686473018 1.1847759065022574, 5.0884577380438 1.1793745483065377, 5.1 1.1732050807568877, 5.11111404660392 1.1662939224605091, 5.121752285801744 1.158670668058247, 5.1318691630200135 1.1503679614957956, 5.141421356237309 1.1414213562373094, 5.150367961495795 1.1318691630200137, 5.158670668058247 1.1217522858017441, 5.166293922460509 1.1111140466039204, 5.173205080756888 1.1, 5.179374548306537 1.0884577380438003, 5.184775906502257 1.076536686473018, 5.189386025899021 1.0642878930606323, 5.193185165257813 1.0517638090205041, 5.196157056080646 1.0390180644032256, 5.198288972274762 1.0261052384440104, 5.199571784647721 1.0130806258460288, 5.2 1, 5.199571784647721 0.9869193741539714, 5.198288972274762 0.9738947615559896, 5.196157056080646 0.9609819355967744, 5.193185165257813 0.9482361909794959, 5.189386025899021 0.9357121069393677, 5.184775906502257 0.9234633135269821, 5.179374548306538 0.9115422619561997, 5.173205080756888 0.9, 5.166293922460509 0.8888859533960796, 5.158670668058247 0.8782477141982559, 5.150367961495795 0.8681308369799863, 5.141421356237309 0.8585786437626906, 5.1318691630200135 0.8496320385042045, 5.121752285801744 0.841329331941753, 5.11111404660392 0.833706077539491, 5.1 0.8267949192431123, 5.0884577380438 0.8206254516934623, 5.076536686473018 0.8152240934977427, 5.064287893060633 0.8106139741009789, 5.051763809020504 0.8068148347421864, 5.039018064403225 0.803842943919354, 5.026105238444011 0.801711027725238, 5.013080625846029 0.8004282153522794, 5 0.8)), ((3 3.8, 2.9869193741539712 3.8004282153522793, 2.9738947615559894 3.8017110277252377, 2.9609819355967746 3.8038429439193537, 2.948236190979496 3.806814834742186, 2.9357121069393677 3.810613974100979, 2.923463313526982 3.8152240934977426, 2.9115422619561997 3.8206254516934623, 2.9 3.8267949192431123, 2.8888859533960796 3.833706077539491, 2.878247714198256 3.841329331941753, 2.8681308369799865 3.8496320385042044, 2.8585786437626908 3.8585786437626908, 2.8496320385042044 3.8681308369799865, 2.841329331941753 3.878247714198256, 2.833706077539491 3.8888859533960796, 2.8267949192431123 3.9, 2.8206254516934623 3.9115422619561997, 2.8152240934977426 3.923463313526982, 2.810613974100979 3.9357121069393677, 2.8068148347421866 3.948236190979496, 2.8038429439193537 3.9609819355967746, 2.801711027725238 3.97389476155599, 2.8004282153522793 3.9869193741539712, 2.8 4, 2.8004282153522793 4.013080625846029, 2.801711027725238 4.026105238444011, 2.8038429439193537 4.039018064403225, 2.8068148347421866 4.051763809020504, 2.8106139741009786 4.064287893060632, 2.8152240934977426 4.076536686473018, 2.8206254516934623 4.0884577380438, 2.8267949192431123 4.1, 2.833706077539491 4.11111404660392, 2.841329331941753 4.121752285801744, 2.8496320385042044 4.1318691630200135, 2.8585786437626908 4.141421356237309, 2.8681308369799865 4.150367961495795, 2.878247714198256 4.158670668058247, 2.8888859533960796 4.166293922460509, 2.9000000000000004 4.173205080756888, 2.9115422619561997 4.179374548306537, 2.923463313526982 4.184775906502257, 2.9357121069393677 4.189386025899021, 2.948236190979496 4.193185165257813, 2.9609819355967746 4.196157056080646, 2.9738947615559894 4.198288972274762, 2.9869193741539712 4.199571784647721, 3 4.2, 4 4.2, 4.013080625846029 4.199571784647721, 4.026105238444011 4.198288972274762, 4.039018064403225 4.196157056080646, 4.051763809020504 4.193185165257813, 4.064287893060632 4.189386025899021, 4.076536686473018 4.184775906502257, 4.0884577380438 4.179374548306537, 4.1 4.173205080756888, 4.11111404660392 4.166293922460509, 4.121752285801744 4.158670668058247, 4.1318691630200135 4.150367961495795, 4.141421356237309 4.141421356237309, 4.150367961495795 4.1318691630200135, 4.158670668058247 4.121752285801744, 4.166293922460509 4.11111404660392, 4.173205080756888 4.1, 4.179374548306537 4.0884577380438, 4.184775906502257 4.076536686473018, 4.189386025899021 4.064287893060632, 4.193185165257813 4.051763809020504, 4.196157056080646 4.039018064403225, 4.198288972274762 4.026105238444011, 4.199571784647721 4.013080625846029, 4.2 4, 4.199571784647721 3.9869193741539712, 4.198288972274762 3.97389476155599, 4.196157056080646 3.960981935596774, 4.193185165257813 3.948236190979496, 4.189386025899021 3.9357121069393677, 4.184775906502257 3.923463313526982, 4.179374548306537 3.9115422619561997, 4.173205080756888 3.9, 4.166293922460509 3.8888859533960796, 4.158670668058247 3.878247714198256, 4.150367961495795 3.868130836979986, 4.141421356237309 3.8585786437626903, 4.1318691630200135 3.8496320385042044, 4.121752285801744 3.841329331941753, 4.11111404660392 3.833706077539491, 4.1 3.8267949192431123, 4.0884577380438 3.8206254516934623, 4.076536686473018 3.8152240934977426, 4.064287893060632 3.810613974100979, 4.051763809020504 3.8068148347421866, 4.039018064403225 3.803842943919354, 4.026105238444011 3.801711027725238, 4.013080625846029 3.8004282153522793, 4 3.8, 3 3.8)))"); // empty geometry assertFunction("ST_Buffer(ST_GeometryFromText('POINT EMPTY'), 1)", GEOMETRY, null); @@ -193,6 +199,16 @@ public void testSTBuffer() // infinity() and nan() distance assertFunction("ST_AsText(ST_Buffer(ST_Point(0, 0), infinity()))", VARCHAR, "MULTIPOLYGON EMPTY"); assertInvalidFunction("ST_Buffer(ST_Point(0, 0), nan())", "distance is NaN"); + + // For small polygons, there was a bug in ESRI that throw an NPE. This + // was fixed (https://github.com/Esri/geometry-api-java/pull/243) to + // return an empty geometry instead. Ideally, these would return + // something approximately like `ST_Buffer(ST_Centroid(geometry))`. + assertFunction("ST_IsEmpty(ST_Buffer(ST_Buffer(ST_Point(177.50102959662, 64.726807421691), 0.0000000001), 0.00005))", + BOOLEAN, true); + assertFunction("ST_IsEmpty(ST_Buffer(ST_GeometryFromText(" + + "'POLYGON ((177.0 64.0, 177.0000000001 64.0, 177.0000000001 64.0000000001, 177.0 64.0000000001, 177.0 64.0))'" + + "), 0.01))", BOOLEAN, true); } @Test @@ -203,13 +219,30 @@ public void testSTCentroid() assertCentroid("MULTIPOINT (1 2, 2 4, 3 6, 4 8)", new Point(2.5, 5)); assertCentroid("LINESTRING (1 1, 2 2, 3 3)", new Point(2, 2)); assertCentroid("MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))", new Point(3, 2)); - assertCentroid("POLYGON ((1 1, 1 4, 4 4, 4 1))", new Point(2.5, 2.5)); - assertCentroid("POLYGON ((1 1, 5 1, 3 4))", new Point(3, 2)); - assertCentroid("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((2 4, 2 6, 6 6, 6 4)))", new Point(3.3333333333333335, 4)); + assertCentroid("POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))", new Point(2.5, 2.5)); + assertCentroid("POLYGON ((1 1, 5 1, 3 4, 1 1))", new Point(3, 2)); + assertCentroid("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))", new Point(3.3333333333333335, 4)); assertCentroid("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))", new Point(2.5416666666666665, 2.5416666666666665)); // invalid geometry - assertInvalidFunction("ST_Centroid(ST_GeometryFromText('MULTIPOLYGON (((4.903234300000006 52.08474289999999, 4.903234265193165 52.084742934806826, 4.903234299999999 52.08474289999999, 4.903234300000006 52.08474289999999)))'))", "Cannot compute centroid: .* Use ST_IsValid to confirm that input geometry is valid or compute centroid for a bounding box using ST_Envelope."); + assertApproximateCentroid("MULTIPOLYGON (((4.903234300000006 52.08474289999999, 4.903234265193165 52.084742934806826, 4.903234299999999 52.08474289999999, 4.903234300000006 52.08474289999999)))", new Point(4.9032343, 52.0847429), 1e-7); + + // Numerical stability tests + assertApproximateCentroid( + "MULTIPOLYGON (((153.492818 -28.13729, 153.492821 -28.137291, 153.492816 -28.137289, 153.492818 -28.13729)))", + new Point(153.49282, -28.13729), 1e-5); + assertApproximateCentroid( + "MULTIPOLYGON (((153.112475 -28.360526, 153.1124759 -28.360527, 153.1124759 -28.360526, 153.112475 -28.360526)))", + new Point(153.112475, -28.360526), 1e-5); + assertApproximateCentroid( + "POLYGON ((4.903234300000006 52.08474289999999, 4.903234265193165 52.084742934806826, 4.903234299999999 52.08474289999999, 4.903234300000006 52.08474289999999))", + new Point(4.9032343, 52.0847429), 1e-6); + assertApproximateCentroid( + "MULTIPOLYGON (((4.903234300000006 52.08474289999999, 4.903234265193165 52.084742934806826, 4.903234299999999 52.08474289999999, 4.903234300000006 52.08474289999999)))", + new Point(4.9032343, 52.0847429), 1e-6); + assertApproximateCentroid( + "POLYGON ((-81.0387349 29.20822, -81.039974 29.210597, -81.0410331 29.2101579, -81.0404758 29.2090879, -81.0404618 29.2090609, -81.040433 29.209005, -81.0404269 29.208993, -81.0404161 29.2089729, -81.0398001 29.20779, -81.0387349 29.20822), (-81.0404229 29.208986, -81.04042 29.2089809, -81.0404269 29.208993, -81.0404229 29.208986))", + new Point(-81.039885, 29.209191), 1e-6); } private void assertCentroid(String wkt, Point centroid) @@ -217,6 +250,14 @@ private void assertCentroid(String wkt, Point centroid) assertFunction(format("ST_AsText(ST_Centroid(ST_GeometryFromText('%s')))", wkt), VARCHAR, new OGCPoint(centroid, null).asText()); } + private void assertApproximateCentroid(String wkt, Point expectedCentroid, double epsilon) + { + OGCPoint actualCentroid = (OGCPoint) EsriGeometrySerde.deserialize( + stCentroid(EsriGeometrySerde.serialize(OGCGeometry.fromText(wkt)))); + assertEquals(actualCentroid.X(), expectedCentroid.getX(), epsilon); + assertEquals(actualCentroid.Y(), expectedCentroid.getY(), epsilon); + } + @Test public void testSTConvexHull() { @@ -229,45 +270,45 @@ public void testSTConvexHull() assertConvexHull("MULTIPOLYGON EMPTY", "MULTIPOLYGON EMPTY"); assertConvexHull("GEOMETRYCOLLECTION EMPTY", "GEOMETRYCOLLECTION EMPTY"); assertConvexHull("GEOMETRYCOLLECTION (POINT (1 1), POINT EMPTY)", "POINT (1 1)"); - assertConvexHull("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (1 1), GEOMETRYCOLLECTION (POINT (1 5), POINT (4 5), GEOMETRYCOLLECTION (POINT (3 4), POINT EMPTY))))", "POLYGON ((1 1, 4 5, 1 5, 1 1))"); + assertConvexHull("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (1 1), GEOMETRYCOLLECTION (POINT (1 5), POINT (4 5), GEOMETRYCOLLECTION (POINT (3 4), POINT EMPTY))))", "POLYGON ((1 1, 1 5, 4 5, 1 1))"); // test single geometry assertConvexHull("POINT (1 1)", "POINT (1 1)"); - assertConvexHull("LINESTRING (1 1, 1 9, 2 2)", "POLYGON ((1 1, 2 2, 1 9, 1 1))"); + assertConvexHull("LINESTRING (1 1, 1 9, 2 2)", "POLYGON ((1 1, 1 9, 2 2, 1 1))"); // convex single geometry - assertConvexHull("LINESTRING (1 1, 1 9, 2 2, 1 1)", "POLYGON ((1 1, 2 2, 1 9, 1 1))"); - assertConvexHull("POLYGON ((0 0, 0 3, 2 4, 4 2, 3 0, 0 0))", "POLYGON ((0 0, 3 0, 4 2, 2 4, 0 3, 0 0))"); + assertConvexHull("LINESTRING (1 1, 1 9, 2 2, 1 1)", "POLYGON ((1 1, 1 9, 2 2, 1 1))"); + assertConvexHull("POLYGON ((0 0, 0 3, 2 4, 4 2, 3 0, 0 0))", "POLYGON ((0 0, 0 3, 2 4, 4 2, 3 0, 0 0))"); // non-convex geometry - assertConvexHull("LINESTRING (1 1, 1 9, 2 2, 1 1, 4 0)", "POLYGON ((1 1, 4 0, 1 9, 1 1))"); - assertConvexHull("POLYGON ((0 0, 0 3, 4 4, 1 1, 3 0))", "POLYGON ((0 0, 3 0, 4 4, 0 3, 0 0))"); + assertConvexHull("LINESTRING (1 1, 1 9, 2 2, 1 1, 4 0)", "POLYGON ((1 1, 1 9, 4 0, 1 1))"); + assertConvexHull("POLYGON ((0 0, 0 3, 4 4, 1 1, 3 0, 0 0))", "POLYGON ((0 0, 0 3, 4 4, 3 0, 0 0))"); // all points are on the same line assertConvexHull("LINESTRING (20 20, 30 30)", "LINESTRING (20 20, 30 30)"); assertConvexHull("MULTILINESTRING ((0 0, 3 3), (1 1, 2 2), (2 2, 4 4), (5 5, 8 8))", "LINESTRING (0 0, 8 8)"); assertConvexHull("MULTIPOINT (0 1, 1 2, 2 3, 3 4, 4 5, 5 6)", "LINESTRING (0 1, 5 6)"); - assertConvexHull("GEOMETRYCOLLECTION (POINT (0 0), LINESTRING (1 1, 4 4, 2 2), POINT (10 10), POLYGON ((5 5, 7 7)), POINT (2 2), LINESTRING (6 6, 9 9), POLYGON ((1 1)))", "LINESTRING (0 0, 10 10)"); + assertConvexHull("GEOMETRYCOLLECTION (POINT (0 0), LINESTRING (1 1, 4 4, 2 2), POINT (10 10), POLYGON ((5 5, 7 7, 6 6, 5 5)), POINT (2 2), LINESTRING (6 6, 9 9))", "LINESTRING (0 0, 10 10)"); assertConvexHull("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (2 2), POINT (1 1)), POINT (3 3))", "LINESTRING (3 3, 1 1)"); // not all points are on the same line - assertConvexHull("MULTILINESTRING ((1 1, 5 1, 6 6), (2 4, 4 0), (2 -4, 4 4), (3 -2, 4 -3))", "POLYGON ((1 1, 2 -4, 4 -3, 5 1, 6 6, 2 4, 1 1))"); - assertConvexHull("MULTIPOINT (0 2, 1 0, 3 0, 4 0, 4 2, 2 2, 2 4)", "POLYGON ((0 2, 1 0, 4 0, 4 2, 2 4, 0 2))"); - assertConvexHull("MULTIPOLYGON (((0 3, 2 0, 3 6), (2 1, 2 3, 5 3, 5 1), (1 7, 2 4, 4 2, 5 6, 3 8)))", "POLYGON ((0 3, 2 0, 5 1, 5 6, 3 8, 1 7, 0 3))"); - assertConvexHull("GEOMETRYCOLLECTION (POINT (2 3), LINESTRING (2 8, 7 10), POINT (8 10), POLYGON ((4 4, 4 8, 9 8, 6 6, 6 4, 8 3, 6 1)), POINT (4 2), LINESTRING (3 6, 5 5), POLYGON ((7 5, 7 6, 8 6, 8 5)))", "POLYGON ((2 3, 6 1, 8 3, 9 8, 8 10, 7 10, 2 8, 2 3))"); - assertConvexHull("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (2 3), LINESTRING (2 8, 7 10), GEOMETRYCOLLECTION (POINT (8 10))), POLYGON ((4 4, 4 8, 9 8, 6 6, 6 4, 8 3, 6 1)), POINT (4 2), LINESTRING (3 6, 5 5), POLYGON ((7 5, 7 6, 8 6, 8 5)))", "POLYGON ((2 3, 6 1, 8 3, 9 8, 8 10, 7 10, 2 8, 2 3))"); + assertConvexHull("MULTILINESTRING ((1 1, 5 1, 6 6), (2 4, 4 0), (2 -4, 4 4), (3 -2, 4 -3))", "POLYGON ((1 1, 2 4, 6 6, 5 1, 4 -3, 2 -4, 1 1))"); + assertConvexHull("MULTIPOINT (0 2, 1 0, 3 0, 4 0, 4 2, 2 2, 2 4)", "POLYGON ((0 2, 2 4, 4 2, 4 0, 1 0, 0 2))"); + assertConvexHull("MULTIPOLYGON (((0 3, 2 0, 3 6, 0 3), (2 1, 2 3, 5 3, 5 1, 2 1), (1 7, 2 4, 4 2, 5 6, 3 8, 1 7)))", "POLYGON ((0 3, 1 7, 3 8, 5 6, 5 1, 2 0, 0 3))"); + assertConvexHull("GEOMETRYCOLLECTION (POINT (2 3), LINESTRING (2 8, 7 10), POINT (8 10), POLYGON ((4 4, 4 8, 9 8, 6 6, 6 4, 8 3, 6 1, 4 4)), POINT (4 2), LINESTRING (3 6, 5 5), POLYGON ((7 5, 7 6, 8 6, 8 5, 7 5)))", "POLYGON ((2 3, 2 8, 7 10, 8 10, 9 8, 8 3, 6 1, 2 3))"); + assertConvexHull("GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (2 3), LINESTRING (2 8, 7 10), GEOMETRYCOLLECTION (POINT (8 10))), POLYGON ((4 4, 4 8, 9 8, 6 6, 6 4, 8 3, 6 1, 4 4)), POINT (4 2), LINESTRING (3 6, 5 5), POLYGON ((7 5, 7 6, 8 6, 8 5, 7 5)))", "POLYGON ((2 3, 2 8, 7 10, 8 10, 9 8, 8 3, 6 1, 2 3))"); // single-element multi-geometries and geometry collections - assertConvexHull("MULTILINESTRING ((1 1, 5 1, 6 6))", "POLYGON ((1 1, 5 1, 6 6, 1 1))"); - assertConvexHull("MULTILINESTRING ((1 1, 5 1, 1 4, 5 4))", "POLYGON ((1 1, 5 1, 5 4, 1 4, 1 1))"); + assertConvexHull("MULTILINESTRING ((1 1, 5 1, 6 6))", "POLYGON ((1 1, 6 6, 5 1, 1 1))"); + assertConvexHull("MULTILINESTRING ((1 1, 5 1, 1 4, 5 4))", "POLYGON ((1 1, 1 4, 5 4, 5 1, 1 1))"); assertConvexHull("MULTIPOINT (0 2)", "POINT (0 2)"); - assertConvexHull("MULTIPOLYGON (((0 3, 2 0, 3 6)))", "POLYGON ((0 3, 2 0, 3 6, 0 3))"); - assertConvexHull("MULTIPOLYGON (((0 0, 4 0, 4 4, 0 4, 2 2)))", "POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0))"); + assertConvexHull("MULTIPOLYGON (((0 3, 3 6, 2 0, 0 3)))", "POLYGON ((0 3, 3 6, 2 0, 0 3))"); + assertConvexHull("MULTIPOLYGON (((0 0, 4 0, 4 4, 0 4, 2 2, 0 0)))", "POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))"); assertConvexHull("GEOMETRYCOLLECTION (POINT (2 3))", "POINT (2 3)"); - assertConvexHull("GEOMETRYCOLLECTION (LINESTRING (1 1, 5 1, 6 6))", "POLYGON ((1 1, 5 1, 6 6, 1 1))"); - assertConvexHull("GEOMETRYCOLLECTION (LINESTRING (1 1, 5 1, 1 4, 5 4))", "POLYGON ((1 1, 5 1, 5 4, 1 4, 1 1))"); - assertConvexHull("GEOMETRYCOLLECTION (POLYGON ((0 3, 2 0, 3 6)))", "POLYGON ((0 3, 2 0, 3 6, 0 3))"); - assertConvexHull("GEOMETRYCOLLECTION (POLYGON ((0 0, 4 0, 4 4, 0 4, 2 2)))", "POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0))"); + assertConvexHull("GEOMETRYCOLLECTION (LINESTRING (1 1, 5 1, 6 6))", "POLYGON ((1 1, 6 6, 5 1, 1 1))"); + assertConvexHull("GEOMETRYCOLLECTION (LINESTRING (1 1, 5 1, 1 4, 5 4))", "POLYGON ((1 1, 1 4, 5 4, 5 1, 1 1))"); + assertConvexHull("GEOMETRYCOLLECTION (POLYGON ((0 3, 3 6, 2 0, 0 3)))", "POLYGON ((0 3, 3 6, 2 0, 0 3))"); + assertConvexHull("GEOMETRYCOLLECTION (POLYGON ((0 0, 4 0, 4 4, 0 4, 2 2, 0 0)))", "POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))"); } private void assertConvexHull(String inputWKT, String expectWKT) @@ -278,7 +319,7 @@ private void assertConvexHull(String inputWKT, String expectWKT) @Test public void testSTCoordDim() { - assertFunction("ST_CoordDim(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'))", TINYINT, (byte) 2); + assertFunction("ST_CoordDim(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'))", TINYINT, (byte) 2); assertFunction("ST_CoordDim(ST_GeometryFromText('POLYGON EMPTY'))", TINYINT, (byte) 2); assertFunction("ST_CoordDim(ST_GeometryFromText('LINESTRING EMPTY'))", TINYINT, (byte) 2); assertFunction("ST_CoordDim(ST_GeometryFromText('POINT (1 4)'))", TINYINT, (byte) 2); @@ -288,7 +329,7 @@ public void testSTCoordDim() public void testSTDimension() { assertFunction("ST_Dimension(ST_GeometryFromText('POLYGON EMPTY'))", TINYINT, (byte) 2); - assertFunction("ST_Dimension(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'))", TINYINT, (byte) 2); + assertFunction("ST_Dimension(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'))", TINYINT, (byte) 2); assertFunction("ST_Dimension(ST_GeometryFromText('LINESTRING EMPTY'))", TINYINT, (byte) 1); assertFunction("ST_Dimension(ST_GeometryFromText('POINT (1 4)'))", TINYINT, (byte) 0); } @@ -298,7 +339,9 @@ public void testSTIsClosed() { assertFunction("ST_IsClosed(ST_GeometryFromText('LINESTRING (1 1, 2 2, 1 3, 1 1)'))", BOOLEAN, true); assertFunction("ST_IsClosed(ST_GeometryFromText('LINESTRING (1 1, 2 2, 1 3)'))", BOOLEAN, false); - assertInvalidFunction("ST_IsClosed(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'))", "ST_IsClosed only applies to LINE_STRING or MULTI_LINE_STRING. Input type is: POLYGON"); + assertFunction("ST_IsClosed(ST_GeometryFromText('MULTILINESTRING ((1 1, 2 2, 1 3, 1 1), (4 4, 5 5))'))", BOOLEAN, false); + assertFunction("ST_IsClosed(ST_GeometryFromText('MULTILINESTRING ((1 1, 2 2, 1 3, 1 1), (4 4, 5 4, 5 5, 4 5, 4 4))'))", BOOLEAN, true); + assertInvalidFunction("ST_IsClosed(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'))", "ST_IsClosed only applies to LINE_STRING or MULTI_LINE_STRING. Input type is: POLYGON"); } @Test @@ -330,19 +373,19 @@ public void testSTIsSimple() assertSimpleGeometry("MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))"); assertNotSimpleGeometry("MULTILINESTRING ((1 1, 5 1), (2 4, 4 0))"); assertSimpleGeometry("POLYGON EMPTY"); - assertSimpleGeometry("POLYGON ((2 0, 2 1, 3 1))"); - assertSimpleGeometry("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((2 4, 2 6, 6 6, 6 4)))"); + assertSimpleGeometry("POLYGON ((2 0, 2 1, 3 1, 2 0))"); + assertSimpleGeometry("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))"); } @Test public void testSimplifyGeometry() { // Eliminate unnecessary points on the same line. - assertFunction("ST_AsText(simplify_geometry(ST_GeometryFromText('POLYGON ((1 0, 2 1, 3 1, 3 1, 4 1, 1 0))'), 1.5))", VARCHAR, "POLYGON ((1 0, 4 1, 2 1, 1 0))"); + assertFunction("ST_AsText(simplify_geometry(ST_GeometryFromText('POLYGON ((1 0, 2 1, 3 1, 3 1, 4 1, 1 0))'), 1.5))", VARCHAR, "POLYGON ((1 0, 2 1, 4 1, 1 0))"); // Use distanceTolerance to control fidelity. - assertFunction("ST_AsText(simplify_geometry(ST_GeometryFromText('POLYGON ((1 0, 1 1, 2 1, 2 3, 3 3, 3 1, 4 1, 4 0, 1 0))'), 1.0))", VARCHAR, "POLYGON ((1 0, 4 0, 3 3, 2 3, 1 0))"); - assertFunction("ST_AsText(simplify_geometry(ST_GeometryFromText('POLYGON ((1 0, 1 1, 2 1, 2 3, 3 3, 3 1, 4 1, 4 0, 1 0))'), 0.5))", VARCHAR, "POLYGON ((1 0, 4 0, 4 1, 3 1, 3 3, 2 3, 2 1, 1 1, 1 0))"); + assertFunction("ST_AsText(simplify_geometry(ST_GeometryFromText('POLYGON ((1 0, 1 1, 2 1, 2 3, 3 3, 3 1, 4 1, 4 0, 1 0))'), 1.0))", VARCHAR, "POLYGON ((1 0, 2 3, 3 3, 4 0, 1 0))"); + assertFunction("ST_AsText(simplify_geometry(ST_GeometryFromText('POLYGON ((1 0, 1 1, 2 1, 2 3, 3 3, 3 1, 4 1, 4 0, 1 0))'), 0.5))", VARCHAR, "POLYGON ((1 0, 1 1, 2 1, 2 3, 3 3, 3 1, 4 1, 4 0, 1 0))"); // Negative distance tolerance is invalid. assertInvalidFunction("ST_AsText(simplify_geometry(ST_GeometryFromText('" + "POLYGON ((1 0, 1 1, 2 1, 2 3, 3 3, 3 1, 4 1, 4 0, 1 0))" + "'), -0.5))", "distanceTolerance is negative"); @@ -366,7 +409,7 @@ public void testSTIsValid() assertValidGeometry("LINESTRING (0 0, 1 2, 3 4)"); assertValidGeometry("MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))"); assertValidGeometry("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))"); - assertValidGeometry("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((2 4, 2 6, 6 6, 6 4)))"); + assertValidGeometry("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))"); assertValidGeometry("GEOMETRYCOLLECTION (POINT (1 2), LINESTRING (0 0, 1 2, 3 4), POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0)))"); // invalid geometries @@ -406,7 +449,7 @@ public void testSTLength() assertFunction("ST_Length(ST_GeometryFromText('LINESTRING EMPTY'))", DOUBLE, 0.0); assertFunction("ST_Length(ST_GeometryFromText('LINESTRING (0 0, 2 2)'))", DOUBLE, 2.8284271247461903); assertFunction("ST_Length(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'))", DOUBLE, 6.0); - assertInvalidFunction("ST_Length(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'))", "ST_Length only applies to LINE_STRING or MULTI_LINE_STRING. Input type is: POLYGON"); + assertInvalidFunction("ST_Length(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'))", "ST_Length only applies to LINE_STRING or MULTI_LINE_STRING. Input type is: POLYGON"); } @Test @@ -427,8 +470,32 @@ public void testLineLocatePoint() assertFunction("line_locate_point(ST_GeometryFromText('LINESTRING EMPTY'), ST_Point(0, 1))", DOUBLE, null); assertFunction("line_locate_point(ST_GeometryFromText('LINESTRING (0 0, 0 1, 2 1)'), ST_GeometryFromText('POINT EMPTY'))", DOUBLE, null); - assertInvalidFunction("line_locate_point(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'), ST_Point(0.4, 1))", "First argument to line_locate_point must be a LineString or a MultiLineString. Got: Polygon"); - assertInvalidFunction("line_locate_point(ST_GeometryFromText('LINESTRING (0 0, 0 1, 2 1)'), ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'))", "Second argument to line_locate_point must be a Point. Got: Polygon"); + assertInvalidFunction("line_locate_point(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'), ST_Point(0.4, 1))", "First argument to line_locate_point must be a LineString or a MultiLineString. Got: Polygon"); + assertInvalidFunction("line_locate_point(ST_GeometryFromText('LINESTRING (0 0, 0 1, 2 1)'), ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'))", "Second argument to line_locate_point must be a Point. Got: Polygon"); + } + + @Test + public void testLineInterpolatePoint() + { + assertLineInterpolatePoint("LINESTRING EMPTY", 0.5, "POINT EMPTY"); + assertLineInterpolatePoint("LINESTRING (0 0, 0 1)", 0.2, "POINT (0 0.2)"); + assertLineInterpolatePoint("LINESTRING (0 0, 0 1)", 0.0, "POINT (0 0)"); + assertLineInterpolatePoint("LINESTRING (0 0, 0 1)", 1.0, "POINT (0 1)"); + assertLineInterpolatePoint("LINESTRING (0 0, 0 1, 3 1)", 0.0625, "POINT (0 0.25)"); + assertLineInterpolatePoint("LINESTRING (0 0, 0 1, 3 1)", 0.75, "POINT (2 1)"); + assertLineInterpolatePoint("LINESTRING (1 3, 5 4)", 0.0, "POINT (1 3)"); + assertLineInterpolatePoint("LINESTRING (1 3, 5 4)", 0.25, "POINT (2 3.25)"); + assertLineInterpolatePoint("LINESTRING (1 3, 5 4)", 1.0, "POINT (5 4)"); + + assertInvalidFunction("line_interpolate_point(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'), 0.5)", "line_interpolate_point only applies to LINE_STRING. Input type is: POLYGON"); + assertInvalidFunction("line_interpolate_point(ST_GeometryFromText('MULTILINESTRING ((0 0, 0 1), (2 2, 4 2))'), 0.0)", "line_interpolate_point only applies to LINE_STRING. Input type is: MULTI_LINE_STRING"); + assertInvalidFunction("line_interpolate_point(ST_GeometryFromText('LINESTRING (0 0, 0 1, 2 1)'), -1)", "line_interpolate_point: Fraction must be between 0 and 1, but is -1.0"); + assertInvalidFunction("line_interpolate_point(ST_GeometryFromText('LINESTRING (0 0, 0 1, 2 1)'), 1.5)", "line_interpolate_point: Fraction must be between 0 and 1, but is 1.5"); + } + + private void assertLineInterpolatePoint(String sourceWkt, double distance, String expectedWkt) + { + assertFunction(format("ST_AsText(line_interpolate_point(ST_GeometryFromText('%s'), %s))", sourceWkt, distance), VARCHAR, expectedWkt); } @Test @@ -442,10 +509,10 @@ public void testSTMax() assertFunction("ST_YMax(ST_GeometryFromText('LINESTRING (8 4, 5 7)'))", DOUBLE, 7.0); assertFunction("ST_XMax(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'))", DOUBLE, 5.0); assertFunction("ST_YMax(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'))", DOUBLE, 4.0); - assertFunction("ST_XMax(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1))'))", DOUBLE, 3.0); - assertFunction("ST_YMax(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1))'))", DOUBLE, 1.0); - assertFunction("ST_XMax(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((2 4, 2 6, 6 6, 6 4)))'))", DOUBLE, 6.0); - assertFunction("ST_YMax(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((2 4, 2 6, 6 10, 6 4)))'))", DOUBLE, 10.0); + assertFunction("ST_XMax(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1, 2 0))'))", DOUBLE, 3.0); + assertFunction("ST_YMax(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1, 2 0))'))", DOUBLE, 1.0); + assertFunction("ST_XMax(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))'))", DOUBLE, 6.0); + assertFunction("ST_YMax(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 10, 6 4, 2 4)))'))", DOUBLE, 10.0); assertFunction("ST_XMax(ST_GeometryFromText('POLYGON EMPTY'))", DOUBLE, null); assertFunction("ST_YMax(ST_GeometryFromText('POLYGON EMPTY'))", DOUBLE, null); assertFunction("ST_XMax(ST_GeometryFromText('GEOMETRYCOLLECTION (POINT (5 1), LINESTRING (3 4, 4 4))'))", DOUBLE, 5.0); @@ -465,10 +532,10 @@ public void testSTMin() assertFunction("ST_YMin(ST_GeometryFromText('LINESTRING (8 4, 5 7)'))", DOUBLE, 4.0); assertFunction("ST_XMin(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'))", DOUBLE, 1.0); assertFunction("ST_YMin(ST_GeometryFromText('MULTILINESTRING ((1 2, 5 3), (2 4, 4 4))'))", DOUBLE, 2.0); - assertFunction("ST_XMin(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1))'))", DOUBLE, 2.0); - assertFunction("ST_YMin(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1))'))", DOUBLE, 0.0); - assertFunction("ST_XMin(ST_GeometryFromText('MULTIPOLYGON (((1 10, 1 3, 3 3, 3 10)), ((2 4, 2 6, 6 6, 6 4)))'))", DOUBLE, 1.0); - assertFunction("ST_YMin(ST_GeometryFromText('MULTIPOLYGON (((1 10, 1 3, 3 3, 3 10)), ((2 4, 2 6, 6 10, 6 4)))'))", DOUBLE, 3.0); + assertFunction("ST_XMin(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1, 2 0))'))", DOUBLE, 2.0); + assertFunction("ST_YMin(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1, 2 0))'))", DOUBLE, 0.0); + assertFunction("ST_XMin(ST_GeometryFromText('MULTIPOLYGON (((1 10, 1 3, 3 3, 3 10, 1 10)), ((2 4, 2 6, 6 6, 6 4, 2 4)))'))", DOUBLE, 1.0); + assertFunction("ST_YMin(ST_GeometryFromText('MULTIPOLYGON (((1 10, 1 3, 3 3, 3 10, 1 10)), ((2 4, 2 6, 6 10, 6 4, 2 4)))'))", DOUBLE, 3.0); assertFunction("ST_XMin(ST_GeometryFromText('POLYGON EMPTY'))", DOUBLE, null); assertFunction("ST_YMin(ST_GeometryFromText('POLYGON EMPTY'))", DOUBLE, null); assertFunction("ST_XMin(ST_GeometryFromText('GEOMETRYCOLLECTION (POINT (5 1), LINESTRING (3 4, 4 4))'))", DOUBLE, 3.0); @@ -501,7 +568,7 @@ public void testSTNumPoints() assertNumPoints("LINESTRING (8 4, 5 7)", 2); assertNumPoints("MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))", 4); assertNumPoints("POLYGON ((0 0, 8 0, 0 8, 0 0), (1 1, 1 5, 5 1, 1 1))", 6); - assertNumPoints("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((2 4, 2 6, 6 6, 6 4)))", 8); + assertNumPoints("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))", 8); assertNumPoints("GEOMETRYCOLLECTION (POINT (1 2), LINESTRING (8 4, 5 7), POLYGON EMPTY)", 3); } @@ -515,7 +582,7 @@ public void testSTIsRing() { assertFunction("ST_IsRing(ST_GeometryFromText('LINESTRING (8 4, 4 8)'))", BOOLEAN, false); assertFunction("ST_IsRing(ST_GeometryFromText('LINESTRING (0 0, 1 1, 0 2, 0 0)'))", BOOLEAN, true); - assertInvalidFunction("ST_IsRing(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1))'))", "ST_IsRing only applies to LINE_STRING. Input type is: POLYGON"); + assertInvalidFunction("ST_IsRing(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1, 2 0))'))", "ST_IsRing only applies to LINE_STRING. Input type is: POLYGON"); } @Test @@ -523,8 +590,8 @@ public void testSTStartEndPoint() { assertFunction("ST_AsText(ST_StartPoint(ST_GeometryFromText('LINESTRING (8 4, 4 8, 5 6)')))", VARCHAR, "POINT (8 4)"); assertFunction("ST_AsText(ST_EndPoint(ST_GeometryFromText('LINESTRING (8 4, 4 8, 5 6)')))", VARCHAR, "POINT (5 6)"); - assertInvalidFunction("ST_StartPoint(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1))'))", "ST_StartPoint only applies to LINE_STRING. Input type is: POLYGON"); - assertInvalidFunction("ST_EndPoint(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1))'))", "ST_EndPoint only applies to LINE_STRING. Input type is: POLYGON"); + assertInvalidFunction("ST_StartPoint(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1, 2 0))'))", "ST_StartPoint only applies to LINE_STRING. Input type is: POLYGON"); + assertInvalidFunction("ST_EndPoint(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1, 2 0))'))", "ST_EndPoint only applies to LINE_STRING. Input type is: POLYGON"); } @Test @@ -535,7 +602,7 @@ public void testSTPoints() assertSTPoints("LINESTRING (8 4, 3 9, 8 4)", "8 4", "3 9", "8 4"); assertSTPoints("LINESTRING (8 4, 3 9, 5 6)", "8 4", "3 9", "5 6"); assertSTPoints("LINESTRING (8 4, 3 9, 5 6, 3 9, 8 4)", "8 4", "3 9", "5 6", "3 9", "8 4"); - assertInvalidFunction("ST_Points(ST_GeometryFromText('POLYGON ((8 4, 3 9, 5 6))'))", "ST_Points only applies to LINE_STRING. Input type is: POLYGON"); + assertInvalidFunction("ST_Points(ST_GeometryFromText('POLYGON ((8 4, 3 9, 5 6, 8 4))'))", "ST_Points only applies to LINE_STRING. Input type is: POLYGON"); } private void assertSTPoints(String wkt, String... expected) @@ -550,33 +617,33 @@ public void testSTXY() assertFunction("ST_Y(ST_GeometryFromText('POINT EMPTY'))", DOUBLE, null); assertFunction("ST_X(ST_GeometryFromText('POINT (1 2)'))", DOUBLE, 1.0); assertFunction("ST_Y(ST_GeometryFromText('POINT (1 2)'))", DOUBLE, 2.0); - assertInvalidFunction("ST_Y(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1))'))", "ST_Y only applies to POINT. Input type is: POLYGON"); + assertInvalidFunction("ST_Y(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1, 2 0))'))", "ST_Y only applies to POINT. Input type is: POLYGON"); } @Test public void testSTBoundary() { - assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('POINT (1 2)')))", VARCHAR, "MULTIPOINT EMPTY"); - assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('MULTIPOINT (1 2, 2 4, 3 6, 4 8)')))", VARCHAR, "MULTIPOINT EMPTY"); + assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('POINT (1 2)')))", VARCHAR, "GEOMETRYCOLLECTION EMPTY"); + assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('MULTIPOINT (1 2, 2 4, 3 6, 4 8)')))", VARCHAR, "GEOMETRYCOLLECTION EMPTY"); assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('LINESTRING EMPTY')))", VARCHAR, "MULTIPOINT EMPTY"); assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('LINESTRING (8 4, 5 7)')))", VARCHAR, "MULTIPOINT ((8 4), (5 7))"); assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('LINESTRING (100 150,50 60, 70 80, 160 170)')))", VARCHAR, "MULTIPOINT ((100 150), (160 170))"); - assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))')))", VARCHAR, "MULTIPOINT ((1 1), (5 1), (2 4), (4 4))"); - assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('POLYGON ((1 1, 4 1, 1 4))')))", VARCHAR, "MULTILINESTRING ((1 1, 4 1, 1 4, 1 1))"); - assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))')))", VARCHAR, "MULTILINESTRING ((1 1, 3 1, 3 3, 1 3, 1 1), (0 0, 2 0, 2 2, 0 2, 0 0))"); + assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))')))", VARCHAR, "MULTIPOINT ((1 1), (2 4), (4 4), (5 1))"); + assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('POLYGON ((1 1, 4 1, 1 4, 1 1))')))", VARCHAR, "LINESTRING (1 1, 1 4, 4 1, 1 1)"); + assertFunction("ST_AsText(ST_Boundary(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))')))", VARCHAR, "MULTILINESTRING ((1 1, 1 3, 3 3, 3 1, 1 1), (0 0, 0 2, 2 2, 2 0, 0 0))"); } @Test public void testSTEnvelope() { - assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('MULTIPOINT (1 2, 2 4, 3 6, 4 8)')))", VARCHAR, "POLYGON ((1 2, 4 2, 4 8, 1 8, 1 2))"); + assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('MULTIPOINT (1 2, 2 4, 3 6, 4 8)')))", VARCHAR, "POLYGON ((1 2, 1 8, 4 8, 4 2, 1 2))"); assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('LINESTRING EMPTY')))", VARCHAR, "POLYGON EMPTY"); - assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('LINESTRING (1 1, 2 2, 1 3)')))", VARCHAR, "POLYGON ((1 1, 2 1, 2 3, 1 3, 1 1))"); - assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('LINESTRING (8 4, 5 7)')))", VARCHAR, "POLYGON ((5 4, 8 4, 8 7, 5 7, 5 4))"); - assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))')))", VARCHAR, "POLYGON ((1 1, 5 1, 5 4, 1 4, 1 1))"); - assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('POLYGON ((1 1, 4 1, 1 4))')))", VARCHAR, "POLYGON ((1 1, 4 1, 4 4, 1 4, 1 1))"); - assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))')))", VARCHAR, "POLYGON ((0 0, 3 0, 3 3, 0 3, 0 0))"); - assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('GEOMETRYCOLLECTION (POINT (5 1), LINESTRING (3 4, 4 4))')))", VARCHAR, "POLYGON ((3 1, 5 1, 5 4, 3 4, 3 1))"); + assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('LINESTRING (1 1, 2 2, 1 3)')))", VARCHAR, "POLYGON ((1 1, 1 3, 2 3, 2 1, 1 1))"); + assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('LINESTRING (8 4, 5 7)')))", VARCHAR, "POLYGON ((5 4, 5 7, 8 7, 8 4, 5 4))"); + assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))')))", VARCHAR, "POLYGON ((1 1, 1 4, 5 4, 5 1, 1 1))"); + assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('POLYGON ((1 1, 4 1, 1 4, 1 1))')))", VARCHAR, "POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))"); + assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))')))", VARCHAR, "POLYGON ((0 0, 0 3, 3 3, 3 0, 0 0))"); + assertFunction("ST_AsText(ST_Envelope(ST_GeometryFromText('GEOMETRYCOLLECTION (POINT (5 1), LINESTRING (3 4, 4 4))')))", VARCHAR, "POLYGON ((3 1, 3 4, 5 4, 5 1, 3 1))"); } @Test @@ -587,8 +654,8 @@ public void testSTEnvelopeAsPts() assertEnvelopeAsPts("LINESTRING (1 1, 2 2, 1 3)", new Point(1, 1), new Point(2, 3)); assertEnvelopeAsPts("LINESTRING (8 4, 5 7)", new Point(5, 4), new Point(8, 7)); assertEnvelopeAsPts("MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))", new Point(1, 1), new Point(5, 4)); - assertEnvelopeAsPts("POLYGON ((1 1, 4 1, 1 4))", new Point(1, 1), new Point(4, 4)); - assertEnvelopeAsPts("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))", new Point(0, 0), new Point(3, 3)); + assertEnvelopeAsPts("POLYGON ((1 1, 4 1, 1 4, 1 1))", new Point(1, 1), new Point(4, 4)); + assertEnvelopeAsPts("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))", new Point(0, 0), new Point(3, 3)); assertEnvelopeAsPts("GEOMETRYCOLLECTION (POINT (5 1), LINESTRING (3 4, 4 4))", new Point(3, 1), new Point(5, 4)); assertEnvelopeAsPts("POINT (1 2)", new Point(1, 2), new Point(1, 2)); } @@ -598,6 +665,24 @@ private void assertEnvelopeAsPts(String wkt, Point lowerLeftCorner, Point upperR assertFunction(format("transform(ST_EnvelopeAsPts(ST_GeometryFromText('%s')), x -> ST_AsText(x))", wkt), new ArrayType(VARCHAR), ImmutableList.of(new OGCPoint(lowerLeftCorner, null).asText(), new OGCPoint(upperRightCorner, null).asText())); } + @Test + public void testExpandEnvelope() + { + assertFunction("ST_IsEmpty(expand_envelope(ST_GeometryFromText('POINT EMPTY'), 1))", BOOLEAN, true); + assertFunction("ST_IsEmpty(expand_envelope(ST_GeometryFromText('POLYGON EMPTY'), 1))", BOOLEAN, true); + assertFunction("ST_AsText(expand_envelope(ST_Envelope(ST_Point(1, 10)), 3))", VARCHAR, "POLYGON ((-2 7, -2 13, 4 13, 4 7, -2 7))"); + assertFunction("ST_AsText(expand_envelope(ST_Point(1, 10), 3))", VARCHAR, "POLYGON ((-2 7, -2 13, 4 13, 4 7, -2 7))"); + assertFunction("ST_AsText(expand_envelope(ST_GeometryFromText('LINESTRING (1 10, 3 15)'), 2))", VARCHAR, "POLYGON ((-1 8, -1 17, 5 17, 5 8, -1 8))"); + assertFunction("ST_AsText(expand_envelope(ST_GeometryFromText('GEOMETRYCOLLECTION (POINT (5 1), LINESTRING (3 4, 4 4))'), 1))", VARCHAR, "POLYGON ((2 0, 2 5, 6 5, 6 0, 2 0))"); + // JTS has an envelope expanded by infinity to be empty, which is weird. + // PostGIS returns an infinite envelope, which is a tricky concept. + // We'll leave it like this until it becomes a problem. + assertFunction("ST_AsText(expand_envelope(ST_Point(0, 0), infinity()))", VARCHAR, "POLYGON EMPTY"); + assertInvalidFunction("ST_AsText(expand_envelope(ST_Point(0, 0), nan()))", "expand_envelope: distance is NaN"); + assertInvalidFunction("ST_AsText(expand_envelope(ST_Point(0, 0), -1))", "expand_envelope: distance -1.0 is negative"); + assertInvalidFunction("ST_AsText(expand_envelope(ST_Point(0, 0), -infinity()))", "expand_envelope: distance -Infinity is negative"); + } + @Test public void testSTDifference() { @@ -605,8 +690,8 @@ public void testSTDifference() assertFunction("ST_AsText(ST_Difference(ST_GeometryFromText('MULTIPOINT (50 100, 50 200)'), ST_GeometryFromText('POINT (50 100)')))", VARCHAR, "POINT (50 200)"); assertFunction("ST_AsText(ST_Difference(ST_GeometryFromText('LINESTRING (50 100, 50 200)'), ST_GeometryFromText('LINESTRING (50 50, 50 150)')))", VARCHAR, "LINESTRING (50 150, 50 200)"); assertFunction("ST_AsText(ST_Difference(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('MULTILINESTRING ((2 1, 4 1), (3 3, 7 3))')))", VARCHAR, "MULTILINESTRING ((1 1, 2 1), (4 1, 5 1), (2 4, 4 4))"); - assertFunction("ST_AsText(ST_Difference(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'), ST_GeometryFromText('POLYGON ((2 2, 2 5, 5 5, 5 2))')))", VARCHAR, "POLYGON ((1 1, 4 1, 4 2, 2 2, 2 4, 1 4, 1 1))"); - assertFunction("ST_AsText(ST_Difference(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))'), ST_GeometryFromText('POLYGON ((0 1, 3 1, 3 3, 0 3, 0 1))')))", VARCHAR, "POLYGON ((1 1, 0 1, 0 0, 2 0, 2 1, 1 1))"); + assertFunction("ST_AsText(ST_Difference(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'), ST_GeometryFromText('POLYGON ((2 2, 2 5, 5 5, 5 2, 2 2))')))", VARCHAR, "POLYGON ((1 1, 1 4, 2 4, 2 2, 4 2, 4 1, 1 1))"); + assertFunction("ST_AsText(ST_Difference(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))'), ST_GeometryFromText('POLYGON ((0 1, 3 1, 3 3, 0 3, 0 1))')))", VARCHAR, "POLYGON ((1 1, 2 1, 2 0, 0 0, 0 1, 1 1))"); } @Test @@ -618,8 +703,8 @@ public void testSTDistance() assertFunction("ST_Distance(ST_GeometryFromText('MULTIPOINT (50 100, 50 200)'), ST_GeometryFromText('Point (50 100)'))", DOUBLE, 0.0); assertFunction("ST_Distance(ST_GeometryFromText('LINESTRING (50 100, 50 200)'), ST_GeometryFromText('LINESTRING (10 10, 20 20)'))", DOUBLE, 85.44003745317531); assertFunction("ST_Distance(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('LINESTRING (10 20, 20 50)'))", DOUBLE, 17.08800749063506); - assertFunction("ST_Distance(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('POLYGON ((4 4, 4 5, 5 5, 5 4))'))", DOUBLE, 1.4142135623730951); - assertFunction("ST_Distance(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))'), ST_GeometryFromText('POLYGON ((10 100, 30 10))'))", DOUBLE, 27.892651361962706); + assertFunction("ST_Distance(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))'), ST_GeometryFromText('POLYGON ((4 4, 4 5, 5 5, 5 4, 4 4))'))", DOUBLE, 1.4142135623730951); + assertFunction("ST_Distance(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))'), ST_GeometryFromText('POLYGON ((10 100, 30 10, 30 100, 10 100))'))", DOUBLE, 27.892651361962706); assertFunction("ST_Distance(ST_GeometryFromText('POINT EMPTY'), ST_Point(150, 150))", DOUBLE, null); assertFunction("ST_Distance(ST_Point(50, 100), ST_GeometryFromText('POINT EMPTY'))", DOUBLE, null); @@ -627,16 +712,16 @@ public void testSTDistance() assertFunction("ST_Distance(ST_GeometryFromText('MULTIPOINT EMPTY'), ST_GeometryFromText('Point (50 100)'))", DOUBLE, null); assertFunction("ST_Distance(ST_GeometryFromText('LINESTRING (50 100, 50 200)'), ST_GeometryFromText('LINESTRING EMPTY'))", DOUBLE, null); assertFunction("ST_Distance(ST_GeometryFromText('MULTILINESTRING EMPTY'), ST_GeometryFromText('LINESTRING (10 20, 20 50)'))", DOUBLE, null); - assertFunction("ST_Distance(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('POLYGON EMPTY'))", DOUBLE, null); - assertFunction("ST_Distance(ST_GeometryFromText('MULTIPOLYGON EMPTY'), ST_GeometryFromText('POLYGON ((10 100, 30 10))'))", DOUBLE, null); + assertFunction("ST_Distance(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))'), ST_GeometryFromText('POLYGON EMPTY'))", DOUBLE, null); + assertFunction("ST_Distance(ST_GeometryFromText('MULTIPOLYGON EMPTY'), ST_GeometryFromText('POLYGON ((10 100, 30 10, 30 100, 10 100))'))", DOUBLE, null); } @Test public void testSTExteriorRing() { assertFunction("ST_AsText(ST_ExteriorRing(ST_GeometryFromText('POLYGON EMPTY')))", VARCHAR, null); - assertFunction("ST_AsText(ST_ExteriorRing(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 1))')))", VARCHAR, "LINESTRING (1 1, 4 1, 1 4, 1 1)"); - assertFunction("ST_AsText(ST_ExteriorRing(ST_GeometryFromText('POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))')))", VARCHAR, "LINESTRING (0 0, 5 0, 5 5, 0 5, 0 0)"); + assertFunction("ST_AsText(ST_ExteriorRing(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 1, 1 1))')))", VARCHAR, "LINESTRING (1 1, 1 4, 4 1, 1 1)"); + assertFunction("ST_AsText(ST_ExteriorRing(ST_GeometryFromText('POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))')))", VARCHAR, "LINESTRING (0 0, 0 5, 5 5, 5 0, 0 0)"); assertInvalidFunction("ST_AsText(ST_ExteriorRing(ST_GeometryFromText('LINESTRING (1 1, 2 2, 1 3)')))", "ST_ExteriorRing only applies to POLYGON. Input type is: LINE_STRING"); assertInvalidFunction("ST_AsText(ST_ExteriorRing(ST_GeometryFromText('MULTIPOLYGON (((1 1, 2 2, 1 3, 1 1)), ((4 4, 5 5, 4 6, 4 4)))')))", "ST_ExteriorRing only applies to POLYGON. Input type is: MULTI_POLYGON"); } @@ -648,20 +733,20 @@ public void testSTIntersection() assertFunction("ST_AsText(ST_Intersection(ST_GeometryFromText('MULTIPOINT (50 100, 50 200)'), ST_GeometryFromText('Point (50 100)')))", VARCHAR, "POINT (50 100)"); assertFunction("ST_AsText(ST_Intersection(ST_GeometryFromText('LINESTRING (50 100, 50 200)'), ST_GeometryFromText('LINESTRING (20 150, 100 150)')))", VARCHAR, "POINT (50 150)"); assertFunction("ST_AsText(ST_Intersection(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))')))", VARCHAR, "GEOMETRYCOLLECTION (POINT (5 1), LINESTRING (3 4, 4 4))"); - assertFunction("ST_AsText(ST_Intersection(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('POLYGON ((4 4, 4 5, 5 5, 5 4))')))", VARCHAR, "MULTIPOLYGON EMPTY"); - assertFunction("ST_AsText(ST_Intersection(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))'), ST_GeometryFromText('POLYGON ((0 1, 3 1, 3 3, 0 3))')))", VARCHAR, "GEOMETRYCOLLECTION (LINESTRING (1 1, 2 1), MULTIPOLYGON (((0 1, 1 1, 1 2, 0 2, 0 1)), ((2 1, 3 1, 3 3, 1 3, 1 2, 2 2, 2 1))))"); - assertFunction("ST_AsText(ST_Intersection(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'), ST_GeometryFromText('LINESTRING (2 0, 2 3)')))", VARCHAR, "LINESTRING (2 1, 2 3)"); + assertFunction("ST_AsText(ST_Intersection(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))'), ST_GeometryFromText('POLYGON ((4 4, 4 5, 5 5, 5 4, 4 4))')))", VARCHAR, "MULTIPOLYGON EMPTY"); + assertFunction("ST_AsText(ST_Intersection(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))'), ST_GeometryFromText('POLYGON ((0 1, 3 1, 3 3, 0 3, 0 1))')))", VARCHAR, "GEOMETRYCOLLECTION (LINESTRING (1 1, 2 1), MULTIPOLYGON (((0 1, 0 2, 1 2, 1 1, 0 1)), ((2 1, 2 2, 1 2, 1 3, 3 3, 3 1, 2 1))))"); + assertFunction("ST_AsText(ST_Intersection(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'), ST_GeometryFromText('LINESTRING (2 0, 2 3)')))", VARCHAR, "LINESTRING (2 1, 2 3)"); assertFunction("ST_AsText(ST_Intersection(ST_GeometryFromText('POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))'), ST_GeometryFromText('LINESTRING (0 0, 1 -1, 1 2)')))", VARCHAR, "GEOMETRYCOLLECTION (POINT (0 0), LINESTRING (1 0, 1 1))"); // test intersection of envelopes - assertEnvelopeIntersection("POLYGON ((0 0, 5 0, 5 5, 0 5, 0 0))", "POLYGON ((0 0, 5 0, 5 5, 0 5, 0 0))", "POLYGON ((0 0, 5 0, 5 5, 0 5, 0 0))"); - assertEnvelopeIntersection("POLYGON ((0 0, 5 0, 5 5, 0 5, 0 0))", "POLYGON ((-1 4, 1 4, 1 6, -1 6, -1 4))", "POLYGON ((0 4, 1 4, 1 5, 0 5, 0 4))"); - assertEnvelopeIntersection("POLYGON ((0 0, 5 0, 5 5, 0 5, 0 0))", "POLYGON ((1 4, 2 4, 2 6, 1 6, 1 4))", "POLYGON ((1 4, 2 4, 2 5, 1 5, 1 4))"); - assertEnvelopeIntersection("POLYGON ((0 0, 5 0, 5 5, 0 5, 0 0))", "POLYGON ((4 4, 6 4, 6 6, 4 6, 4 4))", "POLYGON ((4 4, 5 4, 5 5, 4 5, 4 4))"); - assertEnvelopeIntersection("POLYGON ((0 0, 5 0, 5 5, 0 5, 0 0))", "POLYGON ((10 10, 11 10, 11 11, 10 11, 10 10))", "POLYGON EMPTY"); - assertEnvelopeIntersection("POLYGON ((0 0, 5 0, 5 5, 0 5, 0 0))", "POLYGON ((-1 -1, 0 -1, 0 1, -1 1, -1 -1))", "LINESTRING (0 0, 0 1)"); - assertEnvelopeIntersection("POLYGON ((0 0, 5 0, 5 5, 0 5, 0 0))", "POLYGON ((1 -1, 2 -1, 2 0, 1 0, 1 -1))", "LINESTRING (1 0, 2 0)"); - assertEnvelopeIntersection("POLYGON ((0 0, 5 0, 5 5, 0 5, 0 0))", "POLYGON ((-1 -1, 0 -1, 0 0, -1 0, -1 -1))", "POINT (0 0)"); + assertEnvelopeIntersection("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))", "POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))", "POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))"); + assertEnvelopeIntersection("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))", "POLYGON ((-1 4, 1 4, 1 6, -1 6, -1 4))", "POLYGON ((0 4, 0 5, 1 5, 1 4, 0 4))"); + assertEnvelopeIntersection("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))", "POLYGON ((1 4, 2 4, 2 6, 1 6, 1 4))", "POLYGON ((1 4, 1 5, 2 5, 2 4, 1 4))"); + assertEnvelopeIntersection("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))", "POLYGON ((4 4, 6 4, 6 6, 4 6, 4 4))", "POLYGON ((4 4, 4 5, 5 5, 5 4, 4 4))"); + assertEnvelopeIntersection("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))", "POLYGON ((10 10, 11 10, 11 11, 10 11, 10 10))", "POLYGON EMPTY"); + assertEnvelopeIntersection("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))", "POLYGON ((-1 -1, 0 -1, 0 1, -1 1, -1 -1))", "LINESTRING (0 0, 0 1)"); + assertEnvelopeIntersection("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))", "POLYGON ((1 -1, 2 -1, 2 0, 1 0, 1 -1))", "LINESTRING (1 0, 2 0)"); + assertEnvelopeIntersection("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))", "POLYGON ((-1 -1, 0 -1, 0 0, -1 0, -1 -1))", "POINT (0 0)"); } private void assertEnvelopeIntersection(String envelope, String otherEnvelope, String intersection) @@ -676,141 +761,8 @@ public void testSTSymmetricDifference() assertFunction("ST_AsText(ST_SymDifference(ST_GeometryFromText('MULTIPOINT (50 100, 60 200)'), ST_GeometryFromText('MULTIPOINT (60 200, 70 150)')))", VARCHAR, "MULTIPOINT ((50 100), (70 150))"); assertFunction("ST_AsText(ST_SymDifference(ST_GeometryFromText('LINESTRING (50 100, 50 200)'), ST_GeometryFromText('LINESTRING (50 50, 50 150)')))", VARCHAR, "MULTILINESTRING ((50 50, 50 100), (50 150, 50 200))"); assertFunction("ST_AsText(ST_SymDifference(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))')))", VARCHAR, "MULTILINESTRING ((5 0, 5 1), (1 1, 5 1), (5 1, 5 4), (2 4, 3 4), (4 4, 5 4), (5 4, 6 4))"); - assertFunction("ST_AsText(ST_SymDifference(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'), ST_GeometryFromText('POLYGON ((2 2, 2 5, 5 5, 5 2))')))", VARCHAR, "MULTIPOLYGON (((1 1, 4 1, 4 2, 2 2, 2 4, 1 4, 1 1)), ((4 2, 5 2, 5 5, 2 5, 2 4, 4 4, 4 2)))"); - assertFunction("ST_AsText(ST_SymDifference(ST_GeometryFromText('MULTIPOLYGON (((0 0 , 0 2, 2 2, 2 0)), ((2 2, 2 4, 4 4, 4 2)))'), ST_GeometryFromText('POLYGON ((0 0, 0 3, 3 3, 3 0))')))", VARCHAR, "MULTIPOLYGON (((2 0, 3 0, 3 2, 2 2, 2 0)), ((0 2, 2 2, 2 3, 0 3, 0 2)), ((3 2, 4 2, 4 4, 2 4, 2 3, 3 3, 3 2)))"); - } - - @Test - public void testStContains() - { - assertFunction("ST_Contains(ST_GeometryFromText(null), ST_GeometryFromText('POINT (25 25)'))", BOOLEAN, null); - assertFunction("ST_Contains(ST_GeometryFromText('POINT (20 20)'), ST_GeometryFromText('POINT (25 25)'))", BOOLEAN, false); - assertFunction("ST_Contains(ST_GeometryFromText('MULTIPOINT (20 20, 25 25)'), ST_GeometryFromText('POINT (25 25)'))", BOOLEAN, true); - assertFunction("ST_Contains(ST_GeometryFromText('LINESTRING (20 20, 30 30)'), ST_GeometryFromText('POINT (25 25)'))", BOOLEAN, true); - assertFunction("ST_Contains(ST_GeometryFromText('LINESTRING (20 20, 30 30)'), ST_GeometryFromText('MULTIPOINT (25 25, 31 31)'))", BOOLEAN, false); - assertFunction("ST_Contains(ST_GeometryFromText('LINESTRING (20 20, 30 30)'), ST_GeometryFromText('LINESTRING (25 25, 27 27)'))", BOOLEAN, true); - assertFunction("ST_Contains(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('MULTILINESTRING ((3 4, 4 4), (2 1, 6 1))'))", BOOLEAN, false); - assertFunction("ST_Contains(ST_GeometryFromText('POLYGON ((0 0, 0 4, 4 4, 4 0))'), ST_GeometryFromText('POLYGON ((1 1, 1 2, 2 2, 2 1))'))", BOOLEAN, true); - assertFunction("ST_Contains(ST_GeometryFromText('POLYGON ((0 0, 0 4, 4 4, 4 0))'), ST_GeometryFromText('POLYGON ((-1 -1, -1 2, 2 2, 2 -1))'))", BOOLEAN, false); - assertFunction("ST_Contains(ST_GeometryFromText('MULTIPOLYGON (((0 0 , 0 2, 2 2, 2 0)), ((2 2, 2 4, 4 4, 4 2)))'), ST_GeometryFromText('POLYGON ((2 2, 2 3, 3 3, 3 2))'))", BOOLEAN, true); - assertFunction("ST_Contains(ST_GeometryFromText('LINESTRING (20 20, 30 30)'), ST_GeometryFromText('POLYGON ((0 0, 0 4, 4 4, 4 0))'))", BOOLEAN, false); - assertFunction("ST_Contains(ST_GeometryFromText('LINESTRING EMPTY'), ST_GeometryFromText('POLYGON ((0 0, 0 4, 4 4, 4 0))'))", BOOLEAN, false); - assertFunction("ST_Contains(ST_GeometryFromText('LINESTRING (20 20, 30 30)'), ST_GeometryFromText('POLYGON EMPTY'))", BOOLEAN, false); - } - - @Test - public void testSTCrosses() - { - assertFunction("ST_Crosses(ST_GeometryFromText('POINT (20 20)'), ST_GeometryFromText('POINT (25 25)'))", BOOLEAN, false); - assertFunction("ST_Crosses(ST_GeometryFromText('LINESTRING (20 20, 30 30)'), ST_GeometryFromText('POINT (25 25)'))", BOOLEAN, false); - assertFunction("ST_Crosses(ST_GeometryFromText('LINESTRING (20 20, 30 30)'), ST_GeometryFromText('MULTIPOINT (25 25, 31 31)'))", BOOLEAN, true); - assertFunction("ST_Crosses(ST_GeometryFromText('LINESTRING(0 0, 1 1)'), ST_GeometryFromText('LINESTRING (1 0, 0 1)'))", BOOLEAN, true); - assertFunction("ST_Crosses(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'), ST_GeometryFromText('POLYGON ((2 2, 2 5, 5 5, 5 2))'))", BOOLEAN, false); - assertFunction("ST_Crosses(ST_GeometryFromText('MULTIPOLYGON (((0 0 , 0 2, 2 2, 2 0)), ((2 2, 2 4, 4 4, 4 2)))'), ST_GeometryFromText('POLYGON ((2 2, 2 3, 3 3, 3 2))'))", BOOLEAN, false); - assertFunction("ST_Crosses(ST_GeometryFromText('LINESTRING (-2 -2, 6 6)'), ST_GeometryFromText('POLYGON ((0 0, 0 4, 4 4, 4 0))'))", BOOLEAN, true); - assertFunction("ST_Crosses(ST_GeometryFromText('POINT (20 20)'), ST_GeometryFromText('POINT (20 20)'))", BOOLEAN, false); - assertFunction("ST_Crosses(ST_GeometryFromText('POLYGON ((0 0, 0 4, 4 4, 4 0))'), ST_GeometryFromText('POLYGON ((0 0, 0 4, 4 4, 4 0))'))", BOOLEAN, false); - assertFunction("ST_Crosses(ST_GeometryFromText('POLYGON ((0 0, 0 4, 4 4, 4 0))'), ST_GeometryFromText('LINESTRING (0 0, 0 4, 4 4, 4 0)'))", BOOLEAN, false); - } - - @Test - public void testSTDisjoint() - { - assertFunction("ST_Disjoint(ST_GeometryFromText('POINT (50 100)'), ST_GeometryFromText('POINT (150 150)'))", BOOLEAN, true); - assertFunction("ST_Disjoint(ST_GeometryFromText('MULTIPOINT (50 100, 50 200)'), ST_GeometryFromText('POINT (50 100)'))", BOOLEAN, false); - assertFunction("ST_Disjoint(ST_GeometryFromText('LINESTRING (0 0, 0 1)'), ST_GeometryFromText('LINESTRING (1 1, 1 0)'))", BOOLEAN, true); - assertFunction("ST_Disjoint(ST_GeometryFromText('LINESTRING (2 1, 1 2)'), ST_GeometryFromText('LINESTRING (3 1, 1 3)'))", BOOLEAN, true); - assertFunction("ST_Disjoint(ST_GeometryFromText('LINESTRING (1 1, 3 3)'), ST_GeometryFromText('LINESTRING (3 1, 1 3)'))", BOOLEAN, false); - assertFunction("ST_Disjoint(ST_GeometryFromText('LINESTRING (50 100, 50 200)'), ST_GeometryFromText('LINESTRING (20 150, 100 150)'))", BOOLEAN, false); - assertFunction("ST_Disjoint(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))'))", BOOLEAN, false); - assertFunction("ST_Disjoint(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('POLYGON ((4 4, 4 5, 5 5, 5 4))'))", BOOLEAN, true); - assertFunction("ST_Disjoint(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))'), ST_GeometryFromText('POLYGON ((0 1, 3 1, 3 3, 0 3))'))", BOOLEAN, false); - } - - @Test - public void testSTEquals() - { - assertFunction("ST_Equals(ST_GeometryFromText('POINT (50 100)'), ST_GeometryFromText('POINT (150 150)'))", BOOLEAN, false); - assertFunction("ST_Equals(ST_GeometryFromText('MULTIPOINT (50 100, 50 200)'), ST_GeometryFromText('POINT (50 100)'))", BOOLEAN, false); - assertFunction("ST_Equals(ST_GeometryFromText('LINESTRING (0 0, 0 1)'), ST_GeometryFromText('LINESTRING (1 1, 1 0)'))", BOOLEAN, false); - assertFunction("ST_Equals(ST_GeometryFromText('LINESTRING (0 0, 2 2)'), ST_GeometryFromText('LINESTRING (0 0, 2 2)'))", BOOLEAN, true); - assertFunction("ST_Equals(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))'))", BOOLEAN, false); - assertFunction("ST_Equals(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('POLYGON ((3 3, 3 1, 1 1, 1 3))'))", BOOLEAN, true); - assertFunction("ST_Equals(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))'), ST_GeometryFromText('POLYGON ((0 1, 3 1, 3 3, 0 3))'))", BOOLEAN, false); - } - - @Test - public void testSTIntersects() - { - assertFunction("ST_Intersects(ST_GeometryFromText('POINT (50 100)'), ST_GeometryFromText('POINT (150 150)'))", BOOLEAN, false); - assertFunction("ST_Intersects(ST_GeometryFromText('MULTIPOINT (50 100, 50 200)'), ST_GeometryFromText('POINT (50 100)'))", BOOLEAN, true); - assertFunction("ST_Intersects(ST_GeometryFromText('LINESTRING (0 0, 0 1)'), ST_GeometryFromText('LINESTRING (1 1, 1 0)'))", BOOLEAN, false); - assertFunction("ST_Intersects(ST_GeometryFromText('LINESTRING (50 100, 50 200)'), ST_GeometryFromText('LINESTRING (20 150, 100 150)'))", BOOLEAN, true); - assertFunction("ST_Intersects(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))'))", BOOLEAN, true); - assertFunction("ST_Intersects(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('POLYGON ((4 4, 4 5, 5 5, 5 4))'))", BOOLEAN, false); - assertFunction("ST_Intersects(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))'), ST_GeometryFromText('POLYGON ((0 1, 3 1, 3 3, 0 3))'))", BOOLEAN, true); - assertFunction("ST_Intersects(ST_GeometryFromText('POLYGON ((16.5 54, 16.5 54.1, 16.51 54.1, 16.8 54))'), ST_GeometryFromText('LINESTRING (16.6 53, 16.6 56)'))", BOOLEAN, true); - assertFunction("ST_Intersects(ST_GeometryFromText('POLYGON ((16.5 54, 16.5 54.1, 16.51 54.1, 16.8 54))'), ST_GeometryFromText('LINESTRING (16.6667 54.05, 16.8667 54.05)'))", BOOLEAN, false); - assertFunction("ST_Intersects(ST_GeometryFromText('POLYGON ((16.5 54, 16.5 54.1, 16.51 54.1, 16.8 54))'), ST_GeometryFromText('LINESTRING (16.6667 54.25, 16.8667 54.25)'))", BOOLEAN, false); - } - - @Test - public void testSTOverlaps() - { - assertFunction("ST_Overlaps(ST_GeometryFromText('POINT (50 100)'), ST_GeometryFromText('POINT (150 150)'))", BOOLEAN, false); - assertFunction("ST_Overlaps(ST_GeometryFromText('POINT (50 100)'), ST_GeometryFromText('POINT (50 100)'))", BOOLEAN, false); - assertFunction("ST_Overlaps(ST_GeometryFromText('MULTIPOINT (50 100, 50 200)'), ST_GeometryFromText('POINT (50 100)'))", BOOLEAN, false); - assertFunction("ST_Overlaps(ST_GeometryFromText('LINESTRING (0 0, 0 1)'), ST_GeometryFromText('LINESTRING (1 1, 1 0)'))", BOOLEAN, false); - assertFunction("ST_Overlaps(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))'))", BOOLEAN, true); - assertFunction("ST_Overlaps(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'), ST_GeometryFromText('POLYGON ((3 3, 3 5, 5 5, 5 3))'))", BOOLEAN, true); - assertFunction("ST_Overlaps(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'), ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'))", BOOLEAN, false); - assertFunction("ST_Overlaps(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'), ST_GeometryFromText('LINESTRING (1 1, 4 4)'))", BOOLEAN, false); - assertFunction("ST_Overlaps(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('POLYGON ((4 4, 4 5, 5 5, 5 4))'))", BOOLEAN, false); - assertFunction("ST_Overlaps(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))'), ST_GeometryFromText('POLYGON ((0 1, 3 1, 3 3, 0 3))'))", BOOLEAN, true); - } - - @Test - public void testSTRelate() - { - assertFunction("ST_Relate(ST_GeometryFromText('LINESTRING (0 0, 3 3)'), ST_GeometryFromText('LINESTRING (1 1, 4 1)'), '****T****')", BOOLEAN, false); - assertFunction("ST_Relate(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1))'), ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'), '****T****')", BOOLEAN, true); - assertFunction("ST_Relate(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1))'), ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'), 'T********')", BOOLEAN, false); - } - - @Test - public void testSTTouches() - { - assertFunction("ST_Touches(ST_GeometryFromText('POINT (50 100)'), ST_GeometryFromText('POINT (150 150)'))", BOOLEAN, false); - assertFunction("ST_Touches(ST_GeometryFromText('MULTIPOINT (50 100, 50 200)'), ST_GeometryFromText('POINT (50 100)'))", BOOLEAN, false); - assertFunction("ST_Touches(ST_GeometryFromText('LINESTRING (50 100, 50 200)'), ST_GeometryFromText('LINESTRING (20 150, 100 150)'))", BOOLEAN, false); - assertFunction("ST_Touches(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))'))", BOOLEAN, false); - assertFunction("ST_Touches(ST_GeometryFromText('POINT (1 2)'), ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'))", BOOLEAN, true); - assertFunction("ST_Touches(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('POLYGON ((4 4, 4 5, 5 5, 5 4))'))", BOOLEAN, false); - assertFunction("ST_Touches(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('LINESTRING (0 0, 1 1)'))", BOOLEAN, true); - assertFunction("ST_Touches(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('POLYGON ((3 3, 3 5, 5 5, 5 3))'))", BOOLEAN, true); - assertFunction("ST_Touches(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))'), ST_GeometryFromText('POLYGON ((0 1, 3 1, 3 3, 0 3))'))", BOOLEAN, false); - } - - @Test - public void testSTWithin() - { - assertFunction("ST_Within(ST_GeometryFromText('POINT (50 100)'), ST_GeometryFromText('POINT (150 150)'))", BOOLEAN, false); - assertFunction("ST_Within(ST_GeometryFromText('POINT (50 100)'), ST_GeometryFromText('MULTIPOINT (50 100, 50 200)'))", BOOLEAN, true); - assertFunction("ST_Within(ST_GeometryFromText('LINESTRING (50 100, 50 200)'), ST_GeometryFromText('LINESTRING (50 50, 50 250)'))", BOOLEAN, true); - assertFunction("ST_Within(ST_GeometryFromText('MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'), ST_GeometryFromText('MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))'))", BOOLEAN, false); - assertFunction("ST_Within(ST_GeometryFromText('POINT (3 2)'), ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'))", BOOLEAN, true); - assertFunction("ST_Within(ST_GeometryFromText('POLYGON ((1 1, 1 3, 3 3, 3 1))'), ST_GeometryFromText('POLYGON ((0 0, 0 4, 4 4, 4 0))'))", BOOLEAN, true); - assertFunction("ST_Within(ST_GeometryFromText('LINESTRING (1 1, 3 3)'), ST_GeometryFromText('POLYGON ((0 0, 0 4, 4 4, 4 0))'))", BOOLEAN, true); - assertFunction("ST_Within(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))'), ST_GeometryFromText('POLYGON ((0 1, 3 1, 3 3, 0 3))'))", BOOLEAN, false); - assertFunction("ST_Within(ST_GeometryFromText('POLYGON ((1 1, 1 5, 5 5, 5 1))'), ST_GeometryFromText('POLYGON ((0 0, 0 4, 4 4, 4 0))'))", BOOLEAN, false); - } - - @Test - public void testInvalidWKT() - { - assertInvalidFunction("ST_LineFromText('LINESTRING (0 0, 1)')", INVALID_FUNCTION_ARGUMENT, "Invalid WKT: LINESTRING (0 0, 1)"); - assertInvalidFunction("ST_GeometryFromText('POLYGON(0 0)')", INVALID_FUNCTION_ARGUMENT, "Invalid WKT: POLYGON(0 0)"); - assertInvalidFunction("ST_Polygon('POLYGON(-1 1, 1 -1)')", INVALID_FUNCTION_ARGUMENT, "Invalid WKT: POLYGON(-1 1, 1 -1)"); + assertFunction("ST_AsText(ST_SymDifference(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'), ST_GeometryFromText('POLYGON ((2 2, 2 5, 5 5, 5 2, 2 2))')))", VARCHAR, "MULTIPOLYGON (((1 1, 1 4, 2 4, 2 2, 4 2, 4 1, 1 1)), ((4 2, 4 4, 2 4, 2 5, 5 5, 5 2, 4 2)))"); + assertFunction("ST_AsText(ST_SymDifference(ST_GeometryFromText('MULTIPOLYGON (((0 0, 0 2, 2 2, 2 0, 0 0)), ((2 2, 2 4, 4 4, 4 2, 2 2)))'), ST_GeometryFromText('POLYGON ((0 0, 0 3, 3 3, 3 0, 0 0))')))", VARCHAR, "MULTIPOLYGON (((2 0, 2 2, 3 2, 3 0, 2 0)), ((0 2, 0 3, 2 3, 2 2, 0 2)), ((3 2, 3 3, 2 3, 2 4, 4 4, 4 2, 3 2)))"); } @Test @@ -843,14 +795,14 @@ public void testSTInteriorRings() assertInvalidInteriorRings("LINESTRING EMPTY", "LINE_STRING"); assertInvalidInteriorRings("MULTIPOINT (30 20, 60 70)", "MULTI_POINT"); assertInvalidInteriorRings("MULTILINESTRING ((1 10, 100 1000), (2 2, 1 0, 5 6))", "MULTI_LINE_STRING"); - assertInvalidInteriorRings("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((0 0, 0 2, 2 2, 2 0)))", "MULTI_POLYGON"); + assertInvalidInteriorRings("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))", "MULTI_POLYGON"); assertInvalidInteriorRings("GEOMETRYCOLLECTION (POINT (1 1), POINT (2 3), LINESTRING (5 8, 13 21))", "GEOMETRY_COLLECTION"); assertFunction("ST_InteriorRings(ST_GeometryFromText('POLYGON EMPTY'))", new ArrayType(GEOMETRY), null); - assertInteriorRings("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))"); - assertInteriorRings("POLYGON ((0 0, 0 3, 3 3, 3 0, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))", "LINESTRING (1 1, 1 2, 2 2, 2 1, 1 1)"); - assertInteriorRings("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1), (3 3, 3 4, 4 4, 4 3, 3 3))", - "LINESTRING (1 1, 1 2, 2 2, 2 1, 1 1)", "LINESTRING (3 3, 3 4, 4 4, 4 3, 3 3)"); + assertInteriorRings("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))"); + assertInteriorRings("POLYGON ((0 0, 0 3, 3 3, 3 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))", "LINESTRING (1 1, 2 1, 2 2, 1 2, 1 1)"); + assertInteriorRings("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1), (3 3, 4 3, 4 4, 3 4, 3 3))", + "LINESTRING (1 1, 2 1, 2 2, 1 2, 1 1)", "LINESTRING (3 3, 4 3, 4 4, 3 4, 3 3)"); } private void assertInteriorRings(String wkt, String... expected) @@ -878,7 +830,7 @@ public void testSTNumGeometries() assertSTNumGeometries("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", 1); assertSTNumGeometries("MULTIPOINT (1 2, 2 4, 3 6, 4 8)", 4); assertSTNumGeometries("MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))", 2); - assertSTNumGeometries("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((2 4, 2 6, 6 6, 6 4)))", 2); + assertSTNumGeometries("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))", 2); assertSTNumGeometries("GEOMETRYCOLLECTION(POINT(2 3), LINESTRING (2 3, 3 4))", 2); } @@ -905,9 +857,9 @@ public void testSTUnion() "MULTIPOINT ((1 2), (3 4))", "LINESTRING (0 0, 2 2, 4 4)", "MULTILINESTRING ((0 0, 2 2, 4 4), (5 5, 7 7, 9 9))", - "POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", - "MULTIPOLYGON (((1 1, 3 1, 3 3, 1 3, 1 1)), ((2 4, 6 4, 6 6, 2 6, 2 4)))", - "GEOMETRYCOLLECTION (LINESTRING (0 5, 5 5), POLYGON ((1 1, 3 1, 3 3, 1 3, 1 1)))"); + "POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", + "MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))", + "GEOMETRYCOLLECTION (LINESTRING (0 5, 5 5), POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1)))"); // empty geometry for (String emptyWkt : emptyWkts) { @@ -926,24 +878,24 @@ public void testSTUnion() assertUnion("MULTIPOINT ((1 2))", "MULTIPOINT ((1 2), (3 4))", "MULTIPOINT ((1 2), (3 4))"); assertUnion("LINESTRING (0 1, 1 2)", "LINESTRING (1 2, 3 4)", "LINESTRING (0 1, 1 2, 3 4)"); assertUnion("MULTILINESTRING ((0 0, 2 2, 4 4), (5 5, 7 7, 9 9))", "MULTILINESTRING ((5 5, 7 7, 9 9), (11 11, 13 13, 15 15))", "MULTILINESTRING ((0 0, 2 2, 4 4), (5 5, 7 7, 9 9), (11 11, 13 13, 15 15))"); - assertUnion("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", "POLYGON ((1 0, 2 0, 2 1, 1 1, 1 0))", "POLYGON ((0 0, 1 0, 2 0, 2 1, 1 1, 0 1, 0 0))"); - assertUnion("MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", "MULTIPOLYGON (((1 0, 2 0, 2 1, 1 1, 1 0)))", "POLYGON ((0 0, 1 0, 2 0, 2 1, 1 1, 0 1, 0 0))"); - assertUnion("GEOMETRYCOLLECTION (POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0)), POINT (1 2))", "GEOMETRYCOLLECTION (POLYGON ((1 0, 2 0, 2 1, 1 1, 1 0)), MULTIPOINT ((1 2), (3 4)))", "GEOMETRYCOLLECTION (MULTIPOINT ((1 2), (3 4)), POLYGON ((0 0, 1 0, 2 0, 2 1, 1 1, 0 1, 0 0)))"); + assertUnion("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", "POLYGON ((1 0, 2 0, 2 1, 1 1, 1 0))", "POLYGON ((0 0, 0 1, 1 1, 2 1, 2 0, 1 0, 0 0))"); + assertUnion("MULTIPOLYGON (((0 0, 0 1, 1 1, 1 0, 0 0)))", "MULTIPOLYGON (((1 0, 2 0, 2 1, 1 1, 1 0)))", "POLYGON ((0 0, 0 1, 1 1, 2 1, 2 0, 1 0, 0 0))"); + assertUnion("GEOMETRYCOLLECTION (POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0)), POINT (1 2))", "GEOMETRYCOLLECTION (POLYGON ((1 0, 2 0, 2 1, 1 1, 1 0)), MULTIPOINT ((1 2), (3 4)))", "GEOMETRYCOLLECTION (MULTIPOINT ((1 2), (3 4)), POLYGON ((0 0, 0 1, 1 1, 2 1, 2 0, 1 0, 0 0)))"); // within union assertUnion("MULTIPOINT ((20 20), (25 25))", "POINT (25 25)", "MULTIPOINT ((20 20), (25 25))"); assertUnion("LINESTRING (20 20, 30 30)", "POINT (25 25)", "LINESTRING (20 20, 25 25, 30 30)"); assertUnion("LINESTRING (20 20, 30 30)", "LINESTRING (25 25, 27 27)", "LINESTRING (20 20, 25 25, 27 27, 30 30)"); - assertUnion("POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0))", "POLYGON ((1 1, 1 2, 2 2, 2 1, 1 1))", "POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0))"); - assertUnion("MULTIPOLYGON (((0 0 , 0 2, 2 2, 2 0)), ((2 2, 2 4, 4 4, 4 2)))", "POLYGON ((2 2, 2 3, 3 3, 3 2))", "MULTIPOLYGON (((2 2, 3 2, 4 2, 4 4, 2 4, 2 3, 2 2)), ((0 0, 2 0, 2 2, 0 2, 0 0)))"); - assertUnion("GEOMETRYCOLLECTION (POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0)), MULTIPOINT ((20 20), (25 25)))", "GEOMETRYCOLLECTION (POLYGON ((1 1, 1 2, 2 2, 2 1, 1 1)), POINT (25 25))", "GEOMETRYCOLLECTION (MULTIPOINT ((20 20), (25 25)), POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0)))"); + assertUnion("POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))", "POLYGON ((1 1, 1 2, 2 2, 2 1, 1 1))", "POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))"); + assertUnion("MULTIPOLYGON (((0 0, 0 2, 2 2, 2 0, 0 0)), ((2 2, 2 4, 4 4, 4 2, 2 2)))", "POLYGON ((2 2, 2 3, 3 3, 3 2, 2 2))", "MULTIPOLYGON (((2 2, 2 3, 2 4, 4 4, 4 2, 3 2, 2 2)), ((0 0, 0 2, 2 2, 2 0, 0 0)))"); + assertUnion("GEOMETRYCOLLECTION (POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0)), MULTIPOINT ((20 20), (25 25)))", "GEOMETRYCOLLECTION (POLYGON ((1 1, 1 2, 2 2, 2 1, 1 1)), POINT (25 25))", "GEOMETRYCOLLECTION (MULTIPOINT ((20 20), (25 25)), POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0)))"); // overlap union assertUnion("LINESTRING (1 1, 3 1)", "LINESTRING (2 1, 4 1)", "LINESTRING (1 1, 2 1, 3 1, 4 1)"); assertUnion("MULTILINESTRING ((1 1, 3 1))", "MULTILINESTRING ((2 1, 4 1))", "LINESTRING (1 1, 2 1, 3 1, 4 1)"); - assertUnion("POLYGON ((1 1, 3 1, 3 3, 1 3, 1 1))", "POLYGON ((2 2, 4 2, 4 4, 2 4, 2 2))", "POLYGON ((1 1, 3 1, 3 2, 4 2, 4 4, 2 4, 2 3, 1 3, 1 1))"); - assertUnion("MULTIPOLYGON (((1 1, 3 1, 3 3, 1 3, 1 1)))", "MULTIPOLYGON (((2 2, 4 2, 4 4, 2 4, 2 2)))", "POLYGON ((1 1, 3 1, 3 2, 4 2, 4 4, 2 4, 2 3, 1 3, 1 1))"); - assertUnion("GEOMETRYCOLLECTION (POLYGON ((1 1, 3 1, 3 3, 1 3, 1 1)), LINESTRING (1 1, 3 1))", "GEOMETRYCOLLECTION (POLYGON ((2 2, 4 2, 4 4, 2 4, 2 2)), LINESTRING (2 1, 4 1))", "GEOMETRYCOLLECTION (LINESTRING (3 1, 4 1), POLYGON ((1 1, 2 1, 3 1, 3 2, 4 2, 4 4, 2 4, 2 3, 1 3, 1 1)))"); + assertUnion("POLYGON ((1 1, 3 1, 3 3, 1 3, 1 1))", "POLYGON ((2 2, 4 2, 4 4, 2 4, 2 2))", "POLYGON ((1 1, 1 3, 2 3, 2 4, 4 4, 4 2, 3 2, 3 1, 1 1))"); + assertUnion("MULTIPOLYGON (((1 1, 3 1, 3 3, 1 3, 1 1)))", "MULTIPOLYGON (((2 2, 4 2, 4 4, 2 4, 2 2)))", "POLYGON ((1 1, 1 3, 2 3, 2 4, 4 4, 4 2, 3 2, 3 1, 1 1))"); + assertUnion("GEOMETRYCOLLECTION (POLYGON ((1 1, 3 1, 3 3, 1 3, 1 1)), LINESTRING (1 1, 3 1))", "GEOMETRYCOLLECTION (POLYGON ((2 2, 4 2, 4 4, 2 4, 2 2)), LINESTRING (2 1, 4 1))", "GEOMETRYCOLLECTION (LINESTRING (3 1, 4 1), POLYGON ((1 1, 1 3, 2 3, 2 4, 4 4, 4 2, 3 2, 3 1, 2 1, 1 1)))"); } private void assertUnion(String leftWkt, String rightWkt, String expectWkt) @@ -979,9 +931,9 @@ public void testSTGeometryN() assertSTGeometryN("LINESTRING(77.29 29.07, 77.42 29.26, 77.27 29.31, 77.29 29.07)", 1, "LINESTRING (77.29 29.07, 77.42 29.26, 77.27 29.31, 77.29 29.07)"); assertSTGeometryN("LINESTRING(77.29 29.07, 77.42 29.26, 77.27 29.31, 77.29 29.07)", 2, null); assertSTGeometryN("LINESTRING(77.29 29.07, 77.42 29.26, 77.27 29.31, 77.29 29.07)", -1, null); - assertSTGeometryN("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", 1, "POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))"); - assertSTGeometryN("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", 2, null); - assertSTGeometryN("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", -1, null); + assertSTGeometryN("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", 1, "POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))"); + assertSTGeometryN("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", 2, null); + assertSTGeometryN("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", -1, null); assertSTGeometryN("MULTIPOINT (1 2, 2 4, 3 6, 4 8)", 1, "POINT (1 2)"); assertSTGeometryN("MULTIPOINT (1 2, 2 4, 3 6, 4 8)", 2, "POINT (2 4)"); assertSTGeometryN("MULTIPOINT (1 2, 2 4, 3 6, 4 8)", 0, null); @@ -992,11 +944,11 @@ public void testSTGeometryN() assertSTGeometryN("MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))", 0, null); assertSTGeometryN("MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))", 3, null); assertSTGeometryN("MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))", -1, null); - assertSTGeometryN("MULTIPOLYGON (((1 1, 3 1, 3 3, 1 3, 1 1)), ((2 4, 6 4, 6 6, 2 6, 2 4)))", 1, "POLYGON ((1 1, 3 1, 3 3, 1 3, 1 1))"); - assertSTGeometryN("MULTIPOLYGON (((1 1, 3 1, 3 3, 1 3, 1 1)), ((2 4, 6 4, 6 6, 2 6, 2 4)))", 2, "POLYGON ((2 4, 6 4, 6 6, 2 6, 2 4))"); - assertSTGeometryN("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((2 4, 2 6, 6 6, 6 4)))", 0, null); - assertSTGeometryN("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((2 4, 2 6, 6 6, 6 4)))", 3, null); - assertSTGeometryN("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((2 4, 2 6, 6 6, 6 4)))", -1, null); + assertSTGeometryN("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))", 1, "POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))"); + assertSTGeometryN("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))", 2, "POLYGON ((2 4, 2 6, 6 6, 6 4, 2 4))"); + assertSTGeometryN("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))", 0, null); + assertSTGeometryN("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))", 3, null); + assertSTGeometryN("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))", -1, null); assertSTGeometryN("GEOMETRYCOLLECTION(POINT(2 3), LINESTRING (2 3, 3 4))", 1, "POINT (2 3)"); assertSTGeometryN("GEOMETRYCOLLECTION(POINT(2 3), LINESTRING (2 3, 3 4))", 2, "LINESTRING (2 3, 3 4)"); assertSTGeometryN("GEOMETRYCOLLECTION(POINT(2 3), LINESTRING (2 3, 3 4))", 3, null); @@ -1031,13 +983,13 @@ public void testSTLineString() assertFunction("ST_LineString(array[])", GEOMETRY, "LINESTRING EMPTY"); // Only points can be passed - assertInvalidFunction("ST_LineString(array[ST_Point(7,8), ST_GeometryFromText('LINESTRING (1 2, 3 4)')])", INVALID_FUNCTION_ARGUMENT, "ST_LineString takes only an array of valid points, LineString was passed"); + assertInvalidFunction("ST_LineString(array[ST_Point(7,8), ST_GeometryFromText('LINESTRING (1 2, 3 4)')])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: geometry is not a point: LINE_STRING at index 2"); // Nulls points are invalid - assertInvalidFunction("ST_LineString(array[NULL])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: null point at index 1"); - assertInvalidFunction("ST_LineString(array[ST_Point(1,2), NULL])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: null point at index 2"); - assertInvalidFunction("ST_LineString(array[ST_Point(1, 2), NULL, ST_Point(3, 4)])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: null point at index 2"); - assertInvalidFunction("ST_LineString(array[ST_Point(1, 2), NULL, ST_Point(3, 4), NULL])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: null point at index 2"); + assertInvalidFunction("ST_LineString(array[NULL])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: null at index 1"); + assertInvalidFunction("ST_LineString(array[ST_Point(1,2), NULL])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: null at index 2"); + assertInvalidFunction("ST_LineString(array[ST_Point(1, 2), NULL, ST_Point(3, 4)])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: null at index 2"); + assertInvalidFunction("ST_LineString(array[ST_Point(1, 2), NULL, ST_Point(3, 4), NULL])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: null at index 2"); // Empty points are invalid assertInvalidFunction("ST_LineString(array[ST_GeometryFromText('POINT EMPTY')])", INVALID_FUNCTION_ARGUMENT, "Invalid input to ST_LineString: empty point at index 1"); @@ -1065,7 +1017,7 @@ public void testMultiPoint() assertFunction("ST_MultiPoint(array[])", GEOMETRY, null); // Only points can be passed - assertInvalidMultiPoint("geometry is not a point: LineString at index 2", "POINT (7 8)", "LINESTRING (1 2, 3 4)"); + assertInvalidMultiPoint("geometry is not a point: LINE_STRING at index 2", "POINT (7 8)", "LINESTRING (1 2, 3 4)"); // Null point raises exception assertInvalidFunction("ST_MultiPoint(array[null])", "Invalid input to ST_MultiPoint: null at index 1"); @@ -1114,8 +1066,8 @@ public void testSTPointN() assertInvalidPointN("POINT (1 2)", "POINT"); assertInvalidPointN("MULTIPOINT (1 1, 2 2)", "MULTI_POINT"); assertInvalidPointN("MULTILINESTRING ((1 1, 2 2), (3 3, 4 4))", "MULTI_LINE_STRING"); - assertInvalidPointN("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", "POLYGON"); - assertInvalidPointN("MULTIPOLYGON (((1 1, 1 4, 4 4, 4 1)), ((1 1, 1 4, 4 4, 4 1)))", "MULTI_POLYGON"); + assertInvalidPointN("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", "POLYGON"); + assertInvalidPointN("MULTIPOLYGON (((1 1, 1 4, 4 4, 4 1, 1 1)), ((1 1, 1 4, 4 4, 4 1, 1 1)))", "MULTI_POLYGON"); assertInvalidPointN("GEOMETRYCOLLECTION(POINT(4 6),LINESTRING(4 6, 7 10))", "GEOMETRY_COLLECTION"); } @@ -1136,12 +1088,14 @@ public void testSTGeometries() assertFunction("ST_Geometries(ST_GeometryFromText('POINT EMPTY'))", new ArrayType(GEOMETRY), null); assertSTGeometries("POINT (1 5)", "POINT (1 5)"); assertSTGeometries("LINESTRING (77.29 29.07, 77.42 29.26, 77.27 29.31, 77.29 29.07)", "LINESTRING (77.29 29.07, 77.42 29.26, 77.27 29.31, 77.29 29.07)"); - assertSTGeometries("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", "POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))"); + assertSTGeometries("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", "POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))"); assertSTGeometries("MULTIPOINT (1 2, 4 8, 16 32)", "POINT (1 2)", "POINT (4 8)", "POINT (16 32)"); assertSTGeometries("MULTILINESTRING ((1 1, 2 2))", "LINESTRING (1 1, 2 2)"); - assertSTGeometries("MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)), ((1 1, 3 1, 3 3, 1 3, 1 1)))", - "POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", "POLYGON ((1 1, 3 1, 3 3, 1 3, 1 1))"); + assertSTGeometries("MULTIPOLYGON (((0 0, 0 1, 1 1, 1 0, 0 0)), ((1 1, 3 1, 3 3, 1 3, 1 1)))", + "POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", "POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))"); assertSTGeometries("GEOMETRYCOLLECTION (POINT (2 3), LINESTRING (2 3, 3 4))", "POINT (2 3)", "LINESTRING (2 3, 3 4)"); + assertSTGeometries("GEOMETRYCOLLECTION(MULTIPOINT(0 0, 1 1), GEOMETRYCOLLECTION(MULTILINESTRING((2 2, 3 3))))", + "MULTIPOINT ((0 0), (1 1))", "GEOMETRYCOLLECTION (MULTILINESTRING ((2 2, 3 3)))"); } private void assertSTGeometries(String wkt, String... expected) @@ -1156,15 +1110,15 @@ public void testSTInteriorRingN() assertInvalidInteriorRingN("LINESTRING (1 2, 2 3, 3 4)", 1, "LINE_STRING"); assertInvalidInteriorRingN("MULTIPOINT (1 1, 2 3, 5 8)", -1, "MULTI_POINT"); assertInvalidInteriorRingN("MULTILINESTRING ((2 4, 4 2), (3 5, 5 3))", 0, "MULTI_LINE_STRING"); - assertInvalidInteriorRingN("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1)), ((2 4, 2 6, 6 6, 6 4)))", 2, "MULTI_POLYGON"); + assertInvalidInteriorRingN("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))", 2, "MULTI_POLYGON"); assertInvalidInteriorRingN("GEOMETRYCOLLECTION (POINT (2 2), POINT (10 20))", 1, "GEOMETRY_COLLECTION"); - assertInteriorRingN("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", 1, null); - assertInteriorRingN("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", 2, null); - assertInteriorRingN("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", -1, null); - assertInteriorRingN("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", 0, null); - assertInteriorRingN("POLYGON ((0 0, 0 3, 3 3, 3 0, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))", 1, "LINESTRING (1 1, 1 2, 2 2, 2 1, 1 1)"); - assertInteriorRingN("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1), (3 3, 3 4, 4 4, 4 3, 3 3))", 2, "LINESTRING (3 3, 3 4, 4 4, 4 3, 3 3)"); + assertInteriorRingN("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", 1, null); + assertInteriorRingN("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", 2, null); + assertInteriorRingN("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", -1, null); + assertInteriorRingN("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))", 0, null); + assertInteriorRingN("POLYGON ((0 0, 0 3, 3 3, 3 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))", 1, "LINESTRING (1 1, 2 1, 2 2, 1 2, 1 1)"); + assertInteriorRingN("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1), (3 3, 4 3, 4 4, 3 4, 3 3))", 2, "LINESTRING (3 3, 4 3, 4 4, 3 4, 3 3)"); } private void assertInteriorRingN(String wkt, int index, String expected) @@ -1182,14 +1136,25 @@ public void testSTGeometryType() { assertFunction("ST_GeometryType(ST_Point(1, 4))", VARCHAR, "ST_Point"); assertFunction("ST_GeometryType(ST_GeometryFromText('LINESTRING (1 1, 2 2)'))", VARCHAR, "ST_LineString"); - assertFunction("ST_GeometryType(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1))'))", VARCHAR, "ST_Polygon"); + assertFunction("ST_GeometryType(ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'))", VARCHAR, "ST_Polygon"); assertFunction("ST_GeometryType(ST_GeometryFromText('MULTIPOINT (1 1, 2 2)'))", VARCHAR, "ST_MultiPoint"); assertFunction("ST_GeometryType(ST_GeometryFromText('MULTILINESTRING ((1 1, 2 2), (3 3, 4 4))'))", VARCHAR, "ST_MultiLineString"); - assertFunction("ST_GeometryType(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 4, 4 4, 4 1)), ((1 1, 1 4, 4 4, 4 1)))'))", VARCHAR, "ST_MultiPolygon"); + assertFunction("ST_GeometryType(ST_GeometryFromText('MULTIPOLYGON (((1 1, 1 4, 4 4, 4 1, 1 1)), ((1 1, 1 4, 4 4, 4 1, 1 1)))'))", VARCHAR, "ST_MultiPolygon"); assertFunction("ST_GeometryType(ST_GeometryFromText('GEOMETRYCOLLECTION(POINT(4 6),LINESTRING(4 6, 7 10))'))", VARCHAR, "ST_GeomCollection"); assertFunction("ST_GeometryType(ST_Envelope(ST_GeometryFromText('LINESTRING (1 1, 2 2)')))", VARCHAR, "ST_Polygon"); } + @Test + public void testSTGeometryFromText() + { + assertInvalidFunction("ST_GeometryFromText('xyz')", "Invalid WKT: Unknown geometry type: XYZ (line 1)"); + assertInvalidFunction("ST_GeometryFromText('LINESTRING (-71.3839245 42.3128124)')", "Invalid WKT: Invalid number of points in LineString (found 1 - must be 0 or >= 2)"); + assertInvalidFunction("ST_GeometryFromText('POLYGON ((-13.719076 9.508430, -13.723493 9.510049, -13.719076 9.508430))')", "Invalid WKT: Invalid number of points in LinearRing (found 3 - must be 0 or >= 4)"); + assertInvalidFunction("ST_GeometryFromText('POLYGON ((-13.637339 9.617113, -13.637339 9.617113))')", "Invalid WKT: Invalid number of points in LinearRing (found 2 - must be 0 or >= 4)"); + assertInvalidFunction("ST_GeometryFromText('POLYGON(0 0)')", INVALID_FUNCTION_ARGUMENT, "Invalid WKT: Expected EMPTY or ( but found '0' (line 1)"); + assertInvalidFunction("ST_GeometryFromText('POLYGON((0 0))')", INVALID_FUNCTION_ARGUMENT, "Invalid WKT: Invalid number of points in LineString (found 1 - must be 0 or >= 2)"); + } + @Test public void testSTGeometryFromBinary() { @@ -1209,10 +1174,10 @@ public void testSTGeometryFromBinary() assertGeomFromBinary("MULTIPOINT ((1 2), (3 4))"); assertGeomFromBinary("LINESTRING (0 0, 1 2, 3 4)"); assertGeomFromBinary("MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))"); - assertGeomFromBinary("POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))"); - assertGeomFromBinary("POLYGON ((0 0, 3 0, 3 3, 0 3, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))"); - assertGeomFromBinary("MULTIPOLYGON (((1 1, 3 1, 3 3, 1 3, 1 1)), ((2 4, 6 4, 6 6, 2 6, 2 4)))"); - assertGeomFromBinary("GEOMETRYCOLLECTION (POINT (1 2), LINESTRING (0 0, 1 2, 3 4), POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0)))"); + assertGeomFromBinary("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))"); + assertGeomFromBinary("POLYGON ((0 0, 0 3, 3 3, 3 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1))"); + assertGeomFromBinary("MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((2 4, 2 6, 6 6, 6 4, 2 4)))"); + assertGeomFromBinary("GEOMETRYCOLLECTION (POINT (1 2), LINESTRING (0 0, 1 2, 3 4), POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0)))"); // array of geometries assertFunction("transform(array[ST_AsBinary(ST_Point(1, 2)), ST_AsBinary(ST_Point(3, 4))], wkb -> ST_AsText(ST_GeomFromBinary(wkb)))", new ArrayType(VARCHAR), ImmutableList.of("POINT (1 2)", "POINT (3 4)")); diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestGeoRelations.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestGeoRelations.java new file mode 100644 index 0000000000000..70272a5b46670 --- /dev/null +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestGeoRelations.java @@ -0,0 +1,286 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.plugin.geospatial; + +import com.facebook.presto.operator.scalar.AbstractTestFunctions; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import org.testng.internal.collections.Pair; + +import java.util.List; + +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static java.lang.String.format; + +public class TestGeoRelations + extends AbstractTestFunctions +{ + // A set of geometries such that: + // 0, 1: Within (1, 0: Contains) + // 0, 2: Touches + // 1, 2: Overlaps + // 0, 3: Touches + // 1, 3: Crosses + // 1, 4: Touches + // 1, 5: Touches + // 2, 3: Contains + // 2, 4: Crosses + // 2, 5: Crosses + // 3, 4: Crosses + // 3, 5: Touches + // 4, 5: Contains + // 1, 6: Contains + // 2, 6: Contains + // 1, 7: Touches + // 2, 7: Contains + // 3, 6: Contains + // 3, 7: Contains + // 4, 7: Contains + // 5, 7: Touches + public static final List RELATION_GEOMETRIES_WKT = ImmutableList.of( + "'POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))'", // 0 + "'POLYGON ((0 0, 0 2, 2 2, 2 0, 0 0))'", // 1 + "'POLYGON ((1 0, 1 1, 3 1, 3 0, 1 0))'", // 2 + "'LINESTRING (1 0.5, 2.5 0.5)'", // 3 + "'LINESTRING (2 0, 2 2)'", // 4 + "'LINESTRING (2 0.5, 2 2)'", // 5 + "'POINT (1.5 0.5)'", // 6 + "'POINT (2 0.5)'"); // 7 + + public static final List> EQUALS_PAIRS = ImmutableList.of( + Pair.of(0, 0), + Pair.of(1, 1), + Pair.of(2, 2), + Pair.of(3, 3), + Pair.of(4, 4), + Pair.of(5, 5), + Pair.of(6, 6), + Pair.of(7, 7)); + + public static final List> CONTAINS_PAIRS = ImmutableList.of( + Pair.of(1, 0), + Pair.of(2, 3), + Pair.of(4, 5), + Pair.of(1, 6), + Pair.of(2, 6), + Pair.of(2, 7), + Pair.of(3, 6), + Pair.of(3, 7), + Pair.of(4, 7)); + + public static final List> TOUCHES_PAIRS = ImmutableList.of( + Pair.of(0, 2), + Pair.of(0, 3), + Pair.of(1, 4), + Pair.of(1, 5), + Pair.of(3, 5), + Pair.of(1, 7), + Pair.of(5, 7)); + + public static final List> OVERLAPS_PAIRS = ImmutableList.of( + Pair.of(1, 2)); + + public static final List> CROSSES_PAIRS = ImmutableList.of( + Pair.of(1, 3), + Pair.of(2, 4), + Pair.of(2, 5), + Pair.of(3, 4)); + + @BeforeClass + protected void registerFunctions() + { + GeoPlugin plugin = new GeoPlugin(); + registerTypes(plugin); + registerFunctions(plugin); + } + + @Test + public void testStContains() + { + assertRelation("ST_Contains", "null", "'POINT (25 25)'", null); + assertRelation("ST_Contains", "'POINT (20 20)'", "'POINT (25 25)'", false); + assertRelation("ST_Contains", "'MULTIPOINT (20 20, 25 25)'", "'POINT (25 25)'", true); + assertRelation("ST_Contains", "'LINESTRING (20 20, 30 30)'", "'POINT (25 25)'", true); + assertRelation("ST_Contains", "'LINESTRING (20 20, 30 30)'", "'MULTIPOINT (25 25, 31 31)'", false); + assertRelation("ST_Contains", "'LINESTRING (20 20, 30 30)'", "'LINESTRING (25 25, 27 27)'", true); + assertRelation("ST_Contains", "'MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'", "'MULTILINESTRING ((3 4, 4 4), (2 1, 6 1))'", false); + assertRelation("ST_Contains", "'POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))'", "'POLYGON ((1 1, 1 2, 2 2, 2 1, 1 1))'", true); + assertRelation("ST_Contains", "'POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))'", "'POLYGON ((-1 -1, -1 2, 2 2, 2 -1, -1 -1))'", false); + assertRelation("ST_Contains", "'MULTIPOLYGON (((0 0, 0 2, 2 2, 2 0, 0 0)), ((2 2, 2 4, 4 4, 4 2, 2 2)))'", "'POLYGON ((2 2, 2 3, 3 3, 3 2, 2 2))'", true); + assertRelation("ST_Contains", "'LINESTRING (20 20, 30 30)'", "'POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))'", false); + assertRelation("ST_Contains", "'LINESTRING EMPTY'", "'POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))'", false); + assertRelation("ST_Contains", "'LINESTRING (20 20, 30 30)'", "'POLYGON EMPTY'", false); + } + + @Test + public void testSTCrosses() + { + assertRelation("ST_Crosses", "'POINT (20 20)'", "'POINT (25 25)'", false); + assertRelation("ST_Crosses", "'LINESTRING (20 20, 30 30)'", "'POINT (25 25)'", false); + assertRelation("ST_Crosses", "'LINESTRING (20 20, 30 30)'", "'MULTIPOINT (25 25, 31 31)'", true); + assertRelation("ST_Crosses", "'LINESTRING(0 0, 1 1)'", "'LINESTRING (1 0, 0 1)'", true); + assertRelation("ST_Crosses", "'POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'", "'POLYGON ((2 2, 2 5, 5 5, 5 2, 2 2))'", false); + assertRelation("ST_Crosses", "'MULTIPOLYGON (((0 0, 0 2, 2 2, 2 0, 0 0)), ((2 2, 2 4, 4 4, 4 2, 2 2)))'", "'POLYGON ((2 2, 2 3, 3 3, 3 2, 2 2))'", false); + assertRelation("ST_Crosses", "'LINESTRING (-2 -2, 6 6)'", "'POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))'", true); + assertRelation("ST_Crosses", "'POINT (20 20)'", "'POINT (20 20)'", false); + assertRelation("ST_Crosses", "'POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))'", "'POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))'", false); + assertRelation("ST_Crosses", "'POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))'", "'LINESTRING (0 0, 0 4, 4 4, 4 0)'", false); + } + + @Test + public void testSTDisjoint() + { + assertRelation("ST_Disjoint", "'POINT (50 100)'", "'POINT (150 150)'", true); + assertRelation("ST_Disjoint", "'MULTIPOINT (50 100, 50 200)'", "'POINT (50 100)'", false); + assertRelation("ST_Disjoint", "'LINESTRING (0 0, 0 1)'", "'LINESTRING (1 1, 1 0)'", true); + assertRelation("ST_Disjoint", "'LINESTRING (2 1, 1 2)'", "'LINESTRING (3 1, 1 3)'", true); + assertRelation("ST_Disjoint", "'LINESTRING (1 1, 3 3)'", "'LINESTRING (3 1, 1 3)'", false); + assertRelation("ST_Disjoint", "'LINESTRING (50 100, 50 200)'", "'LINESTRING (20 150, 100 150)'", false); + assertRelation("ST_Disjoint", "'MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'", "'MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))'", false); + assertRelation("ST_Disjoint", "'POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))'", "'POLYGON ((4 4, 4 5, 5 5, 5 4, 4 4))'", true); + assertRelation("ST_Disjoint", "'MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))'", "'POLYGON ((0 1, 3 1, 3 3, 0 3, 0 1))'", false); + } + + @Test + public void testSTEquals() + { + assertRelation("ST_Equals", "'POINT (50 100)'", "'POINT (150 150)'", false); + assertRelation("ST_Equals", "'MULTIPOINT (50 100, 50 200)'", "'POINT (50 100)'", false); + assertRelation("ST_Equals", "'LINESTRING (0 0, 0 1)'", "'LINESTRING (1 1, 1 0)'", false); + assertRelation("ST_Equals", "'LINESTRING (0 0, 2 2)'", "'LINESTRING (0 0, 2 2)'", true); + assertRelation("ST_Equals", "'MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'", "'MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))'", false); + assertRelation("ST_Equals", "'POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))'", "'POLYGON ((3 3, 3 1, 1 1, 1 3, 3 3))'", true); + assertRelation("ST_Equals", "'MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))'", "'POLYGON ((0 1, 3 1, 3 3, 0 3, 0 1))'", false); + } + + @Test + public void testSTIntersects() + { + assertRelation("ST_Intersects", "'POINT (50 100)'", "'POINT (150 150)'", false); + assertRelation("ST_Intersects", "'MULTIPOINT (50 100, 50 200)'", "'POINT (50 100)'", true); + assertRelation("ST_Intersects", "'LINESTRING (0 0, 0 1)'", "'LINESTRING (1 1, 1 0)'", false); + assertRelation("ST_Intersects", "'LINESTRING (50 100, 50 200)'", "'LINESTRING (20 150, 100 150)'", true); + assertRelation("ST_Intersects", "'MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'", "'MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))'", true); + assertRelation("ST_Intersects", "'POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))'", "'POLYGON ((4 4, 4 5, 5 5, 5 4, 4 4))'", false); + assertRelation("ST_Intersects", "'MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))'", "'POLYGON ((0 1, 3 1, 3 3, 0 3, 0 1))'", true); + assertRelation("ST_Intersects", "'POLYGON ((16.5 54, 16.5 54.1, 16.51 54.1, 16.8 54, 16.5 54))'", "'LINESTRING (16.6 53, 16.6 56)'", true); + assertRelation("ST_Intersects", "'POLYGON ((16.5 54, 16.5 54.1, 16.51 54.1, 16.8 54, 16.5 54))'", "'LINESTRING (16.6667 54.05, 16.8667 54.05)'", false); + assertRelation("ST_Intersects", "'POLYGON ((16.5 54, 16.5 54.1, 16.51 54.1, 16.8 54, 16.5 54))'", "'LINESTRING (16.6667 54.25, 16.8667 54.25)'", false); + } + + @Test + public void testSTOverlaps() + { + assertRelation("ST_Overlaps", "'POINT (50 100)'", "'POINT (150 150)'", false); + assertRelation("ST_Overlaps", "'POINT (50 100)'", "'POINT (50 100)'", false); + assertRelation("ST_Overlaps", "'MULTIPOINT (50 100, 50 200)'", "'POINT (50 100)'", false); + assertRelation("ST_Overlaps", "'LINESTRING (0 0, 0 1)'", "'LINESTRING (1 1, 1 0)'", false); + assertRelation("ST_Overlaps", "'MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'", "'MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))'", true); + assertRelation("ST_Overlaps", "'POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'", "'POLYGON ((3 3, 3 5, 5 5, 5 3, 3 3))'", true); + assertRelation("ST_Overlaps", "'POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'", "'POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'", false); + assertRelation("ST_Overlaps", "'POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'", "'LINESTRING (1 1, 4 4)'", false); + assertRelation("ST_Overlaps", "'POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))'", "'POLYGON ((4 4, 4 5, 5 5, 5 4, 4 4))'", false); + assertRelation("ST_Overlaps", "'MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))'", "'POLYGON ((0 1, 3 1, 3 3, 0 3, 0 1))'", true); + } + + @Test + public void testSTRelate() + { + assertFunction("ST_Relate(ST_GeometryFromText('LINESTRING (0 0, 3 3)'), ST_GeometryFromText('LINESTRING (1 1, 4 1)'), '****T****')", BOOLEAN, false); + assertFunction("ST_Relate(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1, 2 0))'), ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'), '****T****')", BOOLEAN, true); + assertFunction("ST_Relate(ST_GeometryFromText('POLYGON ((2 0, 2 1, 3 1, 2 0))'), ST_GeometryFromText('POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'), 'T********')", BOOLEAN, false); + } + + @Test + public void testSTTouches() + { + assertRelation("ST_Touches", "'POINT (50 100)'", "'POINT (150 150)'", false); + assertRelation("ST_Touches", "'MULTIPOINT (50 100, 50 200)'", "'POINT (50 100)'", false); + assertRelation("ST_Touches", "'LINESTRING (50 100, 50 200)'", "'LINESTRING (20 150, 100 150)'", false); + assertRelation("ST_Touches", "'MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'", "'MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))'", false); + assertRelation("ST_Touches", "'POINT (1 2)'", "'POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'", true); + assertRelation("ST_Touches", "'POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))'", "'POLYGON ((4 4, 4 5, 5 5, 5 4, 4 4))'", false); + assertRelation("ST_Touches", "'POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))'", "'LINESTRING (0 0, 1 1)'", true); + assertRelation("ST_Touches", "'POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))'", "'POLYGON ((3 3, 3 5, 5 5, 5 3, 3 3))'", true); + assertRelation("ST_Touches", "'MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))'", "'POLYGON ((0 1, 3 1, 3 3, 0 3, 0 1))'", false); + } + + @Test + public void testSTWithin() + { + assertRelation("ST_Within", "'POINT (50 100)'", "'POINT (150 150)'", false); + assertRelation("ST_Within", "'POINT (50 100)'", "'MULTIPOINT (50 100, 50 200)'", true); + assertRelation("ST_Within", "'LINESTRING (50 100, 50 200)'", "'LINESTRING (50 50, 50 250)'", true); + assertRelation("ST_Within", "'MULTILINESTRING ((1 1, 5 1), (2 4, 4 4))'", "'MULTILINESTRING ((3 4, 6 4), (5 0, 5 4))'", false); + assertRelation("ST_Within", "'POINT (3 2)'", "'POLYGON ((1 1, 1 4, 4 4, 4 1, 1 1))'", true); + assertRelation("ST_Within", "'POLYGON ((1 1, 1 3, 3 3, 3 1, 1 1))'", "'POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))'", true); + assertRelation("ST_Within", "'LINESTRING (1 1, 3 3)'", "'POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))'", true); + assertRelation("ST_Within", "'MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((0 0, 0 2, 2 2, 2 0, 0 0)))'", "'POLYGON ((0 1, 3 1, 3 3, 0 3, 0 1))'", false); + assertRelation("ST_Within", "'POLYGON ((1 1, 1 5, 5 5, 5 1, 1 1))'", "'POLYGON ((0 0, 0 4, 4 4, 4 0, 0 0))'", false); + } + + @Test + public void testContainsWithin() + { + for (int i = 0; i < RELATION_GEOMETRIES_WKT.size(); i++) { + for (int j = 0; j < RELATION_GEOMETRIES_WKT.size(); j++) { + boolean ok = (i == j) || CONTAINS_PAIRS.contains(Pair.of(i, j)); + assertRelation("ST_Contains", RELATION_GEOMETRIES_WKT.get(i), RELATION_GEOMETRIES_WKT.get(j), ok); + // Within is just the inverse of contain + assertRelation("ST_Within", RELATION_GEOMETRIES_WKT.get(j), RELATION_GEOMETRIES_WKT.get(i), ok); + } + } + } + + @Test + public void testEquals() + { + testSymmetricRelations("ST_Equals", EQUALS_PAIRS); + } + + @Test + public void testTouches() + { + testSymmetricRelations("ST_Touches", TOUCHES_PAIRS); + } + + @Test + public void testOverlaps() + { + testSymmetricRelations("ST_Overlaps", OVERLAPS_PAIRS); + } + + @Test + public void testCrosses() + { + testSymmetricRelations("ST_Crosses", CROSSES_PAIRS); + } + + private void testSymmetricRelations(String relation, List> pairs) + { + for (int i = 0; i < RELATION_GEOMETRIES_WKT.size(); i++) { + for (int j = 0; j < RELATION_GEOMETRIES_WKT.size(); j++) { + boolean ok = pairs.contains(Pair.of(i, j)) || pairs.contains(Pair.of(j, i)); + assertRelation(relation, RELATION_GEOMETRIES_WKT.get(i), RELATION_GEOMETRIES_WKT.get(j), ok); + } + } + } + + private void assertRelation(String relation, String leftWkt, String rightWkt, Boolean expected) + { + assertFunction(format("%s(ST_GeometryFromText(%s), ST_GeometryFromText(%s))", relation, leftWkt, rightWkt), BOOLEAN, expected); + } +} diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestRewriteSpatialPartitioningAggregation.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestRewriteSpatialPartitioningAggregation.java index 8ad78c88a0901..f6e7ffb1fef85 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestRewriteSpatialPartitioningAggregation.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestRewriteSpatialPartitioningAggregation.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.plugin.geospatial; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.sql.planner.iterative.rule.RewriteSpatialPartitioningAggregation; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; import com.facebook.presto.sql.planner.iterative.rule.test.RuleAssert; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoinOperator.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoinOperator.java index 566585784fb4c..6c7798247f5f6 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoinOperator.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoinOperator.java @@ -55,11 +55,12 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.geospatial.KdbTree.Node.newInternal; import static com.facebook.presto.geospatial.KdbTree.Node.newLeaf; -import static com.facebook.presto.operator.OperatorAssertion.assertOperatorEquals; +import static com.facebook.presto.operator.OperatorAssertion.assertOperatorEqualsIgnoreOrder; import static com.facebook.presto.plugin.geospatial.GeoFunctions.stGeometryFromText; import static com.facebook.presto.plugin.geospatial.GeoFunctions.stPoint; import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY; @@ -69,7 +70,6 @@ import static com.facebook.presto.sql.planner.plan.SpatialJoinNode.Type.INNER; import static com.facebook.presto.sql.planner.plan.SpatialJoinNode.Type.LEFT; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newScheduledThreadPool; import static java.util.concurrent.TimeUnit.SECONDS; import static org.testng.Assert.assertEquals; @@ -191,7 +191,7 @@ private void assertSpatialJoin(TaskContext taskContext, Type joinType, RowPagesB DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext(); PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(driverContext, (build, probe, r) -> build.contains(probe), Optional.empty(), Optional.empty(), buildPages); OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), joinType, probePages.getTypes(), Ints.asList(1), 0, Optional.empty(), pagesSpatialIndexFactory); - assertOperatorEquals(joinOperatorFactory, driverContext, probePages.build(), expected); + assertOperatorEqualsIgnoreOrder(joinOperatorFactory, driverContext, probePages.build(), expected); } @Test @@ -358,7 +358,7 @@ public void testDistanceQuery() .row("10_1", "10_0") .build(); - assertOperatorEquals(joinOperatorFactory, driverContext, probePages.build(), expected); + assertOperatorEqualsIgnoreOrder(joinOperatorFactory, driverContext, probePages.build(), expected); } @Test @@ -391,7 +391,7 @@ public void testDistributedSpatialJoin() PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(driverContext, (build, probe, r) -> build.contains(probe), Optional.empty(), Optional.of(2), Optional.of(KDB_TREE_JSON), Optional.empty(), buildPages); OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, probePages.getTypes(), Ints.asList(1), 0, Optional.of(2), pagesSpatialIndexFactory); - assertOperatorEquals(joinOperatorFactory, driverContext, probePages.build(), expected); + assertOperatorEqualsIgnoreOrder(joinOperatorFactory, driverContext, probePages.build(), expected); } @Test @@ -417,7 +417,7 @@ public void testDistributedSpatialSelfJoin() PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(driverContext, (build, probe, r) -> build.intersects(probe), Optional.empty(), Optional.of(2), Optional.of(KDB_TREE_JSON), Optional.empty(), pages); OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, pages.getTypes(), Ints.asList(1), 0, Optional.of(2), pagesSpatialIndexFactory); - assertOperatorEquals(joinOperatorFactory, driverContext, pages.build(), expected); + assertOperatorEqualsIgnoreOrder(joinOperatorFactory, driverContext, pages.build(), expected); } private PagesSpatialIndexFactory buildIndex(DriverContext driverContext, SpatialPredicate spatialRelationshipTest, Optional radiusChannel, Optional filterFunction, RowPagesBuilder buildPages) diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoinPlanning.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoinPlanning.java index ce95cfce8b207..a6cc14c5d7b03 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoinPlanning.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoinPlanning.java @@ -228,24 +228,42 @@ private void assertInvalidSpatialPartitioning(Session session, String sql, Strin } @Test - public void testSpatialJoinIntersects() + public void testSpatialJoinBinaryRelations() + { + testBroadcastJoinRelationship("st_intersects"); + testBroadcastJoinRelationship("st_crosses"); + testBroadcastJoinRelationship("st_equals"); + testBroadcastJoinRelationship("st_overlaps"); + testBroadcastJoinRelationship("st_touches"); + + testDistributedJoinRelationship("st_intersects"); + testDistributedJoinRelationship("st_crosses"); + testDistributedJoinRelationship("st_equals"); + testDistributedJoinRelationship("st_overlaps"); + testDistributedJoinRelationship("st_touches"); + } + + private void testBroadcastJoinRelationship(String relation) { // broadcast assertPlan("SELECT b.name, a.name " + "FROM (VALUES ('POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))', 'a')) AS a (wkt, name), (VALUES ('POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))', 'a')) AS b (wkt, name) " + - "WHERE ST_Intersects(ST_GeometryFromText(a.wkt), ST_GeometryFromText(b.wkt))", + "WHERE " + relation + "(ST_GeometryFromText(a.wkt), ST_GeometryFromText(b.wkt))", anyTree( - spatialJoin("st_intersects(geometry_a, geometry_b)", + spatialJoin(relation + "(geometry_a, geometry_b)", project(ImmutableMap.of("geometry_a", expression("ST_GeometryFromText(cast(wkt_a as varchar))")), anyTree(values(ImmutableMap.of("wkt_a", 0)))), anyTree(project(ImmutableMap.of("geometry_b", expression("ST_GeometryFromText(cast(wkt_b as varchar))")), anyTree(values(ImmutableMap.of("wkt_b", 0)))))))); + } + private void testDistributedJoinRelationship(String relation) + { // distributed assertDistributedPlan("SELECT b.name, a.name " + "FROM (VALUES ('POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))', 'a')) AS a (wkt, name), (VALUES ('POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))', 'a')) AS b (wkt, name) " + - "WHERE ST_Intersects(ST_GeometryFromText(a.wkt), ST_GeometryFromText(b.wkt))", + "WHERE " + relation + "(ST_GeometryFromText(a.wkt), ST_GeometryFromText(b.wkt))", withSpatialPartitioning("default.kdb_tree"), anyTree( - spatialJoin("st_intersects(geometry_a, geometry_b)", Optional.of(KDB_TREE_JSON), + spatialJoin(relation + "(geometry_a, geometry_b)", Optional.of(KDB_TREE_JSON), anyTree(unnest( project(ImmutableMap.of("partitions", expression(format("spatial_partitions(cast('%s' as kdbtree), geometry_a)", KDB_TREE_JSON))), project(ImmutableMap.of("geometry_a", expression("ST_GeometryFromText(cast(wkt_a as varchar))")), anyTree(values(ImmutableMap.of("wkt_a", 0))))))), diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoins.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoins.java index fbd92ff71e35c..7c40638014ecf 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoins.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoins.java @@ -15,23 +15,33 @@ import com.facebook.presto.Session; import com.facebook.presto.hive.HdfsConfiguration; -import com.facebook.presto.hive.HdfsConfigurationUpdater; +import com.facebook.presto.hive.HdfsConfigurationInitializer; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.HiveHdfsConfiguration; import com.facebook.presto.hive.HivePlugin; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.file.FileHiveMetastore; import com.facebook.presto.spi.security.PrincipalType; import com.facebook.presto.tests.AbstractTestQueryFramework; import com.facebook.presto.tests.DistributedQueryRunner; +import com.google.common.collect.ImmutableSet; import org.testng.annotations.Test; +import org.testng.internal.collections.Pair; import java.io.File; +import java.util.List; import java.util.Optional; import static com.facebook.presto.SystemSessionProperties.SPATIAL_PARTITIONING_TABLE_NAME; +import static com.facebook.presto.plugin.geospatial.TestGeoRelations.CONTAINS_PAIRS; +import static com.facebook.presto.plugin.geospatial.TestGeoRelations.CROSSES_PAIRS; +import static com.facebook.presto.plugin.geospatial.TestGeoRelations.EQUALS_PAIRS; +import static com.facebook.presto.plugin.geospatial.TestGeoRelations.OVERLAPS_PAIRS; +import static com.facebook.presto.plugin.geospatial.TestGeoRelations.RELATION_GEOMETRIES_WKT; +import static com.facebook.presto.plugin.geospatial.TestGeoRelations.TOUCHES_PAIRS; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static java.lang.String.format; @@ -59,6 +69,18 @@ public class TestSpatialJoins "(7.1, 7.2, 'z', 3), " + "(null, 1.2, 'null', 4)"; + private static String getRelationalGeometriesSql() + { + StringBuilder sql = new StringBuilder("VALUES "); + for (int i = 0; i < RELATION_GEOMETRIES_WKT.size(); i++) { + sql.append(format("(%s, %s)", RELATION_GEOMETRIES_WKT.get(i), i)); + if (i != RELATION_GEOMETRIES_WKT.size() - 1) { + sql.append(", "); + } + } + return sql.toString(); + } + public TestSpatialJoins() { super(() -> createQueryRunner()); @@ -77,8 +99,9 @@ private static DistributedQueryRunner createQueryRunner() File baseDir = queryRunner.getCoordinator().getBaseDataDir().resolve("hive_data").toFile(); HiveClientConfig hiveClientConfig = new HiveClientConfig(); - HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationUpdater(hiveClientConfig)); - HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication()); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig, metastoreClientConfig), ImmutableSet.of()); + HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); FileHiveMetastore metastore = new FileHiveMetastore(hdfsEnvironment, baseDir.toURI().toString(), "test"); metastore.createDatabase(Database.builder() @@ -324,4 +347,42 @@ public void testBroadcastSpatialLeftJoin() "ON a.name > b.name AND ST_Intersects(ST_GeometryFromText(b.wkt), ST_GeometryFromText(a.wkt))", "SELECT * FROM VALUES ('a', null), ('b', null), ('c', 'a'), ('c', 'b'), ('d', null), ('empty', null), ('null', null)"); } + + private void testRelationshipSpatialJoin(Session session, String relation, List> expectedPairs) + { + StringBuilder expected = new StringBuilder("SELECT * FROM VALUES "); + for (int i = 0; i < expectedPairs.size(); i++) { + Pair pair = expectedPairs.get(i); + expected.append(format("(%d, %d)", pair.first(), pair.second())); + if (i != expectedPairs.size() - 1) { + expected.append(", "); + } + } + + String whereClause; + switch (relation) { + case "ST_Contains": whereClause = "WHERE a.id != b.id"; + break; + case "ST_Equals": whereClause = ""; + break; + default: whereClause = "WHERE a.id < b.id"; + break; + } + + assertQuery(session, + format("SELECT a.id, b.id FROM (%s) AS a (wkt, id) JOIN (%s) AS b (wkt, id) " + + "ON %s(ST_GeometryFromText(a.wkt), ST_GeometryFromText(b.wkt)) %s", + getRelationalGeometriesSql(), getRelationalGeometriesSql(), relation, whereClause), + expected.toString()); + } + + @Test + public void testRelationshipBroadcastSpatialJoin() + { + testRelationshipSpatialJoin(getSession(), "ST_Equals", EQUALS_PAIRS); + testRelationshipSpatialJoin(getSession(), "ST_Contains", CONTAINS_PAIRS); + testRelationshipSpatialJoin(getSession(), "ST_Touches", TOUCHES_PAIRS); + testRelationshipSpatialJoin(getSession(), "ST_Overlaps", OVERLAPS_PAIRS); + testRelationshipSpatialJoin(getSession(), "ST_Crosses", CROSSES_PAIRS); + } } diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialPartitioningInternalAggregation.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialPartitioningInternalAggregation.java index b75e74bd84c06..468d10d2921c7 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialPartitioningInternalAggregation.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialPartitioningInternalAggregation.java @@ -27,6 +27,7 @@ import com.facebook.presto.operator.aggregation.InternalAggregationFunction; import com.facebook.presto.operator.scalar.AbstractTestFunctions; import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.google.common.collect.ImmutableList; @@ -39,16 +40,18 @@ import java.util.Optional; import static com.facebook.presto.geospatial.KdbTree.buildKdbTree; -import static com.facebook.presto.geospatial.serde.GeometrySerde.serialize; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.serialize; import static com.facebook.presto.operator.aggregation.AggregationTestUtils.createGroupByIdBlock; import static com.facebook.presto.operator.aggregation.AggregationTestUtils.getFinalBlock; import static com.facebook.presto.operator.aggregation.AggregationTestUtils.getGroupValue; import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.google.common.math.DoubleMath.roundToInt; import static java.math.RoundingMode.CEILING; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; public class TestSpatialPartitioningInternalAggregation extends AbstractTestFunctions @@ -93,6 +96,28 @@ public void test(int partitionCount) assertEquals(groupValue, expectedValue); } + @Test + public void testEmptyPartitionException() + { + InternalAggregationFunction function = getFunction(); + + Block geometryBlock = GEOMETRY.createBlockBuilder(null, 0).build(); + Block partitionCountBlock = BlockAssertions.createRLEBlock(10, 0); + Page page = new Page(geometryBlock, partitionCountBlock); + + AccumulatorFactory accumulatorFactory = function.bind(Ints.asList(0, 1, 2), Optional.empty()); + Accumulator accumulator = accumulatorFactory.createAccumulator(); + accumulator.addInput(page); + try { + getFinalBlock(accumulator); + fail("Should fail creating spatial partition with no rows."); + } + catch (PrestoException e) { + assertEquals(e.getErrorCode(), INVALID_FUNCTION_ARGUMENT.toErrorCode()); + assertEquals(e.getMessage(), "No rows supplied to spatial partition."); + } + } + private InternalAggregationFunction getFunction() { FunctionManager functionManager = functionAssertions.getMetadata().getFunctionManager(); diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSphericalGeoFunctions.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSphericalGeoFunctions.java index 4ea2936825d8f..491057508a207 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSphericalGeoFunctions.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSphericalGeoFunctions.java @@ -47,7 +47,7 @@ protected void registerFunctions() for (Type type : plugin.getTypes()) { functionAssertions.getTypeRegistry().addType(type); } - functionAssertions.getMetadata().addFunctions(extractFunctions(plugin.getFunctions())); + functionAssertions.getMetadata().registerBuiltInFunctions(extractFunctions(plugin.getFunctions())); } @Test @@ -97,13 +97,13 @@ public void testToAndFromSphericalGeography() assertToAndFromSphericalGeography("MULTIPOINT ((-40.2 28.9), (-40.2 31.9))"); assertToAndFromSphericalGeography("LINESTRING (-40.2 28.9, -40.2 31.9, -37.2 31.9)"); assertToAndFromSphericalGeography("MULTILINESTRING ((-40.2 28.9, -40.2 31.9), (-40.2 31.9, -37.2 31.9))"); - assertToAndFromSphericalGeography("POLYGON ((-40.2 28.9, -37.2 28.9, -37.2 31.9, -40.2 31.9, -40.2 28.9))"); - assertToAndFromSphericalGeography("POLYGON ((-40.2 28.9, -37.2 28.9, -37.2 31.9, -40.2 31.9, -40.2 28.9), " + - "(-39.2 29.9, -39.2 30.9, -38.2 30.9, -38.2 29.9, -39.2 29.9))"); - assertToAndFromSphericalGeography("MULTIPOLYGON (((-40.2 28.9, -37.2 28.9, -37.2 31.9, -40.2 31.9, -40.2 28.9)), " + - "((-39.2 29.9, -38.2 29.9, -38.2 30.9, -39.2 30.9, -39.2 29.9)))"); + assertToAndFromSphericalGeography("POLYGON ((-40.2 28.9, -40.2 31.9, -37.2 31.9, -37.2 28.9, -40.2 28.9))"); + assertToAndFromSphericalGeography("POLYGON ((-40.2 28.9, -40.2 31.9, -37.2 31.9, -37.2 28.9, -40.2 28.9), " + + "(-39.2 29.9, -38.2 29.9, -38.2 30.9, -39.2 30.9, -39.2 29.9))"); + assertToAndFromSphericalGeography("MULTIPOLYGON (((-40.2 28.9, -40.2 31.9, -37.2 31.9, -37.2 28.9, -40.2 28.9)), " + + "((-39.2 29.9, -39.2 30.9, -38.2 30.9, -38.2 29.9, -39.2 29.9)))"); assertToAndFromSphericalGeography("GEOMETRYCOLLECTION (POINT (-40.2 28.9), LINESTRING (-40.2 28.9, -40.2 31.9, -37.2 31.9), " + - "POLYGON ((-40.2 28.9, -37.2 28.9, -37.2 31.9, -40.2 31.9, -40.2 28.9)))"); + "POLYGON ((-40.2 28.9, -40.2 31.9, -37.2 31.9, -37.2 28.9, -40.2 28.9)))"); // geometries containing invalid latitude or longitude values assertInvalidLongitude("POINT (-340.2 28.9)"); @@ -111,7 +111,7 @@ public void testToAndFromSphericalGeography() assertInvalidLongitude("LINESTRING (-40.2 28.9, -40.2 31.9, 237.2 31.9)"); assertInvalidLatitude("MULTILINESTRING ((-40.2 28.9, -40.2 31.9), (-40.2 131.9, -37.2 31.9))"); assertInvalidLongitude("POLYGON ((-40.2 28.9, -40.2 31.9, 237.2 31.9, -37.2 28.9, -40.2 28.9))"); - assertInvalidLatitude("POLYGON ((-40.2 28.9, -40.2 31.9, -37.2 131.9, -37.2 28.9, -40.2 28.9), (-39.2 29.9, -39.2 30.9, -38.2 30.9, -38.2 29.9, -39.2 29.9))"); + assertInvalidLatitude("POLYGON ((-40.2 28.9, -40.2 31.9, -37.2 131.9, -37.2 28.9, -40.2 28.9), (-39.2 29.9, -38.2 29.9, -38.2 30.9, -39.2 30.9, -39.2 29.9))"); assertInvalidLongitude("MULTIPOLYGON (((-40.2 28.9, -40.2 31.9, -37.2 31.9, -37.2 28.9, -40.2 28.9)), " + "((-39.2 29.9, -39.2 30.9, 238.2 30.9, -38.2 29.9, -39.2 29.9)))"); assertInvalidLatitude("GEOMETRYCOLLECTION (POINT (-40.2 28.9), LINESTRING (-40.2 28.9, -40.2 131.9, -37.2 31.9), " + @@ -158,9 +158,6 @@ public void testArea() // Empty polygon assertFunction("ST_Area(to_spherical_geography(ST_GeometryFromText('POLYGON EMPTY')))", DOUBLE, null); - // Invalid polygon (too few vertices) - assertInvalidFunction("ST_Area(to_spherical_geography(ST_GeometryFromText('POLYGON((90 0, 0 0))')))", "Polygon is not valid: a loop contains less then 3 vertices."); - // Invalid data type (point) assertInvalidFunction("ST_Area(to_spherical_geography(ST_GeometryFromText('POINT (0 1)')))", "When applied to SphericalGeography inputs, ST_Area only supports POLYGON or MULTI_POLYGON. Input type is: POINT"); @@ -170,18 +167,18 @@ public void testArea() // A polygon around the North Pole assertArea("POLYGON((-135 85, -45 85, 45 85, 135 85, -135 85))", 619.00E9); - assertArea("POLYGON((0 0, 0 1, 1 1, 1 0))", 123.64E8); + assertArea("POLYGON((0 0, 0 1, 1 1, 1 0, 0 0))", 123.64E8); - assertArea("POLYGON((-122.150124 37.486095, -122.149201 37.486606, -122.145725 37.486580, -122.145923 37.483961 , -122.149324 37.482480 , -122.150837 37.483238, -122.150901 37.485392))", 163290.93943446054); + assertArea("POLYGON((-122.150124 37.486095, -122.149201 37.486606, -122.145725 37.486580, -122.145923 37.483961, -122.149324 37.482480, -122.150837 37.483238, -122.150901 37.485392, -122.150124 37.486095))", 163290.93943446054); double angleOfOneKm = 0.008993201943349; - assertArea(format("POLYGON((0 0, %.15f 0, %.15f %.15f, 0 %.15f))", angleOfOneKm, angleOfOneKm, angleOfOneKm, angleOfOneKm), 1E6); + assertArea(format("POLYGON((0 0, %.15f 0, %.15f %.15f, 0 %.15f, 0 0))", angleOfOneKm, angleOfOneKm, angleOfOneKm, angleOfOneKm), 1E6); // 1/4th of an hemisphere, ie 1/8th of the planet, should be close to 4PiR2/8 = 637.58E11 - assertArea("POLYGON((90 0, 0 0, 0 90))", 637.58E11); + assertArea("POLYGON((90 0, 0 0, 0 90, 90 0))", 637.58E11); //A Polygon with a large hole - assertArea("POLYGON((90 0, 0 0, 0 90), (89 1, 1 1, 1 89))", 348.04E10); + assertArea("POLYGON((90 0, 0 0, 0 90, 90 0), (89 1, 1 1, 1 89, 89 1))", 348.04E10); Path geometryPath = Paths.get(TestSphericalGeoFunctions.class.getClassLoader().getResource("us-states.tsv").getPath()); Map stateGeometries = Files.lines(geometryPath) @@ -210,12 +207,6 @@ public void testLength() // Empty linestring returns null assertLength("LINESTRING EMPTY", null); - // Linestring with one point has length 0 - assertLength("LINESTRING (0 0)", 0.0); - - // Linestring with only one distinct point has length 0 - assertLength("LINESTRING (0 0, 0 0, 0 0)", 0.0); - double length = 4350866.6362; // ST_Length is equivalent to sums of ST_DISTANCE between points in the LineString diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/AbstractTestGeoAggregationFunctions.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/AbstractTestGeoAggregationFunctions.java index 7dfaaf80c0be1..782ec2c337a1e 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/AbstractTestGeoAggregationFunctions.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/AbstractTestGeoAggregationFunctions.java @@ -15,7 +15,7 @@ import com.esri.core.geometry.ogc.OGCGeometry; import com.facebook.presto.block.BlockAssertions; -import com.facebook.presto.geospatial.serde.GeometrySerde; +import com.facebook.presto.geospatial.serde.EsriGeometrySerde; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.operator.aggregation.InternalAggregationFunction; import com.facebook.presto.operator.scalar.AbstractTestFunctions; @@ -48,7 +48,7 @@ public void registerFunctions() for (Type type : plugin.getTypes()) { functionAssertions.getTypeRegistry().addType(type); } - functionAssertions.getMetadata().addFunctions(extractFunctions(plugin.getFunctions())); + functionAssertions.getMetadata().registerBuiltInFunctions(extractFunctions(plugin.getFunctions())); FunctionManager functionManager = functionAssertions.getMetadata().getFunctionManager(); function = functionManager.getAggregateFunctionImplementation( functionManager.lookupFunction(getFunctionName(), fromTypes(GEOMETRY))); @@ -58,7 +58,7 @@ protected void assertAggregatedGeometries(String testDescription, String expecte { List geometrySlices = Arrays.stream(wkts) .map(text -> text == null ? null : OGCGeometry.fromText(text)) - .map(input -> input == null ? null : GeometrySerde.serialize(input)) + .map(input -> input == null ? null : EsriGeometrySerde.serialize(input)) .collect(Collectors.toList()); // Add a custom equality assertion because the resulting geometry may have diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/TestGeometryStateFactory.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/TestGeometryStateFactory.java index 6d42b0b5d17a3..8c861d02c63bf 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/TestGeometryStateFactory.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/aggregation/TestGeometryStateFactory.java @@ -14,7 +14,6 @@ package com.facebook.presto.plugin.geospatial.aggregation; import com.esri.core.geometry.ogc.OGCGeometry; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import static java.lang.String.format; @@ -25,13 +24,7 @@ public class TestGeometryStateFactory { - private GeometryStateFactory factory; - - @BeforeMethod - public void init() - { - factory = new GeometryStateFactory(); - } + private GeometryStateFactory factory = new GeometryStateFactory(); @Test public void testCreateSingleStateEmpty() diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 060a700f1a4c2..0c0fb226a6e10 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-hive-hadoop2 @@ -22,6 +22,11 @@ presto-hive + + com.facebook.presto + presto-hive-metastore + + com.google.guava guava @@ -72,7 +77,7 @@ - io.airlift + com.facebook.airlift testing test @@ -90,6 +95,20 @@ test + + com.facebook.presto + presto-hive-metastore + test-jar + test + + + + com.facebook.presto + presto-main + test-jar + test + + com.facebook.presto presto-main diff --git a/presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystemS3.java b/presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystemS3.java index 2a7bbf2a7b24b..8e1fa073ba3d3 100644 --- a/presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystemS3.java +++ b/presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystemS3.java @@ -16,6 +16,7 @@ import com.facebook.presto.hive.s3.HiveS3Config; import com.facebook.presto.hive.s3.PrestoS3ConfigurationUpdater; import com.facebook.presto.hive.s3.S3ConfigurationUpdater; +import com.google.common.collect.ImmutableSet; import org.apache.hadoop.fs.Path; import static com.google.common.base.Preconditions.checkArgument; @@ -43,12 +44,12 @@ protected void setup(String host, int port, String databaseName, String awsAcces super.setup(host, port, databaseName, this::createHdfsConfiguration, s3SelectPushdownEnabled); } - HdfsConfiguration createHdfsConfiguration(HiveClientConfig config) + HdfsConfiguration createHdfsConfiguration(HiveClientConfig config, MetastoreClientConfig metastoreConfig) { S3ConfigurationUpdater s3Config = new PrestoS3ConfigurationUpdater(new HiveS3Config() .setS3AwsAccessKey(awsAccessKey) .setS3AwsSecretKey(awsSecretKey)); - return new HiveHdfsConfiguration(new HdfsConfigurationUpdater(config, s3Config)); + return new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config, metastoreConfig, s3Config, ignored -> {}), ImmutableSet.of()); } @Override diff --git a/presto-hive-metastore/pom.xml b/presto-hive-metastore/pom.xml new file mode 100644 index 0000000000000..5b6275541f50a --- /dev/null +++ b/presto-hive-metastore/pom.xml @@ -0,0 +1,226 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.231-SNAPSHOT + + + presto-hive-metastore + presto-hive-metastore + + + ${project.parent.basedir} + + + + + com.facebook.presto + presto-spi + + + + com.facebook.airlift + configuration + + + + com.facebook.airlift + concurrent + + + + com.facebook.airlift + log + + + + com.facebook.airlift + json + + + + com.facebook.airlift + stats + + + + io.airlift + units + + + + io.airlift + slice + + + + com.fasterxml.jackson.core + jackson-databind + + + + com.facebook.presto.hive + hive-apache + + + + org.apache.thrift + libthrift + + + + com.amazonaws + aws-java-sdk-core + + + + com.amazonaws + aws-java-sdk-glue + + + + com.google.guava + guava + + + + com.google.inject + guice + + + + javax.inject + javax.inject + + + + javax.validation + validation-api + + + + com.facebook.hive + hive-dwrf + + + + org.openjdk.jol + jol-core + + + + com.fasterxml.jackson.core + jackson-annotations + + + + joda-time + joda-time + + + + io.airlift + joda-to-java-time-bridge + runtime + + + + org.weakref + jmxutils + + + + com.facebook.presto.hadoop + hadoop-apache2 + provided + + + + com.google.code.findbugs + jsr305 + true + provided + + + + org.jetbrains + annotations + provided + + + + + org.openjdk.jmh + jmh-core + test + + + + org.openjdk.jmh + jmh-generator-annprocess + test + + + + com.facebook.presto + presto-main + test + + + + com.facebook.presto + presto-parser + test + + + + org.testng + testng + test + + + + com.facebook.airlift + testing + test + + + + org.slf4j + slf4j-jdk14 + test + + + + org.slf4j + jcl-over-slf4j + test + + + + org.assertj + assertj-core + test + + + + + + ci + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + + + diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ForCachingHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/ForCachingHiveMetastore.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/ForCachingHiveMetastore.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/ForCachingHiveMetastore.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ForRecordingHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/ForRecordingHiveMetastore.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/ForRecordingHiveMetastore.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/ForRecordingHiveMetastore.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfiguration.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HdfsConfiguration.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfiguration.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HdfsConfiguration.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java similarity index 93% rename from presto-hive/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java index c79628a084dd3..9ca2aaf9546dc 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java @@ -45,7 +45,7 @@ public class HdfsEnvironment @Inject public HdfsEnvironment( HdfsConfiguration hdfsConfiguration, - HiveClientConfig config, + MetastoreClientConfig config, HdfsAuthentication hdfsAuthentication) { this.hdfsConfiguration = requireNonNull(hdfsConfiguration, "hdfsConfiguration is null"); @@ -92,6 +92,7 @@ public static class HdfsContext private final Optional queryId; private final Optional schemaName; private final Optional tableName; + private final Optional clientInfo; public HdfsContext(ConnectorIdentity identity) { @@ -100,6 +101,7 @@ public HdfsContext(ConnectorIdentity identity) this.queryId = Optional.empty(); this.schemaName = Optional.empty(); this.tableName = Optional.empty(); + this.clientInfo = Optional.empty(); } public HdfsContext(ConnectorSession session, String schemaName) @@ -111,6 +113,7 @@ public HdfsContext(ConnectorSession session, String schemaName) this.queryId = Optional.of(session.getQueryId()); this.schemaName = Optional.of(schemaName); this.tableName = Optional.empty(); + this.clientInfo = session.getClientInfo(); } public HdfsContext(ConnectorSession session, String schemaName, String tableName) @@ -123,6 +126,7 @@ public HdfsContext(ConnectorSession session, String schemaName, String tableName this.queryId = Optional.of(session.getQueryId()); this.schemaName = Optional.of(schemaName); this.tableName = Optional.of(tableName); + this.clientInfo = session.getClientInfo(); } public ConnectorIdentity getIdentity() @@ -150,6 +154,11 @@ public Optional getTableName() return tableName; } + public Optional getClientInfo() + { + return clientInfo; + } + @Override public String toString() { @@ -160,6 +169,7 @@ public String toString() .add("queryId", queryId.orElse(null)) .add("schemaName", schemaName.orElse(null)) .add("tableName", tableName.orElse(null)) + .add("clientInfo", clientInfo.orElse(null)) .toString(); } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveBasicStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveBasicStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/HiveBasicStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveBasicStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java similarity index 98% rename from presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java index ed0671cd1cc3a..d832fadbd12be 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java @@ -24,7 +24,7 @@ import java.util.Objects; import java.util.Optional; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java similarity index 67% rename from presto-hive/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java index 4e4523d041859..b56dc1d798768 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.hive; -import com.facebook.presto.spi.PrestoException; import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; @@ -32,19 +31,10 @@ import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; import org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hive.hcatalog.data.JsonSerDe; -import java.util.List; - -import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; -import static java.lang.String.format; import static java.util.Objects.requireNonNull; public enum HiveStorageFormat @@ -127,45 +117,4 @@ public DataSize getEstimatedWriterSystemMemoryUsage() { return estimatedWriterSystemMemoryUsage; } - - public void validateColumns(List handles) - { - if (this == AVRO) { - for (HiveColumnHandle handle : handles) { - if (!handle.isPartitionKey()) { - validateAvroType(handle.getHiveType().getTypeInfo(), handle.getName()); - } - } - } - } - - private static void validateAvroType(TypeInfo type, String columnName) - { - if (type.getCategory() == Category.MAP) { - TypeInfo keyType = mapTypeInfo(type).getMapKeyTypeInfo(); - if ((keyType.getCategory() != Category.PRIMITIVE) || - (primitiveTypeInfo(keyType).getPrimitiveCategory() != PrimitiveCategory.STRING)) { - throw new PrestoException(NOT_SUPPORTED, format("Column %s has a non-varchar map key, which is not supported by Avro", columnName)); - } - } - else if (type.getCategory() == Category.PRIMITIVE) { - PrimitiveCategory primitive = primitiveTypeInfo(type).getPrimitiveCategory(); - if (primitive == PrimitiveCategory.BYTE) { - throw new PrestoException(NOT_SUPPORTED, format("Column %s is tinyint, which is not supported by Avro. Use integer instead.", columnName)); - } - if (primitive == PrimitiveCategory.SHORT) { - throw new PrestoException(NOT_SUPPORTED, format("Column %s is smallint, which is not supported by Avro. Use integer instead.", columnName)); - } - } - } - - private static PrimitiveTypeInfo primitiveTypeInfo(TypeInfo typeInfo) - { - return (PrimitiveTypeInfo) typeInfo; - } - - private static MapTypeInfo mapTypeInfo(TypeInfo typeInfo) - { - return (MapTypeInfo) typeInfo; - } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveType.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveType.java similarity index 95% rename from presto-hive/src/main/java/com/facebook/presto/hive/HiveType.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveType.java index ea4fbc9cf99f0..91686c4dffc50 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveType.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveType.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; +import org.openjdk.jol.info.ClassLayout; import java.util.List; import java.util.Locale; @@ -72,6 +73,8 @@ public final class HiveType { + private static final int INSTANCE_SIZE = ClassLayout.parseClass(HiveType.class).instanceSize(); + public static final HiveType HIVE_BOOLEAN = new HiveType(booleanTypeInfo); public static final HiveType HIVE_BYTE = new HiveType(byteTypeInfo); public static final HiveType HIVE_SHORT = new HiveType(shortTypeInfo); @@ -230,7 +233,7 @@ private static TypeSignature getTypeSignature(TypeInfo typeInfo) List structFieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); List structFieldNames = structTypeInfo.getAllStructFieldNames(); if (structFieldTypeInfos.size() != structFieldNames.size()) { - throw new PrestoException(HiveErrorCode.HIVE_INVALID_METADATA, format("Invalid Hive struct type: %s", typeInfo)); + throw new PrestoException(MetastoreErrorCode.HIVE_INVALID_METADATA, format("Invalid Hive struct type: %s", typeInfo)); } ImmutableList.Builder typeSignatureBuilder = ImmutableList.builder(); for (int i = 0; i < structFieldTypeInfos.size(); i++) { @@ -283,4 +286,10 @@ public static Type getPrimitiveType(PrimitiveTypeInfo typeInfo) return null; } } + + public int getEstimatedRetainedSizeInBytes() + { + // Size of TypeInfo is not accounted as TypeInfo's are cached and retained by the TypeInfoFactory + return INSTANCE_SIZE + hiveTypeName.getEstimatedSizeInBytes(); + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeName.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveTypeName.java similarity index 93% rename from presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeName.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveTypeName.java index c22cd0f6bfbef..8ce4a32781f32 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeName.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveTypeName.java @@ -21,8 +21,7 @@ public final class HiveTypeName { - private static final int INSTANCE_SIZE = ClassLayout.parseClass(HivePartitionKey.class).instanceSize() + - ClassLayout.parseClass(String.class).instanceSize(); + private static final int INSTANCE_SIZE = ClassLayout.parseClass(String.class).instanceSize(); private final String value; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveViewNotSupportedException.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveViewNotSupportedException.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/HiveViewNotSupportedException.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveViewNotSupportedException.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/LocationHandle.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/LocationHandle.java similarity index 89% rename from presto-hive/src/main/java/com/facebook/presto/hive/LocationHandle.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/LocationHandle.java index 8c60f6eb3e081..71152a3c06dcd 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/LocationHandle.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/LocationHandle.java @@ -17,6 +17,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.hadoop.fs.Path; +import java.util.Optional; + import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -24,12 +26,14 @@ public class LocationHandle { private final Path targetPath; private final Path writePath; + private final Optional tempPath; private final TableType tableType; private final WriteMode writeMode; public LocationHandle( Path targetPath, Path writePath, + Optional tempPath, TableType tableType, WriteMode writeMode) { @@ -38,6 +42,7 @@ public LocationHandle( } this.targetPath = requireNonNull(targetPath, "targetPath is null"); this.writePath = requireNonNull(writePath, "writePath is null"); + this.tempPath = requireNonNull(tempPath, "tempPath is null"); this.tableType = requireNonNull(tableType, "tableType is null"); this.writeMode = requireNonNull(writeMode, "writeMode is null"); } @@ -46,12 +51,14 @@ public LocationHandle( public LocationHandle( @JsonProperty("targetPath") String targetPath, @JsonProperty("writePath") String writePath, + @JsonProperty("tempPath") Optional tempPath, @JsonProperty("tableType") TableType tableType, @JsonProperty("writeMode") WriteMode writeMode) { this( new Path(requireNonNull(targetPath, "targetPath is null")), new Path(requireNonNull(writePath, "writePath is null")), + requireNonNull(tempPath, "tempPath is null").map(Path::new), tableType, writeMode); } @@ -68,6 +75,12 @@ Path getWritePath() return writePath; } + // This method should only be called by LocationService + Optional getTempPath() + { + return tempPath; + } + // This method should only be called by LocationService public WriteMode getWriteMode() { @@ -92,6 +105,12 @@ public String getJsonSerializableWritePath() return writePath.toString(); } + @JsonProperty("tempPath") + public Optional getJsonSerializableTempPath() + { + return tempPath.map(Path::toString); + } + @JsonProperty("tableType") public TableType getJsonSerializableTableType() { diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientConfig.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientConfig.java new file mode 100644 index 0000000000000..31bc8adafae4e --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientConfig.java @@ -0,0 +1,184 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.airlift.configuration.Config; +import com.google.common.net.HostAndPort; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +import java.util.concurrent.TimeUnit; + +import static java.util.concurrent.TimeUnit.MINUTES; + +public class MetastoreClientConfig +{ + private HostAndPort metastoreSocksProxy; + private Duration metastoreTimeout = new Duration(10, TimeUnit.SECONDS); + private boolean verifyChecksum = true; + + private Duration metastoreCacheTtl = new Duration(0, TimeUnit.SECONDS); + private Duration metastoreRefreshInterval = new Duration(0, TimeUnit.SECONDS); + private long metastoreCacheMaximumSize = 10000; + private long perTransactionMetastoreCacheMaximumSize = 1000; + private int maxMetastoreRefreshThreads = 100; + + private String recordingPath; + private boolean replay; + private Duration recordingDuration = new Duration(0, MINUTES); + + public HostAndPort getMetastoreSocksProxy() + { + return metastoreSocksProxy; + } + + @Config("hive.metastore.thrift.client.socks-proxy") + public MetastoreClientConfig setMetastoreSocksProxy(HostAndPort metastoreSocksProxy) + { + this.metastoreSocksProxy = metastoreSocksProxy; + return this; + } + + @NotNull + public Duration getMetastoreTimeout() + { + return metastoreTimeout; + } + + @Config("hive.metastore-timeout") + public MetastoreClientConfig setMetastoreTimeout(Duration metastoreTimeout) + { + this.metastoreTimeout = metastoreTimeout; + return this; + } + + public boolean isVerifyChecksum() + { + return verifyChecksum; + } + + @Config("hive.dfs.verify-checksum") + public MetastoreClientConfig setVerifyChecksum(boolean verifyChecksum) + { + this.verifyChecksum = verifyChecksum; + return this; + } + + @NotNull + public Duration getMetastoreCacheTtl() + { + return metastoreCacheTtl; + } + + @MinDuration("0ms") + @Config("hive.metastore-cache-ttl") + public MetastoreClientConfig setMetastoreCacheTtl(Duration metastoreCacheTtl) + { + this.metastoreCacheTtl = metastoreCacheTtl; + return this; + } + + @NotNull + public Duration getMetastoreRefreshInterval() + { + return metastoreRefreshInterval; + } + + @MinDuration("1ms") + @Config("hive.metastore-refresh-interval") + public MetastoreClientConfig setMetastoreRefreshInterval(Duration metastoreRefreshInterval) + { + this.metastoreRefreshInterval = metastoreRefreshInterval; + return this; + } + + public long getMetastoreCacheMaximumSize() + { + return metastoreCacheMaximumSize; + } + + @Min(1) + @Config("hive.metastore-cache-maximum-size") + public MetastoreClientConfig setMetastoreCacheMaximumSize(long metastoreCacheMaximumSize) + { + this.metastoreCacheMaximumSize = metastoreCacheMaximumSize; + return this; + } + + public long getPerTransactionMetastoreCacheMaximumSize() + { + return perTransactionMetastoreCacheMaximumSize; + } + + @Min(1) + @Config("hive.per-transaction-metastore-cache-maximum-size") + public MetastoreClientConfig setPerTransactionMetastoreCacheMaximumSize(long perTransactionMetastoreCacheMaximumSize) + { + this.perTransactionMetastoreCacheMaximumSize = perTransactionMetastoreCacheMaximumSize; + return this; + } + + @Min(1) + public int getMaxMetastoreRefreshThreads() + { + return maxMetastoreRefreshThreads; + } + + @Config("hive.metastore-refresh-max-threads") + public MetastoreClientConfig setMaxMetastoreRefreshThreads(int maxMetastoreRefreshThreads) + { + this.maxMetastoreRefreshThreads = maxMetastoreRefreshThreads; + return this; + } + + public String getRecordingPath() + { + return recordingPath; + } + + @Config("hive.metastore-recording-path") + public MetastoreClientConfig setRecordingPath(String recordingPath) + { + this.recordingPath = recordingPath; + return this; + } + + public boolean isReplay() + { + return replay; + } + + @Config("hive.replay-metastore-recording") + public MetastoreClientConfig setReplay(boolean replay) + { + this.replay = replay; + return this; + } + + @NotNull + public Duration getRecordingDuration() + { + return recordingDuration; + } + + @Config("hive.metastore-recoding-duration") + public MetastoreClientConfig setRecordingDuration(Duration recordingDuration) + { + this.recordingDuration = recordingDuration; + return this; + } +} diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientModule.java new file mode 100644 index 0000000000000..0e0cf95a1c1e2 --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientModule.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.google.inject.Binder; +import com.google.inject.Module; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; + +public class MetastoreClientModule + implements Module +{ + public MetastoreClientModule() + { + } + + @Override + public void configure(Binder binder) + { + configBinder(binder).bindConfig(MetastoreClientConfig.class); + } +} diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreErrorCode.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreErrorCode.java new file mode 100644 index 0000000000000..893b85c5273df --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreErrorCode.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.spi.ErrorCode; +import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.spi.ErrorType; + +import static com.facebook.presto.spi.ErrorType.EXTERNAL; +import static com.facebook.presto.spi.ErrorType.INTERNAL_ERROR; +import static com.facebook.presto.spi.ErrorType.USER_ERROR; + +public enum MetastoreErrorCode + implements ErrorCodeSupplier +{ + HIVE_METASTORE_ERROR(0, EXTERNAL), + HIVE_TABLE_OFFLINE(2, USER_ERROR), + HIVE_PARTITION_OFFLINE(6, USER_ERROR), + HIVE_INVALID_PARTITION_VALUE(10, EXTERNAL), + HIVE_INVALID_METADATA(12, EXTERNAL), + HIVE_PATH_ALREADY_EXISTS(15, EXTERNAL), + HIVE_FILESYSTEM_ERROR(16, EXTERNAL), + HIVE_UNSUPPORTED_FORMAT(19, EXTERNAL), + HIVE_PARTITION_DROPPED_DURING_QUERY(31, EXTERNAL), + HIVE_TABLE_DROPPED_DURING_QUERY(35, EXTERNAL), + HIVE_CORRUPTED_COLUMN_STATISTICS(37, EXTERNAL), + HIVE_UNKNOWN_COLUMN_STATISTIC_TYPE(39, INTERNAL_ERROR), + /* Shared error code with HiveErrorCode */; + + private final ErrorCode errorCode; + + public static final int ERROR_CODE_MASK = 0x0100_0000; + + MetastoreErrorCode(int code, ErrorType type) + { + errorCode = new ErrorCode(code + ERROR_CODE_MASK, name(), type); + } + + @Override + public ErrorCode toErrorCode() + { + return errorCode; + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/PartitionNotFoundException.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/PartitionNotFoundException.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/PartitionNotFoundException.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/PartitionNotFoundException.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java similarity index 96% rename from presto-hive/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java index 35d1b56a779c2..6f969c0a325a6 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java @@ -16,7 +16,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_OFFLINE; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PARTITION_OFFLINE; import static com.google.common.base.Strings.isNullOrEmpty; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/RetryDriver.java similarity index 99% rename from presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/RetryDriver.java index 6c08d6f8f4991..bb28df343b376 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/RetryDriver.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.log.Logger; import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; import io.airlift.units.Duration; import java.util.ArrayList; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/SchemaAlreadyExistsException.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/SchemaAlreadyExistsException.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/SchemaAlreadyExistsException.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/SchemaAlreadyExistsException.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/TableAlreadyExistsException.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/TableAlreadyExistsException.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/TableAlreadyExistsException.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/TableAlreadyExistsException.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/TableOfflineException.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/TableOfflineException.java similarity index 95% rename from presto-hive/src/main/java/com/facebook/presto/hive/TableOfflineException.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/TableOfflineException.java index b6188fda97896..174b21784ce91 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/TableOfflineException.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/TableOfflineException.java @@ -16,7 +16,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_TABLE_OFFLINE; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_TABLE_OFFLINE; import static com.google.common.base.Strings.isNullOrEmpty; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/TypeTranslator.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/TypeTranslator.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/TypeTranslator.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/TypeTranslator.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/GenericExceptionAction.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/GenericExceptionAction.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/authentication/GenericExceptionAction.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/GenericExceptionAction.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/HdfsAuthentication.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/HdfsAuthentication.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/authentication/HdfsAuthentication.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/HdfsAuthentication.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/HiveMetastoreAuthentication.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/HiveMetastoreAuthentication.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/authentication/HiveMetastoreAuthentication.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/HiveMetastoreAuthentication.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/NoHiveMetastoreAuthentication.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/NoHiveMetastoreAuthentication.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/authentication/NoHiveMetastoreAuthentication.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/NoHiveMetastoreAuthentication.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/BooleanStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/BooleanStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/BooleanStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/BooleanStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java similarity index 98% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java index 0f994781d96ed..b959f1af9b315 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java @@ -14,9 +14,8 @@ package com.facebook.presto.hive.metastore; import com.facebook.presto.hive.ForCachingHiveMetastore; -import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.HiveType; -import com.facebook.presto.hive.PartitionStatistics; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; @@ -47,7 +46,7 @@ import java.util.concurrent.ExecutorService; import java.util.function.Function; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.metastore.HivePartitionName.hivePartitionName; import static com.facebook.presto.hive.metastore.HiveTableName.hiveTableName; import static com.facebook.presto.hive.metastore.PartitionFilter.partitionFilter; @@ -88,14 +87,14 @@ public class CachingHiveMetastore private final LoadingCache> roleGrantsCache; @Inject - public CachingHiveMetastore(@ForCachingHiveMetastore ExtendedHiveMetastore delegate, @ForCachingHiveMetastore ExecutorService executor, HiveClientConfig hiveClientConfig) + public CachingHiveMetastore(@ForCachingHiveMetastore ExtendedHiveMetastore delegate, @ForCachingHiveMetastore ExecutorService executor, MetastoreClientConfig metastoreClientConfig) { this( delegate, executor, - hiveClientConfig.getMetastoreCacheTtl(), - hiveClientConfig.getMetastoreRefreshInterval(), - hiveClientConfig.getMetastoreCacheMaximumSize()); + metastoreClientConfig.getMetastoreCacheTtl(), + metastoreClientConfig.getMetastoreRefreshInterval(), + metastoreClientConfig.getMetastoreCacheMaximumSize()); } public CachingHiveMetastore(ExtendedHiveMetastore delegate, ExecutorService executor, Duration cacheTtl, Duration refreshInterval, long maximumSize) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/Column.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Column.java similarity index 85% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/Column.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Column.java index 950554c8709c9..3e433ddd8beab 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/Column.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Column.java @@ -16,6 +16,7 @@ import com.facebook.presto.hive.HiveType; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.openjdk.jol.info.ClassLayout; import javax.annotation.concurrent.Immutable; @@ -28,6 +29,8 @@ @Immutable public class Column { + private static final int INSTANCE_SIZE = ClassLayout.parseClass(Column.class).instanceSize(); + private final String name; private final HiveType type; private final Optional comment; @@ -91,4 +94,13 @@ public int hashCode() { return Objects.hash(name, type, comment); } + + public int getEstimatedSizeInBytes() + { + int result = INSTANCE_SIZE; + result += name.length() * Character.BYTES; + result += type.getEstimatedRetainedSizeInBytes(); + result += comment.map(String::length).orElse(0); + return result; + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/Database.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Database.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/Database.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Database.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/DateStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/DateStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/DateStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/DateStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/DecimalStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/DecimalStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/DecimalStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/DecimalStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/DoubleStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/DoubleStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/DoubleStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/DoubleStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java similarity index 98% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java index 9c0b04b7d4698..4207e867c0704 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java @@ -14,7 +14,6 @@ package com.facebook.presto.hive.metastore; import com.facebook.presto.hive.HiveType; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveColumnStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveColumnStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveColumnStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveColumnStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastoreModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveMetastoreModule.java similarity index 92% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastoreModule.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveMetastoreModule.java index b3eb2f9458440..972fdc150e739 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveMetastoreModule.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveMetastoreModule.java @@ -13,16 +13,16 @@ */ package com.facebook.presto.hive.metastore; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.facebook.presto.hive.metastore.file.FileMetastoreModule; import com.facebook.presto.hive.metastore.glue.GlueMetastoreModule; import com.facebook.presto.hive.metastore.thrift.ThriftMetastoreModule; import com.google.inject.Binder; import com.google.inject.Module; -import io.airlift.configuration.AbstractConfigurationAwareModule; import java.util.Optional; -import static io.airlift.configuration.ConditionalModule.installModuleIf; +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; public class HiveMetastoreModule extends AbstractConfigurationAwareModule diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePageSinkMetadata.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HivePageSinkMetadata.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePageSinkMetadata.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HivePageSinkMetadata.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePageSinkMetadataProvider.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HivePageSinkMetadataProvider.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePageSinkMetadataProvider.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HivePageSinkMetadataProvider.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePartitionName.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HivePartitionName.java similarity index 97% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePartitionName.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HivePartitionName.java index 8c1f86704c620..b81f8b24dbd64 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePartitionName.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HivePartitionName.java @@ -23,8 +23,8 @@ import java.util.Objects; import java.util.Optional; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.HiveTableName.hiveTableName; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePrivilegeInfo.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HivePrivilegeInfo.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePrivilegeInfo.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HivePrivilegeInfo.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveTableName.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveTableName.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveTableName.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveTableName.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/IntegerStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/IntegerStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/IntegerStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/IntegerStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/MetastoreConfig.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreConfig.java similarity index 95% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/MetastoreConfig.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreConfig.java index 22535fa45771c..058b566682272 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/MetastoreConfig.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive.metastore; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import javax.validation.constraints.NotNull; diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java new file mode 100644 index 0000000000000..2946984dc970b --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java @@ -0,0 +1,621 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore; + +import com.facebook.presto.hive.HdfsEnvironment; +import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; +import com.facebook.presto.hive.PartitionOfflineException; +import com.facebook.presto.hive.TableOfflineException; +import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.StandardErrorCode; +import com.facebook.presto.spi.TableNotFoundException; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.BooleanType; +import com.facebook.presto.spi.type.CharType; +import com.facebook.presto.spi.type.DateType; +import com.facebook.presto.spi.type.DecimalType; +import com.facebook.presto.spi.type.Decimals; +import com.facebook.presto.spi.type.DoubleType; +import com.facebook.presto.spi.type.IntegerType; +import com.facebook.presto.spi.type.RealType; +import com.facebook.presto.spi.type.SmallintType; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.TimestampType; +import com.facebook.presto.spi.type.TinyintType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.VarbinaryType; +import com.facebook.presto.spi.type.VarcharType; +import com.google.common.base.CharMatcher; +import com.google.common.collect.ImmutableList; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.metastore.ProtectMode; +import org.apache.hadoop.io.Text; +import org.joda.time.DateTimeZone; + +import java.io.IOException; +import java.math.BigInteger; +import java.sql.Date; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_PARTITION_VALUE; +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.base.Strings.padEnd; +import static com.google.common.io.BaseEncoding.base16; +import static java.lang.Float.intBitsToFloat; +import static java.lang.String.format; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.toList; +import static org.apache.hadoop.hive.common.FileUtils.unescapePathName; +import static org.apache.hadoop.hive.metastore.MetaStoreUtils.typeToThriftType; +import static org.apache.hadoop.hive.metastore.ProtectMode.getProtectModeFromString; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_COUNT; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_FIELD_NAME; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_OUTPUT_FORMAT; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_LOCATION; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES; +import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_DDL; +import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB; +import static org.joda.time.DateTimeZone.UTC; + +public class MetastoreUtil +{ + public static final String PRESTO_OFFLINE = "presto_offline"; + public static final String AVRO_SCHEMA_URL_KEY = "avro.schema.url"; + public static final String PRESTO_VIEW_FLAG = "presto_view"; + public static final String PRESTO_QUERY_ID_NAME = "presto_query_id"; + public static final String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__"; + @SuppressWarnings("OctalInteger") + public static final FsPermission ALL_PERMISSIONS = new FsPermission((short) 0777); + + private MetastoreUtil() + { + } + + public static boolean isArrayType(Type type) + { + return type.getTypeSignature().getBase().equals(StandardTypes.ARRAY); + } + + public static boolean isMapType(Type type) + { + return type.getTypeSignature().getBase().equals(StandardTypes.MAP); + } + + public static boolean isRowType(Type type) + { + return type.getTypeSignature().getBase().equals(StandardTypes.ROW); + } + + public static void checkCondition(boolean condition, ErrorCodeSupplier errorCode, String formatString, Object... args) + { + if (!condition) { + throw new PrestoException(errorCode, format(formatString, args)); + } + } + + public static Properties getHiveSchema(Table table) + { + // Mimics function in Hive: MetaStoreUtils.getTableMetadata(Table) + return getHiveSchema( + table.getStorage(), + table.getDataColumns(), + table.getDataColumns(), + table.getParameters(), + table.getDatabaseName(), + table.getTableName(), + table.getPartitionColumns()); + } + + public static Properties getHiveSchema(Partition partition, Table table) + { + // Mimics function in Hive: MetaStoreUtils.getSchema(Partition, Table) + return getHiveSchema( + partition.getStorage(), + partition.getColumns(), + table.getDataColumns(), + table.getParameters(), + table.getDatabaseName(), + table.getTableName(), + table.getPartitionColumns()); + } + + public static Properties getHiveSchema( + Storage storage, + List partitionDataColumns, + List tableDataColumns, + Map tableParameters, + String databaseName, + String tableName, + List partitionKeys) + { + // Mimics function in Hive: + // MetaStoreUtils.getSchema(StorageDescriptor, StorageDescriptor, Map, String, String, List) + + Properties schema = new Properties(); + + schema.setProperty(FILE_INPUT_FORMAT, storage.getStorageFormat().getInputFormat()); + schema.setProperty(FILE_OUTPUT_FORMAT, storage.getStorageFormat().getOutputFormat()); + + schema.setProperty(META_TABLE_NAME, databaseName + "." + tableName); + schema.setProperty(META_TABLE_LOCATION, storage.getLocation()); + + if (storage.getBucketProperty().isPresent()) { + List bucketedBy = storage.getBucketProperty().get().getBucketedBy(); + if (!bucketedBy.isEmpty()) { + schema.setProperty(BUCKET_FIELD_NAME, bucketedBy.get(0)); + } + schema.setProperty(BUCKET_COUNT, Integer.toString(storage.getBucketProperty().get().getBucketCount())); + } + else { + schema.setProperty(BUCKET_COUNT, "0"); + } + + for (Map.Entry param : storage.getSerdeParameters().entrySet()) { + schema.setProperty(param.getKey(), (param.getValue() != null) ? param.getValue() : ""); + } + schema.setProperty(SERIALIZATION_LIB, storage.getStorageFormat().getSerDe()); + + StringBuilder columnNameBuilder = new StringBuilder(); + StringBuilder columnTypeBuilder = new StringBuilder(); + StringBuilder columnCommentBuilder = new StringBuilder(); + boolean first = true; + for (Column column : tableDataColumns) { + if (!first) { + columnNameBuilder.append(","); + columnTypeBuilder.append(":"); + columnCommentBuilder.append('\0'); + } + columnNameBuilder.append(column.getName()); + columnTypeBuilder.append(column.getType()); + columnCommentBuilder.append(column.getComment().orElse("")); + first = false; + } + String columnNames = columnNameBuilder.toString(); + String columnTypes = columnTypeBuilder.toString(); + schema.setProperty(META_TABLE_COLUMNS, columnNames); + schema.setProperty(META_TABLE_COLUMN_TYPES, columnTypes); + schema.setProperty("columns.comments", columnCommentBuilder.toString()); + + schema.setProperty(SERIALIZATION_DDL, toThriftDdl(tableName, partitionDataColumns)); + + String partString = ""; + String partStringSep = ""; + String partTypesString = ""; + String partTypesStringSep = ""; + for (Column partKey : partitionKeys) { + partString += partStringSep; + partString += partKey.getName(); + partTypesString += partTypesStringSep; + partTypesString += partKey.getType().getHiveTypeName().toString(); + if (partStringSep.length() == 0) { + partStringSep = "/"; + partTypesStringSep = ":"; + } + } + if (partString.length() > 0) { + schema.setProperty(META_TABLE_PARTITION_COLUMNS, partString); + schema.setProperty(META_TABLE_PARTITION_COLUMN_TYPES, partTypesString); + } + + if (tableParameters != null) { + for (Map.Entry entry : tableParameters.entrySet()) { + // add non-null parameters to the schema + if (entry.getValue() != null) { + schema.setProperty(entry.getKey(), entry.getValue()); + } + } + } + + return schema; + } + + public static Properties getHiveSchema(Map serdeParameters, Map tableParameters) + { + Properties schema = new Properties(); + for (Map.Entry param : serdeParameters.entrySet()) { + schema.setProperty(param.getKey(), (param.getValue() != null) ? param.getValue() : ""); + } + for (Map.Entry entry : tableParameters.entrySet()) { + // add non-null parameters to the schema + if (entry.getValue() != null) { + schema.setProperty(entry.getKey(), entry.getValue()); + } + } + return schema; + } + + /** + * Recreates partition schema based on the table schema and the column + * coercions map. + * + * partitionColumnCount is needed to handle cases when the partition + * has less columns than the table. + * + * If the partition has more columns than the table does, the partitionSchemaDifference + * map is expected to contain information for the missing columns. + */ + public static List reconstructPartitionSchema(List tableSchema, int partitionColumnCount, Map partitionSchemaDifference) + { + ImmutableList.Builder columns = ImmutableList.builder(); + for (int i = 0; i < partitionColumnCount; i++) { + Column column = partitionSchemaDifference.get(i); + if (column == null) { + checkArgument( + i < tableSchema.size(), + "column descriptor for column with hiveColumnIndex %s not found: tableSchema: %s, partitionSchemaDifference: %s", + i, + tableSchema, + partitionSchemaDifference); + column = tableSchema.get(i); + } + columns.add(column); + } + return columns.build(); + } + + public static ProtectMode getProtectMode(Partition partition) + { + return getProtectMode(partition.getParameters()); + } + + public static ProtectMode getProtectMode(Table table) + { + return getProtectMode(table.getParameters()); + } + + public static String makePartName(List partitionColumns, List values) + { + checkArgument(partitionColumns.size() == values.size(), "Partition value count does not match the partition column count"); + checkArgument(values.stream().allMatch(Objects::nonNull), "partitionValue must not have null elements"); + + List partitionColumnNames = partitionColumns.stream().map(Column::getName).collect(toList()); + return FileUtils.makePartName(partitionColumnNames, values); + } + + public static String getPartitionLocation(Table table, Optional partition) + { + if (!partition.isPresent()) { + return table.getStorage().getLocation(); + } + return partition.get().getStorage().getLocation(); + } + + private static String toThriftDdl(String structName, List columns) + { + // Mimics function in Hive: + // MetaStoreUtils.getDDLFromFieldSchema(String, List) + StringBuilder ddl = new StringBuilder(); + ddl.append("struct "); + ddl.append(structName); + ddl.append(" { "); + boolean first = true; + for (Column column : columns) { + if (first) { + first = false; + } + else { + ddl.append(", "); + } + ddl.append(typeToThriftType(column.getType().getHiveTypeName().toString())); + ddl.append(' '); + ddl.append(column.getName()); + } + ddl.append("}"); + return ddl.toString(); + } + + private static ProtectMode getProtectMode(Map parameters) + { + if (!parameters.containsKey(ProtectMode.PARAMETER_NAME)) { + return new ProtectMode(); + } + else { + return getProtectModeFromString(parameters.get(ProtectMode.PARAMETER_NAME)); + } + } + + public static void verifyOnline(SchemaTableName tableName, Optional partitionName, ProtectMode protectMode, Map parameters) + { + if (protectMode.offline) { + if (partitionName.isPresent()) { + throw new PartitionOfflineException(tableName, partitionName.get(), false, null); + } + throw new TableOfflineException(tableName, false, null); + } + + String prestoOffline = parameters.get(PRESTO_OFFLINE); + if (!isNullOrEmpty(prestoOffline)) { + if (partitionName.isPresent()) { + throw new PartitionOfflineException(tableName, partitionName.get(), true, prestoOffline); + } + throw new TableOfflineException(tableName, true, prestoOffline); + } + } + + public static void verifyCanDropColumn(ExtendedHiveMetastore metastore, String databaseName, String tableName, String columnName) + { + Table table = metastore.getTable(databaseName, tableName) + .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName))); + + if (table.getPartitionColumns().stream().anyMatch(column -> column.getName().equals(columnName))) { + throw new PrestoException(NOT_SUPPORTED, "Cannot drop partition columns"); + } + if (table.getDataColumns().size() <= 1) { + throw new PrestoException(NOT_SUPPORTED, "Cannot drop the only non-partition column in a table"); + } + } + + public static FileSystem getFileSystem(HdfsEnvironment hdfsEnvironment, HdfsContext context, Path path) + { + try { + return hdfsEnvironment.getFileSystem(context, path); + } + catch (IOException e) { + throw new PrestoException(HIVE_FILESYSTEM_ERROR, format("Error getting file system. Path: %s", path), e); + } + } + + public static void renameFile(FileSystem fileSystem, Path source, Path target) + { + try { + if (fileSystem.exists(target) || !fileSystem.rename(source, target)) { + throw new PrestoException(HIVE_FILESYSTEM_ERROR, getRenameErrorMessage(source, target)); + } + } + catch (IOException e) { + throw new PrestoException(HIVE_FILESYSTEM_ERROR, getRenameErrorMessage(source, target), e); + } + } + + public static List toPartitionValues(String partitionName) + { + // mimics Warehouse.makeValsFromName + ImmutableList.Builder resultBuilder = ImmutableList.builder(); + int start = 0; + while (true) { + while (start < partitionName.length() && partitionName.charAt(start) != '=') { + start++; + } + start++; + int end = start; + while (end < partitionName.length() && partitionName.charAt(end) != '/') { + end++; + } + if (start > partitionName.length()) { + break; + } + resultBuilder.add(unescapePathName(partitionName.substring(start, end))); + start = end + 1; + } + return resultBuilder.build(); + } + + public static List extractPartitionValues(String partitionName) + { + ImmutableList.Builder values = ImmutableList.builder(); + + boolean inKey = true; + int valueStart = -1; + for (int i = 0; i < partitionName.length(); i++) { + char current = partitionName.charAt(i); + if (inKey) { + checkArgument(current != '/', "Invalid partition spec: %s", partitionName); + if (current == '=') { + inKey = false; + valueStart = i + 1; + } + } + else if (current == '/') { + checkArgument(valueStart != -1, "Invalid partition spec: %s", partitionName); + values.add(unescapePathName(partitionName.substring(valueStart, i))); + inKey = true; + valueStart = -1; + } + } + checkArgument(!inKey, "Invalid partition spec: %s", partitionName); + values.add(unescapePathName(partitionName.substring(valueStart, partitionName.length()))); + + return values.build(); + } + + public static List createPartitionValues(List partitionColumnTypes, Page partitionColumns, int position) + { + ImmutableList.Builder partitionValues = ImmutableList.builder(); + for (int field = 0; field < partitionColumns.getChannelCount(); field++) { + Object value = getField(partitionColumnTypes.get(field), partitionColumns.getBlock(field), position); + if (value == null) { + partitionValues.add(HIVE_DEFAULT_DYNAMIC_PARTITION); + } + else { + String valueString = value.toString(); + if (!CharMatcher.inRange((char) 0x20, (char) 0x7E).matchesAllOf(valueString)) { + throw new PrestoException(HIVE_INVALID_PARTITION_VALUE, + "Hive partition keys can only contain printable ASCII characters (0x20 - 0x7E). Invalid value: " + + base16().withSeparator(" ", 2).encode(valueString.getBytes(UTF_8))); + } + partitionValues.add(valueString); + } + } + return partitionValues.build(); + } + + public static Object getField(Type type, Block block, int position) + { + if (block.isNull(position)) { + return null; + } + if (BooleanType.BOOLEAN.equals(type)) { + return type.getBoolean(block, position); + } + if (BigintType.BIGINT.equals(type)) { + return type.getLong(block, position); + } + if (IntegerType.INTEGER.equals(type)) { + return (int) type.getLong(block, position); + } + if (SmallintType.SMALLINT.equals(type)) { + return (short) type.getLong(block, position); + } + if (TinyintType.TINYINT.equals(type)) { + return (byte) type.getLong(block, position); + } + if (RealType.REAL.equals(type)) { + return intBitsToFloat((int) type.getLong(block, position)); + } + if (DoubleType.DOUBLE.equals(type)) { + return type.getDouble(block, position); + } + if (type instanceof VarcharType) { + return new Text(type.getSlice(block, position).getBytes()); + } + if (type instanceof CharType) { + CharType charType = (CharType) type; + return new Text(padEnd(type.getSlice(block, position).toStringUtf8(), charType.getLength(), ' ')); + } + if (VarbinaryType.VARBINARY.equals(type)) { + return type.getSlice(block, position).getBytes(); + } + if (DateType.DATE.equals(type)) { + long days = type.getLong(block, position); + return new Date(UTC.getMillisKeepLocal(DateTimeZone.getDefault(), TimeUnit.DAYS.toMillis(days))); + } + if (TimestampType.TIMESTAMP.equals(type)) { + long millisUtc = type.getLong(block, position); + return new Timestamp(millisUtc); + } + if (type instanceof DecimalType) { + DecimalType decimalType = (DecimalType) type; + return getHiveDecimal(decimalType, block, position); + } + if (isArrayType(type)) { + Type elementType = type.getTypeParameters().get(0); + + Block arrayBlock = block.getBlock(position); + + List list = new ArrayList<>(arrayBlock.getPositionCount()); + for (int i = 0; i < arrayBlock.getPositionCount(); i++) { + Object element = getField(elementType, arrayBlock, i); + list.add(element); + } + + return Collections.unmodifiableList(list); + } + if (isMapType(type)) { + Type keyType = type.getTypeParameters().get(0); + Type valueType = type.getTypeParameters().get(1); + + Block mapBlock = block.getBlock(position); + Map map = new HashMap<>(); + for (int i = 0; i < mapBlock.getPositionCount(); i += 2) { + Object key = getField(keyType, mapBlock, i); + Object value = getField(valueType, mapBlock, i + 1); + map.put(key, value); + } + + return Collections.unmodifiableMap(map); + } + if (isRowType(type)) { + Block rowBlock = block.getBlock(position); + + List fieldTypes = type.getTypeParameters(); + checkCondition(fieldTypes.size() == rowBlock.getPositionCount(), StandardErrorCode.GENERIC_INTERNAL_ERROR, "Expected row value field count does not match type field count"); + + List row = new ArrayList<>(rowBlock.getPositionCount()); + for (int i = 0; i < rowBlock.getPositionCount(); i++) { + Object element = getField(fieldTypes.get(i), rowBlock, i); + row.add(element); + } + + return Collections.unmodifiableList(row); + } + throw new PrestoException(NOT_SUPPORTED, "unsupported type: " + type); + } + + public static HiveDecimal getHiveDecimal(DecimalType decimalType, Block block, int position) + { + BigInteger unscaledValue; + if (decimalType.isShort()) { + unscaledValue = BigInteger.valueOf(decimalType.getLong(block, position)); + } + else { + unscaledValue = Decimals.decodeUnscaledValue(decimalType.getSlice(block, position)); + } + return HiveDecimal.create(unscaledValue, decimalType.getScale()); + } + + public static void createDirectory(HdfsContext context, HdfsEnvironment hdfsEnvironment, Path path) + { + try { + if (!hdfsEnvironment.getFileSystem(context, path).mkdirs(path, ALL_PERMISSIONS)) { + throw new IOException("mkdirs returned false"); + } + } + catch (IOException e) { + throw new PrestoException(HIVE_FILESYSTEM_ERROR, "Failed to create directory: " + path, e); + } + + // explicitly set permission since the default umask overrides it on creation + try { + hdfsEnvironment.getFileSystem(context, path).setPermission(path, ALL_PERMISSIONS); + } + catch (IOException e) { + throw new PrestoException(HIVE_FILESYSTEM_ERROR, "Failed to set permission on directory: " + path, e); + } + } + + public static boolean pathExists(HdfsContext context, HdfsEnvironment hdfsEnvironment, Path path) + { + try { + return hdfsEnvironment.getFileSystem(context, path).exists(path); + } + catch (IOException e) { + throw new PrestoException(HIVE_FILESYSTEM_ERROR, "Failed checking path: " + path, e); + } + } + + public static boolean isPrestoView(Table table) + { + return "true".equals(table.getParameters().get(PRESTO_VIEW_FLAG)); + } + + private static String getRenameErrorMessage(Path source, Path target) + { + return format("Error moving data files from %s to final location %s", source, target); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/Partition.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Partition.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/Partition.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Partition.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/PartitionFilter.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionFilter.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/PartitionFilter.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionFilter.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/PartitionStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionStatistics.java similarity index 97% rename from presto-hive/src/main/java/com/facebook/presto/hive/PartitionStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionStatistics.java index 2b0dbb0d9e492..69ad53e90a74b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/PartitionStatistics.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionStatistics.java @@ -12,9 +12,9 @@ * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.hive.metastore; -import com.facebook.presto.hive.metastore.HiveColumnStatistics; +import com.facebook.presto.hive.HiveBasicStatistics; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java similarity index 93% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java index 8f881775fe6db..cd24f67b32cd1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java @@ -13,9 +13,7 @@ */ package com.facebook.presto.hive.metastore; -import com.facebook.presto.hive.PartitionStatistics; - -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/PrincipalPrivileges.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrincipalPrivileges.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/PrincipalPrivileges.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrincipalPrivileges.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/RecordingHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/RecordingHiveMetastore.java similarity index 93% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/RecordingHiveMetastore.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/RecordingHiveMetastore.java index 4c51c2a8f99ac..4e04e56be62c5 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/RecordingHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/RecordingHiveMetastore.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.hive.metastore; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.hive.ForRecordingHiveMetastore; -import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.HiveType; -import com.facebook.presto.hive.PartitionStatistics; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; @@ -29,7 +29,6 @@ import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.json.ObjectMapperProvider; import org.weakref.jmx.Managed; import javax.annotation.concurrent.Immutable; @@ -77,27 +76,27 @@ public class RecordingHiveMetastore private final Cache> roleGrantsCache; @Inject - public RecordingHiveMetastore(@ForRecordingHiveMetastore ExtendedHiveMetastore delegate, HiveClientConfig hiveClientConfig) + public RecordingHiveMetastore(@ForRecordingHiveMetastore ExtendedHiveMetastore delegate, MetastoreClientConfig metastoreClientConfig) throws IOException { this.delegate = requireNonNull(delegate, "delegate is null"); - requireNonNull(hiveClientConfig, "hiveClientConfig is null"); - this.recordingPath = requireNonNull(hiveClientConfig.getRecordingPath(), "recordingPath is null"); - this.replay = hiveClientConfig.isReplay(); - - databaseCache = createCache(hiveClientConfig); - tableCache = createCache(hiveClientConfig); - supportedColumnStatisticsCache = createCache(hiveClientConfig); - tableStatisticsCache = createCache(hiveClientConfig); - partitionStatisticsCache = createCache(hiveClientConfig); - allTablesCache = createCache(hiveClientConfig); - allViewsCache = createCache(hiveClientConfig); - partitionCache = createCache(hiveClientConfig); - partitionNamesCache = createCache(hiveClientConfig); - partitionNamesByPartsCache = createCache(hiveClientConfig); - partitionsByNamesCache = createCache(hiveClientConfig); - tablePrivilegesCache = createCache(hiveClientConfig); - roleGrantsCache = createCache(hiveClientConfig); + requireNonNull(metastoreClientConfig, "hiveClientConfig is null"); + this.recordingPath = requireNonNull(metastoreClientConfig.getRecordingPath(), "recordingPath is null"); + this.replay = metastoreClientConfig.isReplay(); + + databaseCache = createCache(metastoreClientConfig); + tableCache = createCache(metastoreClientConfig); + supportedColumnStatisticsCache = createCache(metastoreClientConfig); + tableStatisticsCache = createCache(metastoreClientConfig); + partitionStatisticsCache = createCache(metastoreClientConfig); + allTablesCache = createCache(metastoreClientConfig); + allViewsCache = createCache(metastoreClientConfig); + partitionCache = createCache(metastoreClientConfig); + partitionNamesCache = createCache(metastoreClientConfig); + partitionNamesByPartsCache = createCache(metastoreClientConfig); + partitionsByNamesCache = createCache(metastoreClientConfig); + tablePrivilegesCache = createCache(metastoreClientConfig); + roleGrantsCache = createCache(metastoreClientConfig); if (replay) { loadRecording(); @@ -127,15 +126,15 @@ void loadRecording() roleGrantsCache.putAll(toMap(recording.getRoleGrants())); } - private static Cache createCache(HiveClientConfig hiveClientConfig) + private static Cache createCache(MetastoreClientConfig metastoreClientConfig) { - if (hiveClientConfig.isReplay()) { + if (metastoreClientConfig.isReplay()) { return CacheBuilder.newBuilder() .build(); } return CacheBuilder.newBuilder() - .expireAfterWrite(hiveClientConfig.getRecordingDuration().toMillis(), MILLISECONDS) + .expireAfterWrite(metastoreClientConfig.getRecordingDuration().toMillis(), MILLISECONDS) .build(); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java similarity index 97% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java index 1dbf8fbb8805a..3e042cad020b4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.hive.metastore; +import com.facebook.airlift.log.Logger; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; import com.facebook.presto.hive.HiveBasicStatistics; import com.facebook.presto.hive.HiveType; import com.facebook.presto.hive.LocationHandle.WriteMode; import com.facebook.presto.hive.PartitionNotFoundException; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.TableAlreadyExistsException; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; @@ -40,7 +40,6 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import io.airlift.log.Logger; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -61,26 +60,27 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_CORRUPTED_COLUMN_STATISTICS; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_PATH_ALREADY_EXISTS; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_TABLE_DROPPED_DURING_QUERY; -import static com.facebook.presto.hive.HiveMetadata.PRESTO_QUERY_ID_NAME; -import static com.facebook.presto.hive.HiveUtil.isPrestoView; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; -import static com.facebook.presto.hive.HiveWriteUtils.createDirectory; -import static com.facebook.presto.hive.HiveWriteUtils.pathExists; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.hive.LocationHandle.WriteMode.DIRECT_TO_TARGET_NEW_DIRECTORY; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_CORRUPTED_COLUMN_STATISTICS; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PATH_ALREADY_EXISTS; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_TABLE_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; +import static com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME; +import static com.facebook.presto.hive.metastore.MetastoreUtil.createDirectory; import static com.facebook.presto.hive.metastore.MetastoreUtil.getFileSystem; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isPrestoView; +import static com.facebook.presto.hive.metastore.MetastoreUtil.pathExists; import static com.facebook.presto.hive.metastore.MetastoreUtil.renameFile; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW; -import static com.facebook.presto.hive.util.Statistics.ReduceOperator.SUBTRACT; -import static com.facebook.presto.hive.util.Statistics.merge; -import static com.facebook.presto.hive.util.Statistics.reduce; +import static com.facebook.presto.hive.metastore.Statistics.ReduceOperator.SUBTRACT; +import static com.facebook.presto.hive.metastore.Statistics.merge; +import static com.facebook.presto.hive.metastore.Statistics.reduce; import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.StandardErrorCode.TRANSACTION_CONFLICT; @@ -92,7 +92,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Futures.whenAllSucceed; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static org.apache.hadoop.hive.common.FileUtils.makePartName; @@ -871,7 +870,14 @@ public synchronized void revokeTablePrivileges(String databaseName, String table setExclusive((delegate, hdfsEnvironment) -> delegate.revokeTablePrivileges(databaseName, tableName, grantee, privileges)); } - public synchronized void declareIntentionToWrite(ConnectorSession session, WriteMode writeMode, Path stagingPathRoot, String filePrefix, SchemaTableName schemaTableName, boolean temporaryTable) + public synchronized void declareIntentionToWrite( + ConnectorSession session, + WriteMode writeMode, + Path stagingPathRoot, + Optional tempPathRoot, + String filePrefix, + SchemaTableName schemaTableName, + boolean temporaryTable) { setShared(); if (writeMode == WriteMode.DIRECT_TO_TARGET_EXISTING_DIRECTORY) { @@ -881,7 +887,7 @@ public synchronized void declareIntentionToWrite(ConnectorSession session, Write } } HdfsContext context = new HdfsContext(session, schemaTableName.getSchemaName(), schemaTableName.getTableName()); - declaredIntentionsToWrite.add(new DeclaredIntentionToWrite(writeMode, context, stagingPathRoot, filePrefix, schemaTableName, temporaryTable)); + declaredIntentionsToWrite.add(new DeclaredIntentionToWrite(writeMode, context, stagingPathRoot, tempPathRoot, filePrefix, schemaTableName, temporaryTable)); } public synchronized void commit() @@ -1048,6 +1054,9 @@ private void commitShared() // Clean up empty staging directories (that may recursively contain empty directories) committer.deleteEmptyStagingDirectories(declaredIntentionsToWrite); + + // Clean up root temp directories + deleteTempPathRootDirectory(declaredIntentionsToWrite, hdfsEnvironment); } } @@ -1361,7 +1370,7 @@ private void deleteEmptyStagingDirectories(List declar if (declaredIntentionToWrite.getMode() != WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY) { continue; } - Path path = declaredIntentionToWrite.getRootPath(); + Path path = declaredIntentionToWrite.getStagingPathRoot(); recursiveDeleteFilesAndLog(declaredIntentionToWrite.getContext(), path, ImmutableList.of(), true, "staging directory cleanup"); } } @@ -1513,6 +1522,7 @@ private void rollbackShared() checkHoldsLock(); deleteTemporaryTableDirectories(declaredIntentionsToWrite, hdfsEnvironment); + deleteTempPathRootDirectory(declaredIntentionsToWrite, hdfsEnvironment); for (DeclaredIntentionToWrite declaredIntentionToWrite : declaredIntentionsToWrite) { switch (declaredIntentionToWrite.getMode()) { @@ -1523,7 +1533,7 @@ private void rollbackShared() } // Note: For STAGE_AND_MOVE_TO_TARGET_DIRECTORY there is no need to cleanup the target directory as it will only be written // to during the commit call and the commit call cleans up after failures. - Path rootPath = declaredIntentionToWrite.getRootPath(); + Path rootPath = declaredIntentionToWrite.getStagingPathRoot(); // In the case of DIRECT_TO_TARGET_NEW_DIRECTORY, if the directory is not guaranteed to be unique // for the query, it is possible that another query or compute engine may see the directory, wrote @@ -1545,7 +1555,7 @@ private void rollbackShared() // Check the base directory of the declared intention // * existing partition may also be in this directory // * this is where new partitions are created - Path baseDirectory = declaredIntentionToWrite.getRootPath(); + Path baseDirectory = declaredIntentionToWrite.getStagingPathRoot(); pathsToClean.add(baseDirectory); SchemaTableName schemaTableName = declaredIntentionToWrite.getSchemaTableName(); @@ -1598,7 +1608,16 @@ private static void deleteTemporaryTableDirectories(List declaredIntentionsToWrite, HdfsEnvironment hdfsEnvironment) + { + for (DeclaredIntentionToWrite declaredIntentionToWrite : declaredIntentionsToWrite) { + if (declaredIntentionToWrite.getTempPathRoot().isPresent()) { + deleteRecursivelyIfExists(declaredIntentionToWrite.getContext(), hdfsEnvironment, declaredIntentionToWrite.getTempPathRoot().get()); } } } @@ -2176,15 +2195,24 @@ private static class DeclaredIntentionToWrite private final WriteMode mode; private final HdfsContext context; private final String filePrefix; - private final Path rootPath; + private final Path stagingPathRoot; + private final Optional tempPathRoot; private final SchemaTableName schemaTableName; private final boolean temporaryTable; - public DeclaredIntentionToWrite(WriteMode mode, HdfsContext context, Path stagingPathRoot, String filePrefix, SchemaTableName schemaTableName, boolean temporaryTable) + public DeclaredIntentionToWrite( + WriteMode mode, + HdfsContext context, + Path stagingPathRoot, + Optional tempPathRoot, + String filePrefix, + SchemaTableName schemaTableName, + boolean temporaryTable) { this.mode = requireNonNull(mode, "mode is null"); this.context = requireNonNull(context, "context is null"); - this.rootPath = requireNonNull(stagingPathRoot, "stagingPathRoot is null"); + this.stagingPathRoot = requireNonNull(stagingPathRoot, "stagingPathRoot is null"); + this.tempPathRoot = requireNonNull(tempPathRoot, "tempPathRoot is null"); this.filePrefix = requireNonNull(filePrefix, "filePrefix is null"); this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null"); this.temporaryTable = temporaryTable; @@ -2205,9 +2233,14 @@ public String getFilePrefix() return filePrefix; } - public Path getRootPath() + public Path getStagingPathRoot() + { + return stagingPathRoot; + } + + public Optional getTempPathRoot() { - return rootPath; + return tempPathRoot; } public SchemaTableName getSchemaTableName() @@ -2227,7 +2260,8 @@ public String toString() .add("mode", mode) .add("context", context) .add("filePrefix", filePrefix) - .add("rootPath", rootPath) + .add("stagingPathRoot", stagingPathRoot) + .add("tempPathRoot", tempPathRoot) .add("schemaTableName", schemaTableName) .add("temporaryTable", temporaryTable) .toString(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java similarity index 97% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java index 019ed67aded6d..61ddf2c26be9a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java @@ -22,7 +22,7 @@ import java.util.Objects; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.spi.block.SortOrder.ASC_NULLS_FIRST; import static com.facebook.presto.spi.block.SortOrder.DESC_NULLS_LAST; import static com.google.common.base.MoreObjects.toStringHelper; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/Statistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Statistics.java similarity index 96% rename from presto-hive/src/main/java/com/facebook/presto/hive/util/Statistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Statistics.java index 71e7bbadbfddd..0e8f47337aff2 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/Statistics.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Statistics.java @@ -11,16 +11,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive.util; +package com.facebook.presto.hive.metastore; import com.facebook.presto.hive.HiveBasicStatistics; -import com.facebook.presto.hive.PartitionStatistics; -import com.facebook.presto.hive.metastore.BooleanStatistics; -import com.facebook.presto.hive.metastore.DateStatistics; -import com.facebook.presto.hive.metastore.DecimalStatistics; -import com.facebook.presto.hive.metastore.DoubleStatistics; -import com.facebook.presto.hive.metastore.HiveColumnStatistics; -import com.facebook.presto.hive.metastore.IntegerStatistics; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; @@ -48,11 +41,11 @@ import java.util.Set; import static com.facebook.presto.hive.HiveBasicStatistics.createZeroStatistics; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_COLUMN_STATISTIC_TYPE; -import static com.facebook.presto.hive.HiveWriteUtils.createPartitionValues; -import static com.facebook.presto.hive.util.Statistics.ReduceOperator.ADD; -import static com.facebook.presto.hive.util.Statistics.ReduceOperator.MAX; -import static com.facebook.presto.hive.util.Statistics.ReduceOperator.MIN; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_UNKNOWN_COLUMN_STATISTIC_TYPE; +import static com.facebook.presto.hive.metastore.MetastoreUtil.createPartitionValues; +import static com.facebook.presto.hive.metastore.Statistics.ReduceOperator.ADD; +import static com.facebook.presto.hive.metastore.Statistics.ReduceOperator.MAX; +import static com.facebook.presto.hive.metastore.Statistics.ReduceOperator.MIN; import static com.facebook.presto.spi.statistics.ColumnStatisticType.MAX_VALUE; import static com.facebook.presto.spi.statistics.ColumnStatisticType.MAX_VALUE_SIZE_IN_BYTES; import static com.facebook.presto.spi.statistics.ColumnStatisticType.MIN_VALUE; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/Storage.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Storage.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/Storage.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Storage.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java similarity index 96% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java index 024f643280367..b29eb45a8a178 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java @@ -22,8 +22,8 @@ import java.util.Objects; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/Table.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Table.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/Table.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Table.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserDatabaseKey.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/UserDatabaseKey.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserDatabaseKey.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/UserDatabaseKey.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserTableKey.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/UserTableKey.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/UserTableKey.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/UserTableKey.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/DatabaseMetadata.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/DatabaseMetadata.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/DatabaseMetadata.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/DatabaseMetadata.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java similarity index 97% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java index 7fa9cfb699fb8..9b2c339842409 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java @@ -13,25 +13,21 @@ */ package com.facebook.presto.hive.metastore.file; -import com.facebook.presto.hive.HdfsConfiguration; -import com.facebook.presto.hive.HdfsConfigurationUpdater; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; import com.facebook.presto.hive.HiveBasicStatistics; -import com.facebook.presto.hive.HiveClientConfig; -import com.facebook.presto.hive.HiveHdfsConfiguration; import com.facebook.presto.hive.HiveType; import com.facebook.presto.hive.PartitionNotFoundException; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.SchemaAlreadyExistsException; import com.facebook.presto.hive.TableAlreadyExistsException; -import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HiveColumnStatistics; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; import com.facebook.presto.hive.metastore.Partition; +import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.hive.metastore.PartitionWithStatistics; import com.facebook.presto.hive.metastore.PrincipalPrivileges; import com.facebook.presto.hive.metastore.Table; @@ -50,7 +46,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.io.ByteStreams; -import io.airlift.json.JsonCodec; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -59,7 +54,6 @@ import javax.annotation.concurrent.ThreadSafe; import javax.inject.Inject; -import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayDeque; @@ -79,12 +73,12 @@ import java.util.Set; import java.util.function.Function; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; -import static com.facebook.presto.hive.HivePartitionManager.extractPartitionValues; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; +import static com.facebook.presto.hive.metastore.MetastoreUtil.extractPartitionValues; import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyCanDropColumn; import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; @@ -126,14 +120,6 @@ public class FileHiveMetastore private final JsonCodec> rolesCodec = JsonCodec.listJsonCodec(String.class); private final JsonCodec> roleGrantsCodec = JsonCodec.listJsonCodec(RoleGrant.class); - public static FileHiveMetastore createTestingFileHiveMetastore(File catalogDirectory) - { - HiveClientConfig hiveClientConfig = new HiveClientConfig(); - HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationUpdater(hiveClientConfig)); - HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication()); - return new FileHiveMetastore(hdfsEnvironment, catalogDirectory.toURI().toString(), "test"); - } - @Inject public FileHiveMetastore(HdfsEnvironment hdfsEnvironment, FileHiveMetastoreConfig config) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastoreConfig.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastoreConfig.java similarity index 92% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastoreConfig.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastoreConfig.java index b0b69fb863f75..31015aadac281 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastoreConfig.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastoreConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive.metastore.file; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import javax.validation.constraints.NotNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java similarity index 96% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java index 51be75ac1d21b..e7a40e5466e69 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java @@ -20,7 +20,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; import static org.weakref.jmx.ObjectNames.generatedNameOf; import static org.weakref.jmx.guice.ExportBinder.newExporter; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java similarity index 99% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java index 97ccc07964ae3..28e65fc06a715 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java @@ -15,10 +15,10 @@ import com.facebook.presto.hive.HiveBucketProperty; import com.facebook.presto.hive.HiveStorageFormat; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.HiveColumnStatistics; import com.facebook.presto.hive.metastore.Partition; +import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.hive.metastore.PartitionWithStatistics; import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.hive.metastore.StorageFormat; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/PermissionMetadata.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PermissionMetadata.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/PermissionMetadata.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PermissionMetadata.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/ForGlueHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/ForGlueHiveMetastore.java new file mode 100644 index 0000000000000..a0c0635db36a9 --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/ForGlueHiveMetastore.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore.glue; + +import javax.inject.Qualifier; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@Qualifier +public @interface ForGlueHiveMetastore {} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueExpressionUtil.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueExpressionUtil.java similarity index 97% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueExpressionUtil.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueExpressionUtil.java index 46065c2123d78..2489c7d804cbe 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueExpressionUtil.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueExpressionUtil.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.Set; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; public final class GlueExpressionUtil { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java similarity index 92% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java index 777786dba946c..029a90757ef57 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java @@ -47,24 +47,25 @@ import com.amazonaws.services.glue.model.PartitionError; import com.amazonaws.services.glue.model.PartitionInput; import com.amazonaws.services.glue.model.PartitionValueList; +import com.amazonaws.services.glue.model.Segment; import com.amazonaws.services.glue.model.TableInput; import com.amazonaws.services.glue.model.UpdateDatabaseRequest; import com.amazonaws.services.glue.model.UpdatePartitionRequest; import com.amazonaws.services.glue.model.UpdateTableRequest; +import com.facebook.airlift.log.Logger; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; import com.facebook.presto.hive.HiveType; -import com.facebook.presto.hive.HiveUtil; -import com.facebook.presto.hive.HiveWriteUtils; import com.facebook.presto.hive.PartitionNotFoundException; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.SchemaAlreadyExistsException; import com.facebook.presto.hive.TableAlreadyExistsException; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; +import com.facebook.presto.hive.metastore.MetastoreUtil; import com.facebook.presto.hive.metastore.Partition; +import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.hive.metastore.PartitionWithStatistics; import com.facebook.presto.hive.metastore.PrincipalPrivileges; import com.facebook.presto.hive.metastore.Table; @@ -84,25 +85,30 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import io.airlift.log.Logger; import org.apache.hadoop.fs.Path; +import javax.annotation.Nullable; import javax.inject.Inject; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.Future; import java.util.function.Function; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; +import static com.facebook.presto.hive.metastore.MetastoreUtil.createDirectory; import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyCanDropColumn; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW; @@ -115,6 +121,8 @@ import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.security.PrincipalType.USER; import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.collect.Comparators.lexicographical; +import static java.util.Comparator.comparing; import static java.util.Objects.requireNonNull; import static java.util.function.UnaryOperator.identity; import static java.util.stream.Collectors.toList; @@ -130,26 +138,38 @@ public class GlueHiveMetastore private static final String WILDCARD_EXPRESSION = ""; private static final int BATCH_GET_PARTITION_MAX_PAGE_SIZE = 1000; private static final int BATCH_CREATE_PARTITION_MAX_PAGE_SIZE = 100; + private static final Comparator PARTITION_COMPARATOR = comparing(Partition::getValues, lexicographical(String.CASE_INSENSITIVE_ORDER)); private final HdfsEnvironment hdfsEnvironment; private final HdfsContext hdfsContext; private final AWSGlueAsync glueClient; private final Optional defaultDir; private final String catalogId; + private final int partitionSegments; + private final Executor executor; @Inject - public GlueHiveMetastore(HdfsEnvironment hdfsEnvironment, GlueHiveMetastoreConfig glueConfig) + public GlueHiveMetastore( + HdfsEnvironment hdfsEnvironment, + GlueHiveMetastoreConfig glueConfig, + @ForGlueHiveMetastore Executor executor) { - this(hdfsEnvironment, glueConfig, createAsyncGlueClient(glueConfig)); + this(hdfsEnvironment, glueConfig, createAsyncGlueClient(glueConfig), executor); } - public GlueHiveMetastore(HdfsEnvironment hdfsEnvironment, GlueHiveMetastoreConfig glueConfig, AWSGlueAsync glueClient) + public GlueHiveMetastore( + HdfsEnvironment hdfsEnvironment, + GlueHiveMetastoreConfig glueConfig, + AWSGlueAsync glueClient, + @ForGlueHiveMetastore Executor executor) { this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.hdfsContext = new HdfsContext(new ConnectorIdentity(DEFAULT_METASTORE_USER, Optional.empty(), Optional.empty())); this.glueClient = requireNonNull(glueClient, "glueClient is null"); this.defaultDir = glueConfig.getDefaultWarehouseDir(); this.catalogId = glueConfig.getCatalogId().orElse(null); + this.partitionSegments = glueConfig.getPartitionSegments(); + this.executor = requireNonNull(executor, "executor is null"); } private static AWSGlueAsync createAsyncGlueClient(GlueHiveMetastoreConfig config) @@ -409,7 +429,7 @@ public void createDatabase(Database database) } if (database.getLocation().isPresent()) { - HiveWriteUtils.createDirectory(hdfsContext, hdfsEnvironment, new Path(database.getLocation().get())); + createDirectory(hdfsContext, hdfsEnvironment, new Path(database.getLocation().get())); } } @@ -649,6 +669,37 @@ public Optional> getPartitionNamesByParts(String databaseName, Stri } private List getPartitions(String databaseName, String tableName, String expression) + { + if (partitionSegments == 1) { + return getPartitions(databaseName, tableName, expression, null); + } + + // Do parallel partition fetch. + CompletionService> completionService = new ExecutorCompletionService<>(executor); + for (int i = 0; i < partitionSegments; i++) { + Segment segment = new Segment().withSegmentNumber(i).withTotalSegments(partitionSegments); + completionService.submit(() -> getPartitions(databaseName, tableName, expression, segment)); + } + + List partitions = new ArrayList<>(); + try { + for (int i = 0; i < partitionSegments; i++) { + Future> futurePartitions = completionService.take(); + partitions.addAll(futurePartitions.get()); + } + } + catch (ExecutionException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new PrestoException(HIVE_METASTORE_ERROR, "Failed to fetch partitions from Glue Data Catalog", e); + } + + partitions.sort(PARTITION_COMPARATOR); + return partitions; + } + + private List getPartitions(String databaseName, String tableName, String expression, @Nullable Segment segment) { try { List partitions = new ArrayList<>(); @@ -660,6 +711,7 @@ private List getPartitions(String databaseName, String tableName, Str .withDatabaseName(databaseName) .withTableName(tableName) .withExpression(expression) + .withSegment(segment) .withNextToken(nextToken)); result.getPartitions() .forEach(partition -> partitions.add(GlueToPrestoConverter.convertPartition(partition))); @@ -701,7 +753,7 @@ public Map> getPartitionsByNames(String databaseName List partitions = batchGetPartition(databaseName, tableName, partitionNames); Map> partitionNameToPartitionValuesMap = partitionNames.stream() - .collect(toMap(identity(), HiveUtil::toPartitionValues)); + .collect(toMap(identity(), MetastoreUtil::toPartitionValues)); Map, Partition> partitionValuesToPartitionMap = partitions.stream() .collect(toMap(Partition::getValues, identity())); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java similarity index 73% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java index b8d0c4a0c6475..979a29c734466 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java @@ -13,9 +13,10 @@ */ package com.facebook.presto.hive.metastore.glue; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import javax.validation.constraints.Max; import javax.validation.constraints.Min; import java.util.Optional; @@ -27,6 +28,8 @@ public class GlueHiveMetastoreConfig private int maxGlueConnections = 5; private Optional defaultWarehouseDir = Optional.empty(); private Optional catalogId = Optional.empty(); + private int partitionSegments = 5; + private int getPartitionThreads = 20; public Optional getGlueRegion() { @@ -93,4 +96,33 @@ public GlueHiveMetastoreConfig setCatalogId(String catalogId) this.catalogId = Optional.ofNullable(catalogId); return this; } + + @Min(1) + @Max(10) + public int getPartitionSegments() + { + return partitionSegments; + } + + @Config("hive.metastore.glue.partitions-segments") + @ConfigDescription("Number of segments for partitioned Glue tables") + public GlueHiveMetastoreConfig setPartitionSegments(int partitionSegments) + { + this.partitionSegments = partitionSegments; + return this; + } + + @Min(1) + public int getGetPartitionThreads() + { + return getPartitionThreads; + } + + @Config("hive.metastore.glue.get-partition-threads") + @ConfigDescription("Number of threads for parallel partition fetches from Glue") + public GlueHiveMetastoreConfig setGetPartitionThreads(int getPartitionThreads) + { + this.getPartitionThreads = getPartitionThreads; + return this; + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java similarity index 64% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java index 64ef56d30d398..d7f6a6da30851 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java @@ -13,13 +13,21 @@ */ package com.facebook.presto.hive.metastore.glue; +import com.facebook.airlift.concurrent.BoundedExecutor; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.google.inject.Binder; import com.google.inject.Module; +import com.google.inject.Provides; import com.google.inject.Scopes; +import com.google.inject.Singleton; -import static io.airlift.configuration.ConfigBinder.configBinder; +import java.util.concurrent.Executor; + +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static java.util.Objects.requireNonNull; +import static java.util.concurrent.Executors.newCachedThreadPool; import static org.weakref.jmx.ObjectNames.generatedNameOf; import static org.weakref.jmx.guice.ExportBinder.newExporter; @@ -41,4 +49,17 @@ public void configure(Binder binder) newExporter(binder).export(ExtendedHiveMetastore.class) .as(generatedNameOf(GlueHiveMetastore.class, connectorId)); } + + @Provides + @Singleton + @ForGlueHiveMetastore + public Executor createExecutor(GlueHiveMetastoreConfig hiveConfig) + { + if (hiveConfig.getGetPartitionThreads() == 1) { + return directExecutor(); + } + return new BoundedExecutor( + newCachedThreadPool(daemonThreadsNamed("hive-glue-%s")), + hiveConfig.getGetPartitionThreads()); + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueInputConverter.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueInputConverter.java similarity index 99% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueInputConverter.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueInputConverter.java index 581fe9beeac6d..d325f6d05cad1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueInputConverter.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueInputConverter.java @@ -18,10 +18,10 @@ import com.amazonaws.services.glue.model.SerDeInfo; import com.amazonaws.services.glue.model.StorageDescriptor; import com.amazonaws.services.glue.model.TableInput; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.Partition; +import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.hive.metastore.PartitionWithStatistics; import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.hive.metastore.Table; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java similarity index 98% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java index 1514b6d2f14a4..e11b9e743c6d3 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java @@ -36,7 +36,7 @@ import java.util.Locale; import java.util.Optional; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.metastore.PrestoTableType.OTHER; import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Strings.nullToEmpty; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java similarity index 98% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java index 7fbeb38d52e08..9a5aa221c05e7 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java @@ -14,12 +14,12 @@ package com.facebook.presto.hive.metastore.thrift; import com.facebook.presto.hive.HiveType; -import com.facebook.presto.hive.HiveUtil; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; +import com.facebook.presto.hive.metastore.MetastoreUtil; import com.facebook.presto.hive.metastore.Partition; +import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.hive.metastore.PartitionWithStatistics; import com.facebook.presto.hive.metastore.PrincipalPrivileges; import com.facebook.presto.hive.metastore.Table; @@ -265,7 +265,7 @@ public Map> getPartitionsByNames(String databaseName return ImmutableMap.of(); } Map> partitionNameToPartitionValuesMap = partitionNames.stream() - .collect(Collectors.toMap(identity(), HiveUtil::toPartitionValues)); + .collect(Collectors.toMap(identity(), MetastoreUtil::toPartitionValues)); Map, Partition> partitionValuesToPartitionMap = delegate.getPartitionsByNames(databaseName, tableName, partitionNames).stream() .map(ThriftMetastoreUtil::fromMetastoreApiPartition) .collect(Collectors.toMap(Partition::getValues, identity())); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveCluster.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveCluster.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveCluster.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveCluster.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java similarity index 97% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java index a53b4b990f6e3..26a161eb7eafd 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive.metastore.thrift; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; +import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.hive.metastore.PartitionWithStatistics; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; @@ -34,7 +34,7 @@ import java.util.Set; import java.util.function.Function; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; public interface HiveMetastore { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreApiStats.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreApiStats.java similarity index 96% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreApiStats.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreApiStats.java index 9a3a1e308372d..e8c5bc28291e2 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreApiStats.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreApiStats.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive.metastore.thrift; -import io.airlift.stats.CounterStat; -import io.airlift.stats.TimeStat; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.TimeStat; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.thrift.TBase; import org.apache.thrift.TException; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java similarity index 85% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java index cc430ef3a917d..4182c2dfbf03d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive.metastore.thrift; -import com.facebook.presto.hive.HiveClientConfig; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; import com.google.common.net.HostAndPort; import io.airlift.units.Duration; @@ -47,9 +47,9 @@ public HiveMetastoreClientFactory( } @Inject - public HiveMetastoreClientFactory(HiveClientConfig config, HiveMetastoreAuthentication metastoreAuthentication) + public HiveMetastoreClientFactory(MetastoreClientConfig metastoreClientConfig, HiveMetastoreAuthentication metastoreAuthentication) { - this(Optional.empty(), Optional.ofNullable(config.getMetastoreSocksProxy()), config.getMetastoreTimeout(), metastoreAuthentication); + this(Optional.empty(), Optional.ofNullable(metastoreClientConfig.getMetastoreSocksProxy()), metastoreClientConfig.getMetastoreTimeout(), metastoreAuthentication); } public HiveMetastoreClient create(HostAndPort address) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/StaticHiveCluster.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/StaticHiveCluster.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/StaticHiveCluster.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/StaticHiveCluster.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/StaticMetastoreConfig.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/StaticMetastoreConfig.java similarity index 94% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/StaticMetastoreConfig.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/StaticMetastoreConfig.java index db972ede83596..fd8c6d09b4d5e 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/StaticMetastoreConfig.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/StaticMetastoreConfig.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.hive.metastore.thrift; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; import javax.validation.constraints.NotNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java similarity index 99% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java index 377fba8468861..3b90b287ae7a2 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java @@ -17,13 +17,13 @@ import com.facebook.presto.hive.HiveType; import com.facebook.presto.hive.HiveViewNotSupportedException; import com.facebook.presto.hive.PartitionNotFoundException; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.RetryDriver; import com.facebook.presto.hive.SchemaAlreadyExistsException; import com.facebook.presto.hive.TableAlreadyExistsException; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.HiveColumnStatistics; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; +import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.hive.metastore.PartitionWithStatistics; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaNotFoundException; @@ -75,10 +75,10 @@ import java.util.stream.Collectors; import static com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; -import static com.facebook.presto.hive.HiveUtil.PRESTO_VIEW_FLAG; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; +import static com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_VIEW_FLAG; import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.createMetastoreColumnStatistics; import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreApiPrincipalType; import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreApiTable; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreStats.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreStats.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreStats.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreStats.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java similarity index 91% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java index 83fd6037a6bbc..0da8535f2bf14 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java @@ -13,17 +13,17 @@ */ package com.facebook.presto.hive.metastore.thrift; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.facebook.presto.hive.ForCachingHiveMetastore; import com.facebook.presto.hive.ForRecordingHiveMetastore; -import com.facebook.presto.hive.HiveClientConfig; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.metastore.CachingHiveMetastore; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.RecordingHiveMetastore; import com.google.inject.Binder; import com.google.inject.Scopes; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; import static org.weakref.jmx.ObjectNames.generatedNameOf; import static org.weakref.jmx.guice.ExportBinder.newExporter; @@ -47,7 +47,7 @@ protected void setup(Binder binder) binder.bind(HiveMetastore.class).to(ThriftHiveMetastore.class).in(Scopes.SINGLETON); - if (buildConfigObject(HiveClientConfig.class).getRecordingPath() != null) { + if (buildConfigObject(MetastoreClientConfig.class).getRecordingPath() != null) { binder.bind(ExtendedHiveMetastore.class) .annotatedWith(ForRecordingHiveMetastore.class) .to(BridgingHiveMetastore.class) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java similarity index 99% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java index 1b89aae4f32f5..84675d1097104 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java @@ -88,9 +88,8 @@ import java.util.function.Predicate; import java.util.stream.Stream; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; -import static com.facebook.presto.hive.HiveMetadata.AVRO_SCHEMA_URL_KEY; import static com.facebook.presto.hive.HiveStorageFormat.AVRO; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createDateColumnStatistics; @@ -103,6 +102,7 @@ import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.SELECT; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.UPDATE; +import static com.facebook.presto.hive.metastore.MetastoreUtil.AVRO_SCHEMA_URL_KEY; import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.OTHER; @@ -300,7 +300,7 @@ public static Stream listApplicableTablePrivileges(SemiTransa Stream principals = Stream.concat( Stream.of(userPrincipal), listApplicableRoles(metastore, userPrincipal) - .map(role -> new PrestoPrincipal(ROLE, role))); + .map(role -> new PrestoPrincipal(ROLE, role))); return listTablePrivileges(metastore, databaseName, tableName, principals); } @@ -944,7 +944,7 @@ public static Set getSupportedColumnStatistics(Type type) return ImmutableSet.of(NUMBER_OF_NON_NULL_VALUES, TOTAL_SIZE_IN_BYTES, MAX_VALUE_SIZE_IN_BYTES); } if (type instanceof ArrayType || type instanceof RowType || type instanceof MapType) { - return ImmutableSet.of(); + return ImmutableSet.of(NUMBER_OF_NON_NULL_VALUES, TOTAL_SIZE_IN_BYTES); } // Throwing here to make sure this method is updated when a new type is added in Hive connector throw new IllegalArgumentException("Unsupported type: " + type); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/Transport.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/Transport.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/Transport.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/Transport.java diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestPartitionOfflineException.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/TestPartitionOfflineException.java similarity index 100% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestPartitionOfflineException.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/TestPartitionOfflineException.java diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestTableOfflineException.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/TestTableOfflineException.java similarity index 100% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestTableOfflineException.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/TestTableOfflineException.java diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java similarity index 99% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java index 3af715d0168fd..697dfc25b3981 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestCachingHiveMetastore.java @@ -30,6 +30,7 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.BAD_DATABASE; import static com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_DATABASE; import static com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_PARTITION1; @@ -37,7 +38,6 @@ import static com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_ROLES; import static com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.TEST_TABLE; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreUtil.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestHiveMetastoreUtil.java similarity index 74% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreUtil.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestHiveMetastoreUtil.java index 981f6ebc11707..2ba6a0c384c91 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreUtil.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestHiveMetastoreUtil.java @@ -27,11 +27,19 @@ import org.testng.annotations.Test; import java.util.List; +import java.util.Optional; import java.util.Properties; +import static com.facebook.presto.hive.HiveType.HIVE_DATE; +import static com.facebook.presto.hive.HiveType.HIVE_DOUBLE; +import static com.facebook.presto.hive.HiveType.HIVE_INT; +import static com.facebook.presto.hive.HiveType.HIVE_STRING; +import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema; +import static com.facebook.presto.hive.metastore.MetastoreUtil.reconstructPartitionSchema; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.testng.Assert.assertEquals; -public class TestMetastoreUtil +public class TestHiveMetastoreUtil { private static final List TEST_SCHEMA = ImmutableList.of( new FieldSchema("col1", "bigint", "comment1"), @@ -139,7 +147,7 @@ public void testPartitionRoundTrip() public void testHiveSchemaTable() { Properties expected = MetaStoreUtils.getTableMetadata(TEST_TABLE_WITH_UNSUPPORTED_FIELDS); - Properties actual = MetastoreUtil.getHiveSchema(ThriftMetastoreUtil.fromMetastoreApiTable(TEST_TABLE_WITH_UNSUPPORTED_FIELDS, TEST_SCHEMA)); + Properties actual = getHiveSchema(ThriftMetastoreUtil.fromMetastoreApiTable(TEST_TABLE_WITH_UNSUPPORTED_FIELDS, TEST_SCHEMA)); assertEquals(actual, expected); } @@ -147,7 +155,7 @@ public void testHiveSchemaTable() public void testHiveSchemaPartition() { Properties expected = MetaStoreUtils.getPartitionMetadata(TEST_PARTITION_WITH_UNSUPPORTED_FIELDS, TEST_TABLE_WITH_UNSUPPORTED_FIELDS); - Properties actual = MetastoreUtil.getHiveSchema(ThriftMetastoreUtil.fromMetastoreApiPartition(TEST_PARTITION_WITH_UNSUPPORTED_FIELDS), ThriftMetastoreUtil.fromMetastoreApiTable(TEST_TABLE_WITH_UNSUPPORTED_FIELDS, TEST_SCHEMA)); + Properties actual = getHiveSchema(ThriftMetastoreUtil.fromMetastoreApiPartition(TEST_PARTITION_WITH_UNSUPPORTED_FIELDS), ThriftMetastoreUtil.fromMetastoreApiTable(TEST_TABLE_WITH_UNSUPPORTED_FIELDS, TEST_SCHEMA)); assertEquals(actual, expected); } @@ -164,4 +172,27 @@ public void testPartitionRoundTripUnsupported() Partition partition = ThriftMetastoreUtil.fromMetastoreApiPartition(TEST_PARTITION_WITH_UNSUPPORTED_FIELDS); ThriftMetastoreUtil.toMetastoreApiPartition(partition); } + + @Test + public void testReconstructPartitionSchema() + { + Column c1 = new Column("_c1", HIVE_STRING, Optional.empty()); + Column c2 = new Column("_c2", HIVE_INT, Optional.empty()); + Column c3 = new Column("_c3", HIVE_DOUBLE, Optional.empty()); + Column c4 = new Column("_c4", HIVE_DATE, Optional.empty()); + + assertEquals(reconstructPartitionSchema(ImmutableList.of(), 0, ImmutableMap.of()), ImmutableList.of()); + assertEquals(reconstructPartitionSchema(ImmutableList.of(c1), 0, ImmutableMap.of()), ImmutableList.of()); + assertEquals(reconstructPartitionSchema(ImmutableList.of(c1), 1, ImmutableMap.of()), ImmutableList.of(c1)); + assertEquals(reconstructPartitionSchema(ImmutableList.of(c1, c2), 1, ImmutableMap.of()), ImmutableList.of(c1)); + assertEquals(reconstructPartitionSchema(ImmutableList.of(c1, c2), 3, ImmutableMap.of(2, c3)), ImmutableList.of(c1, c2, c3)); + assertEquals(reconstructPartitionSchema(ImmutableList.of(c1, c2, c3), 3, ImmutableMap.of(1, c4)), ImmutableList.of(c1, c4, c3)); + + assertThatThrownBy(() -> reconstructPartitionSchema(ImmutableList.of(), 1, ImmutableMap.of())) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> reconstructPartitionSchema(ImmutableList.of(c1), 2, ImmutableMap.of())) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> reconstructPartitionSchema(ImmutableList.of(c1), 2, ImmutableMap.of(0, c2))) + .isInstanceOf(IllegalArgumentException.class); + } } diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreClientConfig.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreClientConfig.java new file mode 100644 index 0000000000000..c2bcb55ba4a46 --- /dev/null +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreClientConfig.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore; + +import com.facebook.airlift.configuration.testing.ConfigAssertions; +import com.facebook.presto.hive.MetastoreClientConfig; +import com.google.common.collect.ImmutableMap; +import com.google.common.net.HostAndPort; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class TestMetastoreClientConfig +{ + @Test + public void testDefaults() + { + ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(MetastoreClientConfig.class) + .setMetastoreSocksProxy(null) + .setMetastoreTimeout(new Duration(10, TimeUnit.SECONDS)) + .setVerifyChecksum(true) + .setMetastoreCacheTtl(new Duration(0, TimeUnit.SECONDS)) + .setMetastoreRefreshInterval(new Duration(0, TimeUnit.SECONDS)) + .setMetastoreCacheMaximumSize(10000) + .setPerTransactionMetastoreCacheMaximumSize(1000) + .setMaxMetastoreRefreshThreads(100) + .setRecordingPath(null) + .setRecordingDuration(new Duration(0, TimeUnit.MINUTES)) + .setReplay(false)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("hive.metastore.thrift.client.socks-proxy", "localhost:1080") + .put("hive.metastore-timeout", "20s") + .put("hive.dfs.verify-checksum", "false") + .put("hive.metastore-cache-ttl", "2h") + .put("hive.metastore-refresh-interval", "30m") + .put("hive.metastore-cache-maximum-size", "5000") + .put("hive.per-transaction-metastore-cache-maximum-size", "500") + .put("hive.metastore-refresh-max-threads", "2500") + .put("hive.metastore-recording-path", "/foo/bar") + .put("hive.metastore-recoding-duration", "42s") + .put("hive.replay-metastore-recording", "true") + .build(); + + MetastoreClientConfig expected = new MetastoreClientConfig() + .setMetastoreSocksProxy(HostAndPort.fromParts("localhost", 1080)) + .setMetastoreTimeout(new Duration(20, TimeUnit.SECONDS)) + .setVerifyChecksum(false) + .setMetastoreCacheTtl(new Duration(2, TimeUnit.HOURS)) + .setMetastoreRefreshInterval(new Duration(30, TimeUnit.MINUTES)) + .setMetastoreCacheMaximumSize(5000) + .setPerTransactionMetastoreCacheMaximumSize(500) + .setMaxMetastoreRefreshThreads(2500) + .setRecordingPath("/foo/bar") + .setRecordingDuration(new Duration(42, TimeUnit.SECONDS)) + .setReplay(true); + + ConfigAssertions.assertFullMapping(properties, expected); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreConfig.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreConfig.java similarity index 82% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreConfig.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreConfig.java index d48b83caba46f..f112242c7c723 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreConfig.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreConfig.java @@ -18,9 +18,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestMetastoreConfig { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java similarity index 97% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java index 1feee934aa138..b4ad968a172dd 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestRecordingHiveMetastore.java @@ -15,9 +15,8 @@ import com.facebook.presto.hive.HiveBasicStatistics; import com.facebook.presto.hive.HiveBucketProperty; -import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.HiveType; -import com.facebook.presto.hive.PartitionStatistics; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege; import com.facebook.presto.hive.metastore.SortingColumn.Order; import com.facebook.presto.spi.security.PrestoPrincipal; @@ -105,7 +104,7 @@ public class TestRecordingHiveMetastore public void testRecordingHiveMetastore() throws IOException { - HiveClientConfig recordingHiveClientConfig = new HiveClientConfig() + MetastoreClientConfig recordingHiveClientConfig = new MetastoreClientConfig() .setRecordingPath(File.createTempFile("recording_test", "json").getAbsolutePath()) .setRecordingDuration(new Duration(10, TimeUnit.MINUTES)); @@ -114,7 +113,7 @@ public void testRecordingHiveMetastore() recordingHiveMetastore.dropDatabase("other_database"); recordingHiveMetastore.writeRecording(); - HiveClientConfig replayingHiveClientConfig = recordingHiveClientConfig + MetastoreClientConfig replayingHiveClientConfig = recordingHiveClientConfig .setReplay(true); recordingHiveMetastore = new RecordingHiveMetastore(new UnimplementedHiveMetastore(), replayingHiveClientConfig); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestStorage.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestStorage.java similarity index 91% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestStorage.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestStorage.java index 82ae9e122877b..eeac10bea9e3d 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestStorage.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestStorage.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.hive.metastore; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import org.testng.annotations.Test; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; public class TestStorage diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/UnimplementedHiveMetastore.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/UnimplementedHiveMetastore.java similarity index 99% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/UnimplementedHiveMetastore.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/UnimplementedHiveMetastore.java index 18c2e9dd9e9c7..c4f822bdaee58 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/UnimplementedHiveMetastore.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/UnimplementedHiveMetastore.java @@ -14,7 +14,6 @@ package com.facebook.presto.hive.metastore; import com.facebook.presto.hive.HiveType; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueExpressionUtil.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueExpressionUtil.java similarity index 100% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueExpressionUtil.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueExpressionUtil.java diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java similarity index 74% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java index 980a3f6bf6e38..eb1caeac9aa4a 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java @@ -18,9 +18,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestGlueHiveMetastoreConfig { @@ -32,7 +32,9 @@ public void testDefaults() .setPinGlueClientToCurrentRegion(false) .setMaxGlueConnections(5) .setDefaultWarehouseDir(null) - .setCatalogId(null)); + .setCatalogId(null) + .setPartitionSegments(5) + .setGetPartitionThreads(20)); } @Test @@ -44,6 +46,8 @@ public void testExplicitPropertyMapping() .put("hive.metastore.glue.max-connections", "10") .put("hive.metastore.glue.default-warehouse-dir", "/location") .put("hive.metastore.glue.catalogid", "0123456789") + .put("hive.metastore.glue.partitions-segments", "10") + .put("hive.metastore.glue.get-partition-threads", "42") .build(); GlueHiveMetastoreConfig expected = new GlueHiveMetastoreConfig() @@ -51,7 +55,9 @@ public void testExplicitPropertyMapping() .setPinGlueClientToCurrentRegion(true) .setMaxGlueConnections(10) .setDefaultWarehouseDir("/location") - .setCatalogId("0123456789"); + .setCatalogId("0123456789") + .setPartitionSegments(10) + .setGetPartitionThreads(42); assertFullMapping(properties, expected); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java similarity index 99% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java index 5a6c64713b334..28fd8defa2793 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.hive.metastore.thrift; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.SchemaAlreadyExistsException; import com.facebook.presto.hive.TableAlreadyExistsException; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; +import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.hive.metastore.PartitionWithStatistics; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaNotFoundException; @@ -53,7 +53,7 @@ import java.util.function.Function; import static com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.toMetastoreApiPartition; import static com.facebook.presto.spi.StandardErrorCode.SCHEMA_NOT_EMPTY; import static com.google.common.base.MoreObjects.toStringHelper; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClient.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClient.java similarity index 100% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClient.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClient.java diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClientFactory.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClientFactory.java similarity index 100% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClientFactory.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/MockHiveMetastoreClientFactory.java diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticHiveCluster.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticHiveCluster.java similarity index 98% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticHiveCluster.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticHiveCluster.java index 8f8abc7b0fd5e..da5f947fdd787 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticHiveCluster.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticHiveCluster.java @@ -20,7 +20,7 @@ import java.util.List; import java.util.Optional; -import static io.airlift.testing.Assertions.assertContains; +import static com.facebook.airlift.testing.Assertions.assertContains; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticMetastoreConfig.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticMetastoreConfig.java similarity index 90% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticMetastoreConfig.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticMetastoreConfig.java index 85ea7a2a083b9..f76c2b3c611fb 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticMetastoreConfig.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestStaticMetastoreConfig.java @@ -20,9 +20,9 @@ import java.net.URI; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static org.testng.Assert.assertEquals; public class TestStaticMetastoreConfig diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftMetastoreUtil.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftHiveMetastoreUtil.java similarity index 99% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftMetastoreUtil.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftHiveMetastoreUtil.java index 9dde3c8849caa..bf9a93a02d8c2 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftMetastoreUtil.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftHiveMetastoreUtil.java @@ -59,7 +59,7 @@ import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME; import static org.testng.Assert.assertEquals; -public class TestThriftMetastoreUtil +public class TestThriftHiveMetastoreUtil { @Test public void testLongStatsToColumnStatistics() diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java similarity index 78% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java index 441f46048e084..570f66c9cb1e3 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive.metastore.thrift; -import com.facebook.presto.hive.HiveClientConfig; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.authentication.NoHiveMetastoreAuthentication; import com.google.common.net.HostAndPort; import org.apache.thrift.TException; @@ -25,12 +25,12 @@ public class TestingHiveCluster implements HiveCluster { - private final HiveClientConfig config; + private final MetastoreClientConfig metastoreClientConfig; private final HostAndPort address; - public TestingHiveCluster(HiveClientConfig config, String host, int port) + public TestingHiveCluster(MetastoreClientConfig metastoreClientConfig, String host, int port) { - this.config = requireNonNull(config, "config is null"); + this.metastoreClientConfig = requireNonNull(metastoreClientConfig, "metastore config is null"); this.address = HostAndPort.fromParts(requireNonNull(host, "host is null"), port); } @@ -38,7 +38,7 @@ public TestingHiveCluster(HiveClientConfig config, String host, int port) public HiveMetastoreClient createMetastoreClient() throws TException { - return new HiveMetastoreClientFactory(config, new NoHiveMetastoreAuthentication()).create(address); + return new HiveMetastoreClientFactory(metastoreClientConfig, new NoHiveMetastoreAuthentication()).create(address); } @Override diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index d3b8bb0961b44..1cf6ddc7c4867 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-hive @@ -37,6 +37,11 @@ presto-parquet + + com.facebook.presto + presto-expressions + + org.apache.parquet parquet-column @@ -70,6 +75,11 @@ presto-memory-context + + com.facebook.presto + presto-hive-metastore + + com.facebook.presto presto-rcfile @@ -97,37 +107,37 @@ - io.airlift + com.facebook.airlift stats - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift event - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift configuration @@ -193,6 +203,26 @@ aws-java-sdk-s3 + + com.google.cloud.bigdataoss + util + + + + com.google.cloud.bigdataoss + gcsio + + + + com.google.cloud.bigdataoss + util-hadoop + + + + com.google.cloud.bigdataoss + gcs-connector + + org.xerial.snappy snappy-java @@ -216,7 +246,7 @@ - io.airlift + com.facebook.airlift log-manager runtime @@ -263,6 +293,13 @@ test + + com.facebook.presto + presto-hive-metastore + test-jar + test + + com.facebook.presto presto-parser @@ -300,7 +337,7 @@ - io.airlift + com.facebook.airlift testing test @@ -349,6 +386,18 @@ + + + org.apache.maven.plugins + maven-dependency-plugin + + + + org.glassfish.jersey.core:jersey-common:jar + org.eclipse.jetty:jetty-server:jar + + + @@ -369,7 +418,12 @@ **/TestHiveClientGlueMetastore.java **/TestHiveDistributedAggregationsWithExchangeMaterialization.java **/TestHiveDistributedQueriesWithExchangeMaterialization.java + **/TestHiveDistributedQueriesWithOptimizedRepartitioning.java **/TestHiveRecoverableGroupedExecution.java + **/TestHivePushdownFilterQueries.java + **/TestHivePushdownIntegrationSmokeTest.java + **/TestHivePushdownDistributedQueries.java + **/TestParquetDistributedQueries.java @@ -393,6 +447,22 @@ + + test-hive-repartitioning + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/TestHiveDistributedQueriesWithOptimizedRepartitioning.java + + + + + + test-hive-glue @@ -425,5 +495,54 @@ + + test-hive-pushdown-filter-queries-basic + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/TestHivePushdownFilterQueries.java + **/TestHivePushdownIntegrationSmokeTest.java + + + + + + + + test-hive-pushdown-filter-queries-advanced + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/TestHivePushdownDistributedQueries.java + + + + + + + + test-hive-parquet + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/TestParquetDistributedQueries.java + + + + + + diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index 00bc863111172..330fc4526b837 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -18,8 +18,8 @@ import com.facebook.presto.hive.HiveSplit.BucketConversion; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Partition; +import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.hive.metastore.Table; -import com.facebook.presto.hive.util.HiveFileIterator; import com.facebook.presto.hive.util.HiveFileIterator.NestedDirectoryNotAllowedException; import com.facebook.presto.hive.util.InternalHiveSplitFactory; import com.facebook.presto.hive.util.ResumableTask; @@ -35,7 +35,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat; import org.apache.hadoop.mapred.FileInputFormat; @@ -65,17 +64,17 @@ import static com.facebook.presto.hive.HiveBucketing.getVirtualBucketNumber; import static com.facebook.presto.hive.HiveColumnHandle.pathColumnHandle; import static com.facebook.presto.hive.HiveErrorCode.HIVE_BAD_DATA; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_BUCKET_FILES; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE; import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR; import static com.facebook.presto.hive.HiveSessionProperties.isForceLocalScheduling; -import static com.facebook.presto.hive.HiveUtil.checkCondition; import static com.facebook.presto.hive.HiveUtil.getFooterCount; import static com.facebook.presto.hive.HiveUtil.getHeaderCount; import static com.facebook.presto.hive.HiveUtil.getInputFormat; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_PARTITION_VALUE; import static com.facebook.presto.hive.S3SelectPushdown.shouldEnablePushdownForTable; +import static com.facebook.presto.hive.metastore.MetastoreUtil.checkCondition; import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema; import static com.facebook.presto.hive.metastore.MetastoreUtil.getPartitionLocation; import static com.facebook.presto.hive.util.ConfigurationUtils.toJobConf; @@ -280,12 +279,15 @@ private ListenableFuture loadPartition(HivePartitionMetadata partition) throws IOException { String partitionName = partition.getHivePartition().getPartitionId(); - Properties schema = getPartitionSchema(table, partition.getPartition()); + Storage storage = partition.getPartition().map(Partition::getStorage).orElse(table.getStorage()); + int partitionDataColumnCount = partition.getPartition() + .map(p -> p.getColumns().size()) + .orElse(table.getDataColumns().size()); List partitionKeys = getPartitionKeys(table, partition.getPartition()); Path path = new Path(getPartitionLocation(table, partition.getPartition())); Configuration configuration = hdfsEnvironment.getConfiguration(hdfsContext, path); - InputFormat inputFormat = getInputFormat(configuration, schema, false); + InputFormat inputFormat = getInputFormat(configuration, storage.getStorageFormat().getInputFormat(), false); FileSystem fs = hdfsEnvironment.getFileSystem(hdfsContext, path); boolean s3SelectPushdownEnabled = shouldEnablePushdownForTable(session, table, path.toString(), partition.getPartition()); @@ -314,7 +316,7 @@ private ListenableFuture loadPartition(HivePartitionMetadata partition) pathDomain, isForceLocalScheduling(session), s3SelectPushdownEnabled, - new HiveSplitPartitionInfo(schema, path.toUri(), partitionKeys, partitionName, partition.getColumnCoercions(), Optional.empty()), + new HiveSplitPartitionInfo(storage, path.toUri(), partitionKeys, partitionName, partitionDataColumnCount, partition.getPartitionSchemaDifference(), Optional.empty()), schedulerUsesHostAddresses); lastResult = addSplitsToSource(targetSplits, splitFactory); if (stopped) { @@ -348,11 +350,12 @@ private ListenableFuture loadPartition(HivePartitionMetadata partition) isForceLocalScheduling(session), s3SelectPushdownEnabled, new HiveSplitPartitionInfo( - schema, + storage, path.toUri(), partitionKeys, partitionName, - partition.getColumnCoercions(), + partitionDataColumnCount, + partition.getPartitionSchemaDifference(), bucketConversionRequiresWorkerParticipation ? bucketConversion : Optional.empty()), schedulerUsesHostAddresses); @@ -371,6 +374,7 @@ private ListenableFuture loadPartition(HivePartitionMetadata partition) // S3 Select pushdown works at the granularity of individual S3 objects, // therefore we must not split files when it is enabled. + Properties schema = getHiveSchema(storage.getSerdeParameters(), table.getParameters()); boolean splittable = getHeaderCount(schema) == 0 && getFooterCount(schema) == 0 && !s3SelectPushdownEnabled; // Bucketed partitions are fully loaded immediately since all files must be loaded to determine the file to bucket mapping @@ -416,7 +420,7 @@ private static boolean shouldUseFileSplitsFromInputFormat(InputFormat inpu private Iterator createInternalHiveSplitIterator(Path path, FileSystem fileSystem, InternalHiveSplitFactory splitFactory, boolean splittable) { - return stream(new HiveFileIterator(path, fileSystem, directoryLister, namenodeStats, recursiveDirWalkerEnabled ? RECURSE : IGNORED)) + return stream(directoryLister.list(fileSystem, path, namenodeStats, recursiveDirWalkerEnabled ? RECURSE : IGNORED)) .map(status -> splitFactory.createInternalHiveSplit(status, splittable)) .filter(Optional::isPresent) .map(Optional::get) @@ -439,9 +443,9 @@ private List getBucketedSplits( checkState(readBucketCount <= tableBucketCount, "readBucketCount(%s) should be less than or equal to tableBucketCount(%s)", readBucketCount, tableBucketCount); // list all files in the partition - ArrayList files = new ArrayList<>(partitionBucketCount); + List fileInfos = new ArrayList<>(partitionBucketCount); try { - Iterators.addAll(files, new HiveFileIterator(path, fileSystem, directoryLister, namenodeStats, FAIL)); + Iterators.addAll(fileInfos, directoryLister.list(fileSystem, path, namenodeStats, FAIL)); } catch (NestedDirectoryNotAllowedException e) { // Fail here to be on the safe side. This seems to be the same as what Hive does @@ -453,18 +457,18 @@ private List getBucketedSplits( } // verify we found one file per bucket - if (files.size() != partitionBucketCount) { + if (fileInfos.size() != partitionBucketCount) { throw new PrestoException( HIVE_INVALID_BUCKET_FILES, format("Hive table '%s' is corrupt. The number of files in the directory (%s) does not match the declared bucket count (%s) for partition: %s", new SchemaTableName(table.getDatabaseName(), table.getTableName()), - files.size(), + fileInfos.size(), partitionBucketCount, partitionName)); } // Sort FileStatus objects (instead of, e.g., fileStatus.getPath().toString). This matches org.apache.hadoop.hive.ql.metadata.Table.getSortedPaths - files.sort(null); + fileInfos.sort(null); // convert files internal splits List splitList = new ArrayList<>(); @@ -497,9 +501,9 @@ private List getBucketedSplits( "partition bucket count: " + partitionBucketCount + ", effective reading bucket count: " + readBucketCount + ")"); } if (!eligibleTableBucketNumbers.isEmpty()) { - LocatedFileStatus file = files.get(partitionBucketNumber); + HiveFileInfo fileInfo = fileInfos.get(partitionBucketNumber); eligibleTableBucketNumbers.stream() - .map(tableBucketNumber -> splitFactory.createInternalHiveSplit(file, readBucketNumber, tableBucketNumber, splittable)) + .map(tableBucketNumber -> splitFactory.createInternalHiveSplit(fileInfo, readBucketNumber, tableBucketNumber, splittable)) .forEach(optionalSplit -> optionalSplit.ifPresent(splitList::add)); } } @@ -509,10 +513,10 @@ private List getBucketedSplits( private List getVirtuallyBucketedSplits(Path path, FileSystem fileSystem, InternalHiveSplitFactory splitFactory, int bucketCount, boolean splittable) { // List all files recursively in the partition and assign virtual bucket number to each of them - return stream(new HiveFileIterator(path, fileSystem, directoryLister, namenodeStats, RECURSE)) - .map(file -> { - int virtualBucketNumber = getVirtualBucketNumber(bucketCount, file.getPath()); - return splitFactory.createInternalHiveSplit(file, virtualBucketNumber, virtualBucketNumber, splittable); + return stream(directoryLister.list(fileSystem, path, namenodeStats, recursiveDirWalkerEnabled ? RECURSE : IGNORED)) + .map(fileInfo -> { + int virtualBucketNumber = getVirtualBucketNumber(bucketCount, fileInfo.getPath()); + return splitFactory.createInternalHiveSplit(fileInfo, virtualBucketNumber, virtualBucketNumber, splittable); }) .filter(Optional::isPresent) .map(Optional::get) @@ -561,14 +565,6 @@ private static List getPartitionKeys(Table table, Optional partition) - { - if (!partition.isPresent()) { - return getHiveSchema(table); - } - return getHiveSchema(partition.get(), table); - } - public static class BucketSplitInfo { private final List bucketColumns; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BucketAdaptation.java b/presto-hive/src/main/java/com/facebook/presto/hive/BucketAdaptation.java new file mode 100644 index 0000000000000..b36da21395fbb --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BucketAdaptation.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import java.util.List; + +public class BucketAdaptation +{ + private final int[] bucketColumnIndices; + private final List bucketColumnHiveTypes; + private final int tableBucketCount; + private final int partitionBucketCount; + private final int bucketToKeep; + + public BucketAdaptation(int[] bucketColumnIndices, List bucketColumnHiveTypes, int tableBucketCount, int partitionBucketCount, int bucketToKeep) + { + this.bucketColumnIndices = bucketColumnIndices; + this.bucketColumnHiveTypes = bucketColumnHiveTypes; + this.tableBucketCount = tableBucketCount; + this.partitionBucketCount = partitionBucketCount; + this.bucketToKeep = bucketToKeep; + } + + public int[] getBucketColumnIndices() + { + return bucketColumnIndices; + } + + public List getBucketColumnHiveTypes() + { + return bucketColumnHiveTypes; + } + + public int getTableBucketCount() + { + return tableBucketCount; + } + + public int getPartitionBucketCount() + { + return partitionBucketCount; + } + + public int getBucketToKeep() + { + return bucketToKeep; + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/CreateEmptyPartitionProcedure.java b/presto-hive/src/main/java/com/facebook/presto/hive/CreateEmptyPartitionProcedure.java index 7a81396aa4c97..1b2e3e9e9c1b4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/CreateEmptyPartitionProcedure.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/CreateEmptyPartitionProcedure.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.hive.LocationService.WriteInfo; import com.facebook.presto.hive.PartitionUpdate.UpdateMode; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; @@ -22,7 +23,6 @@ import com.facebook.presto.spi.procedure.Procedure; import com.facebook.presto.spi.procedure.Procedure.Argument; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import org.apache.hadoop.hive.common.FileUtils; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/DirectoryLister.java b/presto-hive/src/main/java/com/facebook/presto/hive/DirectoryLister.java index 1b2af5c0039a6..26124fe5578e1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/DirectoryLister.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/DirectoryLister.java @@ -14,14 +14,13 @@ package com.facebook.presto.hive; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; -import java.io.IOException; +import java.util.Iterator; + +import static com.facebook.presto.hive.util.HiveFileIterator.NestedDirectoryPolicy; public interface DirectoryLister { - RemoteIterator list(FileSystem fs, Path path) - throws IOException; + Iterator list(FileSystem fileSystem, Path path, NamenodeStats namenodeStats, NestedDirectoryPolicy nestedDirectoryPolicy); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/DynamicConfigurationProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/DynamicConfigurationProvider.java new file mode 100644 index 0000000000000..610db4ab54620 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/DynamicConfigurationProvider.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import org.apache.hadoop.conf.Configuration; + +import java.net.URI; + +import static com.facebook.presto.hive.HdfsEnvironment.HdfsContext; + +public interface DynamicConfigurationProvider +{ + void updateConfiguration(Configuration configuration, HdfsContext context, URI uri); +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/FileFormatDataSourceStats.java b/presto-hive/src/main/java/com/facebook/presto/hive/FileFormatDataSourceStats.java index 71b99c0d0e3db..5de77e87159b8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/FileFormatDataSourceStats.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/FileFormatDataSourceStats.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive; -import io.airlift.stats.DistributionStat; -import io.airlift.stats.TimeStat; +import com.facebook.airlift.stats.DistributionStat; +import com.facebook.airlift.stats.TimeStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/FileOpener.java b/presto-hive/src/main/java/com/facebook/presto/hive/FileOpener.java new file mode 100644 index 0000000000000..2da17bae99310 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/FileOpener.java @@ -0,0 +1,27 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; +import java.util.Optional; + +public interface FileOpener +{ + FSDataInputStream open(FileSystem fileSystem, Path path, Optional extraFileInfo) + throws IOException; +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/FilteringPageSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/FilteringPageSource.java new file mode 100644 index 0000000000000..b7cbafaae5ec0 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/FilteringPageSource.java @@ -0,0 +1,288 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.orc.FilterFunction; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.spi.ConnectorPageSource; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.predicate.Domain; +import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.RowExpressionService; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.DecimalType; +import com.facebook.presto.spi.type.MapType; +import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import io.airlift.slice.Slice; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.IntStream; + +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NOT_NULL; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NULL; +import static com.facebook.presto.orc.TupleDomainFilterUtils.toFilter; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.Chars.isCharType; +import static com.facebook.presto.spi.type.DateType.DATE; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RealType.REAL; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.facebook.presto.spi.type.Varchars.isVarcharType; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.lang.Double.longBitsToDouble; +import static java.lang.Float.intBitsToFloat; +import static java.util.Objects.requireNonNull; +import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET; + +public class FilteringPageSource + implements ConnectorPageSource +{ + private final ConnectorPageSource delegate; + private final TupleDomainFilter[] domainFilters; + private final Type[] columnTypes; + private final Map functionInputs; // key: hiveColumnIndex + private final FilterFunction filterFunction; + private final int outputBlockCount; + + public FilteringPageSource( + List columnMappings, + TupleDomain domainPredicate, + RowExpression remainingPredicate, + TypeManager typeManager, + RowExpressionService rowExpressionService, + ConnectorSession session, + Set originalIndices, + ConnectorPageSource delegate) + { + requireNonNull(rowExpressionService, "rowExpressionService is null"); + requireNonNull(remainingPredicate, "remainingPredicate is null"); + requireNonNull(typeManager, "typeManager is null"); + + this.delegate = requireNonNull(delegate, "delegate is null"); + + domainFilters = new TupleDomainFilter[columnMappings.size()]; + columnTypes = new Type[columnMappings.size()]; + if (!domainPredicate.isAll()) { + Map domains = domainPredicate.transform(HiveColumnHandle::getHiveColumnIndex).getDomains().get(); + for (int i = 0; i < columnMappings.size(); i++) { + HiveColumnHandle columnHandle = columnMappings.get(i).getHiveColumnHandle(); + int hiveColumnIndex = columnHandle.getHiveColumnIndex(); + if (domains.containsKey(hiveColumnIndex)) { + domainFilters[i] = toFilter(domains.get(hiveColumnIndex)); + columnTypes[i] = columnHandle.getHiveType().getType(typeManager); + } + } + } + + this.functionInputs = IntStream.range(0, columnMappings.size()) + .boxed() + .collect(toImmutableMap(i -> columnMappings.get(i).getHiveColumnHandle().getHiveColumnIndex(), Function.identity())); + + Map variableToInput = columnMappings.stream() + .map(HivePageSourceProvider.ColumnMapping::getHiveColumnHandle) + .collect(toImmutableMap( + columnHandle -> new VariableReferenceExpression(columnHandle.getName(), columnHandle.getHiveType().getType(typeManager)), + columnHandle -> new InputReferenceExpression(columnHandle.getHiveColumnIndex(), columnHandle.getHiveType().getType(typeManager)))); + + RowExpression optimizedRemainingPredicate = rowExpressionService.getExpressionOptimizer().optimize(remainingPredicate, OPTIMIZED, session); + if (TRUE_CONSTANT.equals(optimizedRemainingPredicate)) { + this.filterFunction = null; + } + else { + RowExpression expression = replaceExpression(optimizedRemainingPredicate, variableToInput); + this.filterFunction = new FilterFunction(session, + rowExpressionService.getDeterminismEvaluator().isDeterministic(expression), + rowExpressionService.getPredicateCompiler().compilePredicate(session.getSqlFunctionProperties(), expression).get()); + } + + this.outputBlockCount = requireNonNull(originalIndices, "originalIndices is null").size(); + } + + @Override + public Page getNextPage() + { + Page page = delegate.getNextPage(); + if (page == null || page.getPositionCount() == 0) { + return page; + } + + int positionCount = page.getPositionCount(); + int[] positions = new int[positionCount]; + for (int i = 0; i < positionCount; i++) { + positions[i] = i; + } + + for (int i = 0; i < page.getChannelCount(); i++) { + TupleDomainFilter domainFilter = domainFilters[i]; + if (domainFilter != null) { + positionCount = filterBlock(page.getBlock(i), columnTypes[i], domainFilter, positions, positionCount); + if (positionCount == 0) { + return new Page(0); + } + } + } + + if (filterFunction != null) { + RuntimeException[] errors = new RuntimeException[positionCount]; + + int[] inputChannels = filterFunction.getInputChannels(); + Block[] inputBlocks = new Block[inputChannels.length]; + + for (int i = 0; i < inputChannels.length; i++) { + inputBlocks[i] = page.getBlock(this.functionInputs.get(inputChannels[i])); + } + + Page inputPage = new Page(page.getPositionCount(), inputBlocks); + positionCount = filterFunction.filter(inputPage, positions, positionCount, errors); + for (int i = 0; i < positionCount; i++) { + if (errors[i] != null) { + throw errors[i]; + } + } + if (positionCount == 0) { + return new Page(0); + } + } + + if (outputBlockCount == page.getChannelCount()) { + return page.getPositions(positions, 0, positionCount); + } + + Block[] blocks = new Block[outputBlockCount]; + for (int i = 0; i < outputBlockCount; i++) { + blocks[i] = page.getBlock(i); + } + + return new Page(page.getPositionCount(), blocks).getPositions(positions, 0, positionCount); + } + + @Override + public long getSystemMemoryUsage() + { + return delegate.getSystemMemoryUsage(); + } + + @Override + public void close() + throws IOException + { + delegate.close(); + } + + private static int filterBlock(Block block, Type type, TupleDomainFilter filter, int[] positions, int positionCount) + { + int outputPositionsCount = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (block.isNull(position)) { + if (filter.testNull()) { + positions[outputPositionsCount] = position; + outputPositionsCount++; + } + } + else if (testNonNullPosition(block, position, type, filter)) { + positions[outputPositionsCount] = position; + outputPositionsCount++; + } + } + + return outputPositionsCount; + } + + private static boolean testNonNullPosition(Block block, int position, Type type, TupleDomainFilter filter) + { + if (type == BIGINT || type == INTEGER || type == SMALLINT || type == TINYINT || type == TIMESTAMP || type == DATE) { + return filter.testLong(type.getLong(block, position)); + } + + if (type == BOOLEAN) { + return filter.testBoolean(type.getBoolean(block, position)); + } + + if (type == DOUBLE) { + return filter.testDouble(longBitsToDouble(block.getLong(position))); + } + + if (type == REAL) { + return filter.testFloat(intBitsToFloat(block.getInt(position))); + } + + if (type instanceof DecimalType) { + if (((DecimalType) type).isShort()) { + return filter.testLong(block.getLong(position)); + } + else { + return filter.testDecimal(block.getLong(position, 0), block.getLong(position, Long.BYTES)); + } + } + + if (isVarcharType(type) || isCharType(type)) { + Slice slice = block.getSlice(position, 0, block.getSliceLength(position)); + return filter.testBytes((byte[]) slice.getBase(), (int) slice.getAddress() - ARRAY_BYTE_BASE_OFFSET, slice.length()); + } + + if (type instanceof ArrayType || type instanceof MapType || type instanceof RowType) { + if (IS_NULL == filter) { + return block.isNull(position); + } + if (IS_NOT_NULL == filter) { + return !block.isNull(position); + } + } + + throw new UnsupportedOperationException("Unexpected column type " + type); + } + + @Override + public long getCompletedBytes() + { + return delegate.getCompletedBytes(); + } + + @Override + public long getCompletedPositions() + { + return delegate.getCompletedPositions(); + } + + @Override + public long getReadTimeNanos() + { + return delegate.getReadTimeNanos(); + } + + @Override + public boolean isFinished() + { + return delegate.isFinished(); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursorProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursorProvider.java index 81a3dae489c3e..565d2769bf9af 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursorProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursorProvider.java @@ -31,7 +31,7 @@ import java.util.Optional; import java.util.Properties; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static java.util.Objects.requireNonNull; public class GenericHiveRecordCursorProvider diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HadoopDirectoryLister.java b/presto-hive/src/main/java/com/facebook/presto/hive/HadoopDirectoryLister.java index 392691d35ee00..011b5eb84235d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HadoopDirectoryLister.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HadoopDirectoryLister.java @@ -13,20 +13,51 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.util.HiveFileIterator; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import java.io.IOException; +import java.util.Iterator; +import java.util.Optional; + +import static com.facebook.presto.hive.HiveFileInfo.createHiveFileInfo; +import static com.facebook.presto.hive.util.HiveFileIterator.NestedDirectoryPolicy; +import static java.util.Objects.requireNonNull; public class HadoopDirectoryLister implements DirectoryLister { @Override - public RemoteIterator list(FileSystem fs, Path path) - throws IOException + public Iterator list(FileSystem fileSystem, Path path, NamenodeStats namenodeStats, NestedDirectoryPolicy nestedDirectoryPolicy) + { + return new HiveFileIterator(path, p -> new HadoopFileInfoIterator(fileSystem.listLocatedStatus(p)), namenodeStats, nestedDirectoryPolicy); + } + + public static class HadoopFileInfoIterator + implements RemoteIterator { - return fs.listLocatedStatus(path); + private final RemoteIterator locatedFileStatusIterator; + + public HadoopFileInfoIterator(RemoteIterator locatedFileStatusIterator) + { + this.locatedFileStatusIterator = requireNonNull(locatedFileStatusIterator, "locatedFileStatusIterator is null"); + } + + @Override + public boolean hasNext() + throws IOException + { + return locatedFileStatusIterator.hasNext(); + } + + @Override + public HiveFileInfo next() + throws IOException + { + return createHiveFileInfo(locatedFileStatusIterator.next(), Optional.empty()); + } } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HadoopFileOpener.java b/presto-hive/src/main/java/com/facebook/presto/hive/HadoopFileOpener.java new file mode 100644 index 0000000000000..8d2e9cfee2533 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HadoopFileOpener.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; +import java.util.Optional; + +public class HadoopFileOpener + implements FileOpener +{ + @Override + public FSDataInputStream open(FileSystem fileSystem, Path path, Optional extraFileInfo) + throws IOException + { + return fileSystem.open(path); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationUpdater.java b/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationInitializer.java similarity index 89% rename from presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationUpdater.java rename to presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationInitializer.java index da48fdabeb122..991b364614cd5 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationUpdater.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationInitializer.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.gcs.GcsConfigurationInitializer; import com.facebook.presto.hive.s3.S3ConfigurationUpdater; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -44,7 +45,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY; -public class HdfsConfigurationUpdater +public class HdfsConfigurationInitializer { private final HostAndPort socksProxy; private final Duration ipcPingInterval; @@ -55,23 +56,24 @@ public class HdfsConfigurationUpdater private final Configuration resourcesConfiguration; private final int fileSystemMaxCacheSize; private final S3ConfigurationUpdater s3ConfigurationUpdater; + private final GcsConfigurationInitializer gcsConfigurationInitialize; private final boolean isHdfsWireEncryptionEnabled; private int textMaxLineLength; @VisibleForTesting - public HdfsConfigurationUpdater(HiveClientConfig config) + public HdfsConfigurationInitializer(HiveClientConfig config, MetastoreClientConfig metastoreConfig) { - this(config, ignored -> {}); + this(config, metastoreConfig, ignored -> {}, ignored -> {}); } @Inject - public HdfsConfigurationUpdater(HiveClientConfig config, S3ConfigurationUpdater s3ConfigurationUpdater) + public HdfsConfigurationInitializer(HiveClientConfig config, MetastoreClientConfig metastoreConfig, S3ConfigurationUpdater s3ConfigurationUpdater, GcsConfigurationInitializer gcsConfigurationInitialize) { requireNonNull(config, "config is null"); checkArgument(config.getDfsTimeout().toMillis() >= 1, "dfsTimeout must be at least 1 ms"); checkArgument(toIntExact(config.getTextMaxLineLength().toBytes()) >= 1, "textMaxLineLength must be at least 1 byte"); - this.socksProxy = config.getMetastoreSocksProxy(); + this.socksProxy = metastoreConfig.getMetastoreSocksProxy(); this.ipcPingInterval = config.getIpcPingInterval(); this.dfsTimeout = config.getDfsTimeout(); this.dfsConnectTimeout = config.getDfsConnectTimeout(); @@ -83,6 +85,7 @@ public HdfsConfigurationUpdater(HiveClientConfig config, S3ConfigurationUpdater this.textMaxLineLength = toIntExact(config.getTextMaxLineLength().toBytes()); this.s3ConfigurationUpdater = requireNonNull(s3ConfigurationUpdater, "s3ConfigurationUpdater is null"); + this.gcsConfigurationInitialize = requireNonNull(gcsConfigurationInitialize, "gcsConfigurationInitialize is null"); } private static Configuration readConfiguration(List resourcePaths) @@ -134,6 +137,7 @@ public void updateConfiguration(Configuration config) config.setInt(LineRecordReader.MAX_LINE_LENGTH, textMaxLineLength); s3ConfigurationUpdater.updateConfiguration(config); + gcsConfigurationInitialize.updateConfiguration(config); } public static class NoOpDNSToSwitchMapping diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveAnalyzeProperties.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveAnalyzeProperties.java index 0c374efd99dfe..51491161e965b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveAnalyzeProperties.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveAnalyzeProperties.java @@ -25,7 +25,7 @@ import java.util.Map; import java.util.Optional; -import static com.facebook.presto.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; +import static com.facebook.presto.hive.metastore.MetastoreUtil.HIVE_DEFAULT_DYNAMIC_PARTITION; import static com.facebook.presto.spi.StandardErrorCode.INVALID_ANALYZE_PROPERTY; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.google.common.base.MoreObjects.firstNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveBatchPageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveBatchPageSourceFactory.java index eb0f7f9541f79..44dd232d98315 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveBatchPageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveBatchPageSourceFactory.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.predicate.TupleDomain; @@ -21,8 +22,8 @@ import org.joda.time.DateTimeZone; import java.util.List; +import java.util.Map; import java.util.Optional; -import java.util.Properties; public interface HiveBatchPageSourceFactory { @@ -33,8 +34,10 @@ Optional createPageSource( long start, long length, long fileSize, - Properties schema, + Storage storage, + Map tableParameters, List columns, TupleDomain effectivePredicate, - DateTimeZone hiveStorageTimeZone); + DateTimeZone hiveStorageTimeZone, + Optional extraFileInfo); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketing.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketing.java index 3b0fc84a21d68..b19f7b85899f9 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketing.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketing.java @@ -37,18 +37,18 @@ import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; import java.util.Set; import java.util.stream.Collectors; import static com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.HiveUtil.getRegularColumnHandles; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static io.airlift.slice.Slices.utf8Slice; import static java.lang.Double.doubleToLongBits; import static java.lang.Math.toIntExact; @@ -58,7 +58,7 @@ import static java.util.function.Function.identity; import static org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -final class HiveBucketing +public final class HiveBucketing { private static final Set SUPPORTED_TYPES_FOR_BUCKET_FILTER = ImmutableSet.of( HiveType.HIVE_BYTE, @@ -160,11 +160,11 @@ private static int hash(TypeInfo type, Block block, int position) } } case LIST: { - Block elementsBlock = block.getObject(position, Block.class); + Block elementsBlock = block.getBlock(position); return hashOfList((ListTypeInfo) type, elementsBlock); } case MAP: { - Block elementsBlock = block.getObject(position, Block.class); + Block elementsBlock = block.getBlock(position); return hashOfMap((MapTypeInfo) type, elementsBlock); } default: @@ -291,13 +291,14 @@ public static Optional getHiveBucketFilter(Table table, TupleD return Optional.empty(); } - Optional> bindings = TupleDomain.extractFixedValues(effectivePredicate); + Optional>> bindings = TupleDomain.extractFixedValueSets(effectivePredicate); if (!bindings.isPresent()) { return Optional.empty(); } - OptionalInt singleBucket = getHiveBucket(table, bindings.get()); - if (singleBucket.isPresent()) { - return Optional.of(new HiveBucketFilter(ImmutableSet.of(singleBucket.getAsInt()))); + + Optional> buckets = getHiveBuckets(table, bindings.get()); + if (buckets.isPresent()) { + return Optional.of(new HiveBucketFilter(buckets.get())); } if (!effectivePredicate.getDomains().isPresent()) { @@ -321,49 +322,62 @@ public static Optional getHiveBucketFilter(Table table, TupleD return Optional.of(new HiveBucketFilter(builder.build())); } - private static OptionalInt getHiveBucket(Table table, Map bindings) + private static Optional> getHiveBuckets(Table table, Map> bindings) { - if (bindings.isEmpty()) { - return OptionalInt.empty(); - } - List bucketColumns = table.getStorage().getBucketProperty().get().getBucketedBy(); - Map hiveTypes = new HashMap<>(); - for (Column column : table.getDataColumns()) { - hiveTypes.put(column.getName(), column.getType()); + if (bucketColumns.isEmpty()) { + return Optional.empty(); } + Map hiveTypes = table.getDataColumns().stream() + .collect(toImmutableMap(Column::getName, Column::getType)); + // Verify the bucket column types are supported for (String column : bucketColumns) { if (!SUPPORTED_TYPES_FOR_BUCKET_FILTER.contains(hiveTypes.get(column))) { - return OptionalInt.empty(); + return Optional.empty(); } } - // Get bindings for bucket columns - Map bucketBindings = new HashMap<>(); - for (Entry entry : bindings.entrySet()) { - HiveColumnHandle colHandle = (HiveColumnHandle) entry.getKey(); - if (!entry.getValue().isNull() && bucketColumns.contains(colHandle.getName())) { - bucketBindings.put(colHandle.getName(), entry.getValue().getValue()); + Map> nameToBindings = bindings.entrySet().stream() + .collect(toImmutableMap(entry -> ((HiveColumnHandle) entry.getKey()).getName(), Entry::getValue)); + + ImmutableList.Builder> orderedBindingsBuilder = ImmutableList.builder(); + for (String columnName : bucketColumns) { + if (!nameToBindings.containsKey(columnName)) { + return Optional.empty(); } + orderedBindingsBuilder.add(nameToBindings.get(columnName)); } - // Check that we have bindings for all bucket columns - if (bucketBindings.size() != bucketColumns.size()) { - return OptionalInt.empty(); - } + List> orderedBindings = orderedBindingsBuilder.build(); + int bucketCount = table.getStorage().getBucketProperty().get().getBucketCount(); + List types = bucketColumns.stream() + .map(hiveTypes::get) + .map(HiveType::getTypeInfo) + .collect(toImmutableList()); + ImmutableSet.Builder buckets = ImmutableSet.builder(); + getHiveBuckets(new Object[types.size()], 0, orderedBindings, bucketCount, types, buckets); + return Optional.of(buckets.build()); + } - // Get bindings of bucket columns - ImmutableList.Builder typeInfos = ImmutableList.builder(); - Object[] values = new Object[bucketColumns.size()]; - for (int i = 0; i < bucketColumns.size(); i++) { - String column = bucketColumns.get(i); - typeInfos.add(hiveTypes.get(column).getTypeInfo()); - values[i] = bucketBindings.get(column); + private static void getHiveBuckets( + Object[] values, + int valuesCount, + List> bindings, + int bucketCount, + List typeInfos, + ImmutableSet.Builder buckets) + { + if (valuesCount == typeInfos.size()) { + buckets.add(getHiveBucket(bucketCount, typeInfos, values)); + return; } - return OptionalInt.of(getHiveBucket(table.getStorage().getBucketProperty().get().getBucketCount(), typeInfos.build(), values)); + for (NullableValue value : bindings.get(valuesCount)) { + values[valuesCount] = value.getValue(); + getHiveBuckets(values, valuesCount + 1, bindings, bucketCount, typeInfos, buckets); + } } public static class HiveBucketFilter diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java index 86dc4bfb21199..3c5d1ac5576a7 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java @@ -13,15 +13,14 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.DefunctConfig; +import com.facebook.airlift.configuration.LegacyConfig; import com.facebook.presto.hive.s3.S3FileSystemType; import com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import com.google.common.net.HostAndPort; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.DefunctConfig; -import io.airlift.configuration.LegacyConfig; import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.airlift.units.MaxDataSize; @@ -41,7 +40,6 @@ import static com.facebook.presto.hive.HiveStorageFormat.ORC; import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static java.util.concurrent.TimeUnit.MINUTES; @DefunctConfig({ "hive.file-system-cache-ttl", @@ -73,19 +71,10 @@ public class HiveClientConfig private boolean allowCorruptWritesForTesting; - private Duration metastoreCacheTtl = new Duration(0, TimeUnit.SECONDS); - private Duration metastoreRefreshInterval = new Duration(0, TimeUnit.SECONDS); - private long metastoreCacheMaximumSize = 10000; - private long perTransactionMetastoreCacheMaximumSize = 1000; - private int maxMetastoreRefreshThreads = 100; - private HostAndPort metastoreSocksProxy; - private Duration metastoreTimeout = new Duration(10, TimeUnit.SECONDS); - private Duration ipcPingInterval = new Duration(10, TimeUnit.SECONDS); private Duration dfsTimeout = new Duration(60, TimeUnit.SECONDS); private Duration dfsConnectTimeout = new Duration(500, TimeUnit.MILLISECONDS); private int dfsConnectMaxRetries = 5; - private boolean verifyChecksum = true; private String domainSocketPath; private S3FileSystemType s3FileSystemType = S3FileSystemType.PRESTO; @@ -104,6 +93,7 @@ public class HiveClientConfig private boolean useParquetColumnNames; private boolean failOnCorruptedParquetStatistics = true; + private DataSize parquetMaxReadBlockSize = new DataSize(16, MEGABYTE); private boolean assumeCanonicalPartitionKeys; @@ -135,6 +125,9 @@ public class HiveClientConfig private boolean sortedWritingEnabled = true; private boolean ignoreTableBucketing; private int maxBucketsForGroupedExecution = 1_000_000; + // TODO: Clean up this gatekeeper config and related code/session property once the roll out is done. + private boolean sortedWriteToTempPathEnabled; + private int sortedWriteTempPathSubdirectoryCount = 10; private int fileSystemMaxCacheSize = 1000; @@ -147,9 +140,6 @@ public class HiveClientConfig private boolean ignoreCorruptedStatistics; private boolean collectColumnStatisticsOnWrite; - private String recordingPath; - private boolean replay; - private Duration recordingDuration = new Duration(0, MINUTES); private boolean s3SelectPushdownEnabled; private int s3SelectPushdownMaxConnections = 500; @@ -161,6 +151,8 @@ public class HiveClientConfig private HiveCompressionCodec temporaryTableCompressionCodec = HiveCompressionCodec.SNAPPY; private boolean pushdownFilterEnabled; + private boolean rangeFiltersOnSubscriptsEnabled; + private boolean zstdJniDecompressionEnabled; public int getMaxInitialSplits() { @@ -381,98 +373,6 @@ public HiveClientConfig setAllowCorruptWritesForTesting(boolean allowCorruptWrit return this; } - @NotNull - public Duration getMetastoreCacheTtl() - { - return metastoreCacheTtl; - } - - @MinDuration("0ms") - @Config("hive.metastore-cache-ttl") - public HiveClientConfig setMetastoreCacheTtl(Duration metastoreCacheTtl) - { - this.metastoreCacheTtl = metastoreCacheTtl; - return this; - } - - @NotNull - public Duration getMetastoreRefreshInterval() - { - return metastoreRefreshInterval; - } - - @MinDuration("1ms") - @Config("hive.metastore-refresh-interval") - public HiveClientConfig setMetastoreRefreshInterval(Duration metastoreRefreshInterval) - { - this.metastoreRefreshInterval = metastoreRefreshInterval; - return this; - } - - public long getMetastoreCacheMaximumSize() - { - return metastoreCacheMaximumSize; - } - - @Min(1) - @Config("hive.metastore-cache-maximum-size") - public HiveClientConfig setMetastoreCacheMaximumSize(long metastoreCacheMaximumSize) - { - this.metastoreCacheMaximumSize = metastoreCacheMaximumSize; - return this; - } - - public long getPerTransactionMetastoreCacheMaximumSize() - { - return perTransactionMetastoreCacheMaximumSize; - } - - @Min(1) - @Config("hive.per-transaction-metastore-cache-maximum-size") - public HiveClientConfig setPerTransactionMetastoreCacheMaximumSize(long perTransactionMetastoreCacheMaximumSize) - { - this.perTransactionMetastoreCacheMaximumSize = perTransactionMetastoreCacheMaximumSize; - return this; - } - - @Min(1) - public int getMaxMetastoreRefreshThreads() - { - return maxMetastoreRefreshThreads; - } - - @Config("hive.metastore-refresh-max-threads") - public HiveClientConfig setMaxMetastoreRefreshThreads(int maxMetastoreRefreshThreads) - { - this.maxMetastoreRefreshThreads = maxMetastoreRefreshThreads; - return this; - } - - public HostAndPort getMetastoreSocksProxy() - { - return metastoreSocksProxy; - } - - @Config("hive.metastore.thrift.client.socks-proxy") - public HiveClientConfig setMetastoreSocksProxy(HostAndPort metastoreSocksProxy) - { - this.metastoreSocksProxy = metastoreSocksProxy; - return this; - } - - @NotNull - public Duration getMetastoreTimeout() - { - return metastoreTimeout; - } - - @Config("hive.metastore-timeout") - public HiveClientConfig setMetastoreTimeout(Duration metastoreTimeout) - { - this.metastoreTimeout = metastoreTimeout; - return this; - } - @Min(1) public int getMinPartitionBatchSize() { @@ -691,18 +591,6 @@ public HiveClientConfig setS3FileSystemType(S3FileSystemType s3FileSystemType) return this; } - public boolean isVerifyChecksum() - { - return verifyChecksum; - } - - @Config("hive.dfs.verify-checksum") - public HiveClientConfig setVerifyChecksum(boolean verifyChecksum) - { - this.verifyChecksum = verifyChecksum; - return this; - } - public boolean isUseOrcColumnNames() { return useOrcColumnNames; @@ -946,6 +834,19 @@ public HiveClientConfig setFailOnCorruptedParquetStatistics(boolean failOnCorrup return this; } + @NotNull + public DataSize getParquetMaxReadBlockSize() + { + return parquetMaxReadBlockSize; + } + + @Config("hive.parquet.max-read-block-size") + public HiveClientConfig setParquetMaxReadBlockSize(DataSize parquetMaxReadBlockSize) + { + this.parquetMaxReadBlockSize = parquetMaxReadBlockSize; + return this; + } + @Deprecated public boolean isOptimizeMismatchedBucketCount() { @@ -960,6 +861,18 @@ public HiveClientConfig setOptimizeMismatchedBucketCount(boolean optimizeMismatc return this; } + public boolean isZstdJniDecompressionEnabled() + { + return zstdJniDecompressionEnabled; + } + + @Config("hive.zstd-jni-decompression-enabled") + public HiveClientConfig setZstdJniDecompressionEnabled(boolean zstdJniDecompressionEnabled) + { + this.zstdJniDecompressionEnabled = zstdJniDecompressionEnabled; + return this; + } + public enum HiveMetastoreAuthenticationType { NONE, @@ -1104,6 +1017,32 @@ public int getMaxBucketsForGroupedExecution() return maxBucketsForGroupedExecution; } + @Config("hive.sorted-write-to-temp-path-enabled") + @ConfigDescription("Enable writing temp files to temp path when writing to bucketed sorted tables") + public HiveClientConfig setSortedWriteToTempPathEnabled(boolean sortedWriteToTempPathEnabled) + { + this.sortedWriteToTempPathEnabled = sortedWriteToTempPathEnabled; + return this; + } + + public boolean isSortedWriteToTempPathEnabled() + { + return sortedWriteToTempPathEnabled; + } + + @Config("hive.sorted-write-temp-path-subdirectory-count") + @ConfigDescription("Number of directories per partition for temp files generated by writing sorted table") + public HiveClientConfig setSortedWriteTempPathSubdirectoryCount(int sortedWriteTempPathSubdirectoryCount) + { + this.sortedWriteTempPathSubdirectoryCount = sortedWriteTempPathSubdirectoryCount; + return this; + } + + public int getSortedWriteTempPathSubdirectoryCount() + { + return sortedWriteTempPathSubdirectoryCount; + } + public int getFileSystemMaxCacheSize() { return fileSystemMaxCacheSize; @@ -1196,43 +1135,6 @@ public HiveClientConfig setCollectColumnStatisticsOnWrite(boolean collectColumnS return this; } - public String getRecordingPath() - { - return recordingPath; - } - - @Config("hive.metastore-recording-path") - public HiveClientConfig setRecordingPath(String recordingPath) - { - this.recordingPath = recordingPath; - return this; - } - - public boolean isReplay() - { - return replay; - } - - @Config("hive.replay-metastore-recording") - public HiveClientConfig setReplay(boolean replay) - { - this.replay = replay; - return this; - } - - @NotNull - public Duration getRecordingDuration() - { - return recordingDuration; - } - - @Config("hive.metastore-recoding-duration") - public HiveClientConfig setRecordingDuration(Duration recordingDuration) - { - this.recordingDuration = recordingDuration; - return this; - } - public boolean isS3SelectPushdownEnabled() { return s3SelectPushdownEnabled; @@ -1337,4 +1239,17 @@ public HiveClientConfig setPushdownFilterEnabled(boolean pushdownFilterEnabled) this.pushdownFilterEnabled = pushdownFilterEnabled; return this; } + + public boolean isRangeFiltersOnSubscriptsEnabled() + { + return rangeFiltersOnSubscriptsEnabled; + } + + @Config("hive.range-filters-on-subscripts-enabled") + @ConfigDescription("Experimental: enable pushdown of range filters on subscripts (a[2] = 5) into ORC column readers") + public HiveClientConfig setRangeFiltersOnSubscriptsEnabled(boolean rangeFiltersOnSubscriptsEnabled) + { + this.rangeFiltersOnSubscriptsEnabled = rangeFiltersOnSubscriptsEnabled; + return this; + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index fc1a99e6ec314..42149de2c3932 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -13,7 +13,9 @@ */ package com.facebook.presto.hive; -import com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.concurrent.ExecutorServiceAdapter; +import com.facebook.airlift.event.client.EventClient; import com.facebook.presto.hive.orc.DwrfBatchPageSourceFactory; import com.facebook.presto.hive.orc.DwrfSelectivePageSourceFactory; import com.facebook.presto.hive.orc.OrcBatchPageSourceFactory; @@ -22,11 +24,25 @@ import com.facebook.presto.hive.rcfile.RcFilePageSourceFactory; import com.facebook.presto.hive.rule.HivePlanOptimizerProvider; import com.facebook.presto.hive.s3.PrestoS3ClientFactory; +import com.facebook.presto.orc.CacheStatsMBean; +import com.facebook.presto.orc.CachingStripeMetadataSource; +import com.facebook.presto.orc.OrcDataSourceId; +import com.facebook.presto.orc.StorageStripeMetadataSource; +import com.facebook.presto.orc.StripeMetadataSource; +import com.facebook.presto.orc.StripeReader.StripeId; +import com.facebook.presto.orc.StripeReader.StripeStreamId; +import com.facebook.presto.orc.cache.CachingOrcFileTailSource; +import com.facebook.presto.orc.cache.OrcCacheConfig; +import com.facebook.presto.orc.cache.OrcFileTailSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; +import com.facebook.presto.orc.metadata.OrcFileTail; import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider; import com.facebook.presto.spi.connector.ConnectorPageSinkProvider; import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; import com.facebook.presto.spi.connector.ConnectorPlanOptimizerProvider; import com.facebook.presto.spi.connector.ConnectorSplitManager; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.inject.Binder; import com.google.inject.Module; @@ -34,21 +50,21 @@ import com.google.inject.Scopes; import com.google.inject.TypeLiteral; import com.google.inject.multibindings.Multibinder; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.concurrent.ExecutorServiceAdapter; -import io.airlift.event.client.EventClient; +import io.airlift.slice.Slice; +import org.weakref.jmx.MBeanExporter; import javax.inject.Singleton; import java.util.concurrent.ExecutorService; -import java.util.function.Function; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; import static com.google.inject.multibindings.Multibinder.newSetBinder; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static java.lang.Math.toIntExact; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newFixedThreadPool; import static org.weakref.jmx.ObjectNames.generatedNameOf; @@ -71,7 +87,8 @@ public void configure(Binder binder) binder.bind(TypeTranslator.class).toInstance(new HiveTypeTranslator()); binder.bind(CoercionPolicy.class).to(HiveCoercionPolicy.class).in(Scopes.SINGLETON); - binder.bind(HdfsConfigurationUpdater.class).in(Scopes.SINGLETON); + binder.bind(HdfsConfigurationInitializer.class).in(Scopes.SINGLETON); + newSetBinder(binder, DynamicConfigurationProvider.class); binder.bind(HdfsConfiguration.class).to(HiveHdfsConfiguration.class).in(Scopes.SINGLETON); binder.bind(HdfsEnvironment.class).in(Scopes.SINGLETON); binder.bind(DirectoryLister.class).to(HadoopDirectoryLister.class).in(Scopes.SINGLETON); @@ -101,6 +118,7 @@ public void configure(Binder binder) binder.bind(new TypeLiteral>() {}).to(HiveMetadataFactory.class).in(Scopes.SINGLETON); binder.bind(StagingFileCommitter.class).to(HiveStagingFileCommitter.class).in(Scopes.SINGLETON); binder.bind(ZeroRowFileCreator.class).to(HiveZeroRowFileCreator.class).in(Scopes.SINGLETON); + binder.bind(PartitionObjectBuilder.class).to(HivePartitionObjectBuilder.class).in(Scopes.SINGLETON); binder.bind(HiveTransactionManager.class).in(Scopes.SINGLETON); binder.bind(ConnectorSplitManager.class).to(HiveSplitManager.class).in(Scopes.SINGLETON); newExporter(binder).export(ConnectorSplitManager.class).as(generatedNameOf(HiveSplitManager.class, connectorId)); @@ -120,6 +138,10 @@ public void configure(Binder binder) pageSourceFactoryBinder.addBinding().to(ParquetPageSourceFactory.class).in(Scopes.SINGLETON); pageSourceFactoryBinder.addBinding().to(RcFilePageSourceFactory.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(OrcCacheConfig.class, connectorId); + + binder.bind(FileOpener.class).to(HadoopFileOpener.class).in(Scopes.SINGLETON); + Multibinder selectivePageSourceFactoryBinder = newSetBinder(binder, HiveSelectivePageSourceFactory.class); selectivePageSourceFactoryBinder.addBinding().to(OrcSelectivePageSourceFactory.class).in(Scopes.SINGLETON); selectivePageSourceFactoryBinder.addBinding().to(DwrfSelectivePageSourceFactory.class).in(Scopes.SINGLETON); @@ -132,6 +154,7 @@ public void configure(Binder binder) fileWriterFactoryBinder.addBinding().to(RcFileFileWriterFactory.class).in(Scopes.SINGLETON); configBinder(binder).bindConfig(ParquetFileWriterConfig.class); + binder.install(new MetastoreClientModule()); } @ForHiveClient @@ -145,10 +168,10 @@ public ExecutorService createHiveClientExecutor(HiveConnectorId hiveClientId) @ForCachingHiveMetastore @Singleton @Provides - public ExecutorService createCachingHiveMetastoreExecutor(HiveConnectorId hiveClientId, HiveClientConfig hiveClientConfig) + public ExecutorService createCachingHiveMetastoreExecutor(HiveConnectorId hiveClientId, MetastoreClientConfig metastoreClientConfig) { return newFixedThreadPool( - hiveClientConfig.getMaxMetastoreRefreshThreads(), + metastoreClientConfig.getMaxMetastoreRefreshThreads(), daemonThreadsNamed("hive-metastore-" + hiveClientId + "-%s")); } @@ -178,8 +201,47 @@ public ListeningExecutorService createZeroRowFileCreatorExecutor(HiveConnectorId @Singleton @Provides - public Function createMetastoreGetter(HiveTransactionManager transactionManager) + public OrcFileTailSource createOrcFileTailSource(OrcCacheConfig orcCacheConfig, MBeanExporter exporter) + { + OrcFileTailSource orcFileTailSource = new StorageOrcFileTailSource(); + if (orcCacheConfig.isFileTailCacheEnabled()) { + Cache cache = CacheBuilder.newBuilder() + .maximumWeight(orcCacheConfig.getFileTailCacheSize().toBytes()) + .weigher((id, tail) -> ((OrcFileTail) tail).getFooterSize() + ((OrcFileTail) tail).getMetadataSize()) + .expireAfterAccess(orcCacheConfig.getFileTailCacheTtlSinceLastAccess().toMillis(), TimeUnit.MILLISECONDS) + .recordStats() + .build(); + CacheStatsMBean cacheStatsMBean = new CacheStatsMBean(cache); + orcFileTailSource = new CachingOrcFileTailSource(orcFileTailSource, cache); + exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_OrcFileTail"), cacheStatsMBean); + } + return orcFileTailSource; + } + + @Singleton + @Provides + public StripeMetadataSource createStripeMetadataSource(OrcCacheConfig orcCacheConfig, MBeanExporter exporter) { - return transactionHandle -> ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore(); + StripeMetadataSource stripeMetadataSource = new StorageStripeMetadataSource(); + if (orcCacheConfig.isStripeMetadataCacheEnabled()) { + Cache footerCache = CacheBuilder.newBuilder() + .maximumWeight(orcCacheConfig.getStripeFooterCacheSize().toBytes()) + .weigher((id, footer) -> toIntExact(((Slice) footer).getRetainedSize())) + .expireAfterAccess(orcCacheConfig.getStripeFooterCacheTtlSinceLastAccess().toMillis(), TimeUnit.MILLISECONDS) + .recordStats() + .build(); + Cache streamCache = CacheBuilder.newBuilder() + .maximumWeight(orcCacheConfig.getStripeStreamCacheSize().toBytes()) + .weigher((id, stream) -> toIntExact(((Slice) stream).getRetainedSize())) + .expireAfterAccess(orcCacheConfig.getStripeStreamCacheTtlSinceLastAccess().toMillis(), TimeUnit.MILLISECONDS) + .recordStats() + .build(); + CacheStatsMBean footerCacheStatsMBean = new CacheStatsMBean(footerCache); + CacheStatsMBean streamCacheStatsMBean = new CacheStatsMBean(streamCache); + stripeMetadataSource = new CachingStripeMetadataSource(stripeMetadataSource, footerCache, streamCache); + exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_StripeFooter"), footerCacheStatsMBean); + exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_StripeStream"), streamCacheStatsMBean); + } + return stripeMetadataSource; } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercer.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercer.java new file mode 100644 index 0000000000000..741720fc76c47 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercer.java @@ -0,0 +1,555 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.block.ArrayBlock; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.block.ColumnarArray; +import com.facebook.presto.spi.block.ColumnarMap; +import com.facebook.presto.spi.block.ColumnarRow; +import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.block.RowBlock; +import com.facebook.presto.spi.type.MapType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.spi.type.VarcharType; +import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; + +import java.util.List; +import java.util.Optional; +import java.util.function.Function; + +import static com.facebook.presto.hive.HiveType.HIVE_BYTE; +import static com.facebook.presto.hive.HiveType.HIVE_DOUBLE; +import static com.facebook.presto.hive.HiveType.HIVE_FLOAT; +import static com.facebook.presto.hive.HiveType.HIVE_INT; +import static com.facebook.presto.hive.HiveType.HIVE_LONG; +import static com.facebook.presto.hive.HiveType.HIVE_SHORT; +import static com.facebook.presto.hive.HiveUtil.extractStructFieldNames; +import static com.facebook.presto.hive.HiveUtil.extractStructFieldTypes; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isArrayType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isMapType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isRowType; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NOT_NULL; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NULL; +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.spi.block.ColumnarArray.toColumnarArray; +import static com.facebook.presto.spi.block.ColumnarMap.toColumnarMap; +import static com.facebook.presto.spi.block.ColumnarRow.toColumnarRow; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RealType.REAL; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.google.common.base.Preconditions.checkArgument; +import static io.airlift.slice.Slices.utf8Slice; +import static java.lang.Float.intBitsToFloat; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public interface HiveCoercer + extends Function +{ + TupleDomainFilter toCoercingFilter(TupleDomainFilter filter, Subfield subfield); + + Type getToType(); + + static HiveCoercer createCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) + { + Type fromType = typeManager.getType(fromHiveType.getTypeSignature()); + Type toType = typeManager.getType(toHiveType.getTypeSignature()); + if (toType instanceof VarcharType && (fromHiveType.equals(HIVE_BYTE) || fromHiveType.equals(HIVE_SHORT) || fromHiveType.equals(HIVE_INT) || fromHiveType.equals(HIVE_LONG))) { + return new IntegerNumberToVarcharCoercer(fromType, toType); + } + else if (fromType instanceof VarcharType && (toHiveType.equals(HIVE_BYTE) || toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG))) { + return new VarcharToIntegerNumberCoercer(fromType, toType); + } + else if (fromHiveType.equals(HIVE_BYTE) && toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG)) { + return new IntegerNumberUpscaleCoercer(fromType, toType); + } + else if (fromHiveType.equals(HIVE_SHORT) && toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG)) { + return new IntegerNumberUpscaleCoercer(fromType, toType); + } + else if (fromHiveType.equals(HIVE_INT) && toHiveType.equals(HIVE_LONG)) { + return new IntegerNumberUpscaleCoercer(fromType, toType); + } + else if (fromHiveType.equals(HIVE_FLOAT) && toHiveType.equals(HIVE_DOUBLE)) { + return new FloatToDoubleCoercer(); + } + else if (isArrayType(fromType) && isArrayType(toType)) { + return new ListCoercer(typeManager, fromHiveType, toHiveType); + } + else if (isMapType(fromType) && isMapType(toType)) { + return new MapCoercer(typeManager, fromHiveType, toHiveType); + } + else if (isRowType(fromType) && isRowType(toType)) { + return new StructCoercer(typeManager, fromHiveType, toHiveType); + } + + throw new PrestoException(NOT_SUPPORTED, format("Unsupported coercion from %s to %s", fromHiveType, toHiveType)); + } + + class IntegerNumberUpscaleCoercer + implements HiveCoercer + { + private final Type fromType; + private final Type toType; + + public IntegerNumberUpscaleCoercer(Type fromType, Type toType) + { + this.fromType = requireNonNull(fromType, "fromType is null"); + this.toType = requireNonNull(toType, "toType is null"); + } + + @Override + public Block apply(Block block) + { + BlockBuilder blockBuilder = toType.createBlockBuilder(null, block.getPositionCount()); + for (int i = 0; i < block.getPositionCount(); i++) { + if (block.isNull(i)) { + blockBuilder.appendNull(); + continue; + } + toType.writeLong(blockBuilder, fromType.getLong(block, i)); + } + return blockBuilder.build(); + } + + @Override + public TupleDomainFilter toCoercingFilter(TupleDomainFilter filter, Subfield subfield) + { + checkArgument(subfield.getPath().isEmpty(), "Subfields on primitive types are not allowed"); + return filter; + } + + @Override + public Type getToType() + { + return toType; + } + } + + class IntegerNumberToVarcharCoercer + implements HiveCoercer + { + private final Type fromType; + private final Type toType; + + public IntegerNumberToVarcharCoercer(Type fromType, Type toType) + { + this.fromType = requireNonNull(fromType, "fromType is null"); + this.toType = requireNonNull(toType, "toType is null"); + } + + @Override + public Block apply(Block block) + { + BlockBuilder blockBuilder = toType.createBlockBuilder(null, block.getPositionCount()); + for (int i = 0; i < block.getPositionCount(); i++) { + if (block.isNull(i)) { + blockBuilder.appendNull(); + continue; + } + toType.writeSlice(blockBuilder, utf8Slice(String.valueOf(fromType.getLong(block, i)))); + } + return blockBuilder.build(); + } + + @Override + public TupleDomainFilter toCoercingFilter(TupleDomainFilter filter, Subfield subfield) + { + checkArgument(subfield.getPath().isEmpty(), "Subfields on primitive types are not allowed"); + return new CoercingFilter(filter); + } + + private static final class CoercingFilter + extends TupleDomainFilter.AbstractTupleDomainFilter + { + private final TupleDomainFilter delegate; + + public CoercingFilter(TupleDomainFilter delegate) + { + super(delegate.isDeterministic(), !delegate.isDeterministic() || delegate.testNull()); + this.delegate = requireNonNull(delegate, "delegate is null"); + } + + @Override + public boolean testNull() + { + return delegate.testNull(); + } + + @Override + public boolean testLong(long value) + { + byte[] bytes = String.valueOf(value).getBytes(); + return delegate.testBytes(bytes, 0, bytes.length); + } + } + + @Override + public Type getToType() + { + return toType; + } + } + + class VarcharToIntegerNumberCoercer + implements HiveCoercer + { + private final Type fromType; + private final Type toType; + + private final long minValue; + private final long maxValue; + + public VarcharToIntegerNumberCoercer(Type fromType, Type toType) + { + this.fromType = requireNonNull(fromType, "fromType is null"); + this.toType = requireNonNull(toType, "toType is null"); + + if (toType.equals(TINYINT)) { + minValue = Byte.MIN_VALUE; + maxValue = Byte.MAX_VALUE; + } + else if (toType.equals(SMALLINT)) { + minValue = Short.MIN_VALUE; + maxValue = Short.MAX_VALUE; + } + else if (toType.equals(INTEGER)) { + minValue = Integer.MIN_VALUE; + maxValue = Integer.MAX_VALUE; + } + else if (toType.equals(BIGINT)) { + minValue = Long.MIN_VALUE; + maxValue = Long.MAX_VALUE; + } + else { + throw new PrestoException(NOT_SUPPORTED, format("Could not create Coercer from from varchar to %s", toType)); + } + } + + @Override + public Block apply(Block block) + { + BlockBuilder blockBuilder = toType.createBlockBuilder(null, block.getPositionCount()); + for (int i = 0; i < block.getPositionCount(); i++) { + if (block.isNull(i)) { + blockBuilder.appendNull(); + continue; + } + try { + long value = Long.parseLong(fromType.getSlice(block, i).toStringUtf8()); + if (minValue <= value && value <= maxValue) { + toType.writeLong(blockBuilder, value); + } + else { + blockBuilder.appendNull(); + } + } + catch (NumberFormatException e) { + blockBuilder.appendNull(); + } + } + return blockBuilder.build(); + } + + @Override + public TupleDomainFilter toCoercingFilter(TupleDomainFilter filter, Subfield subfield) + { + checkArgument(subfield.getPath().isEmpty(), "Subfields on primitive types are not allowed"); + return new CoercingFilter(filter, minValue, maxValue); + } + + private static final class CoercingFilter + extends TupleDomainFilter.AbstractTupleDomainFilter + { + private final TupleDomainFilter delegate; + private final long minValue; + private final long maxValue; + + public CoercingFilter(TupleDomainFilter delegate, long minValue, long maxValue) + { + super(delegate.isDeterministic(), !delegate.isDeterministic() || delegate.testNull()); + this.delegate = requireNonNull(delegate, "delegate is null"); + this.minValue = minValue; + this.maxValue = maxValue; + } + + @Override + public boolean testNull() + { + return delegate.testNull(); + } + + @Override + public boolean testLength(int length) + { + return true; + } + + @Override + public boolean testBytes(byte[] buffer, int offset, int length) + { + long value; + try { + value = Long.valueOf(new String(buffer, offset, length)); + } + catch (NumberFormatException e) { + return delegate.testNull(); + } + + if (minValue <= value && value <= maxValue) { + return delegate.testLong(value); + } + else { + return delegate.testNull(); + } + } + } + + @Override + public Type getToType() + { + return toType; + } + } + + class FloatToDoubleCoercer + implements HiveCoercer + { + @Override + public Block apply(Block block) + { + BlockBuilder blockBuilder = DOUBLE.createBlockBuilder(null, block.getPositionCount()); + for (int i = 0; i < block.getPositionCount(); i++) { + if (block.isNull(i)) { + blockBuilder.appendNull(); + continue; + } + DOUBLE.writeDouble(blockBuilder, intBitsToFloat((int) REAL.getLong(block, i))); + } + return blockBuilder.build(); + } + + @Override + public TupleDomainFilter toCoercingFilter(TupleDomainFilter filter, Subfield subfield) + { + checkArgument(subfield.getPath().isEmpty(), "Subfields on primitive types are not allowed"); + return filter; + } + @Override + public Type getToType() + { + return DOUBLE; + } + } + + class ListCoercer + implements HiveCoercer + { + private final HiveCoercer elementCoercer; + private final Type toType; + + public ListCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) + { + requireNonNull(typeManager, "typeManage is null"); + requireNonNull(fromHiveType, "fromHiveType is null"); + requireNonNull(toHiveType, "toHiveType is null"); + HiveType fromElementHiveType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); + HiveType toElementHiveType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); + this.elementCoercer = fromElementHiveType.equals(toElementHiveType) ? null : createCoercer(typeManager, fromElementHiveType, toElementHiveType); + this.toType = toHiveType.getType(typeManager); + } + + @Override + public Block apply(Block block) + { + if (elementCoercer == null) { + return block; + } + ColumnarArray arrayBlock = toColumnarArray(block); + Block elementsBlock = elementCoercer.apply(arrayBlock.getElementsBlock()); + boolean[] valueIsNull = new boolean[arrayBlock.getPositionCount()]; + int[] offsets = new int[arrayBlock.getPositionCount() + 1]; + for (int i = 0; i < arrayBlock.getPositionCount(); i++) { + valueIsNull[i] = arrayBlock.isNull(i); + offsets[i + 1] = offsets[i] + arrayBlock.getLength(i); + } + return ArrayBlock.fromElementBlock(arrayBlock.getPositionCount(), Optional.of(valueIsNull), offsets, elementsBlock); + } + + @Override + public TupleDomainFilter toCoercingFilter(TupleDomainFilter filter, Subfield subfield) + { + if (filter == IS_NULL || filter == IS_NOT_NULL) { + return filter; + } + + throw new UnsupportedOperationException("Range filers on array elements are not supported"); + } + + @Override + public Type getToType() + { + return toType; + } + } + + class MapCoercer + implements HiveCoercer + { + private final Type toType; + private final HiveCoercer keyCoercer; + private final HiveCoercer valueCoercer; + + public MapCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) + { + requireNonNull(typeManager, "typeManage is null"); + requireNonNull(fromHiveType, "fromHiveType is null"); + this.toType = requireNonNull(toHiveType, "toHiveType is null").getType(typeManager); + HiveType fromKeyHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); + HiveType fromValueHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); + HiveType toKeyHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); + HiveType toValueHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); + this.keyCoercer = fromKeyHiveType.equals(toKeyHiveType) ? null : createCoercer(typeManager, fromKeyHiveType, toKeyHiveType); + this.valueCoercer = fromValueHiveType.equals(toValueHiveType) ? null : createCoercer(typeManager, fromValueHiveType, toValueHiveType); + } + + @Override + public Block apply(Block block) + { + ColumnarMap mapBlock = toColumnarMap(block); + Block keysBlock = keyCoercer == null ? mapBlock.getKeysBlock() : keyCoercer.apply(mapBlock.getKeysBlock()); + Block valuesBlock = valueCoercer == null ? mapBlock.getValuesBlock() : valueCoercer.apply(mapBlock.getValuesBlock()); + int positionCount = mapBlock.getPositionCount(); + boolean[] valueIsNull = new boolean[positionCount]; + int[] offsets = new int[positionCount + 1]; + for (int i = 0; i < positionCount; i++) { + valueIsNull[i] = mapBlock.isNull(i); + offsets[i + 1] = offsets[i] + mapBlock.getEntryCount(i); + } + return ((MapType) toType).createBlockFromKeyValue(positionCount, Optional.of(valueIsNull), offsets, keysBlock, valuesBlock); + } + + @Override + public TupleDomainFilter toCoercingFilter(TupleDomainFilter filter, Subfield subfield) + { + if (filter == IS_NULL || filter == IS_NOT_NULL) { + return filter; + } + + throw new UnsupportedOperationException("Range filers on map elements are not supported"); + } + + @Override + public Type getToType() + { + return toType; + } + } + + class StructCoercer + implements HiveCoercer + { + private final HiveCoercer[] coercers; + private final Block[] nullBlocks; + private final List toFieldNames; + private final Type toType; + + public StructCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) + { + requireNonNull(typeManager, "typeManage is null"); + requireNonNull(fromHiveType, "fromHiveType is null"); + requireNonNull(toHiveType, "toHiveType is null"); + List fromFieldTypes = extractStructFieldTypes(fromHiveType); + List toFieldTypes = extractStructFieldTypes(toHiveType); + this.coercers = new HiveCoercer[toFieldTypes.size()]; + this.nullBlocks = new Block[toFieldTypes.size()]; + for (int i = 0; i < coercers.length; i++) { + if (i >= fromFieldTypes.size()) { + nullBlocks[i] = toFieldTypes.get(i).getType(typeManager).createBlockBuilder(null, 1).appendNull().build(); + } + else if (!fromFieldTypes.get(i).equals(toFieldTypes.get(i))) { + coercers[i] = createCoercer(typeManager, fromFieldTypes.get(i), toFieldTypes.get(i)); + } + } + this.toFieldNames = extractStructFieldNames(toHiveType); + this.toType = toHiveType.getType(typeManager); + } + + @Override + public Block apply(Block block) + { + ColumnarRow rowBlock = toColumnarRow(block); + Block[] fields = new Block[coercers.length]; + int[] ids = new int[rowBlock.getField(0).getPositionCount()]; + for (int i = 0; i < coercers.length; i++) { + if (coercers[i] != null) { + fields[i] = coercers[i].apply(rowBlock.getField(i)); + } + else if (i < rowBlock.getFieldCount()) { + fields[i] = rowBlock.getField(i); + } + else { + fields[i] = new DictionaryBlock(nullBlocks[i], ids); + } + } + boolean[] valueIsNull = new boolean[rowBlock.getPositionCount()]; + for (int i = 0; i < rowBlock.getPositionCount(); i++) { + valueIsNull[i] = rowBlock.isNull(i); + } + return RowBlock.fromFieldBlocks(valueIsNull.length, Optional.of(valueIsNull), fields); + } + + @Override + public TupleDomainFilter toCoercingFilter(TupleDomainFilter filter, Subfield subfield) + { + if (filter == IS_NULL || filter == IS_NOT_NULL) { + return filter; + } + + if (subfield.getPath().size() > 0) { + String fieldName = ((Subfield.NestedField) subfield.getPath().get(0)).getName(); + for (int i = 0; i < toFieldNames.size(); i++) { + if (fieldName.equals(toFieldNames.get(i))) { + HiveCoercer coercer = coercers[i]; + if (coercer == null) { + // the column value will be null + // -> only isNull method will be called + // -> the original filter will work just fine + return filter; + } + return coercer.toCoercingFilter(filter, subfield.tail(fieldName)); + } + } + throw new IllegalArgumentException("Struct field not found: " + fieldName); + } + + throw new UnsupportedOperationException("Range filers on struct types are not supported"); + } + + @Override + public Type getToType() + { + return toType; + } + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercionRecordCursor.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercionRecordCursor.java index 68b8ea9b1d580..ae95fd3126cb0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercionRecordCursor.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercionRecordCursor.java @@ -37,9 +37,9 @@ import static com.facebook.presto.hive.HiveType.HIVE_LONG; import static com.facebook.presto.hive.HiveType.HIVE_SHORT; import static com.facebook.presto.hive.HiveUtil.extractStructFieldTypes; -import static com.facebook.presto.hive.HiveUtil.isArrayType; -import static com.facebook.presto.hive.HiveUtil.isMapType; -import static com.facebook.presto.hive.HiveUtil.isRowType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isArrayType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isMapType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isRowType; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static io.airlift.slice.Slices.utf8Slice; import static java.lang.Float.intBitsToFloat; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java index 223124f327e32..87e430e3305d1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java @@ -50,6 +50,9 @@ public class HiveColumnHandle private static final String UPDATE_ROW_ID_COLUMN_NAME = "$shard_row_id"; + // Ids <= this can be used for distinguishing between different prefilled columns. + public static final int MAX_PARTITION_KEY_COLUMN_INDEX = -13; + public enum ColumnType { PARTITION_KEY, diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnector.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnector.java index 4c74f7188a1fd..ba567b7395262 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnector.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnector.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.SystemTable; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.connector.Connector; @@ -31,8 +33,6 @@ import com.facebook.presto.spi.transaction.IsolationLevel; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import java.util.List; import java.util.Set; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java index 6d6705e665cb6..6fcd104e80f57 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java @@ -13,7 +13,12 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.event.client.EventModule; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.hive.authentication.HiveAuthenticationModule; +import com.facebook.presto.hive.gcs.HiveGcsModule; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HiveMetastoreModule; import com.facebook.presto.hive.s3.HiveS3Module; @@ -37,7 +42,9 @@ import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorPageSourceProvider; import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorSplitManager; import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeNodePartitioningProvider; +import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.FilterStatsCalculatorService; import com.facebook.presto.spi.procedure.Procedure; import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; @@ -45,10 +52,6 @@ import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.TypeLiteral; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.event.client.EventModule; -import io.airlift.json.JsonModule; import org.weakref.jmx.guice.MBeanModule; import javax.management.MBeanServer; @@ -102,6 +105,7 @@ public Connector create(String catalogName, Map config, Connecto new JsonModule(), new HiveClientModule(catalogName), new HiveS3Module(catalogName), + new HiveGcsModule(), new HiveMetastoreModule(catalogName, metastore), new HiveSecurityModule(), new HiveAuthenticationModule(), @@ -115,7 +119,9 @@ public Connector create(String catalogName, Map config, Connecto binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory()); binder.bind(PageSorter.class).toInstance(context.getPageSorter()); binder.bind(StandardFunctionResolution.class).toInstance(context.getStandardFunctionResolution()); + binder.bind(FunctionMetadataManager.class).toInstance(context.getFunctionMetadataManager()); binder.bind(RowExpressionService.class).toInstance(context.getRowExpressionService()); + binder.bind(FilterStatsCalculatorService.class).toInstance(context.getFilterStatsCalculatorService()); }); Injector injector = app diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveErrorCode.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveErrorCode.java index 3067d542246f3..624717cc7bc72 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveErrorCode.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveErrorCode.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.ErrorType; +import static com.facebook.presto.hive.MetastoreErrorCode.ERROR_CODE_MASK; import static com.facebook.presto.spi.ErrorType.EXTERNAL; import static com.facebook.presto.spi.ErrorType.INTERNAL_ERROR; import static com.facebook.presto.spi.ErrorType.USER_ERROR; @@ -24,26 +25,26 @@ public enum HiveErrorCode implements ErrorCodeSupplier { - HIVE_METASTORE_ERROR(0, EXTERNAL), + // HIVE_METASTORE_ERROR(0) moved to MetastoreErrorCode HIVE_CURSOR_ERROR(1, EXTERNAL), - HIVE_TABLE_OFFLINE(2, USER_ERROR), + // HIVE_TABLE_OFFLINE(2) moved to MetastoreErrorCode HIVE_CANNOT_OPEN_SPLIT(3, EXTERNAL), HIVE_FILE_NOT_FOUND(4, EXTERNAL), HIVE_UNKNOWN_ERROR(5, EXTERNAL), - HIVE_PARTITION_OFFLINE(6, USER_ERROR), + // HIVE_PARTITION_OFFLINE(6) moved to MetastoreErrorCode HIVE_BAD_DATA(7, EXTERNAL), HIVE_PARTITION_SCHEMA_MISMATCH(8, EXTERNAL), HIVE_MISSING_DATA(9, EXTERNAL), - HIVE_INVALID_PARTITION_VALUE(10, EXTERNAL), + // HIVE_INVALID_PARTITION_VALUE(10, EXTERNAL) moved to MetastoreErrorCode HIVE_TIMEZONE_MISMATCH(11, EXTERNAL), - HIVE_INVALID_METADATA(12, EXTERNAL), + // HIVE_INVALID_METADATA(12) moved to MetastoreErrorCode HIVE_INVALID_VIEW_DATA(13, EXTERNAL), HIVE_DATABASE_LOCATION_ERROR(14, EXTERNAL), - HIVE_PATH_ALREADY_EXISTS(15, EXTERNAL), - HIVE_FILESYSTEM_ERROR(16, EXTERNAL), + // HIVE_PATH_ALREADY_EXISTS(15, EXTERNAL) moved to MetastoreErrorCode + // HIVE_FILESYSTEM_ERROR(16) moved to MetastoreErrorCode // code HIVE_WRITER_ERROR(17) is deprecated HIVE_SERDE_NOT_FOUND(18, EXTERNAL), - HIVE_UNSUPPORTED_FORMAT(19, EXTERNAL), + // HIVE_UNSUPPORTED_FORMAT(19) moved to MetastoreErrorCode HIVE_PARTITION_READ_ONLY(20, USER_ERROR), HIVE_TOO_MANY_OPEN_PARTITIONS(21, USER_ERROR), HIVE_CONCURRENT_MODIFICATION_DETECTED(22, EXTERNAL), @@ -55,23 +56,24 @@ public enum HiveErrorCode HIVE_INVALID_BUCKET_FILES(28, EXTERNAL), HIVE_EXCEEDED_PARTITION_LIMIT(29, USER_ERROR), HIVE_WRITE_VALIDATION_FAILED(30, INTERNAL_ERROR), - HIVE_PARTITION_DROPPED_DURING_QUERY(31, EXTERNAL), + // HIVE_PARTITION_DROPPED_DURING_QUERY(31, EXTERNAL) moved to MetastoreErrorCode HIVE_TABLE_READ_ONLY(32, USER_ERROR), HIVE_PARTITION_NOT_READABLE(33, USER_ERROR), HIVE_TABLE_NOT_READABLE(34, USER_ERROR), - HIVE_TABLE_DROPPED_DURING_QUERY(35, EXTERNAL), + // HIVE_TABLE_DROPPED_DURING_QUERY(35, EXTERNAL) moded to MetastoreErrorCode // HIVE_TOO_MANY_BUCKET_SORT_FILES(36) is deprecated - HIVE_CORRUPTED_COLUMN_STATISTICS(37, EXTERNAL), + // HIVE_CORRUPTED_COLUMN_STATISTICS(37, EXTERNAL) moved to MetastoreErrorCode HIVE_EXCEEDED_SPLIT_BUFFERING_LIMIT(38, USER_ERROR), - HIVE_UNKNOWN_COLUMN_STATISTIC_TYPE(39, INTERNAL_ERROR), + // HIVE_UNKNOWN_COLUMN_STATISTIC_TYPE(39, INTERNAL_ERROR) moved to MetastoreErrorCode HIVE_TABLE_BUCKETING_IS_IGNORED(40, USER_ERROR), + HIVE_TRANSACTION_NOT_FOUND(41, INTERNAL_ERROR), /**/; private final ErrorCode errorCode; HiveErrorCode(int code, ErrorType type) { - errorCode = new ErrorCode(code + 0x0100_0000, name(), type); + errorCode = new ErrorCode(code + ERROR_CODE_MASK, name(), type); } @Override diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveEventClient.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveEventClient.java index 6c76b8c7b4cce..5f1edb6788ecb 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveEventClient.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveEventClient.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive; -import io.airlift.event.client.AbstractEventClient; -import io.airlift.log.Logger; +import com.facebook.airlift.event.client.AbstractEventClient; +import com.facebook.airlift.log.Logger; public class HiveEventClient extends AbstractEventClient diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveFileInfo.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveFileInfo.java new file mode 100644 index 0000000000000..7027495c2057d --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveFileInfo.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; + +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class HiveFileInfo + implements Comparable +{ + private final Path path; + private final boolean isDirectory; + private final BlockLocation[] blockLocations; + private final long length; + private final Optional extraFileInfo; + + public static HiveFileInfo createHiveFileInfo(LocatedFileStatus locatedFileStatus, Optional extraFileContext) + { + return new HiveFileInfo( + locatedFileStatus.getPath(), + locatedFileStatus.isDirectory(), + locatedFileStatus.getBlockLocations(), + locatedFileStatus.getLen(), + extraFileContext); + } + + private HiveFileInfo(Path path, boolean isDirectory, BlockLocation[] blockLocations, long length, Optional extraFileInfo) + { + this.path = requireNonNull(path, "path is null"); + this.isDirectory = isDirectory; + this.blockLocations = blockLocations; + this.length = length; + this.extraFileInfo = requireNonNull(extraFileInfo, "extraFileInfo is null"); + } + + public Path getPath() + { + return path; + } + + public boolean isDirectory() + { + return isDirectory; + } + + public BlockLocation[] getBlockLocations() + { + return blockLocations; + } + + public long getLength() + { + return length; + } + + public Optional getExtraFileInfo() + { + return extraFileInfo; + } + + @Override + public int compareTo(Object o) + { + HiveFileInfo other = (HiveFileInfo) o; + return this.getPath().compareTo(other.getPath()); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveHdfsConfiguration.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveHdfsConfiguration.java index ee060d06a01c3..7e7a044c0a0fa 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveHdfsConfiguration.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveHdfsConfiguration.java @@ -14,11 +14,13 @@ package com.facebook.presto.hive; import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; +import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import javax.inject.Inject; import java.net.URI; +import java.util.Set; import static com.facebook.presto.hive.util.ConfigurationUtils.copy; import static com.facebook.presto.hive.util.ConfigurationUtils.getInitialConfiguration; @@ -37,23 +39,34 @@ protected Configuration initialValue() { Configuration configuration = new Configuration(false); copy(INITIAL_CONFIGURATION, configuration); - updater.updateConfiguration(configuration); + initializer.updateConfiguration(configuration); return configuration; } }; - private final HdfsConfigurationUpdater updater; + private final HdfsConfigurationInitializer initializer; + private final Set dynamicProviders; @Inject - public HiveHdfsConfiguration(HdfsConfigurationUpdater updater) + public HiveHdfsConfiguration(HdfsConfigurationInitializer initializer, Set dynamicProviders) { - this.updater = requireNonNull(updater, "updater is null"); + this.initializer = requireNonNull(initializer, "initializer is null"); + this.dynamicProviders = ImmutableSet.copyOf(requireNonNull(dynamicProviders, "dynamicProviders is null")); } @Override public Configuration getConfiguration(HdfsContext context, URI uri) { - // use the same configuration for everything - return hadoopConfiguration.get(); + if (dynamicProviders.isEmpty()) { + // use the same configuration for everything + return hadoopConfiguration.get(); + } + + Configuration config = new Configuration(false); + copy(hadoopConfiguration.get(), config); + for (DynamicConfigurationProvider provider : dynamicProviders) { + provider.updateConfiguration(config, context, uri); + } + return config; } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveLocationService.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveLocationService.java index 55a880fe280c6..69bcb9b46d194 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveLocationService.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveLocationService.java @@ -27,18 +27,19 @@ import java.util.Optional; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_PATH_ALREADY_EXISTS; import static com.facebook.presto.hive.HiveSessionProperties.isTemporaryStagingDirectoryEnabled; import static com.facebook.presto.hive.HiveWriteUtils.createTemporaryPath; import static com.facebook.presto.hive.HiveWriteUtils.getTableDefaultLocation; import static com.facebook.presto.hive.HiveWriteUtils.isS3FileSystem; -import static com.facebook.presto.hive.HiveWriteUtils.pathExists; import static com.facebook.presto.hive.LocationHandle.TableType.EXISTING; import static com.facebook.presto.hive.LocationHandle.TableType.NEW; import static com.facebook.presto.hive.LocationHandle.TableType.TEMPORARY; import static com.facebook.presto.hive.LocationHandle.WriteMode.DIRECT_TO_TARGET_EXISTING_DIRECTORY; import static com.facebook.presto.hive.LocationHandle.WriteMode.DIRECT_TO_TARGET_NEW_DIRECTORY; import static com.facebook.presto.hive.LocationHandle.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PATH_ALREADY_EXISTS; +import static com.facebook.presto.hive.metastore.MetastoreUtil.createDirectory; +import static com.facebook.presto.hive.metastore.MetastoreUtil.pathExists; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.UUID.randomUUID; @@ -55,7 +56,7 @@ public HiveLocationService(HdfsEnvironment hdfsEnvironment) } @Override - public LocationHandle forNewTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, String schemaName, String tableName) + public LocationHandle forNewTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, String schemaName, String tableName, boolean tempPathRequired) { HdfsContext context = new HdfsContext(session, schemaName, tableName); Path targetPath = getTableDefaultLocation(context, metastore, hdfsEnvironment, schemaName, tableName); @@ -64,37 +65,44 @@ public LocationHandle forNewTable(SemiTransactionalHiveMetastore metastore, Conn if (pathExists(context, hdfsEnvironment, targetPath)) { throw new PrestoException(HIVE_PATH_ALREADY_EXISTS, format("Target directory for table '%s.%s' already exists: %s", schemaName, tableName, targetPath)); } - return createLocationHandle(context, session, targetPath, NEW); + return createLocationHandle(context, session, targetPath, NEW, tempPathRequired); } @Override - public LocationHandle forExistingTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, Table table) + public LocationHandle forExistingTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, Table table, boolean tempPathRequired) { HdfsContext context = new HdfsContext(session, table.getDatabaseName(), table.getTableName()); Path targetPath = new Path(table.getStorage().getLocation()); - return createLocationHandle(context, session, targetPath, EXISTING); + return createLocationHandle(context, session, targetPath, EXISTING, tempPathRequired); } @Override - public LocationHandle forTemporaryTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, Table table) + public LocationHandle forTemporaryTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, Table table, boolean tempPathRequired) { String schemaName = table.getDatabaseName(); String tableName = table.getTableName(); HdfsContext context = new HdfsContext(session, schemaName, tableName); Path targetPath = new Path(getTableDefaultLocation(context, metastore, hdfsEnvironment, schemaName, tableName), randomUUID().toString().replaceAll("-", "_")); - return new LocationHandle(targetPath, targetPath, TEMPORARY, DIRECT_TO_TARGET_NEW_DIRECTORY); + return new LocationHandle( + targetPath, + targetPath, + tempPathRequired ? Optional.of(createTemporaryPath(session, context, hdfsEnvironment, targetPath)) : Optional.empty(), + TEMPORARY, + DIRECT_TO_TARGET_NEW_DIRECTORY); } - private LocationHandle createLocationHandle(HdfsContext context, ConnectorSession session, Path targetPath, TableType tableType) + private LocationHandle createLocationHandle(HdfsContext context, ConnectorSession session, Path targetPath, TableType tableType, boolean tempPathRequired) { + Optional tempPath = tempPathRequired ? Optional.of(createTemporaryPath(session, context, hdfsEnvironment, targetPath)) : Optional.empty(); if (shouldUseTemporaryDirectory(session, context, targetPath)) { Path writePath = createTemporaryPath(session, context, hdfsEnvironment, targetPath); - return new LocationHandle(targetPath, writePath, tableType, STAGE_AND_MOVE_TO_TARGET_DIRECTORY); + createDirectory(context, hdfsEnvironment, writePath); + return new LocationHandle(targetPath, writePath, tempPath, tableType, STAGE_AND_MOVE_TO_TARGET_DIRECTORY); } if (tableType.equals(EXISTING)) { - return new LocationHandle(targetPath, targetPath, tableType, DIRECT_TO_TARGET_EXISTING_DIRECTORY); + return new LocationHandle(targetPath, targetPath, tempPath, tableType, DIRECT_TO_TARGET_EXISTING_DIRECTORY); } - return new LocationHandle(targetPath, targetPath, tableType, DIRECT_TO_TARGET_NEW_DIRECTORY); + return new LocationHandle(targetPath, targetPath, tempPath, tableType, DIRECT_TO_TARGET_NEW_DIRECTORY); } private boolean shouldUseTemporaryDirectory(ConnectorSession session, HdfsContext context, Path path) @@ -107,18 +115,19 @@ private boolean shouldUseTemporaryDirectory(ConnectorSession session, HdfsContex @Override public WriteInfo getQueryWriteInfo(LocationHandle locationHandle) { - return new WriteInfo(locationHandle.getTargetPath(), locationHandle.getWritePath(), locationHandle.getWriteMode()); + return new WriteInfo(locationHandle.getTargetPath(), locationHandle.getWritePath(), locationHandle.getTempPath(), locationHandle.getWriteMode()); } @Override public WriteInfo getTableWriteInfo(LocationHandle locationHandle) { - return new WriteInfo(locationHandle.getTargetPath(), locationHandle.getWritePath(), locationHandle.getWriteMode()); + return new WriteInfo(locationHandle.getTargetPath(), locationHandle.getWritePath(), locationHandle.getTempPath(), locationHandle.getWriteMode()); } @Override public WriteInfo getPartitionWriteInfo(LocationHandle locationHandle, Optional partition, String partitionName) { + Optional tempPath = locationHandle.getTempPath().map(path -> new Path(path, randomUUID().toString().replaceAll("-", "_"))); if (partition.isPresent()) { // existing partition WriteMode writeMode = locationHandle.getWriteMode(); @@ -137,13 +146,14 @@ public WriteInfo getPartitionWriteInfo(LocationHandle locationHandle, Optional

RESERVED_ROLES = ImmutableSet.of("all", "default", "none"); @@ -246,7 +277,6 @@ public class HiveMetadata private static final String PARTITIONS_TABLE_SUFFIX = "$partitions"; private static final String PRESTO_TEMPORARY_TABLE_NAME_PREFIX = "__presto_temporary_table_"; - public static final String AVRO_SCHEMA_URL_KEY = "avro.schema.url"; private final boolean allowCorruptWritesForTesting; private final SemiTransactionalHiveMetastore metastore; @@ -256,7 +286,9 @@ public class HiveMetadata private final TypeManager typeManager; private final LocationService locationService; private final StandardFunctionResolution functionResolution; + private final FunctionMetadataManager functionMetadataManager; private final RowExpressionService rowExpressionService; + private final FilterStatsCalculatorService filterStatsCalculatorService; private final TableParameterCodec tableParameterCodec; private final JsonCodec partitionUpdateCodec; private final boolean writesToNonManagedTablesEnabled; @@ -266,6 +298,7 @@ public class HiveMetadata private final HiveStatisticsProvider hiveStatisticsProvider; private final StagingFileCommitter stagingFileCommitter; private final ZeroRowFileCreator zeroRowFileCreator; + private final PartitionObjectBuilder partitionObjectBuilder; public HiveMetadata( SemiTransactionalHiveMetastore metastore, @@ -278,14 +311,17 @@ public HiveMetadata( TypeManager typeManager, LocationService locationService, StandardFunctionResolution functionResolution, + FunctionMetadataManager functionMetadataManager, RowExpressionService rowExpressionService, + FilterStatsCalculatorService filterStatsCalculatorService, TableParameterCodec tableParameterCodec, JsonCodec partitionUpdateCodec, TypeTranslator typeTranslator, String prestoVersion, HiveStatisticsProvider hiveStatisticsProvider, StagingFileCommitter stagingFileCommitter, - ZeroRowFileCreator zeroRowFileCreator) + ZeroRowFileCreator zeroRowFileCreator, + PartitionObjectBuilder partitionObjectBuilder) { this.allowCorruptWritesForTesting = allowCorruptWritesForTesting; @@ -296,7 +332,9 @@ public HiveMetadata( this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.locationService = requireNonNull(locationService, "locationService is null"); this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null"); + this.filterStatsCalculatorService = requireNonNull(filterStatsCalculatorService, "filterStatsCalculatorService is null"); this.tableParameterCodec = requireNonNull(tableParameterCodec, "tableParameterCodec is null"); this.partitionUpdateCodec = requireNonNull(partitionUpdateCodec, "partitionUpdateCodec is null"); this.writesToNonManagedTablesEnabled = writesToNonManagedTablesEnabled; @@ -306,6 +344,7 @@ public HiveMetadata( this.hiveStatisticsProvider = requireNonNull(hiveStatisticsProvider, "hiveStatisticsProvider is null"); this.stagingFileCommitter = requireNonNull(stagingFileCommitter, "stagingFileCommitter is null"); this.zeroRowFileCreator = requireNonNull(zeroRowFileCreator, "zeroRowFileCreator is null"); + this.partitionObjectBuilder = requireNonNull(partitionObjectBuilder, "partitionObjectBuilder is null"); } public SemiTransactionalHiveMetastore getMetastore() @@ -333,7 +372,10 @@ public HiveTableHandle getTableHandle(ConnectorSession session, SchemaTableName throw new PrestoException(StandardErrorCode.NOT_SUPPORTED, format("Unexpected table %s present in Hive metastore", tableName)); } - verifyOnline(tableName, Optional.empty(), getProtectMode(table.get()), table.get().getParameters()); + if (!isOfflineDataDebugModeEnabled(session)) { + verifyOnline(tableName, Optional.empty(), getProtectMode(table.get()), table.get().getParameters()); + } + return new HiveTableHandle(tableName.getSchemaName(), tableName.getTableName()); } @@ -418,7 +460,7 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect Predicate> targetPredicate = convertToPredicate(targetTupleDomain); Constraint targetConstraint = new Constraint<>(targetTupleDomain, targetPredicate); Iterable> records = () -> - stream(partitionManager.getPartitionsIterator(metastore, sourceTableHandle, targetConstraint)) + stream(partitionManager.getPartitionsIterator(metastore, sourceTableHandle, targetConstraint, session)) .map(hivePartition -> IntStream.range(0, partitionColumns.size()) .mapToObj(fieldIdToColumnHandle::get) @@ -585,19 +627,62 @@ public Map> listTableColumns(ConnectorSess } @Override - public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint) + public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Optional tableLayoutHandle, List columnHandles, Constraint constraint) { if (!isStatisticsEnabled(session)) { return TableStatistics.empty(); } - Map columns = getColumnHandles(session, tableHandle) - .entrySet().stream() - .filter(entry -> !((HiveColumnHandle) entry.getValue()).isHidden()) - .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); - Map columnTypes = columns.entrySet().stream() + + // TODO Adjust statistics to take into account required subfields + + if (!tableLayoutHandle.isPresent() || !((HiveTableLayoutHandle) tableLayoutHandle.get()).isPushdownFilterEnabled()) { + Map columns = columnHandles.stream() + .map(HiveColumnHandle.class::cast) + .filter(not(HiveColumnHandle::isHidden)) + .collect(toImmutableMap(HiveColumnHandle::getName, Function.identity())); + + Map columnTypes = columns.entrySet().stream() + .collect(toImmutableMap(Map.Entry::getKey, entry -> getColumnMetadata(session, tableHandle, entry.getValue()).getType())); + List partitions = partitionManager.getPartitions(metastore, tableHandle, constraint, session).getPartitions(); + return hiveStatisticsProvider.getTableStatistics(session, ((HiveTableHandle) tableHandle).getSchemaTableName(), columns, columnTypes, partitions); + } + + verify(!constraint.predicate().isPresent()); + + HiveTableLayoutHandle hiveLayoutHandle = (HiveTableLayoutHandle) tableLayoutHandle.get(); + + Set columnNames = columnHandles.stream() + .map(HiveColumnHandle.class::cast) + .map(HiveColumnHandle::getName) + .collect(toImmutableSet()); + + Set allColumnHandles = ImmutableSet.builder() + .addAll(columnHandles) + .addAll(hiveLayoutHandle.getPredicateColumns().values().stream() + .filter(column -> !columnNames.contains(column.getName())) + .collect(toImmutableList())) + .build(); + + Map allColumns = Maps.uniqueIndex(allColumnHandles, column -> ((HiveColumnHandle) column).getName()); + + Map allColumnTypes = allColumns.entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> getColumnMetadata(session, tableHandle, entry.getValue()).getType())); - List partitions = partitionManager.getPartitions(metastore, tableHandle, constraint, session).getPartitions(); - return hiveStatisticsProvider.getTableStatistics(session, ((HiveTableHandle) tableHandle).getSchemaTableName(), columns, columnTypes, partitions); + + Constraint combinedConstraint = new Constraint<>(constraint.getSummary().intersect(hiveLayoutHandle.getDomainPredicate() + .transform(subfield -> isEntireColumn(subfield) ? subfield.getRootName() : null) + .transform(allColumns::get))); + + SubfieldExtractor subfieldExtractor = new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session); + + RowExpression domainPredicate = rowExpressionService.getDomainTranslator().toPredicate( + hiveLayoutHandle.getDomainPredicate() + .transform(subfield -> subfieldExtractor.toRowExpression(subfield, allColumnTypes.get(subfield.getRootName())))); + RowExpression combinedPredicate = binaryExpression(SpecialFormExpression.Form.AND, ImmutableList.of(hiveLayoutHandle.getRemainingPredicate(), domainPredicate)); + + List partitions = partitionManager.getPartitions(metastore, tableHandle, combinedConstraint, session).getPartitions(); + TableStatistics tableStatistics = hiveStatisticsProvider.getTableStatistics(session, ((HiveTableHandle) tableHandle).getSchemaTableName(), allColumns, allColumnTypes, partitions); + + return filterStatsCalculatorService.filterStats(tableStatistics, combinedPredicate, session, ImmutableBiMap.copyOf(allColumns).inverse(), allColumnTypes); } private List listTables(ConnectorSession session, SchemaTablePrefix prefix) @@ -674,7 +759,7 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties()); Map tableProperties = getEmptyTableProperties(tableMetadata, !partitionedBy.isEmpty(), new HdfsContext(session, schemaName, tableName)); - hiveStorageFormat.validateColumns(columnHandles); + validateColumns(hiveStorageFormat, columnHandles); Map columnHandlesByName = Maps.uniqueIndex(columnHandles, HiveColumnHandle::getName); List partitionColumns = partitionedBy.stream() @@ -696,7 +781,7 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe } else { tableType = MANAGED_TABLE; - LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName); + LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, isTempPathRequired(session, bucketProperty)); targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath(); } @@ -744,8 +829,15 @@ public ConnectorTableHandle createTemporaryTable(ConnectorSession session, List< return new HiveBucketProperty(partitioning.getPartitionColumns(), partitioningHandle.getBucketCount(), ImmutableList.of()); }); - List columnHandles = getColumnHandles(columns, ImmutableSet.of(), typeTranslator); - storageFormat.validateColumns(columnHandles); + List columnHandles = getColumnHandles( + // Hive doesn't support anonymous rows and unknown type + // Since this method doesn't create a real table, it is fine + // to assign dummy field names to the anonymous rows and translate unknown + // type to the boolean type that is binary compatible + translateHiveUnsupportedTypesForTemporaryTable(columns, typeManager), + ImmutableSet.of(), + typeTranslator); + validateColumns(storageFormat, columnHandles); Table table = Table.builder() .setDatabaseName(schemaName) @@ -772,6 +864,47 @@ public ConnectorTableHandle createTemporaryTable(ConnectorSession session, List< return new HiveTableHandle(schemaName, tableName); } + private void validateColumns(HiveStorageFormat hiveStorageFormat, List handles) + { + if (hiveStorageFormat == HiveStorageFormat.AVRO) { + for (HiveColumnHandle handle : handles) { + if (!handle.isPartitionKey()) { + validateAvroType(handle.getHiveType().getTypeInfo(), handle.getName()); + } + } + } + } + + private static void validateAvroType(TypeInfo type, String columnName) + { + if (type.getCategory() == ObjectInspector.Category.MAP) { + TypeInfo keyType = mapTypeInfo(type).getMapKeyTypeInfo(); + if ((keyType.getCategory() != ObjectInspector.Category.PRIMITIVE) || + (primitiveTypeInfo(keyType).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING)) { + throw new PrestoException(NOT_SUPPORTED, format("Column %s has a non-varchar map key, which is not supported by Avro", columnName)); + } + } + else if (type.getCategory() == ObjectInspector.Category.PRIMITIVE) { + PrimitiveObjectInspector.PrimitiveCategory primitive = primitiveTypeInfo(type).getPrimitiveCategory(); + if (primitive == PrimitiveObjectInspector.PrimitiveCategory.BYTE) { + throw new PrestoException(NOT_SUPPORTED, format("Column %s is tinyint, which is not supported by Avro. Use integer instead.", columnName)); + } + if (primitive == PrimitiveObjectInspector.PrimitiveCategory.SHORT) { + throw new PrestoException(NOT_SUPPORTED, format("Column %s is smallint, which is not supported by Avro. Use integer instead.", columnName)); + } + } + } + + private static PrimitiveTypeInfo primitiveTypeInfo(TypeInfo typeInfo) + { + return (PrimitiveTypeInfo) typeInfo; + } + + private static MapTypeInfo mapTypeInfo(TypeInfo typeInfo) + { + return (MapTypeInfo) typeInfo; + } + private Map getEmptyTableProperties(ConnectorTableMetadata tableMetadata, boolean partitioned, HdfsContext hdfsContext) { Builder tableProperties = ImmutableMap.builder(); @@ -1034,7 +1167,7 @@ public void finishStatisticsCollection(ConnectorSession session, ConnectorTableH partitionValuesList = metastore.getPartitionNames(handle.getSchemaName(), handle.getTableName()) .orElseThrow(() -> new TableNotFoundException(((HiveTableHandle) tableHandle).getSchemaTableName())) .stream() - .map(HiveUtil::toPartitionValues) + .map(MetastoreUtil::toPartitionValues) .collect(toImmutableList()); } @@ -1089,7 +1222,7 @@ public HiveOutputTableHandle beginCreateTable(ConnectorSession session, Connecto // unpartitioned tables ignore the partition storage format HiveStorageFormat actualStorageFormat = partitionedBy.isEmpty() ? tableStorageFormat : partitionStorageFormat; - actualStorageFormat.validateColumns(columnHandles); + validateColumns(actualStorageFormat, columnHandles); Map columnHandlesByName = Maps.uniqueIndex(columnHandles, HiveColumnHandle::getName); List partitionColumns = partitionedBy.stream() @@ -1098,7 +1231,7 @@ public HiveOutputTableHandle beginCreateTable(ConnectorSession session, Connecto .collect(toList()); checkPartitionTypesSupported(partitionColumns); - LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName); + LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, isTempPathRequired(session, bucketProperty)); HiveOutputTableHandle result = new HiveOutputTableHandle( schemaName, tableName, @@ -1115,7 +1248,14 @@ public HiveOutputTableHandle beginCreateTable(ConnectorSession session, Connecto tableProperties); WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle); - metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), result.getFilePrefix(), schemaTableName, false); + metastore.declareIntentionToWrite( + session, + writeInfo.getWriteMode(), + writeInfo.getWritePath(), + writeInfo.getTempPath(), + result.getFilePrefix(), + schemaTableName, + false); return result; } @@ -1151,7 +1291,7 @@ public Optional finishCreateTable(ConnectorSession sess partitionUpdates = PartitionUpdate.mergePartitionUpdates(Iterables.concat(partitionUpdates, partitionUpdatesForMissingBuckets)); HdfsContext hdfsContext = new HdfsContext(session, table.getDatabaseName(), table.getTableName()); for (PartitionUpdate partitionUpdate : partitionUpdatesForMissingBuckets) { - Optional partition = table.getPartitionColumns().isEmpty() ? Optional.empty() : Optional.of(buildPartitionObject(session, table, partitionUpdate)); + Optional partition = table.getPartitionColumns().isEmpty() ? Optional.empty() : Optional.of(partitionObjectBuilder.buildPartitionObject(session, table, partitionUpdate, prestoVersion)); zeroRowFileCreator.createFiles( session, hdfsContext, @@ -1189,7 +1329,7 @@ public Optional finishCreateTable(ConnectorSession sess Verify.verify(handle.getPartitionStorageFormat() == handle.getTableStorageFormat()); } for (PartitionUpdate update : partitionUpdates) { - Partition partition = buildPartitionObject(session, table, update); + Partition partition = partitionObjectBuilder.buildPartitionObject(session, table, update, prestoVersion); PartitionStatistics partitionStatistics = createPartitionStatistics( session, update.getStatistics(), @@ -1199,7 +1339,7 @@ public Optional finishCreateTable(ConnectorSession sess session, handle.getSchemaName(), handle.getTableName(), - buildPartitionObject(session, table, update), + partitionObjectBuilder.buildPartitionObject(session, table, update, prestoVersion), update.getWritePath(), partitionStatistics); } @@ -1328,11 +1468,12 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl HiveStorageFormat tableStorageFormat = extractHiveStorageFormat(table.get()); LocationHandle locationHandle; boolean isTemporaryTable = table.get().getTableType().equals(TEMPORARY_TABLE); + boolean tempPathRequired = isTempPathRequired(session, table.map(Table::getStorage).flatMap(Storage::getBucketProperty)); if (isTemporaryTable) { - locationHandle = locationService.forTemporaryTable(metastore, session, table.get()); + locationHandle = locationService.forTemporaryTable(metastore, session, table.get(), tempPathRequired); } else { - locationHandle = locationService.forExistingTable(metastore, session, table.get()); + locationHandle = locationService.forExistingTable(metastore, session, table.get(), tempPathRequired); } HiveInsertTableHandle result = new HiveInsertTableHandle( tableName.getSchemaName(), @@ -1347,7 +1488,14 @@ public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTabl isTemporaryTable ? getTemporaryTableCompressionCodec(session) : getCompressionCodec(session)); WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle); - metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), result.getFilePrefix(), tableName, isTemporaryTable); + metastore.declareIntentionToWrite( + session, + writeInfo.getWriteMode(), + writeInfo.getWritePath(), + writeInfo.getTempPath(), + result.getFilePrefix(), + tableName, + isTemporaryTable); return result; } @@ -1375,7 +1523,7 @@ public Optional finishInsert(ConnectorSession session, partitionUpdates = PartitionUpdate.mergePartitionUpdates(Iterables.concat(partitionUpdates, partitionUpdatesForMissingBuckets)); HdfsContext hdfsContext = new HdfsContext(session, table.get().getDatabaseName(), table.get().getTableName()); for (PartitionUpdate partitionUpdate : partitionUpdatesForMissingBuckets) { - Optional partition = table.get().getPartitionColumns().isEmpty() ? Optional.empty() : Optional.of(buildPartitionObject(session, table.get(), partitionUpdate)); + Optional partition = table.get().getPartitionColumns().isEmpty() ? Optional.empty() : Optional.of(partitionObjectBuilder.buildPartitionObject(session, table.get(), partitionUpdate, prestoVersion)); zeroRowFileCreator.createFiles( session, hdfsContext, @@ -1429,7 +1577,7 @@ else if (partitionUpdate.getUpdateMode() == APPEND) { } else if (partitionUpdate.getUpdateMode() == NEW || partitionUpdate.getUpdateMode() == OVERWRITE) { // insert into new partition or overwrite existing partition - Partition partition = buildPartitionObject(session, table.get(), partitionUpdate); + Partition partition = partitionObjectBuilder.buildPartitionObject(session, table.get(), partitionUpdate, prestoVersion); if (!partition.getStorage().getStorageFormat().getInputFormat().equals(handle.getPartitionStorageFormat().getInputFormat()) && isRespectTableFormat(session)) { throw new PrestoException(HIVE_CONCURRENT_MODIFICATION_DETECTED, "Partition format changed during insert"); } @@ -1455,6 +1603,11 @@ else if (partitionUpdate.getUpdateMode() == NEW || partitionUpdate.getUpdateMode .collect(Collectors.toList()))); } + private static boolean isTempPathRequired(ConnectorSession session, Optional bucketProperty) + { + return isSortedWriteToTempPathEnabled(session) && bucketProperty.map(property -> !property.getSortedBy().isEmpty()).orElse(false); + } + private List getTargetFileNames(List fileWriteInfos) { return fileWriteInfos.stream() @@ -1462,27 +1615,6 @@ private List getTargetFileNames(List fileWriteInfos) .collect(toImmutableList()); } - private Partition buildPartitionObject(ConnectorSession session, Table table, PartitionUpdate partitionUpdate) - { - return Partition.builder() - .setDatabaseName(table.getDatabaseName()) - .setTableName(table.getTableName()) - .setColumns(table.getDataColumns()) - .setValues(extractPartitionValues(partitionUpdate.getName())) - .setParameters(ImmutableMap.builder() - .put(PRESTO_VERSION_NAME, prestoVersion) - .put(PRESTO_QUERY_ID_NAME, session.getQueryId()) - .build()) - .withStorage(storage -> storage - .setStorageFormat(isRespectTableFormat(session) ? - table.getStorage().getStorageFormat() : - fromHiveStorageFormat(HiveSessionProperties.getHiveStorageFormat(session))) - .setLocation(partitionUpdate.getTargetPath().toString()) - .setBucketProperty(table.getStorage().getBucketProperty()) - .setSerdeParameters(table.getStorage().getSerdeParameters())) - .build(); - } - private PartitionStatistics createPartitionStatistics( ConnectorSession session, Map columnTypes, @@ -1552,7 +1684,7 @@ public void createView(ConnectorSession session, SchemaTableName viewName, Strin Optional existing = metastore.getTable(viewName.getSchemaName(), viewName.getTableName()); if (existing.isPresent()) { - if (!replace || !HiveUtil.isPrestoView(existing.get())) { + if (!replace || !MetastoreUtil.isPrestoView(existing.get())) { throw new ViewAlreadyExistsException(viewName); } @@ -1610,7 +1742,7 @@ public Map getViews(ConnectorSession s for (SchemaTableName schemaTableName : tableNames) { Optional
table = metastore.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName()); - if (table.isPresent() && HiveUtil.isPrestoView(table.get())) { + if (table.isPresent() && MetastoreUtil.isPrestoView(table.get())) { views.put(schemaTableName, new ConnectorViewDefinition( schemaTableName, Optional.ofNullable(table.get().getOwner()), @@ -1676,9 +1808,42 @@ static Predicate> convertToPredicate(TupleDomai } @Override - public boolean supportsMetadataDelete(ConnectorSession session, ConnectorTableHandle tableHandle) + public boolean supportsMetadataDelete(ConnectorSession session, ConnectorTableHandle tableHandle, Optional tableLayoutHandle) { - return true; + if (!tableLayoutHandle.isPresent()) { + return true; + } + + // Allow metadata delete for range filters on partition columns. + // TODO Add support for metadata delete for any filter on partition columns. + + HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) tableLayoutHandle.get(); + if (!layoutHandle.isPushdownFilterEnabled()) { + return true; + } + + if (layoutHandle.getPredicateColumns().isEmpty()) { + return true; + } + + if (!TRUE_CONSTANT.equals(layoutHandle.getRemainingPredicate())) { + return false; + } + + TupleDomain domainPredicate = layoutHandle.getDomainPredicate(); + if (domainPredicate.isAll()) { + return true; + } + + Set predicateColumnNames = domainPredicate.getDomains().get().keySet().stream() + .map(Subfield::getRootName) + .collect(toImmutableSet()); + + Set partitionColumnNames = layoutHandle.getPartitionColumns().stream() + .map(HiveColumnHandle::getName) + .collect(toImmutableSet()); + + return partitionColumnNames.containsAll(predicateColumnNames); } @Override @@ -1698,23 +1863,38 @@ public ConnectorPushdownFilterResult pushdownFilter(ConnectorSession session, Co return new ConnectorPushdownFilterResult(getTableLayout(session, currentLayoutHandle.get()), TRUE_CONSTANT); } - if (currentLayoutHandle.isPresent()) { - throw new UnsupportedOperationException("Partial filter pushdown is not supported"); - } - // Split the filter into 3 groups of conjuncts: // - range filters that apply to entire columns, // - range filters that apply to subfields, - // - the rest + // - the rest. Intersect these with possibly pre-existing filters. ExtractionResult decomposedFilter = rowExpressionService.getDomainTranslator() - .fromPredicate(session, filter, new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session)); + .fromPredicate(session, filter, new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session).toColumnExtractor()); + if (currentLayoutHandle.isPresent()) { + HiveTableLayoutHandle currentHiveLayout = (HiveTableLayoutHandle) currentLayoutHandle.get(); + decomposedFilter = intersectExtractionResult(decomposedFilter, new ExtractionResult(currentHiveLayout.getDomainPredicate(), currentHiveLayout.getRemainingPredicate())); + } Map columnHandles = getColumnHandles(session, tableHandle); TupleDomain entireColumnDomain = decomposedFilter.getTupleDomain() .transform(subfield -> isEntireColumn(subfield) ? subfield.getRootName() : null) .transform(columnHandles::get); - // TODO Extract deterministic conjuncts that apply to partition columns and specify these as Contraint#predicate - HivePartitionResult hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, new Constraint<>(entireColumnDomain), session); + if (currentLayoutHandle.isPresent()) { + entireColumnDomain = entireColumnDomain.intersect(((HiveTableLayoutHandle) (currentLayoutHandle.get())).getPartitionColumnPredicate()); + } + + Constraint constraint = new Constraint<>(entireColumnDomain); + + // Extract deterministic conjuncts that apply to partition columns and specify these as Constraint#predicate + if (!TRUE_CONSTANT.equals(decomposedFilter.getRemainingExpression())) { + LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(rowExpressionService.getDeterminismEvaluator(), functionResolution, functionMetadataManager); + RowExpression deterministicPredicate = logicalRowExpressions.filterDeterministicConjuncts(decomposedFilter.getRemainingExpression()); + if (!TRUE_CONSTANT.equals(deterministicPredicate)) { + ConstraintEvaluator evaluator = new ConstraintEvaluator(rowExpressionService, session, columnHandles, deterministicPredicate); + constraint = new Constraint<>(entireColumnDomain, evaluator::isCandidate); + } + } + + HivePartitionResult hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, constraint, session); TupleDomain domainPredicate = withColumnDomains(ImmutableMap.builder() .putAll(hivePartitionResult.getUnenforcedConstraint() @@ -1740,22 +1920,127 @@ public ConnectorPushdownFilterResult pushdownFilter(ConnectorSession session, Co .map(HiveColumnHandle.class::cast) .collect(toImmutableMap(HiveColumnHandle::getName, Functions.identity())); + SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName(); return new ConnectorPushdownFilterResult( getTableLayout( session, new HiveTableLayoutHandle( - ((HiveTableHandle) tableHandle).getSchemaTableName(), - ImmutableList.copyOf(hivePartitionResult.getPartitionColumns()), + tableName, + hivePartitionResult.getPartitionColumns(), + // remove comments to optimize serialization costs + pruneColumnComments(hivePartitionResult.getDataColumns()), + hivePartitionResult.getTableParameters(), hivePartitionResult.getPartitions(), domainPredicate, decomposedFilter.getRemainingExpression(), predicateColumns, hivePartitionResult.getEnforcedConstraint(), hivePartitionResult.getBucketHandle(), - hivePartitionResult.getBucketFilter())), + hivePartitionResult.getBucketFilter(), + true, + createTableLayoutString(session, tableName, hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), decomposedFilter.getRemainingExpression(), domainPredicate))), TRUE_CONSTANT); } + private static class ConstraintEvaluator + { + private final Map assignments; + private final RowExpressionService evaluator; + private final ConnectorSession session; + private final RowExpression expression; + private final Set arguments; + + public ConstraintEvaluator(RowExpressionService evaluator, ConnectorSession session, Map assignments, RowExpression expression) + { + this.assignments = assignments; + this.evaluator = evaluator; + this.session = session; + this.expression = expression; + + arguments = ImmutableSet.copyOf(extractAll(expression)).stream() + .map(VariableReferenceExpression::getName) + .map(assignments::get) + .collect(toImmutableSet()); + } + + private boolean isCandidate(Map bindings) + { + if (intersection(bindings.keySet(), arguments).isEmpty()) { + return true; + } + + Function variableResolver = variable -> { + ColumnHandle column = assignments.get(variable.getName()); + checkArgument(column != null, "Missing column assignment for %s", variable); + + if (!bindings.containsKey(column)) { + return variable; + } + + return bindings.get(column).getValue(); + }; + + // Skip pruning if evaluation fails in a recoverable way. Failing here can cause + // spurious query failures for partitions that would otherwise be filtered out. + Object optimized = null; + try { + optimized = evaluator.getExpressionOptimizer().optimize(expression, OPTIMIZED, session, variableResolver); + } + catch (PrestoException e) { + propagateIfUnhandled(e); + } + + // If any conjuncts evaluate to FALSE or null, then the whole predicate will never be true and so the partition should be pruned + return !Boolean.FALSE.equals(optimized) && optimized != null && (!(optimized instanceof ConstantExpression) || !((ConstantExpression) optimized).isNull()); + } + + private static void propagateIfUnhandled(PrestoException e) + throws PrestoException + { + int errorCode = e.getErrorCode().getCode(); + if (errorCode == DIVISION_BY_ZERO.toErrorCode().getCode() + || errorCode == INVALID_CAST_ARGUMENT.toErrorCode().getCode() + || errorCode == INVALID_FUNCTION_ARGUMENT.toErrorCode().getCode() + || errorCode == NUMERIC_VALUE_OUT_OF_RANGE.toErrorCode().getCode()) { + return; + } + + throw e; + } + } + + private static ExtractionResult intersectExtractionResult(ExtractionResult left, ExtractionResult right) + { + RowExpression newRemainingExpression; + if (right.getRemainingExpression().equals(TRUE_CONSTANT)) { + newRemainingExpression = left.getRemainingExpression(); + } + else if (left.getRemainingExpression().equals(TRUE_CONSTANT)) { + newRemainingExpression = right.getRemainingExpression(); + } + else { + newRemainingExpression = and(left.getRemainingExpression(), right.getRemainingExpression()); + } + return new ExtractionResult(left.getTupleDomain().intersect(right.getTupleDomain()), newRemainingExpression); + } + + private String createTableLayoutString( + ConnectorSession session, + SchemaTableName tableName, + Optional bucketHandle, + Optional bucketFilter, + RowExpression remainingPredicate, + TupleDomain domainPredicate) + { + return toStringHelper(tableName.toString()) + .omitNullValues() + .add("buckets", bucketHandle.map(HiveBucketHandle::getReadBucketCount).orElse(null)) + .add("bucketsToKeep", bucketFilter.map(HiveBucketFilter::getBucketsToKeep).orElse(null)) + .add("filter", TRUE_CONSTANT.equals(remainingPredicate) ? null : rowExpressionService.formatRowExpression(session, remainingPredicate)) + .add("domains", domainPredicate.isAll() ? null : domainPredicate.toString(session)) + .toString(); + } + private static Set extractAll(RowExpression expression) { ImmutableSet.Builder builder = ImmutableSet.builder(); @@ -1797,19 +2082,25 @@ public List getTableLayouts(ConnectorSession session hiveBucketHandle = Optional.of(createVirtualBucketHandle(virtualBucketCount)); } + TupleDomain domainPredicate = hivePartitionResult.getEffectivePredicate().transform(HiveMetadata::toSubfield); return ImmutableList.of(new ConnectorTableLayoutResult( getTableLayout( session, new HiveTableLayoutHandle( handle.getSchemaTableName(), - ImmutableList.copyOf(hivePartitionResult.getPartitionColumns()), + hivePartitionResult.getPartitionColumns(), + // remove comments to optimize serialization costs + pruneColumnComments(hivePartitionResult.getDataColumns()), + hivePartitionResult.getTableParameters(), hivePartitionResult.getPartitions(), - hivePartitionResult.getEffectivePredicate().transform(HiveMetadata::toSubfield), + domainPredicate, TRUE_CONSTANT, predicateColumns, hivePartitionResult.getEnforcedConstraint(), hiveBucketHandle, - hivePartitionResult.getBucketFilter())), + hivePartitionResult.getBucketFilter(), + false, + createTableLayoutString(session, handle.getSchemaTableName(), hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), TRUE_CONSTANT, domainPredicate))), hivePartitionResult.getUnenforcedConstraint())); } @@ -1835,15 +2126,20 @@ private boolean isPushdownFilterEnabled(ConnectorSession session, ConnectorTable return false; } + private List pruneColumnComments(List columns) + { + return columns.stream() + .map(column -> new Column(column.getName(), column.getType(), Optional.empty())) + .collect(toImmutableList()); + } + @Override public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle layoutHandle) { HiveTableLayoutHandle hiveLayoutHandle = (HiveTableLayoutHandle) layoutHandle; - List partitionColumns = hiveLayoutHandle.getPartitionColumns(); + List partitionColumns = ImmutableList.copyOf(hiveLayoutHandle.getPartitionColumns()); List partitions = hiveLayoutHandle.getPartitions().get(); - TupleDomain predicate = createPredicate(partitionColumns, partitions); - Optional discretePredicates = Optional.empty(); if (!partitionColumns.isEmpty()) { // Do not create tuple domains for every partition at the same time! @@ -1872,6 +2168,19 @@ public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTa .collect(toList()))); } + TupleDomain predicate; + Map constants; + if (hiveLayoutHandle.isPushdownFilterEnabled()) { + predicate = hiveLayoutHandle.getDomainPredicate() + .transform(subfield -> isEntireColumn(subfield) ? subfield.getRootName() : null) + .transform(hiveLayoutHandle.getPredicateColumns()::get) + .transform(ColumnHandle.class::cast) + .intersect(createPredicate(partitionColumns, partitions)); + } + else { + predicate = createPredicate(partitionColumns, partitions); + } + return new ConnectorTableLayout( hiveLayoutHandle, Optional.empty(), @@ -1975,13 +2284,17 @@ public ConnectorTableLayoutHandle getAlternativeLayoutHandle(ConnectorSession se return new HiveTableLayoutHandle( hiveLayoutHandle.getSchemaTableName(), hiveLayoutHandle.getPartitionColumns(), + hiveLayoutHandle.getDataColumns(), + hiveLayoutHandle.getTableParameters(), hiveLayoutHandle.getPartitions().get(), hiveLayoutHandle.getDomainPredicate(), hiveLayoutHandle.getRemainingPredicate(), hiveLayoutHandle.getPredicateColumns(), hiveLayoutHandle.getPartitionColumnPredicate(), Optional.of(new HiveBucketHandle(bucketHandle.getColumns(), bucketHandle.getTableBucketCount(), hivePartitioningHandle.getBucketCount())), - hiveLayoutHandle.getBucketFilter()); + hiveLayoutHandle.getBucketFilter(), + hiveLayoutHandle.isPushdownFilterEnabled(), + hiveLayoutHandle.getLayoutString()); } @Override @@ -1990,7 +2303,7 @@ public ConnectorPartitioningHandle getPartitioningHandleForExchange(ConnectorSes return new HivePartitioningHandle( partitionCount, partitionTypes.stream() - .map(type -> toHiveType(typeTranslator, type)) + .map(type -> toHiveType(typeTranslator, translateHiveUnsupportedTypeForTemporaryTable(type, typeManager))) .collect(toImmutableList()), OptionalInt.empty()); } @@ -2246,17 +2559,17 @@ public List listTablePrivileges(ConnectorSession session, SchemaTable } @Override - public void commitPartition(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments) + public CompletableFuture commitPartitionAsync(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments) { HiveOutputTableHandle handle = (HiveOutputTableHandle) tableHandle; - stagingFileCommitter.commitFiles(session, handle.getSchemaName(), handle.getTableName(), getPartitionUpdates(fragments)); + return toCompletableFuture(stagingFileCommitter.commitFiles(session, handle.getSchemaName(), handle.getTableName(), getPartitionUpdates(fragments))); } @Override - public void commitPartition(ConnectorSession session, ConnectorInsertTableHandle tableHandle, Collection fragments) + public CompletableFuture commitPartitionAsync(ConnectorSession session, ConnectorInsertTableHandle tableHandle, Collection fragments) { HiveInsertTableHandle handle = (HiveInsertTableHandle) tableHandle; - stagingFileCommitter.commitFiles(session, handle.getSchemaName(), handle.getTableName(), getPartitionUpdates(fragments)); + return toCompletableFuture(stagingFileCommitter.commitFiles(session, handle.getSchemaName(), handle.getTableName(), getPartitionUpdates(fragments))); } private List buildGrants(SchemaTableName tableName, PrestoPrincipal principal) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadataFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadataFactory.java index 24af2152a6fd3..bd543bf031d66 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadataFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadataFactory.java @@ -13,16 +13,18 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.facebook.presto.hive.metastore.CachingHiveMetastore; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore; import com.facebook.presto.hive.statistics.MetastoreHiveStatisticsProvider; +import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.FilterStatsCalculatorService; import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; import com.google.common.util.concurrent.ListeningExecutorService; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import org.joda.time.DateTimeZone; import javax.inject.Inject; @@ -49,7 +51,9 @@ public class HiveMetadataFactory private final TypeManager typeManager; private final LocationService locationService; private final StandardFunctionResolution functionResolution; + private final FunctionMetadataManager functionMetadataManager; private final RowExpressionService rowExpressionService; + private final FilterStatsCalculatorService filterStatsCalculatorService; private final TableParameterCodec tableParameterCodec; private final JsonCodec partitionUpdateCodec; private final ListeningExecutorService fileRenameExecutor; @@ -57,11 +61,13 @@ public class HiveMetadataFactory private final StagingFileCommitter stagingFileCommitter; private final ZeroRowFileCreator zeroRowFileCreator; private final String prestoVersion; + private final PartitionObjectBuilder partitionObjectBuilder; @Inject @SuppressWarnings("deprecation") public HiveMetadataFactory( HiveClientConfig hiveClientConfig, + MetastoreClientConfig metastoreClientConfig, ExtendedHiveMetastore metastore, HdfsEnvironment hdfsEnvironment, HivePartitionManager partitionManager, @@ -69,13 +75,16 @@ public HiveMetadataFactory( TypeManager typeManager, LocationService locationService, StandardFunctionResolution functionResolution, + FunctionMetadataManager functionMetadataManager, RowExpressionService rowExpressionService, + FilterStatsCalculatorService filterStatsCalculatorService, TableParameterCodec tableParameterCodec, JsonCodec partitionUpdateCodec, TypeTranslator typeTranslator, StagingFileCommitter stagingFileCommitter, ZeroRowFileCreator zeroRowFileCreator, - NodeVersion nodeVersion) + NodeVersion nodeVersion, + PartitionObjectBuilder partitionObjectBuilder) { this( metastore, @@ -87,18 +96,21 @@ public HiveMetadataFactory( hiveClientConfig.isSkipTargetCleanupOnRollback(), hiveClientConfig.getWritesToNonManagedTablesEnabled(), hiveClientConfig.getCreatesOfNonManagedTablesEnabled(), - hiveClientConfig.getPerTransactionMetastoreCacheMaximumSize(), + metastoreClientConfig.getPerTransactionMetastoreCacheMaximumSize(), typeManager, locationService, functionResolution, + functionMetadataManager, rowExpressionService, + filterStatsCalculatorService, tableParameterCodec, partitionUpdateCodec, fileRenameExecutor, typeTranslator, stagingFileCommitter, zeroRowFileCreator, - nodeVersion.toString()); + nodeVersion.toString(), + partitionObjectBuilder); } public HiveMetadataFactory( @@ -115,14 +127,17 @@ public HiveMetadataFactory( TypeManager typeManager, LocationService locationService, StandardFunctionResolution functionResolution, + FunctionMetadataManager functionMetadataManager, RowExpressionService rowExpressionService, + FilterStatsCalculatorService filterStatsCalculatorService, TableParameterCodec tableParameterCodec, JsonCodec partitionUpdateCodec, ListeningExecutorService fileRenameExecutor, TypeTranslator typeTranslator, StagingFileCommitter stagingFileCommitter, ZeroRowFileCreator zeroRowFileCreator, - String prestoVersion) + String prestoVersion, + PartitionObjectBuilder partitionObjectBuilder) { this.allowCorruptWritesForTesting = allowCorruptWritesForTesting; this.skipDeletionForAlter = skipDeletionForAlter; @@ -138,7 +153,9 @@ public HiveMetadataFactory( this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.locationService = requireNonNull(locationService, "locationService is null"); this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null"); + this.filterStatsCalculatorService = requireNonNull(filterStatsCalculatorService, "filterStatsCalculatorService is null"); this.tableParameterCodec = requireNonNull(tableParameterCodec, "tableParameterCodec is null"); this.partitionUpdateCodec = requireNonNull(partitionUpdateCodec, "partitionUpdateCodec is null"); this.fileRenameExecutor = requireNonNull(fileRenameExecutor, "fileRenameExecutor is null"); @@ -146,6 +163,7 @@ public HiveMetadataFactory( this.stagingFileCommitter = requireNonNull(stagingFileCommitter, "stagingFileCommitter is null"); this.zeroRowFileCreator = requireNonNull(zeroRowFileCreator, "zeroRowFileCreator is null"); this.prestoVersion = requireNonNull(prestoVersion, "prestoVersion is null"); + this.partitionObjectBuilder = requireNonNull(partitionObjectBuilder, "partitionObjectBuilder is null"); if (!allowCorruptWritesForTesting && !timeZone.equals(DateTimeZone.getDefault())) { log.warn("Hive writes are disabled. " + @@ -176,13 +194,16 @@ public HiveMetadata get() typeManager, locationService, functionResolution, + functionMetadataManager, rowExpressionService, + filterStatsCalculatorService, tableParameterCodec, partitionUpdateCodec, typeTranslator, prestoVersion, new MetastoreHiveStatisticsProvider(metastore), stagingFileCommitter, - zeroRowFileCreator); + zeroRowFileCreator, + partitionObjectBuilder); } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java index c08c4d3b958a6..5865bcc0a15cf 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSink.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.concurrent.MoreFutures; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.ConnectorPageSink; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Page; @@ -28,9 +31,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import io.airlift.concurrent.MoreFutures; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import it.unimi.dsi.fastutil.objects.Object2IntMap; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java index d9ae4f0d43849..2bcd2f90a12f4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSinkProvider.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.event.client.EventClient; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HivePageSinkMetadataProvider; import com.facebook.presto.hive.metastore.SortingColumn; @@ -30,8 +32,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListeningExecutorService; -import io.airlift.event.client.EventClient; -import io.airlift.json.JsonCodec; import io.airlift.units.DataSize; import javax.inject.Inject; @@ -40,9 +40,9 @@ import java.util.OptionalInt; import java.util.Set; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.hive.metastore.CachingHiveMetastore.memoizeMetastore; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newFixedThreadPool; @@ -77,7 +77,8 @@ public HivePageSinkProvider( ExtendedHiveMetastore metastore, PageIndexerFactory pageIndexerFactory, TypeManager typeManager, - HiveClientConfig config, + HiveClientConfig hiveClientConfig, + MetastoreClientConfig metastoreClientConfig, LocationService locationService, JsonCodec partitionUpdateCodec, NodeManager nodeManager, @@ -94,19 +95,19 @@ public HivePageSinkProvider( this.metastore = requireNonNull(metastore, "metastore is null"); this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); - this.maxOpenPartitions = config.getMaxPartitionsPerWriter(); - this.maxOpenSortFiles = config.getMaxOpenSortFiles(); - this.writerSortBufferSize = requireNonNull(config.getWriterSortBufferSize(), "writerSortBufferSize is null"); - this.immutablePartitions = config.isImmutablePartitions(); + this.maxOpenPartitions = hiveClientConfig.getMaxPartitionsPerWriter(); + this.maxOpenSortFiles = hiveClientConfig.getMaxOpenSortFiles(); + this.writerSortBufferSize = requireNonNull(hiveClientConfig.getWriterSortBufferSize(), "writerSortBufferSize is null"); + this.immutablePartitions = hiveClientConfig.isImmutablePartitions(); this.locationService = requireNonNull(locationService, "locationService is null"); - this.writeVerificationExecutor = listeningDecorator(newFixedThreadPool(config.getWriteValidationThreads(), daemonThreadsNamed("hive-write-validation-%s"))); + this.writeVerificationExecutor = listeningDecorator(newFixedThreadPool(hiveClientConfig.getWriteValidationThreads(), daemonThreadsNamed("hive-write-validation-%s"))); this.partitionUpdateCodec = requireNonNull(partitionUpdateCodec, "partitionUpdateCodec is null"); this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.eventClient = requireNonNull(eventClient, "eventClient is null"); this.hiveSessionProperties = requireNonNull(hiveSessionProperties, "hiveSessionProperties is null"); this.hiveWriterStats = requireNonNull(hiveWriterStats, "stats is null"); this.orcFileWriterFactory = requireNonNull(orcFileWriterFactory, "orcFileWriterFactory is null"); - this.perTransactionMetastoreCacheMaximumSize = config.getPerTransactionMetastoreCacheMaximumSize(); + this.perTransactionMetastoreCacheMaximumSize = metastoreClientConfig.getPerTransactionMetastoreCacheMaximumSize(); } @Override diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSource.java index aeadf94702d4c..c0674b947cd0b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSource.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSource.java @@ -13,30 +13,18 @@ */ package com.facebook.presto.hive; -import com.facebook.presto.hive.HivePageSourceProvider.BucketAdaptation; import com.facebook.presto.hive.HivePageSourceProvider.ColumnMapping; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.block.ArrayBlock; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.spi.block.ColumnarArray; -import com.facebook.presto.spi.block.ColumnarMap; -import com.facebook.presto.spi.block.ColumnarRow; -import com.facebook.presto.spi.block.DictionaryBlock; import com.facebook.presto.spi.block.LazyBlock; import com.facebook.presto.spi.block.LazyBlockLoader; -import com.facebook.presto.spi.block.RowBlock; import com.facebook.presto.spi.block.RunLengthEncodedBlock; -import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; -import com.facebook.presto.spi.type.VarcharType; import com.google.common.annotations.VisibleForTesting; import it.unimi.dsi.fastutil.ints.IntArrayList; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.joda.time.DateTimeZone; @@ -48,33 +36,12 @@ import java.util.function.Function; import static com.facebook.presto.hive.HiveBucketing.getHiveBucket; +import static com.facebook.presto.hive.HiveCoercer.createCoercer; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CURSOR_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_BUCKET_FILES; import static com.facebook.presto.hive.HivePageSourceProvider.ColumnMappingKind.PREFILLED; -import static com.facebook.presto.hive.HiveType.HIVE_BYTE; -import static com.facebook.presto.hive.HiveType.HIVE_DOUBLE; -import static com.facebook.presto.hive.HiveType.HIVE_FLOAT; -import static com.facebook.presto.hive.HiveType.HIVE_INT; -import static com.facebook.presto.hive.HiveType.HIVE_LONG; -import static com.facebook.presto.hive.HiveType.HIVE_SHORT; -import static com.facebook.presto.hive.HiveUtil.extractStructFieldTypes; -import static com.facebook.presto.hive.HiveUtil.isArrayType; -import static com.facebook.presto.hive.HiveUtil.isMapType; -import static com.facebook.presto.hive.HiveUtil.isRowType; import static com.facebook.presto.hive.HiveUtil.typedPartitionKey; -import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; -import static com.facebook.presto.spi.block.ColumnarArray.toColumnarArray; -import static com.facebook.presto.spi.block.ColumnarMap.toColumnarMap; -import static com.facebook.presto.spi.block.ColumnarRow.toColumnarRow; -import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static com.facebook.presto.spi.type.DoubleType.DOUBLE; -import static com.facebook.presto.spi.type.IntegerType.INTEGER; -import static com.facebook.presto.spi.type.RealType.REAL; -import static com.facebook.presto.spi.type.SmallintType.SMALLINT; -import static com.facebook.presto.spi.type.TinyintType.TINYINT; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.slice.Slices.utf8Slice; -import static java.lang.Float.intBitsToFloat; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -134,6 +101,12 @@ public long getCompletedBytes() return delegate.getCompletedBytes(); } + @Override + public long getCompletedPositions() + { + return delegate.getCompletedPositions(); + } + @Override public long getReadTimeNanos() { @@ -247,293 +220,6 @@ ConnectorPageSource getPageSource() return delegate; } - private static Function createCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) - { - Type fromType = typeManager.getType(fromHiveType.getTypeSignature()); - Type toType = typeManager.getType(toHiveType.getTypeSignature()); - if (toType instanceof VarcharType && (fromHiveType.equals(HIVE_BYTE) || fromHiveType.equals(HIVE_SHORT) || fromHiveType.equals(HIVE_INT) || fromHiveType.equals(HIVE_LONG))) { - return new IntegerNumberToVarcharCoercer(fromType, toType); - } - else if (fromType instanceof VarcharType && (toHiveType.equals(HIVE_BYTE) || toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG))) { - return new VarcharToIntegerNumberCoercer(fromType, toType); - } - else if (fromHiveType.equals(HIVE_BYTE) && toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG)) { - return new IntegerNumberUpscaleCoercer(fromType, toType); - } - else if (fromHiveType.equals(HIVE_SHORT) && toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG)) { - return new IntegerNumberUpscaleCoercer(fromType, toType); - } - else if (fromHiveType.equals(HIVE_INT) && toHiveType.equals(HIVE_LONG)) { - return new IntegerNumberUpscaleCoercer(fromType, toType); - } - else if (fromHiveType.equals(HIVE_FLOAT) && toHiveType.equals(HIVE_DOUBLE)) { - return new FloatToDoubleCoercer(); - } - else if (isArrayType(fromType) && isArrayType(toType)) { - return new ListCoercer(typeManager, fromHiveType, toHiveType); - } - else if (isMapType(fromType) && isMapType(toType)) { - return new MapCoercer(typeManager, fromHiveType, toHiveType); - } - else if (isRowType(fromType) && isRowType(toType)) { - return new StructCoercer(typeManager, fromHiveType, toHiveType); - } - - throw new PrestoException(NOT_SUPPORTED, format("Unsupported coercion from %s to %s", fromHiveType, toHiveType)); - } - - private static class IntegerNumberUpscaleCoercer - implements Function - { - private final Type fromType; - private final Type toType; - - public IntegerNumberUpscaleCoercer(Type fromType, Type toType) - { - this.fromType = requireNonNull(fromType, "fromType is null"); - this.toType = requireNonNull(toType, "toType is null"); - } - - @Override - public Block apply(Block block) - { - BlockBuilder blockBuilder = toType.createBlockBuilder(null, block.getPositionCount()); - for (int i = 0; i < block.getPositionCount(); i++) { - if (block.isNull(i)) { - blockBuilder.appendNull(); - continue; - } - toType.writeLong(blockBuilder, fromType.getLong(block, i)); - } - return blockBuilder.build(); - } - } - - private static class IntegerNumberToVarcharCoercer - implements Function - { - private final Type fromType; - private final Type toType; - - public IntegerNumberToVarcharCoercer(Type fromType, Type toType) - { - this.fromType = requireNonNull(fromType, "fromType is null"); - this.toType = requireNonNull(toType, "toType is null"); - } - - @Override - public Block apply(Block block) - { - BlockBuilder blockBuilder = toType.createBlockBuilder(null, block.getPositionCount()); - for (int i = 0; i < block.getPositionCount(); i++) { - if (block.isNull(i)) { - blockBuilder.appendNull(); - continue; - } - toType.writeSlice(blockBuilder, utf8Slice(String.valueOf(fromType.getLong(block, i)))); - } - return blockBuilder.build(); - } - } - - private static class VarcharToIntegerNumberCoercer - implements Function - { - private final Type fromType; - private final Type toType; - - private final long minValue; - private final long maxValue; - - public VarcharToIntegerNumberCoercer(Type fromType, Type toType) - { - this.fromType = requireNonNull(fromType, "fromType is null"); - this.toType = requireNonNull(toType, "toType is null"); - - if (toType.equals(TINYINT)) { - minValue = Byte.MIN_VALUE; - maxValue = Byte.MAX_VALUE; - } - else if (toType.equals(SMALLINT)) { - minValue = Short.MIN_VALUE; - maxValue = Short.MAX_VALUE; - } - else if (toType.equals(INTEGER)) { - minValue = Integer.MIN_VALUE; - maxValue = Integer.MAX_VALUE; - } - else if (toType.equals(BIGINT)) { - minValue = Long.MIN_VALUE; - maxValue = Long.MAX_VALUE; - } - else { - throw new PrestoException(NOT_SUPPORTED, format("Could not create Coercer from from varchar to %s", toType)); - } - } - - @Override - public Block apply(Block block) - { - BlockBuilder blockBuilder = toType.createBlockBuilder(null, block.getPositionCount()); - for (int i = 0; i < block.getPositionCount(); i++) { - if (block.isNull(i)) { - blockBuilder.appendNull(); - continue; - } - try { - long value = Long.parseLong(fromType.getSlice(block, i).toStringUtf8()); - if (minValue <= value && value <= maxValue) { - toType.writeLong(blockBuilder, value); - } - else { - blockBuilder.appendNull(); - } - } - catch (NumberFormatException e) { - blockBuilder.appendNull(); - } - } - return blockBuilder.build(); - } - } - - private static class FloatToDoubleCoercer - implements Function - { - @Override - public Block apply(Block block) - { - BlockBuilder blockBuilder = DOUBLE.createBlockBuilder(null, block.getPositionCount()); - for (int i = 0; i < block.getPositionCount(); i++) { - if (block.isNull(i)) { - blockBuilder.appendNull(); - continue; - } - DOUBLE.writeDouble(blockBuilder, intBitsToFloat((int) REAL.getLong(block, i))); - } - return blockBuilder.build(); - } - } - - private static class ListCoercer - implements Function - { - private final Function elementCoercer; - - public ListCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) - { - requireNonNull(typeManager, "typeManage is null"); - requireNonNull(fromHiveType, "fromHiveType is null"); - requireNonNull(toHiveType, "toHiveType is null"); - HiveType fromElementHiveType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); - HiveType toElementHiveType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); - this.elementCoercer = fromElementHiveType.equals(toElementHiveType) ? null : createCoercer(typeManager, fromElementHiveType, toElementHiveType); - } - - @Override - public Block apply(Block block) - { - if (elementCoercer == null) { - return block; - } - ColumnarArray arrayBlock = toColumnarArray(block); - Block elementsBlock = elementCoercer.apply(arrayBlock.getElementsBlock()); - boolean[] valueIsNull = new boolean[arrayBlock.getPositionCount()]; - int[] offsets = new int[arrayBlock.getPositionCount() + 1]; - for (int i = 0; i < arrayBlock.getPositionCount(); i++) { - valueIsNull[i] = arrayBlock.isNull(i); - offsets[i + 1] = offsets[i] + arrayBlock.getLength(i); - } - return ArrayBlock.fromElementBlock(arrayBlock.getPositionCount(), Optional.of(valueIsNull), offsets, elementsBlock); - } - } - - private static class MapCoercer - implements Function - { - private final Type toType; - private final Function keyCoercer; - private final Function valueCoercer; - - public MapCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) - { - requireNonNull(typeManager, "typeManage is null"); - requireNonNull(fromHiveType, "fromHiveType is null"); - this.toType = requireNonNull(toHiveType, "toHiveType is null").getType(typeManager); - HiveType fromKeyHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); - HiveType fromValueHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); - HiveType toKeyHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); - HiveType toValueHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); - this.keyCoercer = fromKeyHiveType.equals(toKeyHiveType) ? null : createCoercer(typeManager, fromKeyHiveType, toKeyHiveType); - this.valueCoercer = fromValueHiveType.equals(toValueHiveType) ? null : createCoercer(typeManager, fromValueHiveType, toValueHiveType); - } - - @Override - public Block apply(Block block) - { - ColumnarMap mapBlock = toColumnarMap(block); - Block keysBlock = keyCoercer == null ? mapBlock.getKeysBlock() : keyCoercer.apply(mapBlock.getKeysBlock()); - Block valuesBlock = valueCoercer == null ? mapBlock.getValuesBlock() : valueCoercer.apply(mapBlock.getValuesBlock()); - boolean[] valueIsNull = new boolean[mapBlock.getPositionCount()]; - int[] offsets = new int[mapBlock.getPositionCount() + 1]; - for (int i = 0; i < mapBlock.getPositionCount(); i++) { - valueIsNull[i] = mapBlock.isNull(i); - offsets[i + 1] = offsets[i] + mapBlock.getEntryCount(i); - } - return ((MapType) toType).createBlockFromKeyValue(Optional.of(valueIsNull), offsets, keysBlock, valuesBlock); - } - } - - private static class StructCoercer - implements Function - { - private final Function[] coercers; - private final Block[] nullBlocks; - - public StructCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) - { - requireNonNull(typeManager, "typeManage is null"); - requireNonNull(fromHiveType, "fromHiveType is null"); - requireNonNull(toHiveType, "toHiveType is null"); - List fromFieldTypes = extractStructFieldTypes(fromHiveType); - List toFieldTypes = extractStructFieldTypes(toHiveType); - this.coercers = new Function[toFieldTypes.size()]; - this.nullBlocks = new Block[toFieldTypes.size()]; - for (int i = 0; i < coercers.length; i++) { - if (i >= fromFieldTypes.size()) { - nullBlocks[i] = toFieldTypes.get(i).getType(typeManager).createBlockBuilder(null, 1).appendNull().build(); - } - else if (!fromFieldTypes.get(i).equals(toFieldTypes.get(i))) { - coercers[i] = createCoercer(typeManager, fromFieldTypes.get(i), toFieldTypes.get(i)); - } - } - } - - @Override - public Block apply(Block block) - { - ColumnarRow rowBlock = toColumnarRow(block); - Block[] fields = new Block[coercers.length]; - int[] ids = new int[rowBlock.getField(0).getPositionCount()]; - for (int i = 0; i < coercers.length; i++) { - if (coercers[i] != null) { - fields[i] = coercers[i].apply(rowBlock.getField(i)); - } - else if (i < rowBlock.getFieldCount()) { - fields[i] = rowBlock.getField(i); - } - else { - fields[i] = new DictionaryBlock(nullBlocks[i], ids); - } - } - boolean[] valueIsNull = new boolean[rowBlock.getPositionCount()]; - for (int i = 0; i < rowBlock.getPositionCount(); i++) { - valueIsNull[i] = rowBlock.isNull(i); - } - return RowBlock.fromFieldBlocks(valueIsNull.length, Optional.of(valueIsNull), fields); - } - } - private static final class CoercionLazyBlockLoader implements LazyBlockLoader { @@ -596,7 +282,7 @@ private static Page extractColumns(Page page, int[] columns) return new Page(page.getPositionCount(), blocks); } - public static class BucketAdapter + private static class BucketAdapter { public final int[] bucketColumns; public final int bucketToKeep; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java index f85d139dd3a19..096cd996fb41f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java @@ -15,12 +15,16 @@ import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; import com.facebook.presto.hive.HiveSplit.BucketConversion; +import com.facebook.presto.hive.metastore.Column; +import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.RecordCursor; import com.facebook.presto.spi.RecordPageSource; +import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.Subfield; import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; @@ -44,19 +48,22 @@ import java.util.OptionalInt; import java.util.Properties; import java.util.Set; +import java.util.function.Function; +import static com.facebook.presto.hive.HiveCoercer.createCoercer; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.SYNTHESIZED; import static com.facebook.presto.hive.HivePageSourceProvider.ColumnMapping.toColumnHandles; -import static com.facebook.presto.hive.HiveSessionProperties.isPushdownFilterEnabled; import static com.facebook.presto.hive.HiveUtil.getPrefilledColumnValue; -import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.MOST_OPTIMIZED; +import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema; +import static com.facebook.presto.hive.metastore.MetastoreUtil.reconstructPartitionSchema; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Predicates.not; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Maps.uniqueIndex; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; @@ -93,9 +100,11 @@ public HivePageSourceProvider( } @Override - public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, List columns) + public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, ConnectorTableLayoutHandle layout, List columns) { - List hiveColumns = columns.stream() + HiveTableLayoutHandle hiveLayout = (HiveTableLayoutHandle) layout; + + List selectedColumns = columns.stream() .map(HiveColumnHandle.class::cast) .collect(toList()); @@ -104,8 +113,11 @@ public ConnectorPageSource createPageSource(ConnectorTransactionHandle transacti Configuration configuration = hdfsEnvironment.getConfiguration(new HdfsContext(session, hiveSplit.getDatabase(), hiveSplit.getTable()), path); - if (isPushdownFilterEnabled(session)) { - return createSelectivePageSource(selectivePageSourceFactories, configuration, session, hiveSplit, hiveColumns, hiveStorageTimeZone, rowExpressionService); + if (hiveLayout.isPushdownFilterEnabled()) { + Optional selectivePageSource = createSelectivePageSource(selectivePageSourceFactories, configuration, session, hiveSplit, hiveLayout, selectedColumns, hiveStorageTimeZone, rowExpressionService, typeManager); + if (selectivePageSource.isPresent()) { + return selectivePageSource.get(); + } } Optional pageSource = createHivePageSource( @@ -118,40 +130,54 @@ public ConnectorPageSource createPageSource(ConnectorTransactionHandle transacti hiveSplit.getStart(), hiveSplit.getLength(), hiveSplit.getFileSize(), - hiveSplit.getSchema(), - hiveSplit.getDomainPredicate() + hiveSplit.getStorage(), + hiveLayout.getDomainPredicate() .transform(Subfield::getRootName) - .transform(hiveSplit.getPredicateColumns()::get), - hiveColumns, + .transform(hiveLayout.getPredicateColumns()::get), + selectedColumns, + hiveLayout.getPredicateColumns(), hiveSplit.getPartitionKeys(), hiveStorageTimeZone, typeManager, - hiveSplit.getColumnCoercions(), + hiveLayout.getSchemaTableName(), + hiveLayout.getPartitionColumns(), + hiveLayout.getDataColumns(), + hiveLayout.getTableParameters(), + hiveSplit.getPartitionDataColumnCount(), + hiveSplit.getPartitionSchemaDifference(), hiveSplit.getBucketConversion(), - hiveSplit.isS3SelectPushdownEnabled()); + hiveSplit.isS3SelectPushdownEnabled(), + hiveSplit.getExtraFileInfo(), + hiveLayout.getRemainingPredicate(), + hiveLayout.isPushdownFilterEnabled(), + rowExpressionService); if (pageSource.isPresent()) { return pageSource.get(); } throw new IllegalStateException("Could not find a file reader for split " + hiveSplit); } - private static ConnectorPageSource createSelectivePageSource( + private static Optional createSelectivePageSource( Set selectivePageSourceFactories, Configuration configuration, ConnectorSession session, HiveSplit split, + HiveTableLayoutHandle layout, List columns, DateTimeZone hiveStorageTimeZone, - RowExpressionService rowExpressionService) + RowExpressionService rowExpressionService, + TypeManager typeManager) { Set interimColumns = ImmutableSet.builder() - .addAll(split.getPredicateColumns().values()) + .addAll(layout.getPredicateColumns().values()) .addAll(split.getBucketConversion().map(BucketConversion::getBucketColumnHandles).orElse(ImmutableList.of())) .build(); + Set columnNames = columns.stream().map(HiveColumnHandle::getName).collect(toImmutableSet()); + List allColumns = ImmutableList.builder() .addAll(columns) - .addAll(interimColumns.stream().filter(not(columns::contains)).collect(toImmutableList())) + .addAll(interimColumns.stream().filter(column -> !columnNames.contains(column.getName())).collect(toImmutableList())) .build(); Path path = new Path(split.getPath()); @@ -159,22 +185,27 @@ private static ConnectorPageSource createSelectivePageSource( split.getPartitionKeys(), allColumns, ImmutableList.of(), - split.getColumnCoercions(), // TODO Include predicateColumns + split.getPartitionSchemaDifference(), path, split.getTableBucketNumber()); - Optional bucketAdaptation = split.getBucketConversion().map(conversion -> toBucketAdaptation(conversion, columnMappings, split.getTableBucketNumber())); - checkArgument(!bucketAdaptation.isPresent(), "Bucket conversion is not supported yet"); + Optional bucketAdaptation = split.getBucketConversion().map(conversion -> toBucketAdaptation(conversion, columnMappings, split.getTableBucketNumber(), mapping -> mapping.getHiveColumnHandle().getHiveColumnIndex())); Map prefilledValues = columnMappings.stream() .filter(mapping -> mapping.getKind() == ColumnMappingKind.PREFILLED) .collect(toImmutableMap(mapping -> mapping.getHiveColumnHandle().getHiveColumnIndex(), ColumnMapping::getPrefilledValue)); + Map coercers = columnMappings.stream() + .filter(mapping -> mapping.getCoercionFrom().isPresent()) + .collect(toImmutableMap( + mapping -> mapping.getHiveColumnHandle().getHiveColumnIndex(), + mapping -> createCoercer(typeManager, mapping.getCoercionFrom().get(), mapping.getHiveColumnHandle().getHiveType()))); + List outputColumns = columns.stream() .map(HiveColumnHandle::getHiveColumnIndex) .collect(toImmutableList()); - RowExpression optimizedRemainingPredicate = rowExpressionService.getExpressionOptimizer().optimize(split.getRemainingPredicate(), MOST_OPTIMIZED, session); + RowExpression optimizedRemainingPredicate = rowExpressionService.getExpressionOptimizer().optimize(layout.getRemainingPredicate(), OPTIMIZED, session); for (HiveSelectivePageSourceFactory pageSourceFactory : selectivePageSourceFactories) { Optional pageSource = pageSourceFactory.createPageSource( @@ -184,19 +215,22 @@ private static ConnectorPageSource createSelectivePageSource( split.getStart(), split.getLength(), split.getFileSize(), - split.getSchema(), + split.getStorage(), toColumnHandles(columnMappings, true), prefilledValues, + coercers, + bucketAdaptation, outputColumns, - split.getDomainPredicate(), + layout.getDomainPredicate(), optimizedRemainingPredicate, - hiveStorageTimeZone); + hiveStorageTimeZone, + split.getExtraFileInfo()); if (pageSource.isPresent()) { - return pageSource.get(); + return Optional.of(pageSource.get()); } } - throw new IllegalStateException("Could not find a file reader for split " + split); + return Optional.empty(); } public static Optional createHivePageSource( @@ -209,26 +243,58 @@ public static Optional createHivePageSource( long start, long length, long fileSize, - Properties schema, + Storage storage, TupleDomain effectivePredicate, List hiveColumns, + Map predicateColumns, List partitionKeys, DateTimeZone hiveStorageTimeZone, TypeManager typeManager, - Map columnCoercions, + SchemaTableName tableName, + List partitionKeyColumnHandles, + List tableDataColumns, + Map tableParameters, + int partitionDataColumnCount, + Map partitionSchemaDifference, Optional bucketConversion, - boolean s3SelectPushdownEnabled) + boolean s3SelectPushdownEnabled, + Optional extraFileInfo, + RowExpression remainingPredicate, + boolean isPushdownFilterEnabled, + RowExpressionService rowExpressionService) { + List allColumns; + + if (isPushdownFilterEnabled) { + Set columnNames = hiveColumns.stream().map(HiveColumnHandle::getName).collect(toImmutableSet()); + List additionalColumns = predicateColumns.values().stream() + .filter(column -> !columnNames.contains(column.getName())) + .collect(toImmutableList()); + + allColumns = ImmutableList.builder() + .addAll(hiveColumns) + .addAll(additionalColumns) + .build(); + } + else { + allColumns = hiveColumns; + } + List columnMappings = ColumnMapping.buildColumnMappings( partitionKeys, - hiveColumns, + allColumns, bucketConversion.map(BucketConversion::getBucketColumnHandles).orElse(ImmutableList.of()), - columnCoercions, + partitionSchemaDifference, path, tableBucketNumber); + + Set outputIndices = hiveColumns.stream() + .map(HiveColumnHandle::getHiveColumnIndex) + .collect(toImmutableSet()); + List regularAndInterimColumnMappings = ColumnMapping.extractRegularAndInterimColumnMappings(columnMappings); - Optional bucketAdaptation = bucketConversion.map(conversion -> toBucketAdaptation(conversion, regularAndInterimColumnMappings, tableBucketNumber)); + Optional bucketAdaptation = bucketConversion.map(conversion -> toBucketAdaptation(conversion, regularAndInterimColumnMappings, tableBucketNumber, ColumnMapping::getIndex)); for (HiveBatchPageSourceFactory pageSourceFactory : pageSourceFactories) { Optional pageSource = pageSourceFactory.createPageSource( @@ -238,18 +304,32 @@ public static Optional createHivePageSource( start, length, fileSize, - schema, + storage, + tableParameters, toColumnHandles(regularAndInterimColumnMappings, true), effectivePredicate, - hiveStorageTimeZone); + hiveStorageTimeZone, + extraFileInfo); if (pageSource.isPresent()) { - return Optional.of( - new HivePageSource( - columnMappings, - bucketAdaptation, - hiveStorageTimeZone, - typeManager, - pageSource.get())); + HivePageSource hivePageSource = new HivePageSource( + columnMappings, + bucketAdaptation, + hiveStorageTimeZone, + typeManager, + pageSource.get()); + + if (isPushdownFilterEnabled) { + return Optional.of(new FilteringPageSource( + columnMappings, + effectivePredicate, + remainingPredicate, + typeManager, + rowExpressionService, + session, + outputIndices, + hivePageSource)); + } + return Optional.of(hivePageSource); } } @@ -257,6 +337,20 @@ public static Optional createHivePageSource( // GenericHiveRecordCursor will automatically do the coercion without HiveCoercionRecordCursor boolean doCoercion = !(provider instanceof GenericHiveRecordCursorProvider); + List partitionDataColumns = reconstructPartitionSchema(tableDataColumns, partitionDataColumnCount, partitionSchemaDifference); + List partitionKeyColumns = partitionKeyColumnHandles.stream() + .map(handle -> new Column(handle.getName(), handle.getHiveType(), handle.getComment())) + .collect(toImmutableList()); + + Properties schema = getHiveSchema( + storage, + partitionDataColumns, + tableDataColumns, + tableParameters, + tableName.getSchemaName(), + tableName.getTableName(), + partitionKeyColumns); + Optional cursor = provider.createRecordCursor( configuration, session, @@ -295,25 +389,38 @@ public static Optional createHivePageSource( hiveStorageTimeZone, typeManager, delegate); - List columnTypes = hiveColumns.stream() + List columnTypes = allColumns.stream() .map(input -> typeManager.getType(input.getTypeSignature())) .collect(toList()); - return Optional.of(new RecordPageSource(columnTypes, hiveRecordCursor)); + RecordPageSource recordPageSource = new RecordPageSource(columnTypes, hiveRecordCursor); + if (isPushdownFilterEnabled) { + return Optional.of(new FilteringPageSource( + columnMappings, + effectivePredicate, + remainingPredicate, + typeManager, + rowExpressionService, + session, + outputIndices, + recordPageSource)); + } + return Optional.of(recordPageSource); } } return Optional.empty(); } - private static BucketAdaptation toBucketAdaptation(BucketConversion conversion, List columnMappings, OptionalInt tableBucketNumber) + private static BucketAdaptation toBucketAdaptation(BucketConversion conversion, List columnMappings, OptionalInt tableBucketNumber, Function bucketColumnIndexProducer) { Map hiveIndexToBlockIndex = uniqueIndex(columnMappings, columnMapping -> columnMapping.getHiveColumnHandle().getHiveColumnIndex()); int[] bucketColumnIndices = conversion.getBucketColumnHandles().stream() .map(HiveColumnHandle::getHiveColumnIndex) .map(hiveIndexToBlockIndex::get) - .mapToInt(ColumnMapping::getIndex) + .mapToInt(bucketColumnIndexProducer::apply) .toArray(); + List bucketColumnHiveTypes = conversion.getBucketColumnHandles().stream() .map(HiveColumnHandle::getHiveColumnIndex) .map(hiveIndexToBlockIndex::get) @@ -391,14 +498,14 @@ public Optional getCoercionFrom() /** * @param columns columns that need to be returned to engine * @param requiredInterimColumns columns that are needed for processing, but shouldn't be returned to engine (may overlaps with columns) - * @param columnCoercions map from hive column index to hive type + * @param partitionSchemaDifference map from hive column index to hive type * @param bucketNumber empty if table is not bucketed, a number within [0, # bucket in table) otherwise */ public static List buildColumnMappings( List partitionKeys, List columns, List requiredInterimColumns, - Map columnCoercions, + Map partitionSchemaDifference, Path path, OptionalInt bucketNumber) { @@ -407,7 +514,14 @@ public static List buildColumnMappings( Set regularColumnIndices = new HashSet<>(); ImmutableList.Builder columnMappings = ImmutableList.builder(); for (HiveColumnHandle column : columns) { - Optional coercionFrom = Optional.ofNullable(columnCoercions.get(column.getHiveColumnIndex())); + // will be present if the partition has a different schema (column type, column name) for the column + Optional partitionColumn = Optional.ofNullable(partitionSchemaDifference.get(column.getHiveColumnIndex())); + Optional coercionFrom = Optional.empty(); + // we don't care if only the column name has changed + if (partitionColumn.isPresent() && !partitionColumn.get().getType().equals(column.getHiveType())) { + coercionFrom = Optional.of(partitionColumn.get().getType()); + } + if (column.getColumnType() == REGULAR) { checkArgument(regularColumnIndices.add(column.getHiveColumnIndex()), "duplicate hiveColumnIndex in columns list"); columnMappings.add(regular(column, regularIndex, coercionFrom)); @@ -468,47 +582,4 @@ public enum ColumnMappingKind PREFILLED, INTERIM, } - - public static class BucketAdaptation - { - private final int[] bucketColumnIndices; - private final List bucketColumnHiveTypes; - private final int tableBucketCount; - private final int partitionBucketCount; - private final int bucketToKeep; - - public BucketAdaptation(int[] bucketColumnIndices, List bucketColumnHiveTypes, int tableBucketCount, int partitionBucketCount, int bucketToKeep) - { - this.bucketColumnIndices = bucketColumnIndices; - this.bucketColumnHiveTypes = bucketColumnHiveTypes; - this.tableBucketCount = tableBucketCount; - this.partitionBucketCount = partitionBucketCount; - this.bucketToKeep = bucketToKeep; - } - - public int[] getBucketColumnIndices() - { - return bucketColumnIndices; - } - - public List getBucketColumnHiveTypes() - { - return bucketColumnHiveTypes; - } - - public int getTableBucketCount() - { - return tableBucketCount; - } - - public int getPartitionBucketCount() - { - return partitionBucketCount; - } - - public int getBucketToKeep() - { - return bucketToKeep; - } - } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionKey.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionKey.java index 129e10f4adb08..2151cce2cbb75 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionKey.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionKey.java @@ -19,6 +19,7 @@ import java.util.Objects; +import static com.facebook.presto.hive.metastore.MetastoreUtil.HIVE_DEFAULT_DYNAMIC_PARTITION; import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; @@ -27,7 +28,6 @@ public final class HivePartitionKey private static final int INSTANCE_SIZE = ClassLayout.parseClass(HivePartitionKey.class).instanceSize() + ClassLayout.parseClass(String.class).instanceSize() * 2; - public static final String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__"; private final String name; private final String value; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java index 4bf8762d8217d..39f86a57aa683 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java @@ -47,7 +47,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import io.airlift.slice.Slice; -import org.apache.hadoop.hive.common.FileUtils; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.ISODateTimeFormat; @@ -63,11 +62,15 @@ import static com.facebook.presto.hive.HiveBucketing.getHiveBucketFilter; import static com.facebook.presto.hive.HiveBucketing.getHiveBucketHandle; +import static com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME; import static com.facebook.presto.hive.HiveErrorCode.HIVE_EXCEEDED_PARTITION_LIMIT; import static com.facebook.presto.hive.HiveSessionProperties.getMaxBucketsForGroupedExecution; +import static com.facebook.presto.hive.HiveSessionProperties.isOfflineDataDebugModeEnabled; import static com.facebook.presto.hive.HiveSessionProperties.shouldIgnoreTableBucketing; import static com.facebook.presto.hive.HiveUtil.getPartitionKeyColumnHandles; import static com.facebook.presto.hive.HiveUtil.parsePartitionValue; +import static com.facebook.presto.hive.metastore.MetastoreUtil.HIVE_DEFAULT_DYNAMIC_PARTITION; +import static com.facebook.presto.hive.metastore.MetastoreUtil.extractPartitionValues; import static com.facebook.presto.hive.metastore.MetastoreUtil.getProtectMode; import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline; @@ -120,13 +123,13 @@ public HivePartitionManager( this.domainCompactionThreshold = domainCompactionThreshold; } - public Iterable getPartitionsIterator(SemiTransactionalHiveMetastore metastore, ConnectorTableHandle tableHandle, Constraint constraint) + public Iterable getPartitionsIterator(SemiTransactionalHiveMetastore metastore, ConnectorTableHandle tableHandle, Constraint constraint, ConnectorSession session) { HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle; TupleDomain effectivePredicate = constraint.getSummary(); SchemaTableName tableName = hiveTableHandle.getSchemaTableName(); - Table table = getTable(metastore, tableName); + Table table = getTable(metastore, tableName, isOfflineDataDebugModeEnabled(session)); List partitionColumns = getPartitionKeyColumnHandles(table); @@ -148,24 +151,35 @@ public HivePartitionResult getPartitions(SemiTransactionalHiveMetastore metastor TupleDomain effectivePredicate = constraint.getSummary(); SchemaTableName tableName = hiveTableHandle.getSchemaTableName(); - Table table = getTable(metastore, tableName); + Table table = getTable(metastore, tableName, isOfflineDataDebugModeEnabled(session)); List partitionColumns = getPartitionKeyColumnHandles(table); - List partitions = getPartitionsAsList(getPartitionsIterator(metastore, tableHandle, constraint).iterator()); + List partitions = getPartitionsAsList(getPartitionsIterator(metastore, tableHandle, constraint, session).iterator()); // never ignore table bucketing for temporary tables as those are created such explicitly by the engine request boolean shouldIgnoreTableBucketing = !table.getTableType().equals(TEMPORARY_TABLE) && shouldIgnoreTableBucketing(session); Optional hiveBucketHandle = shouldIgnoreTableBucketing ? Optional.empty() : getHiveBucketHandle(table); Optional bucketFilter = shouldIgnoreTableBucketing ? Optional.empty() : getHiveBucketFilter(table, effectivePredicate); - if (hiveBucketHandle.isPresent() && hiveBucketHandle.get().getReadBucketCount() * partitions.size() > getMaxBucketsForGroupedExecution(session)) { + if (!queryUsesHiveBucketColumn(effectivePredicate) + && hiveBucketHandle.isPresent() + && queryAccessesTooManyBuckets(hiveBucketHandle.get(), bucketFilter, partitions, session)) { hiveBucketHandle = Optional.empty(); bucketFilter = Optional.empty(); } if (effectivePredicate.isNone()) { - return new HivePartitionResult(partitionColumns, partitions, TupleDomain.none(), TupleDomain.none(), TupleDomain.none(), hiveBucketHandle, Optional.empty()); + return new HivePartitionResult( + partitionColumns, + table.getDataColumns(), + table.getParameters(), + partitions, + TupleDomain.none(), + TupleDomain.none(), + TupleDomain.none(), + hiveBucketHandle, + Optional.empty()); } TupleDomain compactEffectivePredicate = effectivePredicate.compact(domainCompactionThreshold); @@ -173,10 +187,12 @@ public HivePartitionResult getPartitions(SemiTransactionalHiveMetastore metastor if (partitionColumns.isEmpty()) { return new HivePartitionResult( partitionColumns, + table.getDataColumns(), + table.getParameters(), partitions, compactEffectivePredicate, effectivePredicate, - TupleDomain.none(), + TupleDomain.all(), hiveBucketHandle, bucketFilter); } @@ -184,7 +200,31 @@ public HivePartitionResult getPartitions(SemiTransactionalHiveMetastore metastor // All partition key domains will be fully evaluated, so we don't need to include those TupleDomain remainingTupleDomain = TupleDomain.withColumnDomains(Maps.filterKeys(effectivePredicate.getDomains().get(), not(Predicates.in(partitionColumns)))); TupleDomain enforcedTupleDomain = TupleDomain.withColumnDomains(Maps.filterKeys(effectivePredicate.getDomains().get(), Predicates.in(partitionColumns))); - return new HivePartitionResult(partitionColumns, partitions, compactEffectivePredicate, remainingTupleDomain, enforcedTupleDomain, hiveBucketHandle, bucketFilter); + return new HivePartitionResult( + partitionColumns, + table.getDataColumns(), + table.getParameters(), + partitions, + compactEffectivePredicate, + remainingTupleDomain, + enforcedTupleDomain, + hiveBucketHandle, + bucketFilter); + } + + private boolean queryUsesHiveBucketColumn(TupleDomain effectivePredicate) + { + if (!effectivePredicate.getDomains().isPresent()) { + return false; + } + return effectivePredicate.getDomains().get().keySet().stream().anyMatch(key -> ((HiveColumnHandle) key).getName().equals(BUCKET_COLUMN_NAME)); + } + + private boolean queryAccessesTooManyBuckets(HiveBucketHandle handle, Optional filter, List partitions, ConnectorSession session) + { + int bucketsPerPartition = filter.map(hiveBucketFilter -> hiveBucketFilter.getBucketsToKeep().size()) + .orElseGet(handle::getReadBucketCount); + return bucketsPerPartition * partitions.size() > getMaxBucketsForGroupedExecution(session); } private List getPartitionsAsList(Iterator partitionsIterator) @@ -210,7 +250,7 @@ public HivePartitionResult getPartitions(SemiTransactionalHiveMetastore metastor HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle; SchemaTableName tableName = hiveTableHandle.getSchemaTableName(); - Table table = getTable(metastore, tableName); + Table table = getTable(metastore, tableName, isOfflineDataDebugModeEnabled(session)); List partitionColumns = getPartitionKeyColumnHandles(table); List partitionColumnTypes = partitionColumns.stream() @@ -224,7 +264,16 @@ public HivePartitionResult getPartitions(SemiTransactionalHiveMetastore metastor .collect(toImmutableList()); Optional bucketHandle = shouldIgnoreTableBucketing(session) ? Optional.empty() : getHiveBucketHandle(table); - return new HivePartitionResult(partitionColumns, partitionList, TupleDomain.all(), TupleDomain.all(), TupleDomain.none(), bucketHandle, Optional.empty()); + return new HivePartitionResult( + partitionColumns, + table.getDataColumns(), + table.getParameters(), + partitionList, + TupleDomain.all(), + TupleDomain.all(), + TupleDomain.none(), + bucketHandle, + Optional.empty()); } private Optional parseValuesAndFilterPartition( @@ -252,14 +301,18 @@ private Optional parseValuesAndFilterPartition( return Optional.of(partition); } - private Table getTable(SemiTransactionalHiveMetastore metastore, SchemaTableName tableName) + private Table getTable(SemiTransactionalHiveMetastore metastore, SchemaTableName tableName, boolean offlineDataDebugModeEnabled) { Optional
target = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); if (!target.isPresent()) { throw new TableNotFoundException(tableName); } Table table = target.get(); - verifyOnline(tableName, Optional.empty(), getProtectMode(table), table.getParameters()); + + if (!offlineDataDebugModeEnabled) { + verifyOnline(tableName, Optional.empty(), getProtectMode(table), table.getParameters()); + } + return table; } @@ -276,7 +329,7 @@ private List getFilteredPartitionNames(SemiTransactionalHiveMetastore me Object value = domain.getNullableSingleValue(); Type type = domain.getType(); if (value == null) { - filter.add(HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION); + filter.add(HIVE_DEFAULT_DYNAMIC_PARTITION); } else if (type instanceof CharType) { Slice slice = (Slice) value; @@ -349,32 +402,4 @@ public static HivePartition parsePartition( Map values = builder.build(); return new HivePartition(tableName, partitionName, values); } - - public static List extractPartitionValues(String partitionName) - { - ImmutableList.Builder values = ImmutableList.builder(); - - boolean inKey = true; - int valueStart = -1; - for (int i = 0; i < partitionName.length(); i++) { - char current = partitionName.charAt(i); - if (inKey) { - checkArgument(current != '/', "Invalid partition spec: %s", partitionName); - if (current == '=') { - inKey = false; - valueStart = i + 1; - } - } - else if (current == '/') { - checkArgument(valueStart != -1, "Invalid partition spec: %s", partitionName); - values.add(FileUtils.unescapePathName(partitionName.substring(valueStart, i))); - inKey = true; - valueStart = -1; - } - } - checkArgument(!inKey, "Invalid partition spec: %s", partitionName); - values.add(FileUtils.unescapePathName(partitionName.substring(valueStart, partitionName.length()))); - - return values.build(); - } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionMetadata.java index 40314d3e560d3..87299538d3b5d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionMetadata.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Partition; import java.util.Map; @@ -24,16 +25,16 @@ public class HivePartitionMetadata { private final Optional partition; private final HivePartition hivePartition; - private final Map columnCoercions; + private final Map partitionSchemaDifference; HivePartitionMetadata( HivePartition hivePartition, Optional partition, - Map columnCoercions) + Map partitionSchemaDifference) { this.partition = requireNonNull(partition, "partition is null"); this.hivePartition = requireNonNull(hivePartition, "hivePartition is null"); - this.columnCoercions = requireNonNull(columnCoercions, "columnCoercions is null"); + this.partitionSchemaDifference = requireNonNull(partitionSchemaDifference, "partitionSchemaDifference is null"); } public HivePartition getHivePartition() @@ -49,8 +50,8 @@ public Optional getPartition() return partition; } - public Map getColumnCoercions() + public Map getPartitionSchemaDifference() { - return columnCoercions; + return partitionSchemaDifference; } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionObjectBuilder.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionObjectBuilder.java new file mode 100644 index 0000000000000..3193c2323bba2 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionObjectBuilder.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.hive.metastore.MetastoreUtil; +import com.facebook.presto.hive.metastore.Partition; +import com.facebook.presto.hive.metastore.StorageFormat; +import com.facebook.presto.hive.metastore.Table; +import com.facebook.presto.spi.ConnectorSession; +import com.google.common.collect.ImmutableMap; + +public class HivePartitionObjectBuilder + implements PartitionObjectBuilder +{ + @Override + public Partition buildPartitionObject( + ConnectorSession session, + Table table, + PartitionUpdate partitionUpdate, + String prestoVersion) + { + return Partition.builder() + .setDatabaseName(table.getDatabaseName()) + .setTableName(table.getTableName()) + .setColumns(table.getDataColumns()) + .setValues(MetastoreUtil.extractPartitionValues(partitionUpdate.getName())) + .setParameters(ImmutableMap.builder() + .put(HiveMetadata.PRESTO_VERSION_NAME, prestoVersion) + .put(MetastoreUtil.PRESTO_QUERY_ID_NAME, session.getQueryId()) + .build()) + .withStorage(storage -> storage + .setStorageFormat(HiveSessionProperties.isRespectTableFormat(session) ? + table.getStorage().getStorageFormat() : + StorageFormat.fromHiveStorageFormat(HiveSessionProperties.getHiveStorageFormat(session))) + .setLocation(partitionUpdate.getTargetPath().toString()) + .setBucketProperty(table.getStorage().getBucketProperty()) + .setSerdeParameters(table.getStorage().getSerdeParameters())) + .build(); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionResult.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionResult.java index a824d14f83c6b..75076ef279055 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionResult.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionResult.java @@ -14,10 +14,14 @@ package com.facebook.presto.hive; import com.facebook.presto.hive.HiveBucketing.HiveBucketFilter; +import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.predicate.TupleDomain; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.util.List; +import java.util.Map; import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -33,6 +37,8 @@ public class HivePartitionResult { private final List partitionColumns; + private final List dataColumns; + private final Map tableParameters; private final List partitions; private final TupleDomain effectivePredicate; private final TupleDomain unenforcedConstraint; @@ -42,6 +48,8 @@ public class HivePartitionResult public HivePartitionResult( List partitionColumns, + List dataColumns, + Map tableParameters, List partitions, TupleDomain effectivePredicate, TupleDomain unenforcedConstraint, @@ -50,6 +58,8 @@ public HivePartitionResult( Optional bucketFilter) { this.partitionColumns = requireNonNull(partitionColumns, "partitionColumns is null"); + this.dataColumns = ImmutableList.copyOf(requireNonNull(dataColumns, "dataColumns is null")); + this.tableParameters = ImmutableMap.copyOf(requireNonNull(tableParameters, "tableProperties is null")); this.partitions = requireNonNull(partitions, "partitions is null"); this.effectivePredicate = requireNonNull(effectivePredicate, "effectivePredicate is null"); this.unenforcedConstraint = requireNonNull(unenforcedConstraint, "unenforcedConstraint is null"); @@ -63,6 +73,16 @@ public List getPartitionColumns() return partitionColumns; } + public List getDataColumns() + { + return dataColumns; + } + + public Map getTableParameters() + { + return tableParameters; + } + public List getPartitions() { return partitions; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSelectivePageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSelectivePageSourceFactory.java index 34c70b77967d2..e3816999fd2e5 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSelectivePageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSelectivePageSourceFactory.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Subfield; @@ -25,7 +26,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Properties; public interface HiveSelectivePageSourceFactory { @@ -36,11 +36,14 @@ Optional createPageSource( long start, long length, long fileSize, - Properties schema, + Storage storage, List columns, - Map prefilledValues, // key is hiveColumnIndex - List outputColumns, // element is hiveColumnIndex + Map prefilledValues, // key is hiveColumnIndex + Map coercers, // key is hiveColumnIndex + Optional bucketAdaptation, + List outputColumns, // element is hiveColumnIndex TupleDomain domainPredicate, - RowExpression remainingPredicate, // refers to columns by name; already optimized - DateTimeZone hiveStorageTimeZone); + RowExpression remainingPredicate, // refers to columns by name; already optimized + DateTimeZone hiveStorageTimeZone, + Optional extraFileInfo); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java index a9631e0de8155..6ca47bf33d894 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java @@ -51,6 +51,7 @@ public final class HiveSessionProperties private static final String ORC_TINY_STRIPE_THRESHOLD = "orc_tiny_stripe_threshold"; private static final String ORC_MAX_READ_BLOCK_SIZE = "orc_max_read_block_size"; private static final String ORC_LAZY_READ_SMALL_RANGES = "orc_lazy_read_small_ranges"; + private static final String ORC_ZSTD_JNI_DECOMPRESSION_ENABLED = "orc_zstd_jni_decompression_enabled"; private static final String ORC_STRING_STATISTICS_LIMIT = "orc_string_statistics_limit"; private static final String ORC_OPTIMIZED_WRITER_ENABLED = "orc_optimized_writer_enabled"; private static final String ORC_OPTIMIZED_WRITER_VALIDATE = "orc_optimized_writer_validate"; @@ -65,6 +66,7 @@ public final class HiveSessionProperties private static final String RESPECT_TABLE_FORMAT = "respect_table_format"; private static final String PARQUET_USE_COLUMN_NAME = "parquet_use_column_names"; private static final String PARQUET_FAIL_WITH_CORRUPTED_STATISTICS = "parquet_fail_with_corrupted_statistics"; + private static final String PARQUET_MAX_READ_BLOCK_SIZE = "parquet_max_read_block_size"; private static final String PARQUET_WRITER_BLOCK_SIZE = "parquet_writer_block_size"; private static final String PARQUET_WRITER_PAGE_SIZE = "parquet_writer_page_size"; private static final String MAX_SPLIT_SIZE = "max_split_size"; @@ -72,10 +74,12 @@ public final class HiveSessionProperties public static final String RCFILE_OPTIMIZED_WRITER_ENABLED = "rcfile_optimized_writer_enabled"; private static final String RCFILE_OPTIMIZED_WRITER_VALIDATE = "rcfile_optimized_writer_validate"; private static final String SORTED_WRITING_ENABLED = "sorted_writing_enabled"; + public static final String SORTED_WRITE_TO_TEMP_PATH_ENABLED = "sorted_write_to_temp_path_enabled"; + public static final String SORTED_WRITE_TEMP_PATH_SUBDIRECTORY_COUNT = "sorted_write_temp_path_subdirectory_count"; private static final String STATISTICS_ENABLED = "statistics_enabled"; private static final String PARTITION_STATISTICS_SAMPLE_SIZE = "partition_statistics_sample_size"; private static final String IGNORE_CORRUPTED_STATISTICS = "ignore_corrupted_statistics"; - private static final String COLLECT_COLUMN_STATISTICS_ON_WRITE = "collect_column_statistics_on_write"; + public static final String COLLECT_COLUMN_STATISTICS_ON_WRITE = "collect_column_statistics_on_write"; private static final String OPTIMIZE_MISMATCHED_BUCKET_COUNT = "optimize_mismatched_bucket_count"; private static final String S3_SELECT_PUSHDOWN_ENABLED = "s3_select_pushdown_enabled"; private static final String TEMPORARY_STAGING_DIRECTORY_ENABLED = "temporary_staging_directory_enabled"; @@ -84,8 +88,10 @@ public final class HiveSessionProperties private static final String TEMPORARY_TABLE_STORAGE_FORMAT = "temporary_table_storage_format"; private static final String TEMPORARY_TABLE_COMPRESSION_CODEC = "temporary_table_compression_codec"; public static final String PUSHDOWN_FILTER_ENABLED = "pushdown_filter_enabled"; + public static final String RANGE_FILTERS_ON_SUBSCRIPTS_ENABLED = "range_filters_on_subscripts_enabled"; public static final String VIRTUAL_BUCKET_COUNT = "virtual_bucket_count"; public static final String MAX_BUCKETS_FOR_GROUPED_EXECUTION = "max_buckets_for_grouped_execution"; + public static final String OFFLINE_DATA_DEBUG_MODE_ENABLED = "offline_data_debug_mode_enabled"; private final List> sessionProperties; @@ -256,6 +262,11 @@ public HiveSessionProperties(HiveClientConfig hiveClientConfig, OrcFileWriterCon "Parquet: Fail when scanning Parquet files with corrupted statistics", hiveClientConfig.isFailOnCorruptedParquetStatistics(), false), + dataSizeSessionProperty( + PARQUET_MAX_READ_BLOCK_SIZE, + "Parquet: Maximum size of a block to read", + hiveClientConfig.getParquetMaxReadBlockSize(), + false), dataSizeSessionProperty( PARQUET_WRITER_BLOCK_SIZE, "Parquet: Writer block size", @@ -291,6 +302,16 @@ public HiveSessionProperties(HiveClientConfig hiveClientConfig, OrcFileWriterCon "Enable writing to bucketed sorted tables", hiveClientConfig.isSortedWritingEnabled(), false), + booleanProperty( + SORTED_WRITE_TO_TEMP_PATH_ENABLED, + "Enable writing temp files to temp path when writing to bucketed sorted tables", + hiveClientConfig.isSortedWriteToTempPathEnabled(), + false), + integerProperty( + SORTED_WRITE_TEMP_PATH_SUBDIRECTORY_COUNT, + "Number of directories per partition for temp files generated by writing sorted table", + hiveClientConfig.getSortedWriteTempPathSubdirectoryCount(), + false), booleanProperty( STATISTICS_ENABLED, "Experimental: Expose table statistics", @@ -359,6 +380,11 @@ public HiveSessionProperties(HiveClientConfig hiveClientConfig, OrcFileWriterCon "Experimental: enable complex filter pushdown", hiveClientConfig.isPushdownFilterEnabled(), false), + booleanProperty( + RANGE_FILTERS_ON_SUBSCRIPTS_ENABLED, + "Experimental: enable pushdown of range filters on subscripts (a[2] = 5) into ORC column readers", + hiveClientConfig.isRangeFiltersOnSubscriptsEnabled(), + false), integerProperty( VIRTUAL_BUCKET_COUNT, "Number of virtual bucket assigned for unbucketed tables", @@ -368,7 +394,17 @@ public HiveSessionProperties(HiveClientConfig hiveClientConfig, OrcFileWriterCon MAX_BUCKETS_FOR_GROUPED_EXECUTION, "maximum total buckets to allow using grouped execution", hiveClientConfig.getMaxBucketsForGroupedExecution(), - false)); + false), + booleanProperty( + OFFLINE_DATA_DEBUG_MODE_ENABLED, + "allow reading from tables or partitions that are marked as offline or not readable", + false, + true), + booleanProperty( + ORC_ZSTD_JNI_DECOMPRESSION_ENABLED, + "use JNI based zstd decompression for reading ORC files", + hiveClientConfig.isZstdJniDecompressionEnabled(), + true)); } public List> getSessionProperties() @@ -436,6 +472,11 @@ public static boolean getOrcLazyReadSmallRanges(ConnectorSession session) return session.getProperty(ORC_LAZY_READ_SMALL_RANGES, Boolean.class); } + public static boolean isOrcZstdJniDecompressionEnabled(ConnectorSession session) + { + return session.getProperty(ORC_ZSTD_JNI_DECOMPRESSION_ENABLED, Boolean.class); + } + public static DataSize getOrcStringStatisticsLimit(ConnectorSession session) { return session.getProperty(ORC_STRING_STATISTICS_LIMIT, DataSize.class); @@ -513,6 +554,11 @@ public static boolean isFailOnCorruptedParquetStatistics(ConnectorSession sessio return session.getProperty(PARQUET_FAIL_WITH_CORRUPTED_STATISTICS, Boolean.class); } + public static DataSize getParquetMaxReadBlockSize(ConnectorSession session) + { + return session.getProperty(PARQUET_MAX_READ_BLOCK_SIZE, DataSize.class); + } + public static DataSize getParquetWriterBlockSize(ConnectorSession session) { return session.getProperty(PARQUET_WRITER_BLOCK_SIZE, DataSize.class); @@ -548,6 +594,16 @@ public static boolean isSortedWritingEnabled(ConnectorSession session) return session.getProperty(SORTED_WRITING_ENABLED, Boolean.class); } + public static boolean isSortedWriteToTempPathEnabled(ConnectorSession session) + { + return session.getProperty(SORTED_WRITE_TO_TEMP_PATH_ENABLED, Boolean.class); + } + + public static int getSortedWriteTempPathSubdirectoryCount(ConnectorSession session) + { + return session.getProperty(SORTED_WRITE_TEMP_PATH_SUBDIRECTORY_COUNT, Integer.class); + } + public static boolean isS3SelectPushdownEnabled(ConnectorSession session) { return session.getProperty(S3_SELECT_PUSHDOWN_ENABLED, Boolean.class); @@ -613,6 +669,11 @@ public static boolean isPushdownFilterEnabled(ConnectorSession session) return session.getProperty(PUSHDOWN_FILTER_ENABLED, Boolean.class); } + public static boolean isRangeFiltersOnSubscriptsEnabled(ConnectorSession session) + { + return session.getProperty(RANGE_FILTERS_ON_SUBSCRIPTS_ENABLED, Boolean.class); + } + public static int getVirtualBucketCount(ConnectorSession session) { int virtualBucketCount = session.getProperty(VIRTUAL_BUCKET_COUNT, Integer.class); @@ -622,6 +683,11 @@ public static int getVirtualBucketCount(ConnectorSession session) return virtualBucketCount; } + public static boolean isOfflineDataDebugModeEnabled(ConnectorSession session) + { + return session.getProperty(OFFLINE_DATA_DEBUG_MODE_ENABLED, Boolean.class); + } + public static PropertyMetadata dataSizeSessionProperty(String name, String description, DataSize defaultValue, boolean hidden) { return new PropertyMetadata<>( diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplit.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplit.java index f844c1d6781bb..c422894192435 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplit.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplit.java @@ -13,11 +13,10 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.Column; +import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.HostAddress; -import com.facebook.presto.spi.Subfield; -import com.facebook.presto.spi.predicate.TupleDomain; -import com.facebook.presto.spi.relation.RowExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -28,7 +27,6 @@ import java.util.Objects; import java.util.Optional; import java.util.OptionalInt; -import java.util.Properties; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; @@ -41,21 +39,20 @@ public class HiveSplit private final long start; private final long length; private final long fileSize; - private final Properties schema; + private final Storage storage; private final List partitionKeys; private final List addresses; private final String database; private final String table; private final String partitionName; - private final TupleDomain domainPredicate; - private final RowExpression remainingPredicate; - private final Map predicateColumns; private final OptionalInt readBucketNumber; private final OptionalInt tableBucketNumber; private final boolean forceLocalScheduling; - private final Map columnCoercions; // key: hiveColumnIndex + private final int partitionDataColumnCount; + private final Map partitionSchemaDifference; // key: hiveColumnIndex private final Optional bucketConversion; private final boolean s3SelectPushdownEnabled; + private final Optional extraFileInfo; @JsonCreator public HiveSplit( @@ -66,18 +63,17 @@ public HiveSplit( @JsonProperty("start") long start, @JsonProperty("length") long length, @JsonProperty("fileSize") long fileSize, - @JsonProperty("schema") Properties schema, + @JsonProperty("storage") Storage storage, @JsonProperty("partitionKeys") List partitionKeys, @JsonProperty("addresses") List addresses, @JsonProperty("readBucketNumber") OptionalInt readBucketNumber, @JsonProperty("tableBucketNumber") OptionalInt tableBucketNumber, @JsonProperty("forceLocalScheduling") boolean forceLocalScheduling, - @JsonProperty("domainPredicate") TupleDomain domainPredicate, - @JsonProperty("remainingPredicate") RowExpression remainingPredicate, - @JsonProperty("predicateColumns") Map predicateColumns, - @JsonProperty("columnCoercions") Map columnCoercions, + @JsonProperty("partitionDataColumnCount") int partitionDataColumnCount, + @JsonProperty("partitionSchemaDifference") Map partitionSchemaDifference, @JsonProperty("bucketConversion") Optional bucketConversion, - @JsonProperty("s3SelectPushdownEnabled") boolean s3SelectPushdownEnabled) + @JsonProperty("s3SelectPushdownEnabled") boolean s3SelectPushdownEnabled, + @JsonProperty("extraFileInfo") Optional extraFileInfo) { checkArgument(start >= 0, "start must be positive"); checkArgument(length >= 0, "length must be positive"); @@ -86,16 +82,14 @@ public HiveSplit( requireNonNull(table, "table is null"); requireNonNull(partitionName, "partitionName is null"); requireNonNull(path, "path is null"); - requireNonNull(schema, "schema is null"); + requireNonNull(storage, "storage is null"); requireNonNull(partitionKeys, "partitionKeys is null"); requireNonNull(addresses, "addresses is null"); requireNonNull(readBucketNumber, "readBucketNumber is null"); requireNonNull(tableBucketNumber, "tableBucketNumber is null"); - requireNonNull(columnCoercions, "columnCoercions is null"); + requireNonNull(partitionSchemaDifference, "partitionSchemaDifference is null"); requireNonNull(bucketConversion, "bucketConversion is null"); - requireNonNull(domainPredicate, "domainPredicate is null"); - requireNonNull(remainingPredicate, "remainingPredicate is null"); - requireNonNull(predicateColumns, "predicateColumns is null"); + requireNonNull(extraFileInfo, "extraFileInfo is null"); this.database = database; this.table = table; @@ -104,18 +98,17 @@ public HiveSplit( this.start = start; this.length = length; this.fileSize = fileSize; - this.schema = schema; + this.storage = storage; this.partitionKeys = ImmutableList.copyOf(partitionKeys); this.addresses = ImmutableList.copyOf(addresses); this.readBucketNumber = readBucketNumber; this.tableBucketNumber = tableBucketNumber; this.forceLocalScheduling = forceLocalScheduling; - this.domainPredicate = domainPredicate; - this.remainingPredicate = remainingPredicate; - this.predicateColumns = predicateColumns; - this.columnCoercions = columnCoercions; + this.partitionDataColumnCount = partitionDataColumnCount; + this.partitionSchemaDifference = partitionSchemaDifference; this.bucketConversion = bucketConversion; this.s3SelectPushdownEnabled = s3SelectPushdownEnabled; + this.extraFileInfo = extraFileInfo; } @JsonProperty @@ -161,9 +154,9 @@ public long getFileSize() } @JsonProperty - public Properties getSchema() + public Storage getStorage() { - return schema; + return storage; } @JsonProperty @@ -192,33 +185,21 @@ public OptionalInt getTableBucketNumber() } @JsonProperty - public TupleDomain getDomainPredicate() - { - return domainPredicate; - } - - @JsonProperty - public RowExpression getRemainingPredicate() - { - return remainingPredicate; - } - - @JsonProperty - public Map getPredicateColumns() + public boolean isForceLocalScheduling() { - return predicateColumns; + return forceLocalScheduling; } @JsonProperty - public boolean isForceLocalScheduling() + public int getPartitionDataColumnCount() { - return forceLocalScheduling; + return partitionDataColumnCount; } @JsonProperty - public Map getColumnCoercions() + public Map getPartitionSchemaDifference() { - return columnCoercions; + return partitionSchemaDifference; } @JsonProperty @@ -239,6 +220,12 @@ public boolean isS3SelectPushdownEnabled() return s3SelectPushdownEnabled; } + @JsonProperty + public Optional getExtraFileInfo() + { + return extraFileInfo; + } + @Override public Object getInfo() { @@ -264,8 +251,6 @@ public String toString() .addValue(start) .addValue(length) .addValue(fileSize) - .addValue(domainPredicate) - .addValue(remainingPredicate) .addValue(s3SelectPushdownEnabled) .toString(); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index ec0d7382038c5..c9e831cb427ae 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.hive.HiveBucketing.HiveBucketFilter; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Partition; @@ -36,8 +38,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Ordering; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.stats.CounterStat; import io.airlift.units.DataSize; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -51,14 +51,15 @@ import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; -import java.util.function.Function; import static com.facebook.presto.hive.BackgroundHiveSplitLoader.BucketSplitInfo.createBucketSplitInfo; import static com.facebook.presto.hive.HiveColumnHandle.isPathColumnHandle; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH; +import static com.facebook.presto.hive.HiveErrorCode.HIVE_TRANSACTION_NOT_FOUND; import static com.facebook.presto.hive.HivePartition.UNPARTITIONED_ID; +import static com.facebook.presto.hive.HiveSessionProperties.isOfflineDataDebugModeEnabled; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.metastore.MetastoreUtil.getProtectMode; import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline; @@ -77,10 +78,9 @@ public class HiveSplitManager implements ConnectorSplitManager { - public static final String PRESTO_OFFLINE = "presto_offline"; public static final String OBJECT_NOT_READABLE = "object_not_readable"; - private final Function metastoreProvider; + private final HiveTransactionManager hiveTransactionManager; private final NamenodeStats namenodeStats; private final HdfsEnvironment hdfsEnvironment; private final DirectoryLister directoryLister; @@ -98,7 +98,7 @@ public class HiveSplitManager @Inject public HiveSplitManager( HiveClientConfig hiveClientConfig, - Function metastoreProvider, + HiveTransactionManager hiveTransactionManager, NamenodeStats namenodeStats, HdfsEnvironment hdfsEnvironment, DirectoryLister directoryLister, @@ -106,7 +106,7 @@ public HiveSplitManager( CoercionPolicy coercionPolicy) { this( - metastoreProvider, + hiveTransactionManager, namenodeStats, hdfsEnvironment, directoryLister, @@ -123,7 +123,7 @@ public HiveSplitManager( } public HiveSplitManager( - Function metastoreProvider, + HiveTransactionManager hiveTransactionManager, NamenodeStats namenodeStats, HdfsEnvironment hdfsEnvironment, DirectoryLister directoryLister, @@ -138,7 +138,7 @@ public HiveSplitManager( int splitLoaderConcurrency, boolean recursiveDfsWalkerEnabled) { - this.metastoreProvider = requireNonNull(metastoreProvider, "metastore is null"); + this.hiveTransactionManager = requireNonNull(hiveTransactionManager, "hiveTransactionManager is null"); this.namenodeStats = requireNonNull(namenodeStats, "namenodeStats is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.directoryLister = requireNonNull(directoryLister, "directoryLister is null"); @@ -166,14 +166,20 @@ public ConnectorSplitSource getSplits( SchemaTableName tableName = layout.getSchemaTableName(); // get table metadata - SemiTransactionalHiveMetastore metastore = metastoreProvider.apply((HiveTransactionHandle) transaction); + TransactionalMetadata metadata = hiveTransactionManager.get(transaction); + if (metadata == null) { + throw new PrestoException(HIVE_TRANSACTION_NOT_FOUND, format("Transaction not found: %s", transaction)); + } + SemiTransactionalHiveMetastore metastore = metadata.getMetastore(); Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new TableNotFoundException(tableName)); - // verify table is not marked as non-readable - String tableNotReadable = table.getParameters().get(OBJECT_NOT_READABLE); - if (!isNullOrEmpty(tableNotReadable)) { - throw new HiveNotReadableException(tableName, Optional.empty(), tableNotReadable); + if (!isOfflineDataDebugModeEnabled(session)) { + // verify table is not marked as non-readable + String tableNotReadable = table.getParameters().get(OBJECT_NOT_READABLE); + if (!isNullOrEmpty(tableNotReadable)) { + throw new HiveNotReadableException(tableName, Optional.empty(), tableNotReadable); + } } // get partitions @@ -209,7 +215,7 @@ public ConnectorSplitSource getSplits( if (bucketHandle.isPresent() && !bucketHandle.get().isVirtuallyBucketed()) { hiveBucketProperty = bucketHandle.map(HiveBucketHandle::toTableBucketProperty); } - Iterable hivePartitions = getPartitionMetadata(metastore, table, tableName, partitions, hiveBucketProperty); + Iterable hivePartitions = getPartitionMetadata(metastore, table, tableName, partitions, hiveBucketProperty, session); HiveSplitLoader hiveSplitLoader = new BackgroundHiveSplitLoader( table, @@ -232,9 +238,6 @@ public ConnectorSplitSource getSplits( session, table.getDatabaseName(), table.getTableName(), - layout.getDomainPredicate(), - layout.getRemainingPredicate(), - layout.getPredicateColumns(), maxInitialSplits, maxOutstandingSplits, maxOutstandingSplitsSize, @@ -247,9 +250,6 @@ public ConnectorSplitSource getSplits( session, table.getDatabaseName(), table.getTableName(), - layout.getDomainPredicate(), - layout.getRemainingPredicate(), - layout.getPredicateColumns(), maxInitialSplits, maxOutstandingSplits, maxOutstandingSplitsSize, @@ -262,9 +262,6 @@ public ConnectorSplitSource getSplits( session, table.getDatabaseName(), table.getTableName(), - layout.getDomainPredicate(), - layout.getRemainingPredicate(), - layout.getPredicateColumns(), maxInitialSplits, maxOutstandingSplitsSize, hiveSplitLoader, @@ -296,7 +293,13 @@ public CounterStat getHighMemorySplitSource() return highMemorySplitSourceCounter; } - private Iterable getPartitionMetadata(SemiTransactionalHiveMetastore metastore, Table table, SchemaTableName tableName, List hivePartitions, Optional bucketProperty) + private Iterable getPartitionMetadata( + SemiTransactionalHiveMetastore metastore, + Table table, + SchemaTableName tableName, + List hivePartitions, + Optional bucketProperty, + ConnectorSession session) { if (hivePartitions.isEmpty()) { return ImmutableList.of(); @@ -335,13 +338,15 @@ private Iterable getPartitionMetadata(SemiTransactionalHi } String partName = makePartName(table.getPartitionColumns(), partition.getValues()); - // verify partition is online - verifyOnline(tableName, Optional.of(partName), getProtectMode(partition), partition.getParameters()); + if (!isOfflineDataDebugModeEnabled(session)) { + // verify partition is online + verifyOnline(tableName, Optional.of(partName), getProtectMode(partition), partition.getParameters()); - // verify partition is not marked as non-readable - String partitionNotReadable = partition.getParameters().get(OBJECT_NOT_READABLE); - if (!isNullOrEmpty(partitionNotReadable)) { - throw new HiveNotReadableException(tableName, Optional.of(partName), partitionNotReadable); + // verify partition is not marked as non-readable + String partitionNotReadable = partition.getParameters().get(OBJECT_NOT_READABLE); + if (!isNullOrEmpty(partitionNotReadable)) { + throw new HiveNotReadableException(tableName, Optional.of(partName), partitionNotReadable); + } } // Verify that the partition schema matches the table schema. @@ -354,12 +359,18 @@ private Iterable getPartitionMetadata(SemiTransactionalHi if ((tableColumns == null) || (partitionColumns == null)) { throw new PrestoException(HIVE_INVALID_METADATA, format("Table '%s' or partition '%s' has null columns", tableName, partName)); } - ImmutableMap.Builder columnCoercions = ImmutableMap.builder(); - for (int i = 0; i < min(partitionColumns.size(), tableColumns.size()); i++) { + ImmutableMap.Builder partitionSchemaDifference = ImmutableMap.builder(); + for (int i = 0; i < partitionColumns.size(); i++) { + Column partitionColumn = partitionColumns.get(i); + + if (i >= tableColumns.size()) { + partitionSchemaDifference.put(i, partitionColumn); + continue; + } + HiveType tableType = tableColumns.get(i).getType(); - HiveType partitionType = partitionColumns.get(i).getType(); - if (!tableType.equals(partitionType)) { - if (!coercionPolicy.canCoerce(partitionType, tableType)) { + if (!tableType.equals(partitionColumn.getType())) { + if (!coercionPolicy.canCoerce(partitionColumn.getType(), tableType)) { throw new PrestoException(HIVE_PARTITION_SCHEMA_MISMATCH, format("" + "There is a mismatch between the table and partition schemas. " + "The types are incompatible and cannot be coerced. " + @@ -370,9 +381,14 @@ private Iterable getPartitionMetadata(SemiTransactionalHi tableType, partName, partitionColumns.get(i).getName(), - partitionType)); + partitionColumn.getType())); } - columnCoercions.put(i, partitionType.getHiveTypeName()); + partitionSchemaDifference.put(i, partitionColumn); + continue; + } + + if (!tableColumns.get(i).getName().equals(partitionColumn.getName())) { + partitionSchemaDifference.put(i, partitionColumn); } } @@ -400,7 +416,7 @@ private Iterable getPartitionMetadata(SemiTransactionalHi } } - results.add(new HivePartitionMetadata(hivePartition, Optional.of(partition), columnCoercions.build())); + results.add(new HivePartitionMetadata(hivePartition, Optional.of(partition), partitionSchemaDifference.build())); } return results.build(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitPartitionInfo.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitPartitionInfo.java index e796e7b60ddac..328fecbcbeaf0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitPartitionInfo.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitPartitionInfo.java @@ -14,6 +14,8 @@ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.Column; +import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.spi.PrestoException; import org.openjdk.jol.info.ClassLayout; @@ -22,7 +24,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; @@ -38,36 +39,39 @@ public class HiveSplitPartitionInfo private static final int INSTANCE_SIZE = ClassLayout.parseClass(HiveSplitPartitionInfo.class).instanceSize(); private static final int INTEGER_INSTANCE_SIZE = ClassLayout.parseClass(Integer.class).instanceSize(); - private final Properties schema; + private final Storage storage; private final URI path; private final List partitionKeys; private final String partitionName; - private final Map columnCoercions; + private final int partitionDataColumnCount; + private final Map partitionSchemaDifference; private final Optional bucketConversion; // keep track of how many InternalHiveSplits reference this PartitionInfo. private final AtomicInteger references = new AtomicInteger(0); public HiveSplitPartitionInfo( - Properties schema, + Storage storage, URI path, List partitionKeys, String partitionName, - Map columnCoercions, + int partitionDataColumnCount, + Map partitionSchemaDifference, Optional bucketConversion) { - requireNonNull(schema, "schema is null"); + requireNonNull(storage, "storage is null"); requireNonNull(path, "path is null"); requireNonNull(partitionKeys, "partitionKeys is null"); requireNonNull(partitionName, "partitionName is null"); - requireNonNull(columnCoercions, "columnCoersions is null"); + requireNonNull(partitionSchemaDifference, "partitionSchemaDifference is null"); requireNonNull(bucketConversion, "bucketConversion is null"); - this.schema = schema; + this.storage = storage; this.path = ensurePathHasTrailingSlash(path); this.partitionKeys = partitionKeys; this.partitionName = partitionName; - this.columnCoercions = columnCoercions; + this.partitionDataColumnCount = partitionDataColumnCount; + this.partitionSchemaDifference = partitionSchemaDifference; this.bucketConversion = bucketConversion; } @@ -90,9 +94,9 @@ private static URI ensurePathHasTrailingSlash(URI path) return path; } - public Properties getSchema() + public Storage getStorage() { - return schema; + return storage; } public List getPartitionKeys() @@ -105,9 +109,14 @@ public String getPartitionName() return partitionName; } - public Map getColumnCoercions() + public int getPartitionDataColumnCount() { - return columnCoercions; + return partitionDataColumnCount; + } + + public Map getPartitionSchemaDifference() + { + return partitionSchemaDifference; } public Optional getBucketConversion() @@ -124,9 +133,9 @@ public int getEstimatedSizeInBytes() } result += partitionName.length() * Character.BYTES; - result += sizeOfObjectArray(columnCoercions.size()); - for (HiveTypeName hiveTypeName : columnCoercions.values()) { - result += INTEGER_INSTANCE_SIZE + hiveTypeName.getEstimatedSizeInBytes(); + result += sizeOfObjectArray(partitionSchemaDifference.size()); + for (Column column : partitionSchemaDifference.values()) { + result += INTEGER_INSTANCE_SIZE + column.getEstimatedSizeInBytes(); } return result; } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitSource.java index 87df86c96d607..7c39c172f6b71 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitSource.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitSource.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.hive.InternalHiveSplit.InternalHiveBlock; import com.facebook.presto.hive.util.AsyncQueue; import com.facebook.presto.hive.util.AsyncQueue.BorrowResult; @@ -20,17 +22,12 @@ import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.ConnectorSplitSource; import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.Subfield; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; -import com.facebook.presto.spi.predicate.TupleDomain; -import com.facebook.presto.spi.relation.RowExpression; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; import io.airlift.units.DataSize; import javax.annotation.concurrent.GuardedBy; @@ -51,6 +48,8 @@ import java.util.function.Function; import java.util.function.Predicate; +import static com.facebook.airlift.concurrent.MoreFutures.failedFuture; +import static com.facebook.airlift.concurrent.MoreFutures.toCompletableFuture; import static com.facebook.presto.hive.HiveErrorCode.HIVE_EXCEEDED_SPLIT_BUFFERING_LIMIT; import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILE_NOT_FOUND; import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR; @@ -64,11 +63,8 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.Maps.transformValues; import static com.google.common.util.concurrent.Futures.immediateFuture; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.failedFuture; -import static io.airlift.concurrent.MoreFutures.toCompletableFuture; import static io.airlift.units.DataSize.succinctBytes; import static java.lang.Math.min; import static java.lang.Math.toIntExact; @@ -83,9 +79,6 @@ class HiveSplitSource private final String queryId; private final String databaseName; private final String tableName; - private final TupleDomain domainPredicate; - private final RowExpression remainingPredicate; - private final Map predicateColumns; private final PerBucket queues; private final AtomicInteger bufferedInternalSplitCount = new AtomicInteger(); private final int maxOutstandingSplitsBytes; @@ -107,9 +100,6 @@ private HiveSplitSource( ConnectorSession session, String databaseName, String tableName, - TupleDomain domainPredicate, - RowExpression remainingPredicate, - Map predicateColumns, PerBucket queues, int maxInitialSplits, DataSize maxOutstandingSplitsSize, @@ -121,9 +111,6 @@ private HiveSplitSource( this.queryId = session.getQueryId(); this.databaseName = requireNonNull(databaseName, "databaseName is null"); this.tableName = requireNonNull(tableName, "tableName is null"); - this.domainPredicate = requireNonNull(domainPredicate, "domainPredicate is null"); - this.remainingPredicate = requireNonNull(remainingPredicate, "remainingPredicate is null"); - this.predicateColumns = requireNonNull(predicateColumns, "predicateColumns is null"); this.queues = requireNonNull(queues, "queues is null"); this.maxOutstandingSplitsBytes = toIntExact(maxOutstandingSplitsSize.toBytes()); this.splitLoader = requireNonNull(splitLoader, "splitLoader is null"); @@ -139,9 +126,6 @@ public static HiveSplitSource allAtOnce( ConnectorSession session, String databaseName, String tableName, - TupleDomain domainPredicate, - RowExpression remainingPredicate, - Map predicateColumns, int maxInitialSplits, int maxOutstandingSplits, DataSize maxOutstandingSplitsSize, @@ -153,9 +137,6 @@ public static HiveSplitSource allAtOnce( session, databaseName, tableName, - domainPredicate, - remainingPredicate, - predicateColumns, new PerBucket() { private final AsyncQueue queue = new AsyncQueue<>(maxOutstandingSplits, executor); @@ -204,9 +185,6 @@ public static HiveSplitSource bucketed( ConnectorSession session, String databaseName, String tableName, - TupleDomain domainPredicate, - RowExpression remainingPredicate, - Map predicateColumns, int estimatedOutstandingSplitsPerBucket, int maxInitialSplits, DataSize maxOutstandingSplitsSize, @@ -218,9 +196,6 @@ public static HiveSplitSource bucketed( session, databaseName, tableName, - domainPredicate, - remainingPredicate, - predicateColumns, new PerBucket() { private final Map> queues = new ConcurrentHashMap<>(); @@ -289,9 +264,6 @@ public static HiveSplitSource bucketedRewindable( ConnectorSession session, String databaseName, String tableName, - TupleDomain domainPredicate, - RowExpression remainingPredicate, - Map predicateColumns, int maxInitialSplits, DataSize maxOutstandingSplitsSize, HiveSplitLoader splitLoader, @@ -302,9 +274,6 @@ public static HiveSplitSource bucketedRewindable( session, databaseName, tableName, - domainPredicate, - remainingPredicate, - predicateColumns, new PerBucket() { @GuardedBy("this") @@ -505,18 +474,17 @@ public CompletableFuture getNextBatch(ConnectorPartitionHan internalSplit.getStart(), splitBytes, internalSplit.getFileSize(), - internalSplit.getSchema(), + internalSplit.getPartitionInfo().getStorage(), internalSplit.getPartitionKeys(), block.getAddresses(), internalSplit.getReadBucketNumber(), internalSplit.getTableBucketNumber(), internalSplit.isForceLocalScheduling(), - domainPredicate, - remainingPredicate, - predicateColumns, - transformValues(internalSplit.getColumnCoercions(), HiveTypeName::toHiveType), + internalSplit.getPartitionInfo().getPartitionDataColumnCount(), + internalSplit.getPartitionSchemaDifference(), internalSplit.getBucketConversion(), - internalSplit.isS3SelectPushdownEnabled())); + internalSplit.isS3SelectPushdownEnabled(), + internalSplit.getExtraFileInfo())); internalSplit.increaseStart(splitBytes); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveStagingFileCommitter.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveStagingFileCommitter.java index 829616b3f6261..4bf34d3931909 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveStagingFileCommitter.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveStagingFileCommitter.java @@ -16,7 +16,6 @@ import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; import com.facebook.presto.hive.PartitionUpdate.FileWriteInfo; import com.facebook.presto.spi.ConnectorSession; -import com.facebook.presto.spi.PrestoException; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import org.apache.hadoop.fs.FileSystem; @@ -30,9 +29,9 @@ import static com.facebook.presto.hive.metastore.MetastoreUtil.getFileSystem; import static com.facebook.presto.hive.metastore.MetastoreUtil.renameFile; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.util.concurrent.Futures.catching; import static com.google.common.util.concurrent.Futures.whenAllSucceed; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static java.util.Objects.requireNonNull; public class HiveStagingFileCommitter @@ -51,10 +50,10 @@ public HiveStagingFileCommitter( } @Override - public void commitFiles(ConnectorSession session, String schemaName, String tableName, List partitionUpdates) + public ListenableFuture commitFiles(ConnectorSession session, String schemaName, String tableName, List partitionUpdates) { HdfsContext context = new HdfsContext(session, schemaName, tableName); - List> commitFutures = new ArrayList<>(); + List> commitFutures = new ArrayList<>(); for (PartitionUpdate partitionUpdate : partitionUpdates) { Path path = partitionUpdate.getWritePath(); @@ -63,17 +62,22 @@ public void commitFiles(ConnectorSession session, String schemaName, String tabl checkState(!fileWriteInfo.getWriteFileName().equals(fileWriteInfo.getTargetFileName())); Path source = new Path(path, fileWriteInfo.getWriteFileName()); Path target = new Path(path, fileWriteInfo.getTargetFileName()); - commitFutures.add(fileRenameExecutor.submit(() -> renameFile(fileSystem, source, target))); + commitFutures.add(fileRenameExecutor.submit(() -> { + renameFile(fileSystem, source, target); + return null; + })); } } - ListenableFuture listenableFutureAggregate = whenAllSucceed(commitFutures).call(() -> null, directExecutor()); - try { - getFutureValue(listenableFutureAggregate, PrestoException.class); - } - catch (RuntimeException e) { - listenableFutureAggregate.cancel(true); - throw e; - } + ListenableFuture result = whenAllSucceed(commitFutures).call(() -> null, directExecutor()); + return catching( + result, + RuntimeException.class, + e -> { + checkState(e != null, "Null exception is caught during commitFiles"); + result.cancel(true); + throw e; + }, + directExecutor()); } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableLayoutHandle.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableLayoutHandle.java index 41c4f8b5b85ea..d10e7ffc26ac0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableLayoutHandle.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableLayoutHandle.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive; import com.facebook.presto.hive.HiveBucketing.HiveBucketFilter; +import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.SchemaTableName; @@ -24,6 +25,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import javax.annotation.Nullable; import java.util.List; import java.util.Map; @@ -35,28 +39,41 @@ public final class HiveTableLayoutHandle implements ConnectorTableLayoutHandle { private final SchemaTableName schemaTableName; - private final List partitionColumns; - private final List partitions; + private final List partitionColumns; + private final List dataColumns; + private final Map tableParameters; private final TupleDomain domainPredicate; private final RowExpression remainingPredicate; private final Map predicateColumns; private final TupleDomain partitionColumnPredicate; private final Optional bucketHandle; private final Optional bucketFilter; + private final boolean pushdownFilterEnabled; + private final String layoutString; + + // coordinator-only properties + @Nullable + private final List partitions; @JsonCreator public HiveTableLayoutHandle( @JsonProperty("schemaTableName") SchemaTableName schemaTableName, - @JsonProperty("partitionColumns") List partitionColumns, + @JsonProperty("partitionColumns") List partitionColumns, + @JsonProperty("dataColumns") List dataColumns, + @JsonProperty("tableParameters") Map tableParameters, @JsonProperty("domainPredicate") TupleDomain domainPredicate, @JsonProperty("remainingPredicate") RowExpression remainingPredicate, @JsonProperty("predicateColumns") Map predicateColumns, @JsonProperty("partitionColumnPredicate") TupleDomain partitionColumnPredicate, @JsonProperty("bucketHandle") Optional bucketHandle, - @JsonProperty("bucketFilter") Optional bucketFilter) + @JsonProperty("bucketFilter") Optional bucketFilter, + @JsonProperty("pushdownFilterEnabled") boolean pushdownFilterEnabled, + @JsonProperty("layoutString") String layoutString) { this.schemaTableName = requireNonNull(schemaTableName, "table is null"); this.partitionColumns = ImmutableList.copyOf(requireNonNull(partitionColumns, "partitionColumns is null")); + this.dataColumns = ImmutableList.copyOf(requireNonNull(dataColumns, "dataColumns is null")); + this.tableParameters = ImmutableMap.copyOf(requireNonNull(tableParameters, "tableProperties is null")); this.domainPredicate = requireNonNull(domainPredicate, "domainPredicate is null"); this.remainingPredicate = requireNonNull(remainingPredicate, "remainingPredicate is null"); this.predicateColumns = requireNonNull(predicateColumns, "predicateColumns is null"); @@ -64,21 +81,29 @@ public HiveTableLayoutHandle( this.partitions = null; this.bucketHandle = requireNonNull(bucketHandle, "bucketHandle is null"); this.bucketFilter = requireNonNull(bucketFilter, "bucketFilter is null"); + this.pushdownFilterEnabled = pushdownFilterEnabled; + this.layoutString = requireNonNull(layoutString, "layoutString is null"); } public HiveTableLayoutHandle( SchemaTableName schemaTableName, - List partitionColumns, + List partitionColumns, + List dataColumns, + Map tableParameters, List partitions, TupleDomain domainPredicate, RowExpression remainingPredicate, Map predicateColumns, TupleDomain partitionColumnPredicate, Optional bucketHandle, - Optional bucketFilter) + Optional bucketFilter, + boolean pushdownFilterEnabled, + String layoutString) { this.schemaTableName = requireNonNull(schemaTableName, "table is null"); this.partitionColumns = ImmutableList.copyOf(requireNonNull(partitionColumns, "partitionColumns is null")); + this.dataColumns = ImmutableList.copyOf(requireNonNull(dataColumns, "dataColumns is null")); + this.tableParameters = ImmutableMap.copyOf(requireNonNull(tableParameters, "tableProperties is null")); this.partitions = requireNonNull(partitions, "partitions is null"); this.domainPredicate = requireNonNull(domainPredicate, "domainPredicate is null"); this.remainingPredicate = requireNonNull(remainingPredicate, "remainingPredicate is null"); @@ -86,6 +111,8 @@ public HiveTableLayoutHandle( this.partitionColumnPredicate = requireNonNull(partitionColumnPredicate, "partitionColumnPredicate is null"); this.bucketHandle = requireNonNull(bucketHandle, "bucketHandle is null"); this.bucketFilter = requireNonNull(bucketFilter, "bucketFilter is null"); + this.pushdownFilterEnabled = pushdownFilterEnabled; + this.layoutString = requireNonNull(layoutString, "layoutString is null"); } @JsonProperty @@ -95,11 +122,23 @@ public SchemaTableName getSchemaTableName() } @JsonProperty - public List getPartitionColumns() + public List getPartitionColumns() { return partitionColumns; } + @JsonProperty + public List getDataColumns() + { + return dataColumns; + } + + @JsonProperty + public Map getTableParameters() + { + return tableParameters; + } + /** * Partitions are dropped when HiveTableLayoutHandle is serialized. * @@ -147,14 +186,21 @@ public Optional getBucketFilter() return bucketFilter; } + @JsonProperty + public boolean isPushdownFilterEnabled() + { + return pushdownFilterEnabled; + } + + @JsonProperty + public String getLayoutString() + { + return layoutString; + } + @Override public String toString() { - StringBuilder result = new StringBuilder(); - result.append(schemaTableName.toString()); - if (bucketHandle.isPresent()) { - result.append(" bucket=").append(bucketHandle.get().getReadBucketCount()); - } - return result.toString(); + return layoutString; } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableProperties.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableProperties.java index 3263ce93b97a6..0fd25492606a1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableProperties.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableProperties.java @@ -170,6 +170,9 @@ public static Optional getBucketProperty(Map if (bucketCount < 0) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("%s must be greater than zero", BUCKET_COUNT_PROPERTY)); } + if (bucketCount > 1_000_000) { + throw new PrestoException(INVALID_TABLE_PROPERTY, format("%s should be no more than 1000000", BUCKET_COUNT_PROPERTY)); + } if (bucketedBy.isEmpty() || bucketCount == 0) { throw new PrestoException(INVALID_TABLE_PROPERTY, format("%s and %s must be specified together", BUCKETED_BY_PROPERTY, BUCKET_COUNT_PROPERTY)); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeTranslator.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeTranslator.java index e08726d46dd1d..50727d52efe95 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeTranslator.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeTranslator.java @@ -37,9 +37,9 @@ import static com.facebook.presto.hive.HiveType.HIVE_SHORT; import static com.facebook.presto.hive.HiveType.HIVE_STRING; import static com.facebook.presto.hive.HiveType.HIVE_TIMESTAMP; -import static com.facebook.presto.hive.HiveUtil.isArrayType; -import static com.facebook.presto.hive.HiveUtil.isMapType; -import static com.facebook.presto.hive.HiveUtil.isRowType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isArrayType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isMapType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isRowType; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java index 10f574b3e66e3..33587105056c8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java @@ -19,8 +19,8 @@ import com.facebook.presto.hive.metastore.Table; import com.facebook.presto.hive.util.FooterAwareRecordReader; import com.facebook.presto.orc.OrcReader; +import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorTableHandle; -import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.RecordCursor; import com.facebook.presto.spi.SchemaTableName; @@ -28,8 +28,13 @@ import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.Decimals; +import com.facebook.presto.spi.type.NamedTypeSignature; +import com.facebook.presto.spi.type.RowFieldName; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.spi.type.TypeSignature; +import com.facebook.presto.spi.type.TypeSignatureParameter; import com.facebook.presto.spi.type.VarcharType; import com.google.common.base.Joiner; import com.google.common.base.Splitter; @@ -94,6 +99,7 @@ import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; +import static com.facebook.presto.hive.HiveColumnHandle.MAX_PARTITION_KEY_COLUMN_INDEX; import static com.facebook.presto.hive.HiveColumnHandle.bucketColumnHandle; import static com.facebook.presto.hive.HiveColumnHandle.isBucketColumnHandle; import static com.facebook.presto.hive.HiveColumnHandle.isPathColumnHandle; @@ -101,13 +107,14 @@ import static com.facebook.presto.hive.HiveErrorCode.HIVE_BAD_DATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILE_MISSING_COLUMN_NAMES; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE; import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_VIEW_DATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_SERDE_NOT_FOUND; import static com.facebook.presto.hive.HiveErrorCode.HIVE_TABLE_BUCKETING_IS_IGNORED; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; -import static com.facebook.presto.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_PARTITION_VALUE; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_UNSUPPORTED_FORMAT; +import static com.facebook.presto.hive.metastore.MetastoreUtil.HIVE_DEFAULT_DYNAMIC_PARTITION; +import static com.facebook.presto.hive.metastore.MetastoreUtil.checkCondition; import static com.facebook.presto.hive.util.ConfigurationUtils.copy; import static com.facebook.presto.hive.util.ConfigurationUtils.toJobConf; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; @@ -145,7 +152,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; -import static org.apache.hadoop.hive.common.FileUtils.unescapePathName; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT; import static org.apache.hadoop.hive.serde.serdeConstants.DECIMAL_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB; @@ -157,8 +163,6 @@ public final class HiveUtil { private static final Pattern DEFAULT_HIVE_COLUMN_NAME_PATTERN = Pattern.compile("_col\\d+"); - public static final String PRESTO_VIEW_FLAG = "presto_view"; - private static final String VIEW_PREFIX = "/* Presto View: "; private static final String VIEW_SUFFIX = " */"; @@ -206,7 +210,7 @@ private HiveUtil() // Tell hive the columns we would like to read, this lets hive optimize reading column oriented files setReadColumns(configuration, readHiveColumnIndexes); - InputFormat inputFormat = getInputFormat(configuration, schema, true); + InputFormat inputFormat = getInputFormat(configuration, getInputFormatName(schema), true); JobConf jobConf = toJobConf(configuration); FileSplit fileSplit = new FileSplit(path, start, length, (String[]) null); @@ -279,9 +283,8 @@ public static Optional getCompressionCodec(TextInputFormat inp return Optional.ofNullable(compressionCodecFactory.getCodec(file)); } - static InputFormat getInputFormat(Configuration configuration, Properties schema, boolean symlinkTarget) + static InputFormat getInputFormat(Configuration configuration, String inputFormatName, boolean symlinkTarget) { - String inputFormatName = getInputFormatName(schema); try { JobConf jobConf = toJobConf(configuration); @@ -603,11 +606,6 @@ public static NullableValue parsePartitionValue(String partitionName, String val throw new VerifyException(format("Unhandled type [%s] for partition: %s", type, partitionName)); } - public static boolean isPrestoView(Table table) - { - return "true".equals(table.getParameters().get(PRESTO_VIEW_FLAG)); - } - public static String encodeViewData(String data) { return VIEW_PREFIX + Base64.getEncoder().encodeToString(data.getBytes(UTF_8)) + VIEW_SUFFIX; @@ -640,21 +638,6 @@ public static Optional getDecimalType(String hiveTypeName) } } - public static boolean isArrayType(Type type) - { - return type.getTypeSignature().getBase().equals(StandardTypes.ARRAY); - } - - public static boolean isMapType(Type type) - { - return type.getTypeSignature().getBase().equals(StandardTypes.MAP); - } - - public static boolean isRowType(Type type) - { - return type.getTypeSignature().getBase().equals(StandardTypes.ROW); - } - public static boolean isStructuralType(Type type) { String baseName = type.getTypeSignature().getBase(); @@ -852,53 +835,24 @@ public static List getPartitionKeyColumnHandles(Table table) ImmutableList.Builder columns = ImmutableList.builder(); List partitionKeys = table.getPartitionColumns(); + int partitionColumnIndex = MAX_PARTITION_KEY_COLUMN_INDEX; for (Column field : partitionKeys) { HiveType hiveType = field.getType(); if (!hiveType.isSupportedType()) { throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type %s found in partition keys of table %s.%s", hiveType, table.getDatabaseName(), table.getTableName())); } - columns.add(new HiveColumnHandle(field.getName(), hiveType, hiveType.getTypeSignature(), -1, PARTITION_KEY, field.getComment())); + columns.add(new HiveColumnHandle(field.getName(), hiveType, hiveType.getTypeSignature(), partitionColumnIndex--, PARTITION_KEY, field.getComment())); } return columns.build(); } - public static void checkCondition(boolean condition, ErrorCodeSupplier errorCode, String formatString, Object... args) - { - if (!condition) { - throw new PrestoException(errorCode, format(formatString, args)); - } - } - @Nullable public static String columnExtraInfo(boolean partitionKey) { return partitionKey ? "partition key" : null; } - public static List toPartitionValues(String partitionName) - { - // mimics Warehouse.makeValsFromName - ImmutableList.Builder resultBuilder = ImmutableList.builder(); - int start = 0; - while (true) { - while (start < partitionName.length() && partitionName.charAt(start) != '=') { - start++; - } - start++; - int end = start; - while (end < partitionName.length() && partitionName.charAt(end) != '/') { - end++; - } - if (start > partitionName.length()) { - break; - } - resultBuilder.add(unescapePathName(partitionName.substring(start, end))); - start = end + 1; - } - return resultBuilder.build(); - } - public static String getPrefilledColumnValue(HiveColumnHandle columnHandle, HivePartitionKey partitionKey, Path path, OptionalInt bucketNumber) { if (partitionKey != null) { @@ -938,6 +892,11 @@ public static List extractStructFieldTypes(HiveType hiveType) .collect(toImmutableList()); } + public static List extractStructFieldNames(HiveType hiveType) + { + return ((StructTypeInfo) hiveType.getTypeInfo()).getAllStructFieldNames(); + } + public static int getHeaderCount(Properties schema) { return getPositiveIntegerValue(schema, "skip.header.line.count", "0"); @@ -1029,10 +988,16 @@ public static List getPhysicalHiveColumnHandles(List buildPhysicalNameOrdinalMap(OrcReader reader return physicalNameOrdinalMap.build(); } + + /** + * Translates Presto type that is incompatible (cannot be stored in a Hive table) to a compatible type with the same physical layout. + * This allows to store more data types in a Hive temporary table than the Hive permanent tables support. + */ + public static List translateHiveUnsupportedTypesForTemporaryTable(List columns, TypeManager typeManager) + { + return columns.stream() + .map(column -> new ColumnMetadata( + column.getName(), + translateHiveUnsupportedTypeForTemporaryTable(column.getType(), typeManager), + column.isNullable(), + column.getComment(), + column.getExtraInfo(), + column.isHidden(), + column.getProperties())) + .collect(toImmutableList()); + } + + public static Type translateHiveUnsupportedTypeForTemporaryTable(Type type, TypeManager typeManager) + { + return typeManager.getType(translateHiveUnsupportedTypeSignatureForTemporaryTable(type.getTypeSignature())); + } + + private static TypeSignature translateHiveUnsupportedTypeSignatureForTemporaryTable(TypeSignature typeSignature) + { + List parameters = typeSignature.getParameters(); + + if (typeSignature.getBase().equals("unknown")) { + return new TypeSignature(StandardTypes.BOOLEAN); + } + + if (typeSignature.getBase().equals(StandardTypes.ROW)) { + ImmutableList.Builder updatedParameters = ImmutableList.builder(); + for (int i = 0; i < parameters.size(); i++) { + TypeSignatureParameter typeSignatureParameter = parameters.get(i); + checkArgument(typeSignatureParameter.isNamedTypeSignature(), "unexpected row type signature parameter: %s", typeSignatureParameter); + NamedTypeSignature namedTypeSignature = typeSignatureParameter.getNamedTypeSignature(); + updatedParameters.add(TypeSignatureParameter.of(new NamedTypeSignature( + Optional.of(namedTypeSignature.getFieldName().orElse(new RowFieldName("_field_" + i, false))), + translateHiveUnsupportedTypeSignatureForTemporaryTable(namedTypeSignature.getTypeSignature())))); + } + return new TypeSignature(StandardTypes.ROW, updatedParameters.build()); + } + + if (!parameters.isEmpty()) { + ImmutableList.Builder updatedParameters = ImmutableList.builder(); + for (TypeSignatureParameter parameter : parameters) { + switch (parameter.getKind()) { + case LONG: + case VARIABLE: + updatedParameters.add(parameter); + continue; + case TYPE: + updatedParameters.add(TypeSignatureParameter.of(translateHiveUnsupportedTypeSignatureForTemporaryTable(parameter.getTypeSignature()))); + break; + case NAMED_TYPE: + NamedTypeSignature namedTypeSignature = parameter.getNamedTypeSignature(); + updatedParameters.add(TypeSignatureParameter.of(new NamedTypeSignature( + namedTypeSignature.getFieldName(), + translateHiveUnsupportedTypeSignatureForTemporaryTable(namedTypeSignature.getTypeSignature())))); + break; + default: + throw new IllegalArgumentException("Unexpected parameter type: " + parameter.getKind()); + } + } + return new TypeSignature(typeSignature.getBase(), updatedParameters.build()); + } + + return typeSignature; + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java index ac9eafa9dd766..b1d427c68defa 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java @@ -23,18 +23,15 @@ import com.facebook.presto.hive.metastore.Table; import com.facebook.presto.hive.s3.PrestoS3FileSystem; import com.facebook.presto.spi.ConnectorSession; -import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaNotFoundException; import com.facebook.presto.spi.SchemaTableName; -import com.facebook.presto.spi.StandardErrorCode; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.BigintType; import com.facebook.presto.spi.type.BooleanType; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.DateType; import com.facebook.presto.spi.type.DecimalType; -import com.facebook.presto.spi.type.Decimals; import com.facebook.presto.spi.type.DoubleType; import com.facebook.presto.spi.type.IntegerType; import com.facebook.presto.spi.type.RealType; @@ -44,7 +41,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.VarbinaryType; import com.facebook.presto.spi.type.VarcharType; -import com.google.common.base.CharMatcher; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Shorts; import com.google.common.primitives.SignedBytes; @@ -52,9 +48,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.viewfs.ViewFileSystem; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.ProtectMode; @@ -94,45 +88,37 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Reporter; import org.apache.hive.common.util.ReflectionUtil; -import org.joda.time.DateTimeZone; import java.io.IOException; -import java.math.BigInteger; -import java.sql.Date; -import java.sql.Timestamp; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; -import java.util.concurrent.TimeUnit; import static com.facebook.presto.hive.HiveErrorCode.HIVE_DATABASE_LOCATION_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE; import static com.facebook.presto.hive.HiveErrorCode.HIVE_SERDE_NOT_FOUND; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_DATA_ERROR; -import static com.facebook.presto.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; import static com.facebook.presto.hive.HiveSessionProperties.getTemporaryStagingDirectoryPath; -import static com.facebook.presto.hive.HiveUtil.checkCondition; -import static com.facebook.presto.hive.HiveUtil.isArrayType; -import static com.facebook.presto.hive.HiveUtil.isMapType; -import static com.facebook.presto.hive.HiveUtil.isRowType; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.ParquetRecordWriterUtil.createParquetWriter; +import static com.facebook.presto.hive.metastore.MetastoreUtil.createDirectory; +import static com.facebook.presto.hive.metastore.MetastoreUtil.getField; +import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveDecimal; import static com.facebook.presto.hive.metastore.MetastoreUtil.getProtectMode; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isArrayType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isMapType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isRowType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.pathExists; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.type.Chars.isCharType; -import static com.google.common.base.Strings.padEnd; -import static com.google.common.io.BaseEncoding.base16; import static java.lang.Float.intBitsToFloat; import static java.lang.Math.toIntExact; import static java.lang.String.format; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; import static java.util.UUID.randomUUID; import static java.util.stream.Collectors.toList; @@ -165,13 +151,9 @@ import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getCharTypeInfo; import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getVarcharTypeInfo; import static org.apache.hadoop.mapred.FileOutputFormat.getOutputCompressorClass; -import static org.joda.time.DateTimeZone.UTC; public final class HiveWriteUtils { - @SuppressWarnings("OctalInteger") - private static final FsPermission ALL_PERMISSIONS = new FsPermission((short) 0777); - private HiveWriteUtils() { } @@ -314,119 +296,6 @@ else if (isRowType(type)) { throw new IllegalArgumentException("unsupported type: " + type); } - public static List createPartitionValues(List partitionColumnTypes, Page partitionColumns, int position) - { - ImmutableList.Builder partitionValues = ImmutableList.builder(); - for (int field = 0; field < partitionColumns.getChannelCount(); field++) { - Object value = getField(partitionColumnTypes.get(field), partitionColumns.getBlock(field), position); - if (value == null) { - partitionValues.add(HIVE_DEFAULT_DYNAMIC_PARTITION); - } - else { - String valueString = value.toString(); - if (!CharMatcher.inRange((char) 0x20, (char) 0x7E).matchesAllOf(valueString)) { - throw new PrestoException(HIVE_INVALID_PARTITION_VALUE, - "Hive partition keys can only contain printable ASCII characters (0x20 - 0x7E). Invalid value: " + - base16().withSeparator(" ", 2).encode(valueString.getBytes(UTF_8))); - } - partitionValues.add(valueString); - } - } - return partitionValues.build(); - } - - public static Object getField(Type type, Block block, int position) - { - if (block.isNull(position)) { - return null; - } - if (BooleanType.BOOLEAN.equals(type)) { - return type.getBoolean(block, position); - } - if (BigintType.BIGINT.equals(type)) { - return type.getLong(block, position); - } - if (IntegerType.INTEGER.equals(type)) { - return (int) type.getLong(block, position); - } - if (SmallintType.SMALLINT.equals(type)) { - return (short) type.getLong(block, position); - } - if (TinyintType.TINYINT.equals(type)) { - return (byte) type.getLong(block, position); - } - if (RealType.REAL.equals(type)) { - return intBitsToFloat((int) type.getLong(block, position)); - } - if (DoubleType.DOUBLE.equals(type)) { - return type.getDouble(block, position); - } - if (type instanceof VarcharType) { - return new Text(type.getSlice(block, position).getBytes()); - } - if (type instanceof CharType) { - CharType charType = (CharType) type; - return new Text(padEnd(type.getSlice(block, position).toStringUtf8(), charType.getLength(), ' ')); - } - if (VarbinaryType.VARBINARY.equals(type)) { - return type.getSlice(block, position).getBytes(); - } - if (DateType.DATE.equals(type)) { - long days = type.getLong(block, position); - return new Date(UTC.getMillisKeepLocal(DateTimeZone.getDefault(), TimeUnit.DAYS.toMillis(days))); - } - if (TimestampType.TIMESTAMP.equals(type)) { - long millisUtc = type.getLong(block, position); - return new Timestamp(millisUtc); - } - if (type instanceof DecimalType) { - DecimalType decimalType = (DecimalType) type; - return getHiveDecimal(decimalType, block, position); - } - if (isArrayType(type)) { - Type elementType = type.getTypeParameters().get(0); - - Block arrayBlock = block.getObject(position, Block.class); - - List list = new ArrayList<>(arrayBlock.getPositionCount()); - for (int i = 0; i < arrayBlock.getPositionCount(); i++) { - Object element = getField(elementType, arrayBlock, i); - list.add(element); - } - - return Collections.unmodifiableList(list); - } - if (isMapType(type)) { - Type keyType = type.getTypeParameters().get(0); - Type valueType = type.getTypeParameters().get(1); - - Block mapBlock = block.getObject(position, Block.class); - Map map = new HashMap<>(); - for (int i = 0; i < mapBlock.getPositionCount(); i += 2) { - Object key = getField(keyType, mapBlock, i); - Object value = getField(valueType, mapBlock, i + 1); - map.put(key, value); - } - - return Collections.unmodifiableMap(map); - } - if (isRowType(type)) { - Block rowBlock = block.getObject(position, Block.class); - - List fieldTypes = type.getTypeParameters(); - checkCondition(fieldTypes.size() == rowBlock.getPositionCount(), StandardErrorCode.GENERIC_INTERNAL_ERROR, "Expected row value field count does not match type field count"); - - List row = new ArrayList<>(rowBlock.getPositionCount()); - for (int i = 0; i < rowBlock.getPositionCount(); i++) { - Object element = getField(fieldTypes.get(i), rowBlock, i); - row.add(element); - } - - return Collections.unmodifiableList(row); - } - throw new PrestoException(NOT_SUPPORTED, "unsupported type: " + type); - } - public static void checkTableIsWritable(Table table, boolean writesToNonManagedTablesEnabled) { PrestoTableType tableType = table.getTableType(); @@ -505,16 +374,6 @@ private static Database getDatabase(SemiTransactionalHiveMetastore metastore, St return metastore.getDatabase(database).orElseThrow(() -> new SchemaNotFoundException(database)); } - public static boolean pathExists(HdfsContext context, HdfsEnvironment hdfsEnvironment, Path path) - { - try { - return hdfsEnvironment.getFileSystem(context, path).exists(path); - } - catch (IOException e) { - throw new PrestoException(HIVE_FILESYSTEM_ERROR, "Failed checking path: " + path, e); - } - } - public static boolean isS3FileSystem(HdfsContext context, HdfsEnvironment hdfsEnvironment, Path path) { try { @@ -573,26 +432,6 @@ public static Path createTemporaryPath(ConnectorSession session, HdfsContext con return temporaryPath; } - public static void createDirectory(HdfsContext context, HdfsEnvironment hdfsEnvironment, Path path) - { - try { - if (!hdfsEnvironment.getFileSystem(context, path).mkdirs(path, ALL_PERMISSIONS)) { - throw new IOException("mkdirs returned false"); - } - } - catch (IOException e) { - throw new PrestoException(HIVE_FILESYSTEM_ERROR, "Failed to create directory: " + path, e); - } - - // explicitly set permission since the default umask overrides it on creation - try { - hdfsEnvironment.getFileSystem(context, path).setPermission(path, ALL_PERMISSIONS); - } - catch (IOException e) { - throw new PrestoException(HIVE_FILESYSTEM_ERROR, "Failed to set permission on directory: " + path, e); - } - } - public static boolean isWritableType(HiveType hiveType) { return isWritableType(hiveType.getTypeInfo()); @@ -1048,18 +887,6 @@ public void setField(Block block, int position) } } - private static HiveDecimal getHiveDecimal(DecimalType decimalType, Block block, int position) - { - BigInteger unscaledValue; - if (decimalType.isShort()) { - unscaledValue = BigInteger.valueOf(decimalType.getLong(block, position)); - } - else { - unscaledValue = Decimals.decodeUnscaledValue(decimalType.getSlice(block, position)); - } - return HiveDecimal.create(unscaledValue, decimalType.getScale()); - } - private static class ArrayFieldSetter extends FieldSetter { @@ -1074,7 +901,7 @@ public ArrayFieldSetter(SettableStructObjectInspector rowInspector, Object row, @Override public void setField(Block block, int position) { - Block arrayBlock = block.getObject(position, Block.class); + Block arrayBlock = block.getBlock(position); List list = new ArrayList<>(arrayBlock.getPositionCount()); for (int i = 0; i < arrayBlock.getPositionCount(); i++) { @@ -1102,7 +929,7 @@ public MapFieldSetter(SettableStructObjectInspector rowInspector, Object row, St @Override public void setField(Block block, int position) { - Block mapBlock = block.getObject(position, Block.class); + Block mapBlock = block.getBlock(position); Map map = new HashMap<>(mapBlock.getPositionCount() * 2); for (int i = 0; i < mapBlock.getPositionCount(); i += 2) { Object key = getField(keyType, mapBlock, i); @@ -1128,7 +955,7 @@ public RowFieldSetter(SettableStructObjectInspector rowInspector, Object row, St @Override public void setField(Block block, int position) { - Block rowBlock = block.getObject(position, Block.class); + Block rowBlock = block.getBlock(position); // TODO reuse row object and use FieldSetters, like we do at the top level // Ideally, we'd use the same recursive structure starting from the top, but diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java index 469588442d049..5fe657500ed47 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java @@ -13,12 +13,14 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.event.client.EventClient; import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; import com.facebook.presto.hive.HiveSessionProperties.InsertExistingPartitionsBehavior; import com.facebook.presto.hive.LocationService.WriteInfo; import com.facebook.presto.hive.PartitionUpdate.UpdateMode; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.HivePageSinkMetadataProvider; +import com.facebook.presto.hive.metastore.MetastoreUtil; import com.facebook.presto.hive.metastore.Partition; import com.facebook.presto.hive.metastore.SortingColumn; import com.facebook.presto.hive.metastore.StorageFormat; @@ -37,9 +39,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import io.airlift.event.client.EventClient; import io.airlift.units.DataSize; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; @@ -60,17 +60,18 @@ import java.util.function.Consumer; import static com.facebook.presto.hive.HiveCompressionCodec.NONE; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_READ_ONLY; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_PATH_ALREADY_EXISTS; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_OPEN_ERROR; +import static com.facebook.presto.hive.HiveSessionProperties.getSortedWriteTempPathSubdirectoryCount; +import static com.facebook.presto.hive.HiveSessionProperties.isSortedWriteToTempPathEnabled; import static com.facebook.presto.hive.HiveType.toHiveTypes; -import static com.facebook.presto.hive.HiveWriteUtils.createPartitionValues; import static com.facebook.presto.hive.LocationHandle.WriteMode.DIRECT_TO_TARGET_EXISTING_DIRECTORY; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PATH_ALREADY_EXISTS; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.facebook.presto.hive.PartitionUpdate.FileWriteInfo; +import static com.facebook.presto.hive.metastore.MetastoreUtil.createPartitionValues; import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema; import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE; import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; @@ -115,18 +116,15 @@ public class HiveWriterFactory private final HivePageSinkMetadataProvider pageSinkMetadataProvider; private final TypeManager typeManager; private final HdfsEnvironment hdfsEnvironment; - private final PageSorter pageSorter; private final JobConf conf; private final Table table; - private final DataSize sortBufferSize; - private final int maxOpenSortFiles; + private final Optional sortingFileWriterFactory; private final boolean immutablePartitions; private final InsertExistingPartitionsBehavior insertExistingPartitionsBehavior; private final ConnectorSession session; private final OptionalInt bucketCount; - private final List sortedBy; private final NodeManager nodeManager; private final EventClient eventClient; @@ -134,8 +132,6 @@ public class HiveWriterFactory private final HiveWriterStats hiveWriterStats; - private final OrcFileWriterFactory orcFileWriterFactory; - private final boolean partitionCommitRequired; public HiveWriterFactory( @@ -183,9 +179,6 @@ public HiveWriterFactory( this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); - this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); - this.sortBufferSize = requireNonNull(sortBufferSize, "sortBufferSize is null"); - this.maxOpenSortFiles = maxOpenSortFiles; this.immutablePartitions = immutablePartitions; this.insertExistingPartitionsBehavior = HiveSessionProperties.getInsertExistingPartitionsBehavior(session); if (immutablePartitions) { @@ -232,8 +225,6 @@ public HiveWriterFactory( checkArgument(bucketCount.getAsInt() < MAX_BUCKET_COUNT, "bucketCount must be smaller than " + MAX_BUCKET_COUNT); } - this.sortedBy = ImmutableList.copyOf(requireNonNull(sortedBy, "sortedBy is null")); - this.session = requireNonNull(session, "session is null"); this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.eventClient = requireNonNull(eventClient, "eventClient is null"); @@ -245,6 +236,45 @@ public HiveWriterFactory( this.conf = configureCompression(hdfsEnvironment.getConfiguration(new HdfsContext(session, schemaName, tableName), writePath), compressionCodec); + if (!sortedBy.isEmpty()) { + List types = this.dataColumns.stream() + .map(column -> column.getHiveType().getType(typeManager)) + .collect(toImmutableList()); + + Map columnIndexes = new HashMap<>(); + for (int i = 0; i < this.dataColumns.size(); i++) { + columnIndexes.put(this.dataColumns.get(i).getName(), i); + } + + List sortFields = new ArrayList<>(); + List sortOrders = new ArrayList<>(); + for (SortingColumn column : sortedBy) { + Integer index = columnIndexes.get(column.getColumnName()); + if (index == null) { + throw new PrestoException(HIVE_INVALID_METADATA, format("Sorting column '%s' does exist in table '%s.%s'", column.getColumnName(), schemaName, tableName)); + } + sortFields.add(index); + sortOrders.add(column.getOrder().getSortOrder()); + } + + this.sortingFileWriterFactory = Optional.of(new SortingFileWriterFactory( + hdfsEnvironment, + session, + conf, + types, + sortFields, + sortOrders, + sortBufferSize, + maxOpenSortFiles, + pageSorter, + orcFileWriterFactory, + isSortedWriteToTempPathEnabled(session), + getSortedWriteTempPathSubdirectoryCount(session))); + } + else { + this.sortingFileWriterFactory = Optional.empty(); + } + // make sure the FileSystem is created with the correct Configuration object try { hdfsEnvironment.getFileSystem(session.getUser(), writePath, conf); @@ -255,8 +285,6 @@ public HiveWriterFactory( this.hiveWriterStats = requireNonNull(hiveWriterStats, "hiveWriterStats is null"); - this.orcFileWriterFactory = requireNonNull(orcFileWriterFactory, "orcFileWriterFactory is null"); - this.partitionCommitRequired = partitionCommitRequired; } @@ -316,7 +344,7 @@ public HiveWriter createWriter(Page partitionColumns, int position, OptionalInt if (!writeInfo.getWriteMode().isWritePathSameAsTargetPath()) { // When target path is different from write path, // verify that the target directory for the partition does not already exist - if (HiveWriteUtils.pathExists(new HdfsContext(session, schemaName, tableName), hdfsEnvironment, writeInfo.getTargetPath())) { + if (MetastoreUtil.pathExists(new HdfsContext(session, schemaName, tableName), hdfsEnvironment, writeInfo.getTargetPath())) { throw new PrestoException(HIVE_PATH_ALREADY_EXISTS, format( "Target directory for new partition '%s' of table '%s.%s' already exists: %s", partitionName, @@ -504,46 +532,9 @@ else if (insertExistingPartitionsBehavior == InsertExistingPartitionsBehavior.ER hiveWriter.getRowCount())); }; - if (!sortedBy.isEmpty()) { - FileSystem fileSystem; - try { - fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), path, conf); - } - catch (IOException e) { - throw new PrestoException(HIVE_WRITER_OPEN_ERROR, e); - } - - List types = dataColumns.stream() - .map(column -> column.getHiveType().getType(typeManager)) - .collect(toImmutableList()); - - Map columnIndexes = new HashMap<>(); - for (int i = 0; i < dataColumns.size(); i++) { - columnIndexes.put(dataColumns.get(i).getName(), i); - } - - List sortFields = new ArrayList<>(); - List sortOrders = new ArrayList<>(); - for (SortingColumn column : sortedBy) { - Integer index = columnIndexes.get(column.getColumnName()); - if (index == null) { - throw new PrestoException(HIVE_INVALID_METADATA, format("Sorting column '%s' does exist in table '%s.%s'", column.getColumnName(), schemaName, tableName)); - } - sortFields.add(index); - sortOrders.add(column.getOrder().getSortOrder()); - } - - hiveFileWriter = new SortingFileWriter( - fileSystem, - new Path(path.getParent(), ".tmp-sort." + path.getName()), - hiveFileWriter, - sortBufferSize, - maxOpenSortFiles, - types, - sortFields, - sortOrders, - pageSorter, - (fs, p) -> orcFileWriterFactory.createOrcDataSink(session, fs, p)); + if (sortingFileWriterFactory.isPresent()) { + checkState(bucketNumber.isPresent(), "missing bucket number for sorted table write"); + hiveFileWriter = sortingFileWriterFactory.get().createSortingFileWriter(path, hiveFileWriter, bucketNumber.getAsInt(), writeInfo.getTempPath()); } return new HiveWriter( diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterStats.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterStats.java index 55524f16f5d78..1999269b0a0df 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterStats.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterStats.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive; -import io.airlift.stats.DistributionStat; +import com.facebook.airlift.stats.DistributionStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveZeroRowFileCreator.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveZeroRowFileCreator.java index 66c87fccc8ac7..c1f6ebb34bf31 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveZeroRowFileCreator.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveZeroRowFileCreator.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.log.Logger; import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.spi.ConnectorSession; @@ -26,21 +27,30 @@ import javax.inject.Inject; import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Properties; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_CLOSE_ERROR; import static com.facebook.presto.hive.HiveWriteUtils.initializeSerializer; import static com.facebook.presto.hive.util.ConfigurationUtils.configureCompression; import static com.google.common.util.concurrent.Futures.whenAllSucceed; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.getFutureValue; +import static java.lang.String.format; +import static java.nio.file.Files.deleteIfExists; +import static java.nio.file.Files.readAllBytes; import static java.util.Objects.requireNonNull; +import static java.util.UUID.randomUUID; public class HiveZeroRowFileCreator implements ZeroRowFileCreator { + private static final Logger log = Logger.get(HiveZeroRowFileCreator.class); + private final HdfsEnvironment hdfsEnvironment; private final ListeningExecutorService executor; @@ -56,13 +66,12 @@ public HiveZeroRowFileCreator( @Override public void createFiles(ConnectorSession session, HdfsContext hdfsContext, Path destinationDirectory, List fileNames, StorageFormat format, HiveCompressionCodec compressionCodec, Properties schema) { - JobConf conf = configureCompression(hdfsEnvironment.getConfiguration(hdfsContext, destinationDirectory), compressionCodec); + byte[] fileContent = generateZeroRowFile(session, hdfsContext, schema, format.getSerDe(), format.getOutputFormat(), compressionCodec); + List> commitFutures = new ArrayList<>(); for (String fileName : fileNames) { - commitFutures.add( - executor.submit( - () -> writeZeroRowFile(session, new Path(destinationDirectory, fileName), conf, schema, format.getSerDe(), format.getOutputFormat()))); + commitFutures.add(executor.submit(() -> createFile(hdfsContext, new Path(destinationDirectory, fileName), fileContent))); } ListenableFuture listenableFutureAggregate = whenAllSucceed(commitFutures).call(() -> null, directExecutor()); @@ -75,15 +84,53 @@ public void createFiles(ConnectorSession session, HdfsContext hdfsContext, Path } } - private static void writeZeroRowFile(ConnectorSession session, Path target, JobConf conf, Properties properties, String serDe, String outputFormatName) + private byte[] generateZeroRowFile( + ConnectorSession session, + HdfsContext hdfsContext, + Properties properties, + String serDe, + String outputFormatName, + HiveCompressionCodec compressionCodec) { - // Some serializers such as Avro set a property in the schema. - initializeSerializer(conf, properties, serDe); + String tmpDirectoryPath = System.getProperty("java.io.tmpdir"); + String tmpFileName = format("presto-hive-zero-row-file-creator-%s-%s", session.getQueryId(), randomUUID().toString()); + java.nio.file.Path tmpFilePath = Paths.get(tmpDirectoryPath, tmpFileName); - // The code below is not a try with resources because RecordWriter is not Closeable. - RecordWriter recordWriter = HiveWriteUtils.createRecordWriter(target, conf, properties, outputFormatName, session); try { + Path target = new Path(format("file://%s/%s", tmpDirectoryPath, tmpFileName)); + JobConf conf = configureCompression(hdfsEnvironment.getConfiguration(hdfsContext, target), compressionCodec); + + // Some serializers such as Avro set a property in the schema. + initializeSerializer(conf, properties, serDe); + + // The code below is not a try with resources because RecordWriter is not Closeable. + RecordWriter recordWriter = HiveWriteUtils.createRecordWriter( + target, + conf, + properties, + outputFormatName, + session); recordWriter.close(false); + + return readAllBytes(tmpFilePath); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + finally { + try { + deleteIfExists(tmpFilePath); + } + catch (IOException e) { + log.error(e, "Error deleting temporary file: %s", tmpFilePath); + } + } + } + + private void createFile(HdfsContext hdfsContext, Path path, byte[] content) + { + try (OutputStream outputStream = hdfsEnvironment.getFileSystem(hdfsContext, path).create(path, false)) { + outputStream.write(content); } catch (IOException e) { throw new PrestoException(HIVE_WRITER_CLOSE_ERROR, "Error write zero-row file to Hive", e); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/InternalHiveSplit.java b/presto-hive/src/main/java/com/facebook/presto/hive/InternalHiveSplit.java index d0e1bd2f050df..f6d07c587b28a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/InternalHiveSplit.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/InternalHiveSplit.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive; import com.facebook.presto.hive.HiveSplit.BucketConversion; +import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.spi.HostAddress; import com.google.common.collect.ImmutableList; import org.apache.hadoop.fs.Path; @@ -25,7 +26,6 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; -import java.util.Properties; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; @@ -62,6 +62,7 @@ public class InternalHiveSplit private final boolean forceLocalScheduling; private final boolean s3SelectPushdownEnabled; private final HiveSplitPartitionInfo partitionInfo; + private final Optional extraFileInfo; private long start; private int currentBlockIndex; @@ -77,7 +78,8 @@ public InternalHiveSplit( boolean splittable, boolean forceLocalScheduling, boolean s3SelectPushdownEnabled, - HiveSplitPartitionInfo partitionInfo) + HiveSplitPartitionInfo partitionInfo, + Optional extraFileInfo) { checkArgument(start >= 0, "start must be positive"); checkArgument(end >= 0, "end must be positive"); @@ -86,6 +88,7 @@ public InternalHiveSplit( requireNonNull(readBucketNumber, "readBucketNumber is null"); requireNonNull(tableBucketNumber, "tableBucketNumber is null"); requireNonNull(partitionInfo, "partitionInfo is null"); + requireNonNull(extraFileInfo, "extraFileInfo is null"); this.relativeUri = relativeUri.getBytes(UTF_8); this.start = start; @@ -97,6 +100,7 @@ public InternalHiveSplit( this.forceLocalScheduling = forceLocalScheduling; this.s3SelectPushdownEnabled = s3SelectPushdownEnabled; this.partitionInfo = partitionInfo; + this.extraFileInfo = extraFileInfo; ImmutableList.Builder> addressesBuilder = ImmutableList.builder(); blockEndOffsets = new long[blocks.size()]; @@ -137,11 +141,6 @@ public boolean isS3SelectPushdownEnabled() return s3SelectPushdownEnabled; } - public Properties getSchema() - { - return partitionInfo.getSchema(); - } - public List getPartitionKeys() { return partitionInfo.getPartitionKeys(); @@ -172,9 +171,9 @@ public boolean isForceLocalScheduling() return forceLocalScheduling; } - public Map getColumnCoercions() + public Map getPartitionSchemaDifference() { - return partitionInfo.getColumnCoercions(); + return partitionInfo.getPartitionSchemaDifference(); } public Optional getBucketConversion() @@ -207,6 +206,11 @@ public HiveSplitPartitionInfo getPartitionInfo() return partitionInfo; } + public Optional getExtraFileInfo() + { + return extraFileInfo; + } + public void reset() { currentBlockIndex = 0; @@ -232,6 +236,9 @@ public int getEstimatedSizeInBytes() } } } + if (extraFileInfo.isPresent()) { + result += sizeOf(extraFileInfo.get()); + } return result; } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/LocationService.java b/presto-hive/src/main/java/com/facebook/presto/hive/LocationService.java index 0dc8a1a557d11..b6d42925cf282 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/LocationService.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/LocationService.java @@ -25,11 +25,11 @@ public interface LocationService { - LocationHandle forNewTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, String schemaName, String tableName); + LocationHandle forNewTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, String schemaName, String tableName, boolean tempPathRequired); - LocationHandle forExistingTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, Table table); + LocationHandle forExistingTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, Table table, boolean tempPathRequired); - LocationHandle forTemporaryTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, Table table); + LocationHandle forTemporaryTable(SemiTransactionalHiveMetastore metastore, ConnectorSession session, Table table, boolean tempPathRequired); /** * targetPath and writePath will be root directory of all partition and table paths @@ -49,12 +49,14 @@ class WriteInfo { private final Path targetPath; private final Path writePath; + private final Optional tempPath; private final LocationHandle.WriteMode writeMode; - public WriteInfo(Path targetPath, Path writePath, LocationHandle.WriteMode writeMode) + public WriteInfo(Path targetPath, Path writePath, Optional tempPath, LocationHandle.WriteMode writeMode) { this.targetPath = requireNonNull(targetPath, "targetPath is null"); this.writePath = requireNonNull(writePath, "writePath is null"); + this.tempPath = requireNonNull(tempPath, "tempPath is null"); this.writeMode = requireNonNull(writeMode, "writeMode is null"); } @@ -76,6 +78,14 @@ public Path getWritePath() return writePath; } + /** + * Temporary path for temp files generated during query processing, for example, sorted table writes. + */ + public Optional getTempPath() + { + return tempPath; + } + public LocationHandle.WriteMode getWriteMode() { return writeMode; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/NamenodeStats.java b/presto-hive/src/main/java/com/facebook/presto/hive/NamenodeStats.java index ac1e1ed180668..96ef6acc7cc75 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/NamenodeStats.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/NamenodeStats.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive; -import io.airlift.stats.CounterStat; -import io.airlift.stats.TimeStat; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.TimeStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterConfig.java index f38a547831e28..93e28576aefa1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.configuration.Config; import com.facebook.presto.orc.OrcWriterOptions; -import io.airlift.configuration.Config; import io.airlift.units.DataSize; @SuppressWarnings("unused") diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterFactory.java index bb8dc2f53e05c..c855176c5cfe9 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterFactory.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.MetastoreUtil; import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.hive.orc.HdfsOrcDataSource; import com.facebook.presto.orc.OrcDataSink; @@ -47,7 +48,6 @@ import java.util.concurrent.Callable; import java.util.function.Supplier; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_OPEN_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITE_VALIDATION_FAILED; import static com.facebook.presto.hive.HiveSessionProperties.getOrcMaxBufferSize; @@ -60,6 +60,7 @@ import static com.facebook.presto.hive.HiveSessionProperties.getOrcStreamBufferSize; import static com.facebook.presto.hive.HiveSessionProperties.getOrcStringStatisticsLimit; import static com.facebook.presto.hive.HiveType.toHiveTypes; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.facebook.presto.orc.OrcEncoding.DWRF; import static com.facebook.presto.orc.OrcEncoding.ORC; import static java.util.Locale.ENGLISH; @@ -202,7 +203,7 @@ else if (com.facebook.hive.orc.OrcOutputFormat.class.getName().equals(storageFor fileInputColumnIndexes, ImmutableMap.builder() .put(HiveMetadata.PRESTO_VERSION_NAME, nodeVersion.toString()) - .put(HiveMetadata.PRESTO_QUERY_ID_NAME, session.getQueryId()) + .put(MetastoreUtil.PRESTO_QUERY_ID_NAME, session.getQueryId()) .build(), hiveStorageTimeZone, validationInputFactory, diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/ParquetFileWriterConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/ParquetFileWriterConfig.java index f2945a7d91cc7..f3d30e7ace735 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/ParquetFileWriterConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/ParquetFileWriterConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import io.airlift.units.DataSize; import parquet.hadoop.ParquetWriter; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/PartitionObjectBuilder.java b/presto-hive/src/main/java/com/facebook/presto/hive/PartitionObjectBuilder.java new file mode 100644 index 0000000000000..77d1a60bdfcf2 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/PartitionObjectBuilder.java @@ -0,0 +1,27 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.hive.metastore.Partition; +import com.facebook.presto.hive.metastore.Table; +import com.facebook.presto.spi.ConnectorSession; + +public interface PartitionObjectBuilder +{ + Partition buildPartitionObject( + ConnectorSession session, + Table table, + PartitionUpdate partitionUpdate, + String prestoVersion); +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/RcFileFileWriterFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/RcFileFileWriterFactory.java index 6ade85255e0e8..51a15b07c89be 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/RcFileFileWriterFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/RcFileFileWriterFactory.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.MetastoreUtil; import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.hive.rcfile.HdfsRcFileDataSource; import com.facebook.presto.rcfile.RcFileDataSource; @@ -161,7 +162,7 @@ else if (ColumnarSerDe.class.getName().equals(storageFormat.getSerDe())) { fileInputColumnIndexes, ImmutableMap.builder() .put(HiveMetadata.PRESTO_VERSION_NAME, nodeVersion.toString()) - .put(HiveMetadata.PRESTO_QUERY_ID_NAME, session.getQueryId()) + .put(MetastoreUtil.PRESTO_QUERY_ID_NAME, session.getQueryId()) .build(), validationInputFactory)); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/RebindSafeMBeanServer.java b/presto-hive/src/main/java/com/facebook/presto/hive/RebindSafeMBeanServer.java index d4f2c82d8f69a..21abfc5d29566 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/RebindSafeMBeanServer.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/RebindSafeMBeanServer.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import javax.annotation.concurrent.ThreadSafe; import javax.management.Attribute; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectLineRecordReader.java b/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectLineRecordReader.java index 1ac8d1d884757..cea81f6df5dda 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectLineRecordReader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectLineRecordReader.java @@ -43,14 +43,14 @@ import static com.google.common.base.Throwables.throwIfInstanceOf; import static com.google.common.base.Throwables.throwIfUnchecked; import static java.lang.String.format; +import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; +import static java.net.HttpURLConnection.HTTP_FORBIDDEN; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.hadoop.hive.serde.serdeConstants.FIELD_DELIM; import static org.apache.hadoop.hive.serde.serdeConstants.LINE_DELIM; import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT; -import static org.apache.http.HttpStatus.SC_BAD_REQUEST; -import static org.apache.http.HttpStatus.SC_FORBIDDEN; -import static org.apache.http.HttpStatus.SC_NOT_FOUND; @ThreadSafe public abstract class S3SelectLineRecordReader @@ -137,9 +137,9 @@ private int readLine(Text value) recordsFromS3 = 0; if (e instanceof AmazonS3Exception) { switch (((AmazonS3Exception) e).getStatusCode()) { - case SC_FORBIDDEN: - case SC_NOT_FOUND: - case SC_BAD_REQUEST: + case HTTP_FORBIDDEN: + case HTTP_NOT_FOUND: + case HTTP_BAD_REQUEST: throw new UnrecoverableS3OperationException(selectClient.getBucketName(), selectClient.getKeyName(), e); } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectPushdown.java b/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectPushdown.java index 77988f8a3599e..1470c8a72473f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectPushdown.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectPushdown.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.log.Logger; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Partition; import com.facebook.presto.hive.metastore.Table; import com.facebook.presto.spi.ConnectorSession; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursorProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursorProvider.java index cd899ee0e8b28..eae2358a8f951 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursorProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursorProvider.java @@ -33,8 +33,8 @@ import java.util.Properties; import java.util.Set; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.HiveUtil.getDeserializerClassName; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static java.util.Objects.requireNonNull; public class S3SelectRecordCursorProvider diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/SortingFileWriter.java b/presto-hive/src/main/java/com/facebook/presto/hive/SortingFileWriter.java index f7523599b3c2a..776913bd90a84 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/SortingFileWriter.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/SortingFileWriter.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.log.Logger; import com.facebook.presto.hive.orc.HdfsOrcDataSource; import com.facebook.presto.hive.util.MergingPageIterator; import com.facebook.presto.hive.util.SortBuffer; @@ -28,7 +29,6 @@ import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableList; import com.google.common.io.Closer; -import io.airlift.log.Logger; import io.airlift.units.DataSize; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -43,7 +43,6 @@ import java.util.Optional; import java.util.PriorityQueue; import java.util.Queue; -import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.stream.IntStream; @@ -56,6 +55,7 @@ import static java.lang.Math.min; import static java.util.Comparator.comparing; import static java.util.Objects.requireNonNull; +import static java.util.UUID.randomUUID; public class SortingFileWriter implements HiveFileWriter @@ -73,8 +73,8 @@ public class SortingFileWriter private final HiveFileWriter outputWriter; private final SortBuffer sortBuffer; private final TempFileSinkFactory tempFileSinkFactory; + private final boolean sortedWriteToTempPathEnabled; private final Queue tempFiles = new PriorityQueue<>(comparing(TempFile::getSize)); - private final AtomicLong nextFileId = new AtomicLong(); public SortingFileWriter( FileSystem fileSystem, @@ -86,7 +86,8 @@ public SortingFileWriter( List sortFields, List sortOrders, PageSorter pageSorter, - TempFileSinkFactory tempFileSinkFactory) + TempFileSinkFactory tempFileSinkFactory, + boolean sortedWriteToTempPathEnabled) { checkArgument(maxOpenTempFiles >= 2, "maxOpenTempFiles must be at least two"); this.fileSystem = requireNonNull(fileSystem, "fileSystem is null"); @@ -98,6 +99,7 @@ public SortingFileWriter( this.outputWriter = requireNonNull(outputWriter, "outputWriter is null"); this.sortBuffer = new SortBuffer(maxMemory, types, sortFields, sortOrders, pageSorter); this.tempFileSinkFactory = tempFileSinkFactory; + this.sortedWriteToTempPathEnabled = sortedWriteToTempPathEnabled; } @Override @@ -147,8 +149,10 @@ public void commit() @Override public void rollback() { - for (TempFile file : tempFiles) { - cleanupFile(file.getPath()); + if (!sortedWriteToTempPathEnabled) { + for (TempFile file : tempFiles) { + cleanupFile(file.getPath()); + } } outputWriter.rollback(); @@ -224,11 +228,13 @@ private void mergeFiles(Iterable files, Consumer consumer) new MergingPageIterator(iterators, types, sortFields, sortOrders) .forEachRemaining(consumer); - for (TempFile tempFile : files) { - Path file = tempFile.getPath(); - fileSystem.delete(file, false); - if (fileSystem.exists(file)) { - throw new IOException("Failed to delete temporary file: " + file); + if (!sortedWriteToTempPathEnabled) { + for (TempFile tempFile : files) { + Path file = tempFile.getPath(); + fileSystem.delete(file, false); + if (fileSystem.exists(file)) { + throw new IOException("Failed to delete temporary file: " + file); + } } } } @@ -247,7 +253,9 @@ private void writeTempFile(Consumer consumer) tempFiles.add(new TempFile(tempFile, writer.getWrittenBytes())); } catch (IOException | UncheckedIOException e) { - cleanupFile(tempFile); + if (!sortedWriteToTempPathEnabled) { + cleanupFile(tempFile); + } throw new PrestoException(HIVE_WRITER_DATA_ERROR, "Failed to write temporary file: " + tempFile, e); } } @@ -267,7 +275,7 @@ private void cleanupFile(Path file) private Path getTempFileName() { - return new Path(tempFilePrefix + "." + nextFileId.getAndIncrement()); + return new Path(tempFilePrefix + "." + randomUUID().toString().replaceAll("-", "_")); } private static class TempFile diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/SortingFileWriterFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/SortingFileWriterFactory.java new file mode 100644 index 0000000000000..5cc266bc927a6 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/SortingFileWriterFactory.java @@ -0,0 +1,108 @@ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.PageSorter; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; +import io.airlift.units.DataSize; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.JobConf; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_OPEN_ERROR; +import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class SortingFileWriterFactory +{ + private final HdfsEnvironment hdfsEnvironment; + private final ConnectorSession session; + private final JobConf conf; + private final List types; + private final List sortFields; + private final List sortOrders; + private final DataSize sortBufferSize; + private final int maxOpenSortFiles; + private final PageSorter pageSorter; + private final OrcFileWriterFactory orcFileWriterFactory; + private final boolean sortedWriteToTempPathEnabled; + private final int sortedWriteTempFileSubdirectoryCount; + + public SortingFileWriterFactory( + HdfsEnvironment hdfsEnvironment, + ConnectorSession session, + JobConf conf, + List types, + List sortFields, + List sortOrders, + DataSize sortBufferSize, + int maxOpenSortFiles, + PageSorter pageSorter, + OrcFileWriterFactory orcFileWriterFactory, + boolean sortedWriteToTempPathEnabled, + int sortedWriteTempFileSubdirectoryCount) + { + this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); + this.session = requireNonNull(session, "session is null"); + this.conf = requireNonNull(conf, "conf is null"); + this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); + this.sortFields = ImmutableList.copyOf(requireNonNull(sortFields, "sortFields is null")); + this.sortOrders = ImmutableList.copyOf(requireNonNull(sortOrders, "sortOrders is null")); + this.sortBufferSize = requireNonNull(sortBufferSize, "sortBufferSize is null"); + this.maxOpenSortFiles = maxOpenSortFiles; + this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); + this.orcFileWriterFactory = requireNonNull(orcFileWriterFactory, "orcFileWriterFactory is null"); + this.sortedWriteToTempPathEnabled = sortedWriteToTempPathEnabled; + this.sortedWriteTempFileSubdirectoryCount = sortedWriteTempFileSubdirectoryCount; + } + + public SortingFileWriter createSortingFileWriter(Path path, HiveFileWriter outputWriter, int bucketNumber, Optional tempPath) + { + checkState(tempPath.isPresent() == sortedWriteToTempPathEnabled, "tempPath existence is not consistent with sortedWriteToTempPathEnabled config"); + + FileSystem fileSystem; + try { + fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), path, conf); + } + catch (IOException e) { + throw new PrestoException(HIVE_WRITER_OPEN_ERROR, e); + } + + Path prefix = sortedWriteToTempPathEnabled + ? new Path(tempPath.get(), format(".tmp-sort-%s/.tmp-sort-%s", bucketNumber % sortedWriteTempFileSubdirectoryCount, path.getName())) + : new Path(path.getParent(), ".tmp-sort." + path.getName()); + return new SortingFileWriter( + fileSystem, + prefix, + outputWriter, + sortBufferSize, + maxOpenSortFiles, + types, + sortFields, + sortOrders, + pageSorter, + (fs, p) -> orcFileWriterFactory.createOrcDataSink(session, fs, p), + sortedWriteToTempPathEnabled); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/StagingFileCommitter.java b/presto-hive/src/main/java/com/facebook/presto/hive/StagingFileCommitter.java index cbd26a287b75a..0addf9d87cde5 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/StagingFileCommitter.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/StagingFileCommitter.java @@ -14,10 +14,11 @@ package com.facebook.presto.hive; import com.facebook.presto.spi.ConnectorSession; +import com.google.common.util.concurrent.ListenableFuture; import java.util.List; public interface StagingFileCommitter { - void commitFiles(ConnectorSession session, String schemaName, String tableName, List partitionUpdates); + ListenableFuture commitFiles(ConnectorSession session, String schemaName, String tableName, List partitionUpdates); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/SubfieldExtractor.java b/presto-hive/src/main/java/com/facebook/presto/hive/SubfieldExtractor.java index 383c8ef54f9de..d3bf1ca4c42b4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/SubfieldExtractor.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/SubfieldExtractor.java @@ -15,6 +15,7 @@ import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; @@ -23,21 +24,30 @@ import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; +import io.airlift.slice.Slices; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; +import static com.facebook.presto.hive.HiveSessionProperties.isRangeFiltersOnSubscriptsEnabled; +import static com.facebook.presto.spi.function.OperatorType.SUBSCRIPT; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.DEREFERENCE; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.spi.type.Varchars.isVarcharType; +import static com.google.common.base.Verify.verify; import static java.util.Collections.unmodifiableList; import static java.util.Objects.requireNonNull; public final class SubfieldExtractor - implements DomainTranslator.ColumnExtractor { private final StandardFunctionResolution functionResolution; private final ExpressionOptimizer expressionOptimizer; @@ -53,7 +63,39 @@ public SubfieldExtractor( this.connectorSession = requireNonNull(connectorSession, "connectorSession is null"); } - @Override + public DomainTranslator.ColumnExtractor toColumnExtractor() + { + return (expression, domain) -> { + Type type = expression.getType(); + + // For complex types support only Filter IS_NULL and IS_NOT_NULL + if (isComplexType(type) && !(domain.isOnlyNull() || (domain.getValues().isAll() && !domain.isNullAllowed()))) { + return Optional.empty(); + } + + Optional subfield = extract(expression); + // If the expression involves array or map subscripts, it is considered only if allowed by nested_columns_filter_enabled. + if (hasSubscripts(subfield)) { + if (isRangeFiltersOnSubscriptsEnabled(connectorSession)) { + return subfield; + } + return Optional.empty(); + } + // The expression is a column or a set of nested struct field references. + return subfield; + }; + } + + private static boolean isComplexType(Type type) + { + return type instanceof ArrayType || type instanceof MapType || type instanceof RowType; + } + + private static boolean hasSubscripts(Optional subfield) + { + return subfield.isPresent() && subfield.get().getPath().stream().anyMatch(Subfield.PathElement::isSubscript); + } + public Optional extract(RowExpression expression) { return toSubfield(expression, functionResolution, expressionOptimizer, connectorSession); @@ -79,7 +121,7 @@ private static Optional toSubfield( RowExpression indexExpression = expressionOptimizer.optimize( dereferenceExpression.getArguments().get(1), - ExpressionOptimizer.Level.MOST_OPTIMIZED, + ExpressionOptimizer.Level.OPTIMIZED, connectorSession); if (indexExpression instanceof ConstantExpression) { @@ -100,7 +142,7 @@ private static Optional toSubfield( List arguments = ((CallExpression) expression).getArguments(); RowExpression indexExpression = expressionOptimizer.optimize( arguments.get(1), - ExpressionOptimizer.Level.MOST_OPTIMIZED, + ExpressionOptimizer.Level.OPTIMIZED, connectorSession); if (indexExpression instanceof ConstantExpression) { @@ -123,4 +165,90 @@ private static Optional toSubfield( return Optional.empty(); } } + + public RowExpression toRowExpression(Subfield subfield, Type columnType) + { + List path = subfield.getPath(); + ImmutableList.Builder types = ImmutableList.builder(); + types.add(columnType); + Type type = columnType; + for (int i = 0; i < path.size(); i++) { + if (type instanceof RowType) { + type = getFieldType((RowType) type, ((Subfield.NestedField) path.get(i)).getName()); + types.add(type); + } + else if (type instanceof ArrayType) { + type = ((ArrayType) type).getElementType(); + types.add(type); + } + else if (type instanceof MapType) { + type = ((MapType) type).getValueType(); + types.add(type); + } + else { + verify(false, "Unexpected type: " + type); + } + } + + return toRowExpression(subfield, types.build()); + } + + private RowExpression toRowExpression(Subfield subfield, List types) + { + List path = subfield.getPath(); + if (path.isEmpty()) { + return new VariableReferenceExpression(subfield.getRootName(), types.get(0)); + } + + RowExpression base = toRowExpression(new Subfield(subfield.getRootName(), path.subList(0, path.size() - 1)), types.subList(0, types.size() - 1)); + Type baseType = types.get(types.size() - 2); + Subfield.PathElement pathElement = path.get(path.size() - 1); + if (pathElement instanceof Subfield.LongSubscript) { + Type indexType = baseType instanceof MapType ? ((MapType) baseType).getKeyType() : BIGINT; + FunctionHandle functionHandle = functionResolution.subscriptFunction(baseType, indexType); + ConstantExpression index = new ConstantExpression(((Subfield.LongSubscript) pathElement).getIndex(), indexType); + return new CallExpression(SUBSCRIPT.name(), functionHandle, types.get(types.size() - 1), ImmutableList.of(base, index)); + } + + if (pathElement instanceof Subfield.StringSubscript) { + Type indexType = ((MapType) baseType).getKeyType(); + FunctionHandle functionHandle = functionResolution.subscriptFunction(baseType, indexType); + ConstantExpression index = new ConstantExpression(Slices.utf8Slice(((Subfield.StringSubscript) pathElement).getIndex()), indexType); + return new CallExpression(SUBSCRIPT.name(), functionHandle, types.get(types.size() - 1), ImmutableList.of(base, index)); + } + + if (pathElement instanceof Subfield.NestedField) { + Subfield.NestedField nestedField = (Subfield.NestedField) pathElement; + return new SpecialFormExpression(DEREFERENCE, types.get(types.size() - 1), base, new ConstantExpression(getFieldIndex((RowType) baseType, nestedField.getName()), INTEGER)); + } + + verify(false, "Unexpected path element: " + pathElement); + return null; + } + + private static Type getFieldType(RowType rowType, String fieldName) + { + for (RowType.Field field : rowType.getFields()) { + verify(field.getName().isPresent()); + if (field.getName().get().equals(fieldName)) { + return field.getType(); + } + } + verify(false, "Unexpected field name: " + fieldName); + return null; + } + + private static long getFieldIndex(RowType rowType, String fieldName) + { + List fields = rowType.getFields(); + for (int i = 0; i < fields.size(); i++) { + RowType.Field field = fields.get(i); + verify(field.getName().isPresent()); + if (field.getName().get().equals(fieldName)) { + return i; + } + } + verify(false, "Unexpected field name: " + fieldName); + return -1; + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/TransactionalMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/TransactionalMetadata.java index 66c3498419c97..081c51c3700ce 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/TransactionalMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/TransactionalMetadata.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore; import com.facebook.presto.spi.connector.ConnectorMetadata; public interface TransactionalMetadata @@ -21,4 +22,6 @@ public interface TransactionalMetadata void commit(); void rollback(); + + SemiTransactionalHiveMetastore getMetastore(); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/WriteCompletedEvent.java b/presto-hive/src/main/java/com/facebook/presto/hive/WriteCompletedEvent.java index c1b3591da07ca..ff5fcc933e3c0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/WriteCompletedEvent.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/WriteCompletedEvent.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.hive; -import io.airlift.event.client.EventField; -import io.airlift.event.client.EventField.EventFieldMapping; -import io.airlift.event.client.EventType; +import com.facebook.airlift.event.client.EventField; +import com.facebook.airlift.event.client.EventField.EventFieldMapping; +import com.facebook.airlift.event.client.EventType; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/AuthenticationModules.java b/presto-hive/src/main/java/com/facebook/presto/hive/authentication/AuthenticationModules.java index 8b2ca75883f6c..32e1fca17ba91 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/AuthenticationModules.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/authentication/AuthenticationModules.java @@ -15,7 +15,7 @@ import com.facebook.presto.hive.ForHdfs; import com.facebook.presto.hive.ForHiveMetastore; -import com.facebook.presto.hive.HdfsConfigurationUpdater; +import com.facebook.presto.hive.HdfsConfigurationInitializer; import com.google.inject.Binder; import com.google.inject.Key; import com.google.inject.Module; @@ -24,9 +24,9 @@ import javax.inject.Inject; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static com.facebook.presto.hive.authentication.KerberosHadoopAuthentication.createKerberosHadoopAuthentication; import static com.google.inject.Scopes.SINGLETON; -import static io.airlift.configuration.ConfigBinder.configBinder; public final class AuthenticationModules { @@ -57,11 +57,11 @@ public void configure(Binder binder) @Provides @Singleton @ForHiveMetastore - HadoopAuthentication createHadoopAuthentication(MetastoreKerberosConfig config, HdfsConfigurationUpdater updater) + HadoopAuthentication createHadoopAuthentication(MetastoreKerberosConfig config, HdfsConfigurationInitializer initializer) { String principal = config.getHiveMetastoreClientPrincipal(); String keytabLocation = config.getHiveMetastoreClientKeytab(); - return createCachingKerberosHadoopAuthentication(principal, keytabLocation, updater); + return createCachingKerberosHadoopAuthentication(principal, keytabLocation, initializer); } }; } @@ -102,11 +102,11 @@ public void configure(Binder binder) @Provides @Singleton @ForHdfs - HadoopAuthentication createHadoopAuthentication(HdfsKerberosConfig config, HdfsConfigurationUpdater updater) + HadoopAuthentication createHadoopAuthentication(HdfsKerberosConfig config, HdfsConfigurationInitializer initializer) { String principal = config.getHdfsPrestoPrincipal(); String keytabLocation = config.getHdfsPrestoKeytab(); - return createCachingKerberosHadoopAuthentication(principal, keytabLocation, updater); + return createCachingKerberosHadoopAuthentication(principal, keytabLocation, initializer); } }; } @@ -128,19 +128,19 @@ public void configure(Binder binder) @Provides @Singleton @ForHdfs - HadoopAuthentication createHadoopAuthentication(HdfsKerberosConfig config, HdfsConfigurationUpdater updater) + HadoopAuthentication createHadoopAuthentication(HdfsKerberosConfig config, HdfsConfigurationInitializer initializer) { String principal = config.getHdfsPrestoPrincipal(); String keytabLocation = config.getHdfsPrestoKeytab(); - return createCachingKerberosHadoopAuthentication(principal, keytabLocation, updater); + return createCachingKerberosHadoopAuthentication(principal, keytabLocation, initializer); } }; } - private static HadoopAuthentication createCachingKerberosHadoopAuthentication(String principal, String keytabLocation, HdfsConfigurationUpdater updater) + private static HadoopAuthentication createCachingKerberosHadoopAuthentication(String principal, String keytabLocation, HdfsConfigurationInitializer initializer) { KerberosAuthentication kerberosAuthentication = new KerberosAuthentication(principal, keytabLocation); - KerberosHadoopAuthentication kerberosHadoopAuthentication = createKerberosHadoopAuthentication(kerberosAuthentication, updater); + KerberosHadoopAuthentication kerberosHadoopAuthentication = createKerberosHadoopAuthentication(kerberosAuthentication, initializer); return new CachingKerberosHadoopAuthentication(kerberosHadoopAuthentication); } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/HdfsKerberosConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/authentication/HdfsKerberosConfig.java index 23938da6a6bc4..02444edb614b3 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/HdfsKerberosConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/authentication/HdfsKerberosConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive.authentication; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import javax.validation.constraints.NotNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/HiveAuthenticationModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/authentication/HiveAuthenticationModule.java index b90a0848128f1..0f5653010c4e8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/HiveAuthenticationModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/authentication/HiveAuthenticationModule.java @@ -13,22 +13,22 @@ */ package com.facebook.presto.hive.authentication; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.HiveClientConfig.HdfsAuthenticationType; import com.facebook.presto.hive.HiveClientConfig.HiveMetastoreAuthenticationType; import com.google.inject.Binder; import com.google.inject.Module; -import io.airlift.configuration.AbstractConfigurationAwareModule; import java.util.function.Predicate; +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; import static com.facebook.presto.hive.authentication.AuthenticationModules.kerberosHdfsAuthenticationModule; import static com.facebook.presto.hive.authentication.AuthenticationModules.kerberosHiveMetastoreAuthenticationModule; import static com.facebook.presto.hive.authentication.AuthenticationModules.kerberosImpersonatingHdfsAuthenticationModule; import static com.facebook.presto.hive.authentication.AuthenticationModules.noHdfsAuthenticationModule; import static com.facebook.presto.hive.authentication.AuthenticationModules.noHiveMetastoreAuthenticationModule; import static com.facebook.presto.hive.authentication.AuthenticationModules.simpleImpersonatingHdfsAuthenticationModule; -import static io.airlift.configuration.ConditionalModule.installModuleIf; public class HiveAuthenticationModule extends AbstractConfigurationAwareModule diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/KerberosAuthentication.java b/presto-hive/src/main/java/com/facebook/presto/hive/authentication/KerberosAuthentication.java index 47a85bf9774a2..4475def31e8c3 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/KerberosAuthentication.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/authentication/KerberosAuthentication.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.hive.authentication; +import com.facebook.airlift.log.Logger; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; import javax.security.auth.Subject; import javax.security.auth.kerberos.KerberosPrincipal; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/KerberosHadoopAuthentication.java b/presto-hive/src/main/java/com/facebook/presto/hive/authentication/KerberosHadoopAuthentication.java index b627461e7fca6..391a3a62ae856 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/KerberosHadoopAuthentication.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/authentication/KerberosHadoopAuthentication.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive.authentication; -import com.facebook.presto.hive.HdfsConfigurationUpdater; +import com.facebook.presto.hive.HdfsConfigurationInitializer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; @@ -28,10 +28,10 @@ public class KerberosHadoopAuthentication { private final KerberosAuthentication kerberosAuthentication; - public static KerberosHadoopAuthentication createKerberosHadoopAuthentication(KerberosAuthentication kerberosAuthentication, HdfsConfigurationUpdater updater) + public static KerberosHadoopAuthentication createKerberosHadoopAuthentication(KerberosAuthentication kerberosAuthentication, HdfsConfigurationInitializer initializer) { Configuration configuration = getInitialConfiguration(); - updater.updateConfiguration(configuration); + initializer.updateConfiguration(configuration); // In order to enable KERBEROS authentication method for HDFS // UserGroupInformation.authenticationMethod static field must be set to KERBEROS diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/MetastoreKerberosConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/authentication/MetastoreKerberosConfig.java index 33b64a36966d0..a78c835071ca1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/MetastoreKerberosConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/authentication/MetastoreKerberosConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive.authentication; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import javax.validation.constraints.NotNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/gcs/GcsAccessTokenProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/gcs/GcsAccessTokenProvider.java new file mode 100644 index 0000000000000..085cf7875f457 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/gcs/GcsAccessTokenProvider.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.gcs; + +import com.google.cloud.hadoop.util.AccessTokenProvider; +import org.apache.hadoop.conf.Configuration; + +import static com.google.common.base.Strings.nullToEmpty; + +public class GcsAccessTokenProvider + implements AccessTokenProvider +{ + public static final String GCS_ACCESS_TOKEN_CONF = "presto.gcs.oauth-access-token"; + private Configuration config; + + @Override + public AccessToken getAccessToken() + { + // set tokan expiration time to null, as we do not control the token(it is passed by user) + return new AccessToken(nullToEmpty(config.get(GCS_ACCESS_TOKEN_CONF)), null); + } + + @Override + public void refresh() {} + + @Override + public void setConf(Configuration configuration) + { + this.config = configuration; + } + + @Override + public Configuration getConf() + { + return config; + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/gcs/GcsConfigurationInitializer.java b/presto-hive/src/main/java/com/facebook/presto/hive/gcs/GcsConfigurationInitializer.java new file mode 100644 index 0000000000000..8d51d4f0411ae --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/gcs/GcsConfigurationInitializer.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.gcs; + +import org.apache.hadoop.conf.Configuration; + +public interface GcsConfigurationInitializer +{ + void updateConfiguration(Configuration config); +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/gcs/GcsConfigurationProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/gcs/GcsConfigurationProvider.java new file mode 100644 index 0000000000000..1468dd4df6e7f --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/gcs/GcsConfigurationProvider.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.gcs; + +import com.facebook.presto.hive.DynamicConfigurationProvider; +import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystem; +import org.apache.hadoop.conf.Configuration; + +import java.net.URI; + +import static com.facebook.presto.hive.HdfsEnvironment.HdfsContext; +import static com.facebook.presto.hive.gcs.GcsAccessTokenProvider.GCS_ACCESS_TOKEN_CONF; + +public class GcsConfigurationProvider + implements DynamicConfigurationProvider +{ + private static final String GCS_OAUTH_KEY = "hive.gcs.oauth"; + + @Override + public void updateConfiguration(Configuration configuration, HdfsContext context, URI uri) + { + if (!uri.getScheme().equals(GoogleCloudStorageFileSystem.SCHEME)) { + return; + } + + String accessToken = context.getIdentity().getExtraCredentials().get(GCS_OAUTH_KEY); + if (accessToken != null) { + configuration.set(GCS_ACCESS_TOKEN_CONF, accessToken); + } + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/gcs/HiveGcsConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/gcs/HiveGcsConfig.java new file mode 100644 index 0000000000000..f4c2f3df498e4 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/gcs/HiveGcsConfig.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.gcs; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import io.airlift.units.Duration; + +public class HiveGcsConfig +{ + private boolean useGcsAccessToken; + private String jsonKeyFilePath; + private Duration accessTokenExpirationTime; + + public String getJsonKeyFilePath() + { + return jsonKeyFilePath; + } + + @Config("hive.gcs.json-key-file-path") + @ConfigDescription("JSON key file used to access Google Cloud Storage") + public HiveGcsConfig setJsonKeyFilePath(String jsonKeyFilePath) + { + this.jsonKeyFilePath = jsonKeyFilePath; + return this; + } + + public boolean isUseGcsAccessToken() + { + return useGcsAccessToken; + } + + @Config("hive.gcs.use-access-token") + @ConfigDescription("Use client-provided OAuth token to access Google Cloud Storage") + public HiveGcsConfig setUseGcsAccessToken(boolean useGcsAccessToken) + { + this.useGcsAccessToken = useGcsAccessToken; + return this; + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/gcs/HiveGcsConfigurationInitializer.java b/presto-hive/src/main/java/com/facebook/presto/hive/gcs/HiveGcsConfigurationInitializer.java new file mode 100644 index 0000000000000..ce75fee27b5ff --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/gcs/HiveGcsConfigurationInitializer.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.gcs; + +import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; +import org.apache.hadoop.conf.Configuration; + +import javax.inject.Inject; + +import static com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemBase.AUTHENTICATION_PREFIX; +import static com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemConfiguration.AUTH_SERVICE_ACCOUNT_ENABLE; +import static com.google.cloud.hadoop.util.AccessTokenProviderClassFromConfigFactory.ACCESS_TOKEN_PROVIDER_IMPL_SUFFIX; +import static com.google.cloud.hadoop.util.EntriesCredentialConfiguration.JSON_KEYFILE_SUFFIX; + +public class HiveGcsConfigurationInitializer + implements GcsConfigurationInitializer +{ + private final boolean useGcsAccessToken; + private final String jsonKeyFilePath; + + @Inject + public HiveGcsConfigurationInitializer(HiveGcsConfig config) + { + this.useGcsAccessToken = config.isUseGcsAccessToken(); + this.jsonKeyFilePath = config.getJsonKeyFilePath(); + } + + public void updateConfiguration(Configuration config) + { + config.set("fs.gs.impl", GoogleHadoopFileSystem.class.getName()); + + if (useGcsAccessToken) { + // use oauth token to authenticate with Google Cloud Storage + config.set(AUTH_SERVICE_ACCOUNT_ENABLE.getKey(), "false"); + config.set(AUTHENTICATION_PREFIX + ACCESS_TOKEN_PROVIDER_IMPL_SUFFIX, GcsAccessTokenProvider.class.getName()); + } + else if (jsonKeyFilePath != null) { + // use service account key file + config.set(AUTH_SERVICE_ACCOUNT_ENABLE.getKey(), "true"); + config.set(AUTHENTICATION_PREFIX + JSON_KEYFILE_SUFFIX, jsonKeyFilePath); + } + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/gcs/HiveGcsModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/gcs/HiveGcsModule.java new file mode 100644 index 0000000000000..782de363c0534 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/hive/gcs/HiveGcsModule.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.gcs; + +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.presto.hive.DynamicConfigurationProvider; +import com.google.inject.Binder; +import com.google.inject.Scopes; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.google.inject.multibindings.Multibinder.newSetBinder; + +public class HiveGcsModule + extends AbstractConfigurationAwareModule +{ + @Override + protected void setup(Binder binder) + { + configBinder(binder).bindConfig(HiveGcsConfig.class); + binder.bind(GcsConfigurationInitializer.class).to(HiveGcsConfigurationInitializer.class).in(Scopes.SINGLETON); + + if (buildConfigObject(HiveGcsConfig.class).isUseGcsAccessToken()) { + newSetBinder(binder, DynamicConfigurationProvider.class).addBinding().to(GcsConfigurationProvider.class).in(Scopes.SINGLETON); + } + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java deleted file mode 100644 index 03c14365a86bb..0000000000000 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.hive.metastore; - -import com.facebook.presto.hive.HdfsEnvironment; -import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; -import com.facebook.presto.hive.PartitionOfflineException; -import com.facebook.presto.hive.TableOfflineException; -import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.SchemaTableName; -import com.facebook.presto.spi.TableNotFoundException; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.FileUtils; -import org.apache.hadoop.hive.metastore.ProtectMode; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Properties; - -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; -import static com.facebook.presto.hive.HiveSplitManager.PRESTO_OFFLINE; -import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; -import static java.lang.String.format; -import static java.util.stream.Collectors.toList; -import static org.apache.hadoop.hive.metastore.MetaStoreUtils.typeToThriftType; -import static org.apache.hadoop.hive.metastore.ProtectMode.getProtectModeFromString; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_COUNT; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_FIELD_NAME; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_OUTPUT_FORMAT; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_LOCATION; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES; -import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_DDL; -import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB; - -public class MetastoreUtil -{ - private MetastoreUtil() - { - } - - public static Properties getHiveSchema(Table table) - { - // Mimics function in Hive: MetaStoreUtils.getTableMetadata(Table) - return getHiveSchema( - table.getStorage(), - table.getDataColumns(), - table.getDataColumns(), - table.getParameters(), - table.getDatabaseName(), - table.getTableName(), - table.getPartitionColumns()); - } - - public static Properties getHiveSchema(Partition partition, Table table) - { - // Mimics function in Hive: MetaStoreUtils.getSchema(Partition, Table) - return getHiveSchema( - partition.getStorage(), - partition.getColumns(), - table.getDataColumns(), - table.getParameters(), - table.getDatabaseName(), - table.getTableName(), - table.getPartitionColumns()); - } - - private static Properties getHiveSchema( - Storage sd, - List dataColumns, - List tableDataColumns, - Map parameters, - String databaseName, - String tableName, - List partitionKeys) - { - // Mimics function in Hive: - // MetaStoreUtils.getSchema(StorageDescriptor, StorageDescriptor, Map, String, String, List) - - Properties schema = new Properties(); - - schema.setProperty(FILE_INPUT_FORMAT, sd.getStorageFormat().getInputFormat()); - schema.setProperty(FILE_OUTPUT_FORMAT, sd.getStorageFormat().getOutputFormat()); - - schema.setProperty(META_TABLE_NAME, databaseName + "." + tableName); - schema.setProperty(META_TABLE_LOCATION, sd.getLocation()); - - if (sd.getBucketProperty().isPresent()) { - List bucketedBy = sd.getBucketProperty().get().getBucketedBy(); - if (!bucketedBy.isEmpty()) { - schema.setProperty(BUCKET_FIELD_NAME, bucketedBy.get(0)); - } - schema.setProperty(BUCKET_COUNT, Integer.toString(sd.getBucketProperty().get().getBucketCount())); - } - else { - schema.setProperty(BUCKET_COUNT, "0"); - } - - for (Map.Entry param : sd.getSerdeParameters().entrySet()) { - schema.setProperty(param.getKey(), (param.getValue() != null) ? param.getValue() : ""); - } - schema.setProperty(SERIALIZATION_LIB, sd.getStorageFormat().getSerDe()); - - StringBuilder columnNameBuilder = new StringBuilder(); - StringBuilder columnTypeBuilder = new StringBuilder(); - StringBuilder columnCommentBuilder = new StringBuilder(); - boolean first = true; - for (Column column : tableDataColumns) { - if (!first) { - columnNameBuilder.append(","); - columnTypeBuilder.append(":"); - columnCommentBuilder.append('\0'); - } - columnNameBuilder.append(column.getName()); - columnTypeBuilder.append(column.getType()); - columnCommentBuilder.append(column.getComment().orElse("")); - first = false; - } - String columnNames = columnNameBuilder.toString(); - String columnTypes = columnTypeBuilder.toString(); - schema.setProperty(META_TABLE_COLUMNS, columnNames); - schema.setProperty(META_TABLE_COLUMN_TYPES, columnTypes); - schema.setProperty("columns.comments", columnCommentBuilder.toString()); - - schema.setProperty(SERIALIZATION_DDL, toThriftDdl(tableName, dataColumns)); - - String partString = ""; - String partStringSep = ""; - String partTypesString = ""; - String partTypesStringSep = ""; - for (Column partKey : partitionKeys) { - partString += partStringSep; - partString += partKey.getName(); - partTypesString += partTypesStringSep; - partTypesString += partKey.getType().getHiveTypeName().toString(); - if (partStringSep.length() == 0) { - partStringSep = "/"; - partTypesStringSep = ":"; - } - } - if (partString.length() > 0) { - schema.setProperty(META_TABLE_PARTITION_COLUMNS, partString); - schema.setProperty(META_TABLE_PARTITION_COLUMN_TYPES, partTypesString); - } - - if (parameters != null) { - for (Map.Entry entry : parameters.entrySet()) { - // add non-null parameters to the schema - if (entry.getValue() != null) { - schema.setProperty(entry.getKey(), entry.getValue()); - } - } - } - - return schema; - } - - public static ProtectMode getProtectMode(Partition partition) - { - return getProtectMode(partition.getParameters()); - } - - public static ProtectMode getProtectMode(Table table) - { - return getProtectMode(table.getParameters()); - } - - public static String makePartName(List partitionColumns, List values) - { - checkArgument(partitionColumns.size() == values.size(), "Partition value count does not match the partition column count"); - checkArgument(values.stream().allMatch(Objects::nonNull), "partitionValue must not have null elements"); - - List partitionColumnNames = partitionColumns.stream().map(Column::getName).collect(toList()); - return FileUtils.makePartName(partitionColumnNames, values); - } - - public static String getPartitionLocation(Table table, Optional partition) - { - if (!partition.isPresent()) { - return table.getStorage().getLocation(); - } - return partition.get().getStorage().getLocation(); - } - - private static String toThriftDdl(String structName, List columns) - { - // Mimics function in Hive: - // MetaStoreUtils.getDDLFromFieldSchema(String, List) - StringBuilder ddl = new StringBuilder(); - ddl.append("struct "); - ddl.append(structName); - ddl.append(" { "); - boolean first = true; - for (Column column : columns) { - if (first) { - first = false; - } - else { - ddl.append(", "); - } - ddl.append(typeToThriftType(column.getType().getHiveTypeName().toString())); - ddl.append(' '); - ddl.append(column.getName()); - } - ddl.append("}"); - return ddl.toString(); - } - - private static ProtectMode getProtectMode(Map parameters) - { - if (!parameters.containsKey(ProtectMode.PARAMETER_NAME)) { - return new ProtectMode(); - } - else { - return getProtectModeFromString(parameters.get(ProtectMode.PARAMETER_NAME)); - } - } - - public static void verifyOnline(SchemaTableName tableName, Optional partitionName, ProtectMode protectMode, Map parameters) - { - if (protectMode.offline) { - if (partitionName.isPresent()) { - throw new PartitionOfflineException(tableName, partitionName.get(), false, null); - } - throw new TableOfflineException(tableName, false, null); - } - - String prestoOffline = parameters.get(PRESTO_OFFLINE); - if (!isNullOrEmpty(prestoOffline)) { - if (partitionName.isPresent()) { - throw new PartitionOfflineException(tableName, partitionName.get(), true, prestoOffline); - } - throw new TableOfflineException(tableName, true, prestoOffline); - } - } - - public static void verifyCanDropColumn(ExtendedHiveMetastore metastore, String databaseName, String tableName, String columnName) - { - Table table = metastore.getTable(databaseName, tableName) - .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName))); - - if (table.getPartitionColumns().stream().anyMatch(column -> column.getName().equals(columnName))) { - throw new PrestoException(NOT_SUPPORTED, "Cannot drop partition columns"); - } - if (table.getDataColumns().size() <= 1) { - throw new PrestoException(NOT_SUPPORTED, "Cannot drop the only non-partition column in a table"); - } - } - - public static FileSystem getFileSystem(HdfsEnvironment hdfsEnvironment, HdfsContext context, Path path) - { - try { - return hdfsEnvironment.getFileSystem(context, path); - } - catch (IOException e) { - throw new PrestoException(HIVE_FILESYSTEM_ERROR, format("Error getting file system. Path: %s", path), e); - } - } - - public static void renameFile(FileSystem fileSystem, Path source, Path target) - { - try { - if (fileSystem.exists(target) || !fileSystem.rename(source, target)) { - throw new PrestoException(HIVE_FILESYSTEM_ERROR, getRenameErrorMessage(source, target)); - } - } - catch (IOException e) { - throw new PrestoException(HIVE_FILESYSTEM_ERROR, getRenameErrorMessage(source, target), e); - } - } - - private static String getRenameErrorMessage(Path source, Path target) - { - return format("Error moving data files from %s to final location %s", source, target); - } -} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/DwrfBatchPageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/DwrfBatchPageSourceFactory.java index 5098998c3ebd2..619dc3873fbd8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/DwrfBatchPageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/DwrfBatchPageSourceFactory.java @@ -15,10 +15,15 @@ import com.facebook.hive.orc.OrcSerde; import com.facebook.presto.hive.FileFormatDataSourceStats; +import com.facebook.presto.hive.FileOpener; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HiveBatchPageSourceFactory; import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.HiveColumnHandle; +import com.facebook.presto.hive.metastore.Storage; +import com.facebook.presto.orc.OrcReaderOptions; +import com.facebook.presto.orc.StripeMetadataSource; +import com.facebook.presto.orc.cache.OrcFileTailSource; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; @@ -31,8 +36,8 @@ import javax.inject.Inject; import java.util.List; +import java.util.Map; import java.util.Optional; -import java.util.Properties; import static com.facebook.presto.hive.HiveErrorCode.HIVE_BAD_DATA; import static com.facebook.presto.hive.HiveSessionProperties.getOrcLazyReadSmallRanges; @@ -41,7 +46,7 @@ import static com.facebook.presto.hive.HiveSessionProperties.getOrcMaxReadBlockSize; import static com.facebook.presto.hive.HiveSessionProperties.getOrcStreamBufferSize; import static com.facebook.presto.hive.HiveSessionProperties.getOrcTinyStripeThreshold; -import static com.facebook.presto.hive.HiveUtil.isDeserializerClass; +import static com.facebook.presto.hive.HiveSessionProperties.isOrcZstdJniDecompressionEnabled; import static com.facebook.presto.hive.orc.OrcBatchPageSourceFactory.createOrcPageSource; import static com.facebook.presto.orc.OrcEncoding.DWRF; import static java.util.Objects.requireNonNull; @@ -53,14 +58,27 @@ public class DwrfBatchPageSourceFactory private final HdfsEnvironment hdfsEnvironment; private final FileFormatDataSourceStats stats; private final int domainCompactionThreshold; + private final OrcFileTailSource orcFileTailSource; + private final StripeMetadataSource stripeMetadataSource; + private final FileOpener fileOpener; @Inject - public DwrfBatchPageSourceFactory(TypeManager typeManager, HiveClientConfig config, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats) + public DwrfBatchPageSourceFactory( + TypeManager typeManager, + HiveClientConfig config, + HdfsEnvironment hdfsEnvironment, + FileFormatDataSourceStats stats, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource, + FileOpener fileOpener) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.stats = requireNonNull(stats, "stats is null"); this.domainCompactionThreshold = requireNonNull(config, "config is null").getDomainCompactionThreshold(); + this.orcFileTailSource = requireNonNull(orcFileTailSource, "orcFileTailSource is null"); + this.stripeMetadataSource = requireNonNull(stripeMetadataSource, "stripeMetadataSource is null"); + this.fileOpener = requireNonNull(fileOpener, "fileOpener is null"); } @Override @@ -70,12 +88,14 @@ public Optional createPageSource(Configuration co long start, long length, long fileSize, - Properties schema, + Storage storage, + Map tableParameters, List columns, TupleDomain effectivePredicate, - DateTimeZone hiveStorageTimeZone) + DateTimeZone hiveStorageTimeZone, + Optional extraFileInfo) { - if (!isDeserializerClass(schema, OrcSerde.class)) { + if (!OrcSerde.class.getName().equals(storage.getStorageFormat().getSerDe())) { return Optional.empty(); } @@ -97,14 +117,20 @@ public Optional createPageSource(Configuration co effectivePredicate, hiveStorageTimeZone, typeManager, - getOrcMaxMergeDistance(session), getOrcMaxBufferSize(session), getOrcStreamBufferSize(session), - getOrcTinyStripeThreshold(session), - getOrcMaxReadBlockSize(session), getOrcLazyReadSmallRanges(session), false, stats, - domainCompactionThreshold)); + domainCompactionThreshold, + orcFileTailSource, + stripeMetadataSource, + extraFileInfo, + fileOpener, + new OrcReaderOptions( + getOrcMaxMergeDistance(session), + getOrcTinyStripeThreshold(session), + getOrcMaxReadBlockSize(session), + isOrcZstdJniDecompressionEnabled(session)))); } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/DwrfSelectivePageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/DwrfSelectivePageSourceFactory.java index 040ea29173789..4d9b2bf7c201c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/DwrfSelectivePageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/DwrfSelectivePageSourceFactory.java @@ -14,15 +14,22 @@ package com.facebook.presto.hive.orc; import com.facebook.hive.orc.OrcSerde; +import com.facebook.presto.hive.BucketAdaptation; import com.facebook.presto.hive.FileFormatDataSourceStats; +import com.facebook.presto.hive.FileOpener; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HiveClientConfig; +import com.facebook.presto.hive.HiveCoercer; import com.facebook.presto.hive.HiveColumnHandle; import com.facebook.presto.hive.HiveSelectivePageSourceFactory; +import com.facebook.presto.hive.metastore.Storage; +import com.facebook.presto.orc.StripeMetadataSource; +import com.facebook.presto.orc.cache.OrcFileTailSource; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionService; @@ -36,10 +43,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Properties; import static com.facebook.presto.hive.HiveErrorCode.HIVE_BAD_DATA; -import static com.facebook.presto.hive.HiveUtil.isDeserializerClass; import static com.facebook.presto.hive.orc.OrcSelectivePageSourceFactory.createOrcPageSource; import static com.facebook.presto.orc.OrcEncoding.DWRF; import static java.util.Objects.requireNonNull; @@ -48,19 +53,36 @@ public class DwrfSelectivePageSourceFactory implements HiveSelectivePageSourceFactory { private final TypeManager typeManager; + private final StandardFunctionResolution functionResolution; private final RowExpressionService rowExpressionService; private final HdfsEnvironment hdfsEnvironment; private final FileFormatDataSourceStats stats; private final int domainCompactionThreshold; + private final OrcFileTailSource orcFileTailSource; + private final StripeMetadataSource stripeMetadataSource; + private final FileOpener fileOpener; @Inject - public DwrfSelectivePageSourceFactory(TypeManager typeManager, RowExpressionService rowExpressionService, HiveClientConfig config, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats) + public DwrfSelectivePageSourceFactory( + TypeManager typeManager, + StandardFunctionResolution functionResolution, + RowExpressionService rowExpressionService, + HiveClientConfig config, + HdfsEnvironment hdfsEnvironment, + FileFormatDataSourceStats stats, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource, + FileOpener fileOpener) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.stats = requireNonNull(stats, "stats is null"); this.domainCompactionThreshold = requireNonNull(config, "config is null").getDomainCompactionThreshold(); + this.orcFileTailSource = requireNonNull(orcFileTailSource, "orcFileTailSource is null"); + this.stripeMetadataSource = requireNonNull(stripeMetadataSource, "stripeMetadataSource is null"); + this.fileOpener = requireNonNull(fileOpener, "fileOpener is null"); } @Override @@ -71,15 +93,18 @@ public Optional createPageSource( long start, long length, long fileSize, - Properties schema, + Storage storage, List columns, Map prefilledValues, + Map coercers, + Optional bucketAdaptation, List outputColumns, TupleDomain domainPredicate, RowExpression remainingPredicate, - DateTimeZone hiveStorageTimeZone) + DateTimeZone hiveStorageTimeZone, + Optional extraFileInfo) { - if (!isDeserializerClass(schema, OrcSerde.class)) { + if (!OrcSerde.class.getName().equals(storage.getStorageFormat().getSerDe())) { return Optional.empty(); } @@ -98,15 +123,22 @@ public Optional createPageSource( fileSize, columns, prefilledValues, + coercers, + bucketAdaptation, outputColumns, domainPredicate, remainingPredicate, false, hiveStorageTimeZone, typeManager, + functionResolution, rowExpressionService, false, stats, - domainCompactionThreshold)); + domainCompactionThreshold, + orcFileTailSource, + stripeMetadataSource, + extraFileInfo, + fileOpener)); } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/HdfsOrcDataSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/HdfsOrcDataSource.java index b4d7b3c3b0ee2..2b5d8298f763b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/HdfsOrcDataSource.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/HdfsOrcDataSource.java @@ -22,9 +22,9 @@ import java.io.IOException; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_MISSING_DATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static java.lang.String.format; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSource.java index 29f588f711799..b592fcb6643dd 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSource.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSource.java @@ -23,9 +23,9 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.LazyBlock; import com.facebook.presto.spi.block.LazyBlockLoader; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -46,7 +46,6 @@ public class OrcBatchPageSource implements ConnectorPageSource { - private static final int NULL_ENTRY_SIZE = 0; private final OrcBatchRecordReader recordReader; private final OrcDataSource orcDataSource; @@ -57,6 +56,7 @@ public class OrcBatchPageSource private final int[] hiveColumnIndexes; private int batchId; + private long completedPositions; private boolean closed; private final AggregatedMemoryContext systemMemoryContext; @@ -96,11 +96,7 @@ public OrcBatchPageSource( hiveColumnIndexes[columnIndex] = column.getHiveColumnIndex(); if (!recordReader.isColumnPresent(column.getHiveColumnIndex())) { - BlockBuilder blockBuilder = type.createBlockBuilder(null, MAX_BATCH_SIZE, NULL_ENTRY_SIZE); - for (int i = 0; i < MAX_BATCH_SIZE; i++) { - blockBuilder.appendNull(); - } - constantBlocks[columnIndex] = blockBuilder.build(); + constantBlocks[columnIndex] = RunLengthEncodedBlock.create(type, null, MAX_BATCH_SIZE); } } types = typesBuilder.build(); @@ -115,6 +111,12 @@ public long getCompletedBytes() return orcDataSource.getReadBytes(); } + @Override + public long getCompletedPositions() + { + return completedPositions; + } + @Override public long getReadTimeNanos() { @@ -138,14 +140,15 @@ public Page getNextPage() return null; } + completedPositions += batchSize; + Block[] blocks = new Block[hiveColumnIndexes.length]; for (int fieldId = 0; fieldId < blocks.length; fieldId++) { - Type type = types.get(fieldId); if (constantBlocks[fieldId] != null) { blocks[fieldId] = constantBlocks[fieldId].getRegion(0, batchSize); } else { - blocks[fieldId] = new LazyBlock(batchSize, new OrcBlockLoader(hiveColumnIndexes[fieldId], type)); + blocks[fieldId] = new LazyBlock(batchSize, new OrcBlockLoader(hiveColumnIndexes[fieldId])); } } return new Page(batchSize, blocks); @@ -216,13 +219,11 @@ private final class OrcBlockLoader { private final int expectedBatchId = batchId; private final int columnIndex; - private final Type type; private boolean loaded; - public OrcBlockLoader(int columnIndex, Type type) + public OrcBlockLoader(int columnIndex) { this.columnIndex = columnIndex; - this.type = requireNonNull(type, "type is null"); } @Override @@ -235,7 +236,7 @@ public final void load(LazyBlock lazyBlock) checkState(batchId == expectedBatchId); try { - Block block = recordReader.readBlock(type, columnIndex); + Block block = recordReader.readBlock(columnIndex); lazyBlock.setBlock(block); } catch (OrcCorruptionException e) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSourceFactory.java index f9f9967492cb9..0ecc285eacab3 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSourceFactory.java @@ -14,10 +14,12 @@ package com.facebook.presto.hive.orc; import com.facebook.presto.hive.FileFormatDataSourceStats; +import com.facebook.presto.hive.FileOpener; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HiveBatchPageSourceFactory; import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.HiveColumnHandle; +import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.memory.context.AggregatedMemoryContext; import com.facebook.presto.orc.OrcBatchRecordReader; import com.facebook.presto.orc.OrcDataSource; @@ -25,8 +27,11 @@ import com.facebook.presto.orc.OrcEncoding; import com.facebook.presto.orc.OrcPredicate; import com.facebook.presto.orc.OrcReader; +import com.facebook.presto.orc.OrcReaderOptions; +import com.facebook.presto.orc.StripeMetadataSource; import com.facebook.presto.orc.TupleDomainOrcPredicate; import com.facebook.presto.orc.TupleDomainOrcPredicate.ColumnReference; +import com.facebook.presto.orc.cache.OrcFileTailSource; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.FixedPageSource; @@ -49,8 +54,8 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Optional; -import java.util.Properties; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; @@ -62,8 +67,8 @@ import static com.facebook.presto.hive.HiveSessionProperties.getOrcStreamBufferSize; import static com.facebook.presto.hive.HiveSessionProperties.getOrcTinyStripeThreshold; import static com.facebook.presto.hive.HiveSessionProperties.isOrcBloomFiltersEnabled; +import static com.facebook.presto.hive.HiveSessionProperties.isOrcZstdJniDecompressionEnabled; import static com.facebook.presto.hive.HiveUtil.getPhysicalHiveColumnHandles; -import static com.facebook.presto.hive.HiveUtil.isDeserializerClass; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.OrcEncoding.ORC; import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE; @@ -80,20 +85,49 @@ public class OrcBatchPageSourceFactory private final HdfsEnvironment hdfsEnvironment; private final FileFormatDataSourceStats stats; private final int domainCompactionThreshold; + private final OrcFileTailSource orcFileTailSource; + private final StripeMetadataSource stripeMetadataSource; + private final FileOpener fileOpener; @Inject - public OrcBatchPageSourceFactory(TypeManager typeManager, HiveClientConfig config, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats) + public OrcBatchPageSourceFactory( + TypeManager typeManager, + HiveClientConfig config, + HdfsEnvironment hdfsEnvironment, + FileFormatDataSourceStats stats, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource, + FileOpener fileOpener) { - this(typeManager, requireNonNull(config, "hiveClientConfig is null").isUseOrcColumnNames(), hdfsEnvironment, stats, config.getDomainCompactionThreshold()); + this( + typeManager, + requireNonNull(config, "hiveClientConfig is null").isUseOrcColumnNames(), + hdfsEnvironment, + stats, + config.getDomainCompactionThreshold(), + orcFileTailSource, + stripeMetadataSource, + fileOpener); } - public OrcBatchPageSourceFactory(TypeManager typeManager, boolean useOrcColumnNames, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats, int domainCompactionThreshold) + public OrcBatchPageSourceFactory( + TypeManager typeManager, + boolean useOrcColumnNames, + HdfsEnvironment hdfsEnvironment, + FileFormatDataSourceStats stats, + int domainCompactionThreshold, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource, + FileOpener fileOpener) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.useOrcColumnNames = useOrcColumnNames; this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.stats = requireNonNull(stats, "stats is null"); this.domainCompactionThreshold = domainCompactionThreshold; + this.orcFileTailSource = requireNonNull(orcFileTailSource, "orcFileTailSource is null"); + this.stripeMetadataSource = requireNonNull(stripeMetadataSource, "stripeMetadataSource is null"); + this.fileOpener = requireNonNull(fileOpener, "fileOpener is null"); } @Override @@ -104,12 +138,14 @@ public Optional createPageSource( long start, long length, long fileSize, - Properties schema, + Storage storage, + Map tableParameters, List columns, TupleDomain effectivePredicate, - DateTimeZone hiveStorageTimeZone) + DateTimeZone hiveStorageTimeZone, + Optional extraFileInfo) { - if (!isDeserializerClass(schema, OrcSerde.class)) { + if (!OrcSerde.class.getName().equals(storage.getStorageFormat().getSerDe())) { return Optional.empty(); } @@ -132,15 +168,21 @@ public Optional createPageSource( effectivePredicate, hiveStorageTimeZone, typeManager, - getOrcMaxMergeDistance(session), getOrcMaxBufferSize(session), getOrcStreamBufferSize(session), - getOrcTinyStripeThreshold(session), - getOrcMaxReadBlockSize(session), getOrcLazyReadSmallRanges(session), isOrcBloomFiltersEnabled(session), stats, - domainCompactionThreshold)); + domainCompactionThreshold, + orcFileTailSource, + stripeMetadataSource, + extraFileInfo, + fileOpener, + new OrcReaderOptions( + getOrcMaxMergeDistance(session), + getOrcTinyStripeThreshold(session), + getOrcMaxReadBlockSize(session), + isOrcZstdJniDecompressionEnabled(session)))); } public static OrcBatchPageSource createOrcPageSource( @@ -157,26 +199,28 @@ public static OrcBatchPageSource createOrcPageSource( TupleDomain effectivePredicate, DateTimeZone hiveStorageTimeZone, TypeManager typeManager, - DataSize maxMergeDistance, DataSize maxBufferSize, DataSize streamBufferSize, - DataSize tinyStripeThreshold, - DataSize maxReadBlockSize, boolean lazyReadSmallRanges, boolean orcBloomFiltersEnabled, FileFormatDataSourceStats stats, - int domainCompactionThreshold) + int domainCompactionThreshold, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource, + Optional extraFileInfo, + FileOpener fileOpener, + OrcReaderOptions orcReaderOptions) { checkArgument(domainCompactionThreshold >= 1, "domainCompactionThreshold must be at least 1"); OrcDataSource orcDataSource; try { FileSystem fileSystem = hdfsEnvironment.getFileSystem(sessionUser, path, configuration); - FSDataInputStream inputStream = fileSystem.open(path); + FSDataInputStream inputStream = fileOpener.open(fileSystem, path, extraFileInfo); orcDataSource = new HdfsOrcDataSource( new OrcDataSourceId(path.toString()), fileSize, - maxMergeDistance, + orcReaderOptions.getMaxMergeDistance(), maxBufferSize, streamBufferSize, lazyReadSmallRanges, @@ -193,7 +237,7 @@ public static OrcBatchPageSource createOrcPageSource( AggregatedMemoryContext systemMemoryUsage = newSimpleAggregatedMemoryContext(); try { - OrcReader reader = new OrcReader(orcDataSource, orcEncoding, maxMergeDistance, maxBufferSize, tinyStripeThreshold, maxReadBlockSize); + OrcReader reader = new OrcReader(orcDataSource, orcEncoding, orcFileTailSource, stripeMetadataSource, orcReaderOptions); List physicalColumns = getPhysicalHiveColumnHandles(columns, useOrcColumnNames, reader, path); ImmutableMap.Builder includedColumns = ImmutableMap.builder(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSource.java index fe709f4f168c0..71a108bde6663 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSource.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSource.java @@ -59,6 +59,12 @@ public long getCompletedBytes() return orcDataSource.getReadBytes(); } + @Override + public long getCompletedPositions() + { + return recordReader.getReadPositions(); + } + @Override public long getReadTimeNanos() { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java index b852a83d62a12..c713c64e8b432 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java @@ -13,11 +13,18 @@ */ package com.facebook.presto.hive.orc; +import com.facebook.presto.expressions.DefaultRowExpressionTraversalVisitor; +import com.facebook.presto.hive.BucketAdaptation; import com.facebook.presto.hive.FileFormatDataSourceStats; +import com.facebook.presto.hive.FileOpener; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HiveClientConfig; +import com.facebook.presto.hive.HiveCoercer; import com.facebook.presto.hive.HiveColumnHandle; import com.facebook.presto.hive.HiveSelectivePageSourceFactory; +import com.facebook.presto.hive.HiveType; +import com.facebook.presto.hive.SubfieldExtractor; +import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.memory.context.AggregatedMemoryContext; import com.facebook.presto.orc.FilterFunction; import com.facebook.presto.orc.OrcDataSource; @@ -25,28 +32,36 @@ import com.facebook.presto.orc.OrcEncoding; import com.facebook.presto.orc.OrcPredicate; import com.facebook.presto.orc.OrcReader; +import com.facebook.presto.orc.OrcReaderOptions; import com.facebook.presto.orc.OrcSelectiveRecordReader; +import com.facebook.presto.orc.StripeMetadataSource; import com.facebook.presto.orc.TupleDomainFilter; import com.facebook.presto.orc.TupleDomainFilterUtils; import com.facebook.presto.orc.TupleDomainOrcPredicate; +import com.facebook.presto.orc.cache.OrcFileTailSource; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.FixedPageSource; +import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.predicate.TupleDomain; -import com.facebook.presto.spi.relation.DefaultRowExpressionTraversalVisitor; +import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.Predicate; import com.facebook.presto.spi.relation.PredicateCompiler; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionService; +import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import io.airlift.units.DataSize; @@ -55,6 +70,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.io.orc.OrcSerde; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.joda.time.DateTimeZone; import javax.inject.Inject; @@ -62,17 +78,27 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Properties; import java.util.Set; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.IntStream; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.binaryExpression; +import static com.facebook.presto.expressions.LogicalRowExpressions.extractConjuncts; +import static com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression; +import static com.facebook.presto.hive.HiveBucketing.getHiveBucket; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; +import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_BUCKET_FILES; import static com.facebook.presto.hive.HiveErrorCode.HIVE_MISSING_DATA; import static com.facebook.presto.hive.HiveSessionProperties.getOrcLazyReadSmallRanges; import static com.facebook.presto.hive.HiveSessionProperties.getOrcMaxBufferSize; @@ -81,19 +107,16 @@ import static com.facebook.presto.hive.HiveSessionProperties.getOrcStreamBufferSize; import static com.facebook.presto.hive.HiveSessionProperties.getOrcTinyStripeThreshold; import static com.facebook.presto.hive.HiveSessionProperties.isOrcBloomFiltersEnabled; +import static com.facebook.presto.hive.HiveSessionProperties.isOrcZstdJniDecompressionEnabled; import static com.facebook.presto.hive.HiveUtil.getPhysicalHiveColumnHandles; -import static com.facebook.presto.hive.HiveUtil.isDeserializerClass; import static com.facebook.presto.hive.HiveUtil.typedPartitionKey; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.OrcEncoding.ORC; import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.binaryExpression; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.extractConjuncts; -import static com.facebook.presto.spi.relation.RowExpressionNodeInliner.replaceExpression; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.nullToEmpty; +import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableBiMap.toImmutableBiMap; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; @@ -105,26 +128,63 @@ public class OrcSelectivePageSourceFactory implements HiveSelectivePageSourceFactory { private final TypeManager typeManager; + private final StandardFunctionResolution functionResolution; private final RowExpressionService rowExpressionService; private final boolean useOrcColumnNames; private final HdfsEnvironment hdfsEnvironment; private final FileFormatDataSourceStats stats; private final int domainCompactionThreshold; + private final OrcFileTailSource orcFileTailSource; + private final StripeMetadataSource stripeMetadataSource; + private final FileOpener fileOpener; @Inject - public OrcSelectivePageSourceFactory(TypeManager typeManager, RowExpressionService rowExpressionService, HiveClientConfig config, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats) + public OrcSelectivePageSourceFactory( + TypeManager typeManager, + StandardFunctionResolution functionResolution, + RowExpressionService rowExpressionService, + HiveClientConfig config, + HdfsEnvironment hdfsEnvironment, + FileFormatDataSourceStats stats, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource, + FileOpener fileOpener) { - this(typeManager, rowExpressionService, requireNonNull(config, "hiveClientConfig is null").isUseOrcColumnNames(), hdfsEnvironment, stats, config.getDomainCompactionThreshold()); + this( + typeManager, + functionResolution, + rowExpressionService, + requireNonNull(config, "hiveClientConfig is null").isUseOrcColumnNames(), + hdfsEnvironment, + stats, + config.getDomainCompactionThreshold(), + orcFileTailSource, + stripeMetadataSource, + fileOpener); } - public OrcSelectivePageSourceFactory(TypeManager typeManager, RowExpressionService rowExpressionService, boolean useOrcColumnNames, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats, int domainCompactionThreshold) + public OrcSelectivePageSourceFactory( + TypeManager typeManager, + StandardFunctionResolution functionResolution, + RowExpressionService rowExpressionService, + boolean useOrcColumnNames, + HdfsEnvironment hdfsEnvironment, + FileFormatDataSourceStats stats, + int domainCompactionThreshold, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource, + FileOpener fileOpener) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null"); this.useOrcColumnNames = useOrcColumnNames; this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.stats = requireNonNull(stats, "stats is null"); this.domainCompactionThreshold = domainCompactionThreshold; + this.orcFileTailSource = requireNonNull(orcFileTailSource, "orcFileTailCache is null"); + this.stripeMetadataSource = requireNonNull(stripeMetadataSource, "stripeMetadataSource is null"); + this.fileOpener = requireNonNull(fileOpener, "fileOpener is null"); } @Override @@ -135,15 +195,18 @@ public Optional createPageSource( long start, long length, long fileSize, - Properties schema, + Storage storage, List columns, Map prefilledValues, + Map coercers, + Optional bucketAdaptation, List outputColumns, TupleDomain domainPredicate, RowExpression remainingPredicate, - DateTimeZone hiveStorageTimeZone) + DateTimeZone hiveStorageTimeZone, + Optional extraFileInfo) { - if (!isDeserializerClass(schema, OrcSerde.class)) { + if (!OrcSerde.class.getName().equals(storage.getStorageFormat().getSerDe())) { return Optional.empty(); } @@ -163,16 +226,23 @@ public Optional createPageSource( fileSize, columns, prefilledValues, + coercers, + bucketAdaptation, outputColumns, domainPredicate, remainingPredicate, useOrcColumnNames, hiveStorageTimeZone, typeManager, + functionResolution, rowExpressionService, isOrcBloomFiltersEnabled(session), stats, - domainCompactionThreshold)); + domainCompactionThreshold, + orcFileTailSource, + stripeMetadataSource, + extraFileInfo, + fileOpener)); } public static OrcSelectivePageSource createOrcPageSource( @@ -186,16 +256,23 @@ public static OrcSelectivePageSource createOrcPageSource( long fileSize, List columns, Map prefilledValues, + Map coercers, + Optional bucketAdaptation, List outputColumns, TupleDomain domainPredicate, RowExpression remainingPredicate, boolean useOrcColumnNames, DateTimeZone hiveStorageTimeZone, TypeManager typeManager, + StandardFunctionResolution functionResolution, RowExpressionService rowExpressionService, boolean orcBloomFiltersEnabled, FileFormatDataSourceStats stats, - int domainCompactionThreshold) + int domainCompactionThreshold, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource, + Optional extraFileInfo, + FileOpener fileOpener) { checkArgument(domainCompactionThreshold >= 1, "domainCompactionThreshold must be at least 1"); @@ -204,12 +281,13 @@ public static OrcSelectivePageSource createOrcPageSource( DataSize streamBufferSize = getOrcStreamBufferSize(session); DataSize tinyStripeThreshold = getOrcTinyStripeThreshold(session); DataSize maxReadBlockSize = getOrcMaxReadBlockSize(session); + OrcReaderOptions orcReaderOptions = new OrcReaderOptions(maxMergeDistance, tinyStripeThreshold, maxReadBlockSize, isOrcZstdJniDecompressionEnabled(session)); boolean lazyReadSmallRanges = getOrcLazyReadSmallRanges(session); OrcDataSource orcDataSource; try { FileSystem fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), path, configuration); - FSDataInputStream inputStream = fileSystem.open(path); + FSDataInputStream inputStream = fileOpener.open(fileSystem, path, extraFileInfo); orcDataSource = new HdfsOrcDataSource( new OrcDataSourceId(path.toString()), fileSize, @@ -230,7 +308,7 @@ public static OrcSelectivePageSource createOrcPageSource( AggregatedMemoryContext systemMemoryUsage = newSimpleAggregatedMemoryContext(); try { - OrcReader reader = new OrcReader(orcDataSource, orcEncoding, maxMergeDistance, maxBufferSize, tinyStripeThreshold, maxReadBlockSize); + OrcReader reader = new OrcReader(orcDataSource, orcEncoding, orcFileTailSource, stripeMetadataSource, orcReaderOptions); checkArgument(!domainPredicate.isNone(), "Unexpected NONE domain"); @@ -242,12 +320,15 @@ public static OrcSelectivePageSource createOrcPageSource( Map columnNames = physicalColumns.stream() .collect(toImmutableMap(HiveColumnHandle::getHiveColumnIndex, HiveColumnHandle::getName)); - OrcPredicate orcPredicate = toOrcPredicate(domainPredicate, physicalColumns, typeManager, domainCompactionThreshold, orcBloomFiltersEnabled); + Map mappedCoercers = coercers.entrySet().stream().collect(toImmutableMap(entry -> indexMapping.get(entry.getKey()), Map.Entry::getValue)); + + OrcPredicate orcPredicate = toOrcPredicate(domainPredicate, physicalColumns, mappedCoercers, typeManager, domainCompactionThreshold, orcBloomFiltersEnabled); - Map tupleDomainFilters = toTupleDomainFilters(domainPredicate, ImmutableBiMap.copyOf(columnNames).inverse()); + Map columnIndices = ImmutableBiMap.copyOf(columnNames).inverse(); + Map> tupleDomainFilters = toTupleDomainFilters(domainPredicate, columnIndices, mappedCoercers); - Map> requiredSubfields = physicalColumns.stream() - .collect(toImmutableMap(HiveColumnHandle::getHiveColumnIndex, HiveColumnHandle::getRequiredSubfields)); + List outputIndices = outputColumns.stream().map(indexMapping::get).collect(toImmutableList()); + Map> requiredSubfields = collectRequiredSubfields(physicalColumns, outputIndices, tupleDomainFilters, remainingPredicate, columnIndices, functionResolution, rowExpressionService, session); Map columnTypes = physicalColumns.stream() .collect(toImmutableMap(HiveColumnHandle::getHiveColumnIndex, column -> typeManager.getType(column.getTypeSignature()))); @@ -261,21 +342,33 @@ public static OrcSelectivePageSource createOrcPageSource( .boxed() .collect(toImmutableBiMap(i -> physicalColumns.get(i).getHiveColumnIndex(), Function.identity())); + // use column types from the current table schema; these types might be different from this partition's schema Map variableToInput = columnNames.keySet().stream() .collect(toImmutableMap( - hiveColumnIndex -> new VariableReferenceExpression(columnNames.get(hiveColumnIndex), columnTypes.get(hiveColumnIndex)), - hiveColumnIndex -> new InputReferenceExpression(inputs.get(hiveColumnIndex), columnTypes.get(hiveColumnIndex)))); + hiveColumnIndex -> new VariableReferenceExpression(columnNames.get(hiveColumnIndex), getColumnTypeFromTableSchema(coercers, columnTypes, hiveColumnIndex)), + hiveColumnIndex -> new InputReferenceExpression(inputs.get(hiveColumnIndex), getColumnTypeFromTableSchema(coercers, columnTypes, hiveColumnIndex)))); + + Optional bucketAdapter = bucketAdaptation.map(adaptation -> new BucketAdapter( + Arrays.stream(adaptation.getBucketColumnIndices()) + .map(indexMapping::get) + .map(inputs::get) + .toArray(), + adaptation.getBucketColumnHiveTypes(), + adaptation.getTableBucketCount(), + adaptation.getPartitionBucketCount(), + adaptation.getBucketToKeep())); - List filterFunctions = toFilterFunctions(replaceExpression(remainingPredicate, variableToInput), session, rowExpressionService.getDeterminismEvaluator(), rowExpressionService.getPredicateCompiler()); + List filterFunctions = toFilterFunctions(replaceExpression(remainingPredicate, variableToInput), bucketAdapter, session, rowExpressionService.getDeterminismEvaluator(), rowExpressionService.getPredicateCompiler()); OrcSelectiveRecordReader recordReader = reader.createSelectiveRecordReader( columnTypes, - outputColumns.stream().map(indexMapping::get).collect(toImmutableList()), + outputIndices, tupleDomainFilters, filterFunctions, inputs.inverse(), requiredSubfields, typedPrefilledValues, + Maps.transformValues(mappedCoercers, Function.class::cast), orcPredicate, start, length, @@ -307,17 +400,163 @@ public static OrcSelectivePageSource createOrcPageSource( } } - private static Map toTupleDomainFilters(TupleDomain domainPredicate, Map columnIndices) + private static Type getColumnTypeFromTableSchema(Map coercers, Map columnTypes, int hiveColumnIndex) { - // TODO Add support for filters on subfields - checkArgument(domainPredicate.getDomains().get().keySet().stream() - .allMatch(OrcSelectivePageSourceFactory::isEntireColumn), "Filters on subfields are not supported yet"); - - return Maps.transformValues( - domainPredicate.transform(subfield -> isEntireColumn(subfield) ? columnIndices.get(subfield.getRootName()) : null) - .getDomains() - .get(), - TupleDomainFilterUtils::toFilter); + return coercers.containsKey(hiveColumnIndex) ? coercers.get(hiveColumnIndex).getToType() : columnTypes.get(hiveColumnIndex); + } + + private static Map> collectRequiredSubfields(List physicalColumns, List outputColumns, Map> tupleDomainFilters, RowExpression remainingPredicate, Map columnIndices, StandardFunctionResolution functionResolution, RowExpressionService rowExpressionService, ConnectorSession session) + { + /** + * The logic is: + * + * - columns projected fully are not modified; + * - columns projected partially are updated to include subfields used in the filters or + * to be read in full if entire column is used in a filter + * - columns used for filtering only are updated to prune subfields if filters don't use full column + */ + + Map> outputSubfields = new HashMap<>(); + physicalColumns.stream() + .filter(column -> outputColumns.contains(column.getHiveColumnIndex())) + .forEach(column -> outputSubfields.put(column.getHiveColumnIndex(), new HashSet<>(column.getRequiredSubfields()))); + + Map> predicateSubfields = new HashMap<>(); + SubfieldExtractor subfieldExtractor = new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session); + remainingPredicate.accept( + new RequiredSubfieldsExtractor(subfieldExtractor), + subfield -> predicateSubfields.computeIfAbsent(columnIndices.get(subfield.getRootName()), v -> new HashSet<>()).add(subfield)); + + for (Map.Entry> entry : tupleDomainFilters.entrySet()) { + predicateSubfields.computeIfAbsent(entry.getKey(), v -> new HashSet<>()).addAll(entry.getValue().keySet()); + } + + Map> allSubfields = new HashMap<>(); + for (Map.Entry> entry : outputSubfields.entrySet()) { + int columnIndex = entry.getKey(); + if (entry.getValue().isEmpty()) { + // entire column is projected out + continue; + } + + if (!predicateSubfields.containsKey(columnIndex)) { + // column is not used in filters + allSubfields.put(columnIndex, ImmutableList.copyOf(entry.getValue())); + continue; + } + + List prunedSubfields = pruneSubfields(ImmutableSet.builder() + .addAll(entry.getValue()) + .addAll(predicateSubfields.get(columnIndex)).build()); + + if (prunedSubfields.size() == 1 && isEntireColumn(prunedSubfields.get(0))) { + // entire column is used in a filter + continue; + } + + allSubfields.put(columnIndex, prunedSubfields); + } + + for (Map.Entry> entry : predicateSubfields.entrySet()) { + int columnIndex = entry.getKey(); + if (outputSubfields.containsKey(columnIndex)) { + // this column has been already processed (in the previous loop) + continue; + } + + List prunedSubfields = pruneSubfields(entry.getValue()); + if (prunedSubfields.size() == 1 && isEntireColumn(prunedSubfields.get(0))) { + // entire column is used in a filter + continue; + } + + allSubfields.put(columnIndex, prunedSubfields); + } + + return allSubfields; + } + + // Prunes subfields: if one subfield is a prefix of another subfield, keeps the shortest one. + // Example: {a.b.c, a.b} -> {a.b} + private static List pruneSubfields(Set subfields) + { + verify(!subfields.isEmpty()); + + return subfields.stream() + .filter(subfield -> !prefixExists(subfield, subfields)) + .collect(toImmutableList()); + } + + private static boolean prefixExists(Subfield subfield, Collection subfields) + { + return subfields.stream().anyMatch(path -> path.isPrefix(subfield)); + } + + private static final class RequiredSubfieldsExtractor + extends DefaultRowExpressionTraversalVisitor> + { + private final SubfieldExtractor subfieldExtractor; + + public RequiredSubfieldsExtractor(SubfieldExtractor subfieldExtractor) + { + this.subfieldExtractor = requireNonNull(subfieldExtractor, "subfieldExtractor is null"); + } + + @Override + public Void visitCall(CallExpression call, Consumer context) + { + Optional subfield = subfieldExtractor.extract(call); + if (subfield.isPresent()) { + context.accept(subfield.get()); + return null; + } + + call.getArguments().forEach(argument -> argument.accept(this, context)); + return null; + } + + @Override + public Void visitSpecialForm(SpecialFormExpression specialForm, Consumer context) + { + Optional subfield = subfieldExtractor.extract(specialForm); + if (subfield.isPresent()) { + context.accept(subfield.get()); + return null; + } + + specialForm.getArguments().forEach(argument -> argument.accept(this, context)); + return null; + } + + @Override + public Void visitVariableReference(VariableReferenceExpression reference, Consumer context) + { + Optional subfield = subfieldExtractor.extract(reference); + if (subfield.isPresent()) { + context.accept(subfield.get()); + return null; + } + + return null; + } + } + + private static Map> toTupleDomainFilters(TupleDomain domainPredicate, Map columnIndices, Map coercers) + { + Map filtersBySubfield = Maps.transformValues(domainPredicate.getDomains().get(), TupleDomainFilterUtils::toFilter); + + Map> filtersByColumn = new HashMap<>(); + for (Map.Entry entry : filtersBySubfield.entrySet()) { + Subfield subfield = entry.getKey(); + int columnIndex = columnIndices.get(subfield.getRootName()); + TupleDomainFilter filter = entry.getValue(); + if (coercers.containsKey(columnIndex)) { + filter = coercers.get(columnIndex).toCoercingFilter(filter, subfield); + } + filtersByColumn.computeIfAbsent(columnIndex, k -> new HashMap<>()).put(subfield, filter); + } + + return ImmutableMap.copyOf(filtersByColumn); } private static boolean isEntireColumn(Subfield subfield) @@ -325,7 +564,7 @@ private static boolean isEntireColumn(Subfield subfield) return subfield.getPath().isEmpty(); } - private static OrcPredicate toOrcPredicate(TupleDomain domainPredicate, List physicalColumns, TypeManager typeManager, int domainCompactionThreshold, boolean orcBloomFiltersEnabled) + private static OrcPredicate toOrcPredicate(TupleDomain domainPredicate, List physicalColumns, Map coercers, TypeManager typeManager, int domainCompactionThreshold, boolean orcBloomFiltersEnabled) { ImmutableList.Builder> columnReferences = ImmutableList.builder(); for (HiveColumnHandle column : physicalColumns) { @@ -336,7 +575,10 @@ private static OrcPredicate toOrcPredicate(TupleDomain domainPredicate } Map columnsByName = uniqueIndex(physicalColumns, HiveColumnHandle::getName); - TupleDomain entireColumnDomains = domainPredicate.transform(subfield -> isEntireColumn(subfield) ? columnsByName.get(subfield.getRootName()) : null); + TupleDomain entireColumnDomains = domainPredicate + .transform(subfield -> isEntireColumn(subfield) ? columnsByName.get(subfield.getRootName()) : null) + // filter out columns with coercions to avoid type mismatch errors between column stats in the file and values domain + .transform(column -> coercers.containsKey(column.getHiveColumnIndex()) ? null : column); return new TupleDomainOrcPredicate<>(entireColumnDomains, columnReferences.build(), orcBloomFiltersEnabled, Optional.of(domainCompactionThreshold)); } @@ -344,15 +586,21 @@ private static OrcPredicate toOrcPredicate(TupleDomain domainPredicate * Split filter expression into groups of conjuncts that depend on the same set of inputs, * then compile each group into FilterFunction. */ - private static List toFilterFunctions(RowExpression filter, ConnectorSession session, DeterminismEvaluator determinismEvaluator, PredicateCompiler predicateCompiler) + private static List toFilterFunctions(RowExpression filter, Optional bucketAdapter, ConnectorSession session, DeterminismEvaluator determinismEvaluator, PredicateCompiler predicateCompiler) { + ImmutableList.Builder filterFunctions = ImmutableList.builder(); + + bucketAdapter.map(predicate -> new FilterFunction(session, true, predicate)) + .ifPresent(filterFunctions::add); + if (TRUE_CONSTANT.equals(filter)) { - return ImmutableList.of(); + return filterFunctions.build(); } List conjuncts = extractConjuncts(filter); if (conjuncts.size() == 1) { - return ImmutableList.of(new FilterFunction(session, determinismEvaluator.isDeterministic(filter), predicateCompiler.compilePredicate(filter).get())); + filterFunctions.add(new FilterFunction(session, determinismEvaluator.isDeterministic(filter), predicateCompiler.compilePredicate(session.getSqlFunctionProperties(), filter).get())); + return filterFunctions.build(); } // Use LinkedHashMap to preserve user-specified order of conjuncts. This will be the initial order in which filters are applied. @@ -361,10 +609,12 @@ private static List toFilterFunctions(RowExpression filter, Conn inputsToConjuncts.computeIfAbsent(extractInputs(conjunct), k -> new ArrayList<>()).add(conjunct); } - return inputsToConjuncts.values().stream() + inputsToConjuncts.values().stream() .map(expressions -> binaryExpression(AND, expressions)) - .map(predicate -> new FilterFunction(session, determinismEvaluator.isDeterministic(predicate), predicateCompiler.compilePredicate(predicate).get())) - .collect(toImmutableList()); + .map(predicate -> new FilterFunction(session, determinismEvaluator.isDeterministic(predicate), predicateCompiler.compilePredicate(session.getSqlFunctionProperties(), predicate).get())) + .forEach(filterFunctions::add); + + return filterFunctions.build(); } private static Set extractInputs(RowExpression expression) @@ -389,4 +639,44 @@ private static String splitError(Throwable t, Path path, long start, long length { return format("Error opening Hive split %s (offset=%s, length=%s): %s", path, start, length, t.getMessage()); } + + private static class BucketAdapter + implements Predicate + { + public final int[] bucketColumns; + public final int bucketToKeep; + public final int tableBucketCount; + public final int partitionBucketCount; // for sanity check only + private final List typeInfoList; + + public BucketAdapter(int[] bucketColumnIndices, List bucketColumnHiveTypes, int tableBucketCount, int partitionBucketCount, int bucketToKeep) + { + this.bucketColumns = requireNonNull(bucketColumnIndices, "bucketColumnIndices is null"); + this.bucketToKeep = bucketToKeep; + this.typeInfoList = requireNonNull(bucketColumnHiveTypes, "bucketColumnHiveTypes is null").stream() + .map(HiveType::getTypeInfo) + .collect(toImmutableList()); + this.tableBucketCount = tableBucketCount; + this.partitionBucketCount = partitionBucketCount; + } + + @Override + public int[] getInputChannels() + { + return bucketColumns; + } + + @Override + public boolean evaluate(ConnectorSession session, Page page, int position) + { + int bucket = getHiveBucket(tableBucketCount, typeInfoList, page, position); + if ((bucket - bucketToKeep) % partitionBucketCount != 0) { + throw new PrestoException(HIVE_INVALID_BUCKET_FILES, format( + "A row that is supposed to be in bucket %s is encountered. Only rows in bucket %s (modulo %s) are expected", + bucket, bucketToKeep % partitionBucketCount, partitionBucketCount)); + } + + return bucket == bucketToKeep; + } + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/HdfsParquetDataSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/HdfsParquetDataSource.java index a1bc6852f9cf8..2fde5c238fdc4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/HdfsParquetDataSource.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/HdfsParquetDataSource.java @@ -25,7 +25,7 @@ import java.io.IOException; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static com.google.common.base.Strings.nullToEmpty; import static java.lang.String.format; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSource.java index f4d63ce765920..b021df97223a4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSource.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSource.java @@ -24,7 +24,6 @@ import com.facebook.presto.spi.block.LazyBlock; import com.facebook.presto.spi.block.LazyBlockLoader; import com.facebook.presto.spi.block.RunLengthEncodedBlock; -import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -35,13 +34,11 @@ import java.io.UncheckedIOException; import java.util.List; import java.util.Optional; -import java.util.Properties; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_BAD_DATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CURSOR_ERROR; import static com.facebook.presto.hive.parquet.ParquetPageSourceFactory.getParquetType; -import static com.facebook.presto.parquet.ParquetTypeUtils.getFieldIndex; import static com.facebook.presto.parquet.ParquetTypeUtils.lookupColumnByName; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; @@ -50,48 +47,32 @@ public class ParquetPageSource implements ConnectorPageSource { - private static final int MAX_VECTOR_LENGTH = 1024; - private final ParquetReader parquetReader; - private final MessageType fileSchema; // for debugging heap dump private final List columnNames; private final List types; private final List> fields; - private final Block[] constantBlocks; - private final int[] hiveColumnIndexes; - private int batchId; + private long completedPositions; private boolean closed; - private final boolean useParquetColumnNames; public ParquetPageSource( ParquetReader parquetReader, MessageType fileSchema, MessageColumnIO messageColumnIO, TypeManager typeManager, - Properties splitSchema, List columns, - TupleDomain effectivePredicate, boolean useParquetColumnNames) { - requireNonNull(splitSchema, "splitSchema is null"); requireNonNull(columns, "columns is null"); - requireNonNull(effectivePredicate, "effectivePredicate is null"); + requireNonNull(fileSchema, "fileSchema is null"); this.parquetReader = requireNonNull(parquetReader, "parquetReader is null"); - this.fileSchema = requireNonNull(fileSchema, "fileSchema is null"); - this.useParquetColumnNames = useParquetColumnNames; - - int size = columns.size(); - this.constantBlocks = new Block[size]; - this.hiveColumnIndexes = new int[size]; ImmutableList.Builder namesBuilder = ImmutableList.builder(); ImmutableList.Builder typesBuilder = ImmutableList.builder(); ImmutableList.Builder> fieldsBuilder = ImmutableList.builder(); - for (int columnIndex = 0; columnIndex < size; columnIndex++) { - HiveColumnHandle column = columns.get(columnIndex); + for (HiveColumnHandle column : columns) { checkState(column.getColumnType() == REGULAR, "column type must be regular"); String name = column.getName(); @@ -99,16 +80,14 @@ public ParquetPageSource( namesBuilder.add(name); typesBuilder.add(type); - hiveColumnIndexes[columnIndex] = column.getHiveColumnIndex(); - if (getParquetType(column, fileSchema, useParquetColumnNames) == null) { - constantBlocks[columnIndex] = RunLengthEncodedBlock.create(type, null, MAX_VECTOR_LENGTH); - fieldsBuilder.add(Optional.empty()); - } - else { + if (getParquetType(type, fileSchema, useParquetColumnNames, column).isPresent()) { String columnName = useParquetColumnNames ? name : fileSchema.getFields().get(column.getHiveColumnIndex()).getName(); fieldsBuilder.add(constructField(type, lookupColumnByName(messageColumnIO, columnName))); } + else { + fieldsBuilder.add(Optional.empty()); + } } types = typesBuilder.build(); fields = fieldsBuilder.build(); @@ -121,6 +100,12 @@ public long getCompletedBytes() return parquetReader.getDataSource().getReadBytes(); } + @Override + public long getCompletedPositions() + { + return completedPositions; + } + @Override public long getReadTimeNanos() { @@ -151,27 +136,16 @@ public Page getNextPage() return null; } - Block[] blocks = new Block[hiveColumnIndexes.length]; + completedPositions += batchSize; + + Block[] blocks = new Block[fields.size()]; for (int fieldId = 0; fieldId < blocks.length; fieldId++) { - if (constantBlocks[fieldId] != null) { - blocks[fieldId] = constantBlocks[fieldId].getRegion(0, batchSize); + Optional field = fields.get(fieldId); + if (field.isPresent()) { + blocks[fieldId] = new LazyBlock(batchSize, new ParquetBlockLoader(field.get())); } else { - Type type = types.get(fieldId); - Optional field = fields.get(fieldId); - int fieldIndex; - if (useParquetColumnNames) { - fieldIndex = getFieldIndex(fileSchema, columnNames.get(fieldId)); - } - else { - fieldIndex = hiveColumnIndexes[fieldId]; - } - if (fieldIndex != -1 && field.isPresent()) { - blocks[fieldId] = new LazyBlock(batchSize, new ParquetBlockLoader(field.get())); - } - else { - blocks[fieldId] = RunLengthEncodedBlock.create(type, null, batchSize); - } + blocks[fieldId] = RunLengthEncodedBlock.create(types.get(fieldId), null, batchSize); } } return new Page(batchSize, blocks); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSourceFactory.java index 4530c1b1dbfa1..f12e47e97774c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSourceFactory.java @@ -14,9 +14,11 @@ package com.facebook.presto.hive.parquet; import com.facebook.presto.hive.FileFormatDataSourceStats; +import com.facebook.presto.hive.FileOpener; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HiveBatchPageSourceFactory; import com.facebook.presto.hive.HiveColumnHandle; +import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.memory.context.AggregatedMemoryContext; import com.facebook.presto.parquet.ParquetCorruptionException; import com.facebook.presto.parquet.ParquetDataSource; @@ -27,12 +29,16 @@ import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.Subfield; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import io.airlift.units.DataSize; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; @@ -42,7 +48,9 @@ import org.apache.parquet.hadoop.metadata.FileMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.joda.time.DateTimeZone; import javax.inject.Inject; @@ -52,29 +60,43 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.Optional; -import java.util.Properties; import java.util.Set; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_BAD_DATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; import static com.facebook.presto.hive.HiveErrorCode.HIVE_MISSING_DATA; +import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH; +import static com.facebook.presto.hive.HiveSessionProperties.getParquetMaxReadBlockSize; import static com.facebook.presto.hive.HiveSessionProperties.isFailOnCorruptedParquetStatistics; import static com.facebook.presto.hive.HiveSessionProperties.isUseParquetColumnNames; -import static com.facebook.presto.hive.HiveUtil.getDeserializerClassName; import static com.facebook.presto.hive.parquet.HdfsParquetDataSource.buildHdfsParquetDataSource; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.parquet.ParquetTypeUtils.getColumnIO; import static com.facebook.presto.parquet.ParquetTypeUtils.getDescriptors; import static com.facebook.presto.parquet.ParquetTypeUtils.getParquetTypeByName; +import static com.facebook.presto.parquet.ParquetTypeUtils.getSubfieldType; import static com.facebook.presto.parquet.predicate.PredicateUtils.buildPredicate; import static com.facebook.presto.parquet.predicate.PredicateUtils.predicateMatches; +import static com.facebook.presto.spi.type.StandardTypes.ARRAY; +import static com.facebook.presto.spi.type.StandardTypes.BIGINT; +import static com.facebook.presto.spi.type.StandardTypes.CHAR; +import static com.facebook.presto.spi.type.StandardTypes.DATE; +import static com.facebook.presto.spi.type.StandardTypes.DECIMAL; +import static com.facebook.presto.spi.type.StandardTypes.INTEGER; +import static com.facebook.presto.spi.type.StandardTypes.MAP; +import static com.facebook.presto.spi.type.StandardTypes.REAL; +import static com.facebook.presto.spi.type.StandardTypes.ROW; +import static com.facebook.presto.spi.type.StandardTypes.SMALLINT; +import static com.facebook.presto.spi.type.StandardTypes.TIMESTAMP; +import static com.facebook.presto.spi.type.StandardTypes.TINYINT; +import static com.facebook.presto.spi.type.StandardTypes.VARBINARY; +import static com.facebook.presto.spi.type.StandardTypes.VARCHAR; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.nullToEmpty; import static java.lang.String.format; import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE; public class ParquetPageSourceFactory @@ -88,13 +110,15 @@ public class ParquetPageSourceFactory private final TypeManager typeManager; private final HdfsEnvironment hdfsEnvironment; private final FileFormatDataSourceStats stats; + private final FileOpener fileOpener; @Inject - public ParquetPageSourceFactory(TypeManager typeManager, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats) + public ParquetPageSourceFactory(TypeManager typeManager, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats, FileOpener fileOpener) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.stats = requireNonNull(stats, "stats is null"); + this.fileOpener = requireNonNull(fileOpener, "fileOpener is null"); } @Override @@ -105,12 +129,14 @@ public Optional createPageSource( long start, long length, long fileSize, - Properties schema, + Storage storage, + Map tableParameters, List columns, TupleDomain effectivePredicate, - DateTimeZone hiveStorageTimeZone) + DateTimeZone hiveStorageTimeZone, + Optional extraFileInfo) { - if (!PARQUET_SERDE_CLASS_NAMES.contains(getDeserializerClassName(schema))) { + if (!PARQUET_SERDE_CLASS_NAMES.contains(storage.getStorageFormat().getSerDe())) { return Optional.empty(); } @@ -122,13 +148,15 @@ public Optional createPageSource( start, length, fileSize, - schema, columns, isUseParquetColumnNames(session), isFailOnCorruptedParquetStatistics(session), + getParquetMaxReadBlockSize(session), typeManager, effectivePredicate, - stats)); + stats, + extraFileInfo, + fileOpener)); } public static ParquetPageSource createParquetPageSource( @@ -139,32 +167,36 @@ public static ParquetPageSource createParquetPageSource( long start, long length, long fileSize, - Properties schema, List columns, boolean useParquetColumnNames, boolean failOnCorruptedParquetStatistics, + DataSize maxReadBlockSize, TypeManager typeManager, TupleDomain effectivePredicate, - FileFormatDataSourceStats stats) + FileFormatDataSourceStats stats, + Optional extraFileInfo, + FileOpener fileOpener) { AggregatedMemoryContext systemMemoryContext = newSimpleAggregatedMemoryContext(); ParquetDataSource dataSource = null; try { FileSystem fileSystem = hdfsEnvironment.getFileSystem(user, path, configuration); - FSDataInputStream inputStream = fileSystem.open(path); + FSDataInputStream inputStream = fileOpener.open(fileSystem, path, extraFileInfo); ParquetMetadata parquetMetadata = MetadataReader.readFooter(inputStream, path, fileSize); FileMetaData fileMetaData = parquetMetadata.getFileMetaData(); MessageType fileSchema = fileMetaData.getSchema(); dataSource = buildHdfsParquetDataSource(inputStream, path, fileSize, stats); - List fields = columns.stream() + Optional message = columns.stream() .filter(column -> column.getColumnType() == REGULAR) - .map(column -> getParquetType(column, fileSchema, useParquetColumnNames)) - .filter(Objects::nonNull) - .collect(toList()); + .map(column -> getColumnType(typeManager.getType(column.getTypeSignature()), fileSchema, useParquetColumnNames, column)) + .filter(Optional::isPresent) + .map(Optional::get) + .map(type -> new MessageType(fileSchema.getName(), type)) + .reduce(MessageType::union); - MessageType requestedSchema = new MessageType(fileSchema.getName(), fields); + MessageType requestedSchema = message.orElse(new MessageType(fileSchema.getName(), ImmutableList.of())); ImmutableList.Builder footerBlocks = ImmutableList.builder(); for (BlockMetaData block : parquetMetadata.getBlocks()) { @@ -189,16 +221,15 @@ public static ParquetPageSource createParquetPageSource( messageColumnIO, blocks.build(), dataSource, - systemMemoryContext); + systemMemoryContext, + maxReadBlockSize); return new ParquetPageSource( parquetReader, fileSchema, messageColumnIO, typeManager, - schema, columns, - effectivePredicate, useParquetColumnNames); } catch (Exception e) { @@ -249,15 +280,134 @@ public static TupleDomain getParquetTupleDomain(Map getParquetType(Type prestoType, MessageType messageType, boolean useParquetColumnNames, HiveColumnHandle column) { + org.apache.parquet.schema.Type type = null; if (useParquetColumnNames) { - return getParquetTypeByName(column.getName(), messageType); + type = getParquetTypeByName(column.getName(), messageType); } + else if (column.getHiveColumnIndex() < messageType.getFieldCount()) { + type = messageType.getType(column.getHiveColumnIndex()); + } + + if (type == null) { + return Optional.empty(); + } + + if (!checkSchemaMatch(type, prestoType)) { + String parquetTypeName; + if (type.isPrimitive()) { + parquetTypeName = type.asPrimitiveType().getPrimitiveTypeName().toString(); + } + else { + GroupType group = type.asGroupType(); + StringBuilder builder = new StringBuilder(); + group.writeToStringBuilder(builder, ""); + parquetTypeName = builder.toString(); + } + throw new PrestoException(HIVE_PARTITION_SCHEMA_MISMATCH, format("The column %s is declared as type %s, but the Parquet file declares the column as type %s", + column.getName(), + column.getHiveType(), + parquetTypeName)); + } + return Optional.of(type); + } + + private static boolean checkSchemaMatch(org.apache.parquet.schema.Type parquetType, Type type) + { + String prestoType = type.getTypeSignature().getBase(); + if (parquetType instanceof GroupType) { + GroupType groupType = parquetType.asGroupType(); + switch (prestoType) { + case ROW: + if (groupType.getFields().size() == type.getTypeParameters().size()) { + for (int i = 0; i < groupType.getFields().size(); i++) { + if (!checkSchemaMatch(groupType.getFields().get(i), type.getTypeParameters().get(i))) { + return false; + } + } + return true; + } + return false; + case MAP: + if (groupType.getFields().size() != 1) { + return false; + } + org.apache.parquet.schema.Type mapKeyType = groupType.getFields().get(0); + if (mapKeyType instanceof GroupType) { + GroupType mapGroupType = mapKeyType.asGroupType(); + return mapGroupType.getFields().size() == 2 && + checkSchemaMatch(mapGroupType.getFields().get(0), type.getTypeParameters().get(0)) && + checkSchemaMatch(mapGroupType.getFields().get(1), type.getTypeParameters().get(1)); + } + return false; + case ARRAY: + /* array has a standard 3-level structure with middle level repeated group with a single field: + * optional group my_list (LIST) { + * repeated group element { + * required type field; + * }; + * } + * Backward-compatibility support for 2-level arrays: + * optional group my_list (LIST) { + * repeated type field; + * } + * field itself could be primitive or group + */ + if (groupType.getFields().size() != 1) { + return false; + } + org.apache.parquet.schema.Type bagType = groupType.getFields().get(0); + if (bagType.isPrimitive()) { + return checkSchemaMatch(bagType.asPrimitiveType(), type.getTypeParameters().get(0)); + } + GroupType bagGroupType = bagType.asGroupType(); + return checkSchemaMatch(bagGroupType, type.getTypeParameters().get(0)) || + (bagGroupType.getFields().size() == 1 && checkSchemaMatch(bagGroupType.getFields().get(0), type.getTypeParameters().get(0))); + default: + return false; + } + } + + checkArgument(parquetType.isPrimitive(), "Unexpected parquet type for column: %s " + parquetType.getName()); + PrimitiveTypeName parquetTypeName = parquetType.asPrimitiveType().getPrimitiveTypeName(); + switch (parquetTypeName) { + case INT64: + return prestoType.equals(BIGINT) || prestoType.equals(DECIMAL) || prestoType.equals(TIMESTAMP); + case INT32: + return prestoType.equals(INTEGER) || prestoType.equals(SMALLINT) || prestoType.equals(DATE) || prestoType.equals(DECIMAL) || prestoType.equals(TINYINT); + case BOOLEAN: + return prestoType.equals(StandardTypes.BOOLEAN); + case FLOAT: + return prestoType.equals(REAL); + case DOUBLE: + return prestoType.equals(StandardTypes.DOUBLE); + case BINARY: + return prestoType.equals(VARBINARY) || prestoType.equals(VARCHAR) || prestoType.startsWith(CHAR) || prestoType.equals(DECIMAL); + case INT96: + return prestoType.equals(TIMESTAMP); + case FIXED_LEN_BYTE_ARRAY: + return prestoType.equals(DECIMAL); + default: + throw new IllegalArgumentException("Unexpected parquet type name: " + parquetTypeName); + } + } - if (column.getHiveColumnIndex() < messageType.getFieldCount()) { - return messageType.getType(column.getHiveColumnIndex()); + public static Optional getColumnType(Type prestoType, MessageType messageType, boolean useParquetColumnNames, HiveColumnHandle column) + { + if (useParquetColumnNames && !column.getRequiredSubfields().isEmpty()) { + MessageType result = null; + for (Subfield subfield : column.getRequiredSubfields()) { + MessageType type = getSubfieldType(messageType, subfield); + if (result == null) { + result = type; + } + else { + result = result.union(type); + } + } + return Optional.of(result); } - return null; + return getParquetType(prestoType, messageType, useParquetColumnNames, column); } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/rcfile/RcFilePageSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/rcfile/RcFilePageSource.java index e0d34a1d31ca9..1de869b585e42 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/rcfile/RcFilePageSource.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/rcfile/RcFilePageSource.java @@ -56,6 +56,7 @@ public class RcFilePageSource private final int[] hiveColumnIndexes; private int pageId; + private long completedPositions; private boolean closed; @@ -110,6 +111,12 @@ public long getCompletedBytes() return rcFileReader.getBytesRead(); } + @Override + public long getCompletedPositions() + { + return completedPositions; + } + @Override public long getReadTimeNanos() { @@ -136,6 +143,8 @@ public Page getNextPage() return null; } + completedPositions += currentPageSize; + Block[] blocks = new Block[hiveColumnIndexes.length]; for (int fieldId = 0; fieldId < blocks.length; fieldId++) { if (constantBlocks[fieldId] != null) { @@ -146,9 +155,7 @@ public Page getNextPage() } } - Page page = new Page(currentPageSize, blocks); - - return page; + return new Page(currentPageSize, blocks); } catch (PrestoException e) { closeWithSuppression(e); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/rcfile/RcFilePageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/rcfile/RcFilePageSourceFactory.java index 8088d16f9478a..9f5eb82253bd6 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/rcfile/RcFilePageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/rcfile/RcFilePageSourceFactory.java @@ -14,9 +14,11 @@ package com.facebook.presto.hive.rcfile; import com.facebook.presto.hive.FileFormatDataSourceStats; +import com.facebook.presto.hive.FileOpener; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HiveBatchPageSourceFactory; import com.facebook.presto.hive.HiveColumnHandle; +import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.rcfile.AircompressorCodecFactory; import com.facebook.presto.rcfile.HadoopCodecFactory; import com.facebook.presto.rcfile.RcFileCorruptionException; @@ -49,13 +51,14 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Properties; import static com.facebook.presto.hive.HiveErrorCode.HIVE_BAD_DATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; import static com.facebook.presto.hive.HiveErrorCode.HIVE_MISSING_DATA; -import static com.facebook.presto.hive.HiveUtil.getDeserializerClassName; +import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema; import static com.facebook.presto.rcfile.text.TextRcFileEncoding.DEFAULT_NULL_SEQUENCE; import static com.facebook.presto.rcfile.text.TextRcFileEncoding.DEFAULT_SEPARATORS; import static com.google.common.base.Strings.nullToEmpty; @@ -80,13 +83,15 @@ public class RcFilePageSourceFactory private final TypeManager typeManager; private final HdfsEnvironment hdfsEnvironment; private final FileFormatDataSourceStats stats; + private final FileOpener fileOpener; @Inject - public RcFilePageSourceFactory(TypeManager typeManager, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats) + public RcFilePageSourceFactory(TypeManager typeManager, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats, FileOpener fileOpener) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.stats = requireNonNull(stats, "stats is null"); + this.fileOpener = requireNonNull(fileOpener, "fileOpener is null"); } @Override @@ -97,18 +102,19 @@ public Optional createPageSource( long start, long length, long fileSize, - Properties schema, + Storage storage, + Map tableParameters, List columns, TupleDomain effectivePredicate, - DateTimeZone hiveStorageTimeZone) + DateTimeZone hiveStorageTimeZone, + Optional extraFileInfo) { RcFileEncoding rcFileEncoding; - String deserializerClassName = getDeserializerClassName(schema); - if (deserializerClassName.equals(LazyBinaryColumnarSerDe.class.getName())) { + if (LazyBinaryColumnarSerDe.class.getName().equals(storage.getStorageFormat().getSerDe())) { rcFileEncoding = new BinaryRcFileEncoding(); } - else if (deserializerClassName.equals(ColumnarSerDe.class.getName())) { - rcFileEncoding = createTextVectorEncoding(schema, hiveStorageTimeZone); + else if (ColumnarSerDe.class.getName().equals(storage.getStorageFormat().getSerDe())) { + rcFileEncoding = createTextVectorEncoding(getHiveSchema(storage.getSerdeParameters(), tableParameters), hiveStorageTimeZone); } else { return Optional.empty(); @@ -121,7 +127,7 @@ else if (deserializerClassName.equals(ColumnarSerDe.class.getName())) { FSDataInputStream inputStream; try { FileSystem fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), path, configuration); - inputStream = fileSystem.open(path); + inputStream = fileOpener.open(fileSystem, path, extraFileInfo); } catch (Exception e) { if (nullToEmpty(e.getMessage()).trim().equals("Filesystem closed") || diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/rule/HivePlanOptimizerProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/rule/HivePlanOptimizerProvider.java index f149c4eee8497..22ba73b70c439 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/rule/HivePlanOptimizerProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/rule/HivePlanOptimizerProvider.java @@ -23,7 +23,13 @@ public class HivePlanOptimizerProvider implements ConnectorPlanOptimizerProvider { @Override - public Set getConnectorPlanOptimizers() + public Set getLogicalPlanOptimizers() + { + return ImmutableSet.of(); + } + + @Override + public Set getPhysicalPlanOptimizers() { return ImmutableSet.of(); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/s3/HiveS3Config.java b/presto-hive/src/main/java/com/facebook/presto/hive/s3/HiveS3Config.java index 50bdaab7026fa..027f62300a6f2 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/s3/HiveS3Config.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/s3/HiveS3Config.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.hive.s3; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.ConfigSecuritySensitive; import com.google.common.base.StandardSystemProperty; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.ConfigSecuritySensitive; import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.airlift.units.MinDataSize; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/s3/HiveS3Module.java b/presto-hive/src/main/java/com/facebook/presto/hive/s3/HiveS3Module.java index 9288179167a79..588ea984f3e28 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/s3/HiveS3Module.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/s3/HiveS3Module.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.hive.s3; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.facebook.presto.hive.HiveClientConfig; import com.google.inject.Binder; import com.google.inject.Scopes; -import io.airlift.configuration.AbstractConfigurationAwareModule; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.JavaUtils; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; import static org.weakref.jmx.ObjectNames.generatedNameOf; import static org.weakref.jmx.guice.ExportBinder.newExporter; @@ -52,6 +52,9 @@ else if (type == S3FileSystemType.EMRFS) { validateEmrFsClass(); binder.bind(S3ConfigurationUpdater.class).to(EmrFsS3ConfigurationUpdater.class).in(Scopes.SINGLETON); } + else if (type == S3FileSystemType.HADOOP_DEFAULT) { + // configuration is done using Hadoop configuration files + } else { throw new RuntimeException("Unknown file system type: " + type); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystem.java b/presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystem.java index 8c89b9070092d..8669a8c060c7b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystem.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystem.java @@ -48,11 +48,11 @@ import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import com.amazonaws.services.s3.transfer.Upload; +import com.facebook.airlift.log.Logger; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.AbstractSequentialIterator; import com.google.common.collect.Iterators; import com.google.common.io.Closer; -import io.airlift.log.Logger; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.apache.hadoop.conf.Configurable; @@ -73,6 +73,7 @@ import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; import java.io.Closeable; +import java.io.EOFException; import java.io.File; import java.io.FileNotFoundException; import java.io.FilterOutputStream; @@ -88,6 +89,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import static com.amazonaws.regions.Regions.US_EAST_1; import static com.amazonaws.services.s3.Headers.SERVER_SIDE_ENCRYPTION; @@ -123,7 +125,7 @@ import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USER_AGENT_SUFFIX; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USE_INSTANCE_CREDENTIALS; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Preconditions.checkPositionIndexes; import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.base.Throwables.throwIfInstanceOf; @@ -134,14 +136,16 @@ import static java.lang.Math.max; import static java.lang.Math.toIntExact; import static java.lang.String.format; +import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; +import static java.net.HttpURLConnection.HTTP_FORBIDDEN; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.nio.file.Files.createDirectories; import static java.nio.file.Files.createTempFile; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.SECONDS; -import static org.apache.http.HttpStatus.SC_BAD_REQUEST; -import static org.apache.http.HttpStatus.SC_FORBIDDEN; -import static org.apache.http.HttpStatus.SC_NOT_FOUND; -import static org.apache.http.HttpStatus.SC_REQUESTED_RANGE_NOT_SATISFIABLE; +import static org.apache.hadoop.fs.FSExceptionMessages.CANNOT_SEEK_PAST_EOF; +import static org.apache.hadoop.fs.FSExceptionMessages.NEGATIVE_SEEK; +import static org.apache.hadoop.fs.FSExceptionMessages.STREAM_IS_CLOSED; public class PrestoS3FileSystem extends FileSystem @@ -156,6 +160,7 @@ public class PrestoS3FileSystem private static final DataSize MAX_SKIP_SIZE = new DataSize(1, MEGABYTE); private static final String PATH_SEPARATOR = "/"; private static final Duration BACKOFF_MIN_SLEEP = new Duration(1, SECONDS); + private static final int HTTP_RANGE_NOT_SATISFIABLE = 416; private URI uri; private Path workingDirectory; @@ -550,7 +555,7 @@ private boolean isGlacierObject(S3ObjectSummary object) */ @VisibleForTesting static class UnrecoverableS3OperationException - extends RuntimeException + extends IOException { public UnrecoverableS3OperationException(Path path, Throwable cause) { @@ -590,10 +595,10 @@ private ObjectMetadata getS3ObjectMetadata(Path path, String bucketName, String STATS.newGetMetadataError(); if (e instanceof AmazonS3Exception) { switch (((AmazonS3Exception) e).getStatusCode()) { - case SC_NOT_FOUND: + case HTTP_NOT_FOUND: return null; - case SC_FORBIDDEN: - case SC_BAD_REQUEST: + case HTTP_FORBIDDEN: + case HTTP_BAD_REQUEST: throw new UnrecoverableS3OperationException(path, e); } } @@ -799,7 +804,8 @@ private static class PrestoS3InputStream private final Duration maxBackoffTime; private final Duration maxRetryTime; - private boolean closed; + private final AtomicBoolean closed = new AtomicBoolean(); + private InputStream in; private long streamPosition; private long nextReadPosition; @@ -819,15 +825,90 @@ public PrestoS3InputStream(AmazonS3 s3, String host, Path path, int maxAttempts, @Override public void close() { - closed = true; + closed.set(true); closeStream(); } + @Override + public int read(long position, byte[] buffer, int offset, int length) + throws IOException + { + checkClosed(); + if (position < 0) { + throw new EOFException(NEGATIVE_SEEK); + } + checkPositionIndexes(offset, offset + length, buffer.length); + if (length == 0) { + return 0; + } + + try { + return retry() + .maxAttempts(maxAttempts) + .exponentialBackoff(BACKOFF_MIN_SLEEP, maxBackoffTime, maxRetryTime, 2.0) + .stopOn(InterruptedException.class, UnrecoverableS3OperationException.class, EOFException.class) + .onRetry(STATS::newGetObjectRetry) + .run("getS3Object", () -> { + InputStream stream; + try { + GetObjectRequest request = new GetObjectRequest(host, keyFromPath(path)) + .withRange(position, (position + length) - 1); + stream = s3.getObject(request).getObjectContent(); + } + catch (RuntimeException e) { + STATS.newGetObjectError(); + if (e instanceof AmazonS3Exception) { + switch (((AmazonS3Exception) e).getStatusCode()) { + case HTTP_RANGE_NOT_SATISFIABLE: + throw new EOFException(CANNOT_SEEK_PAST_EOF); + case HTTP_FORBIDDEN: + case HTTP_NOT_FOUND: + case HTTP_BAD_REQUEST: + throw new UnrecoverableS3OperationException(path, e); + } + } + throw e; + } + + STATS.connectionOpened(); + try { + int read = 0; + while (read < length) { + int n = stream.read(buffer, offset + read, length - read); + if (n <= 0) { + if (read > 0) { + return read; + } + return -1; + } + read += n; + } + return read; + } + catch (Throwable t) { + STATS.newReadError(t); + abortStream(stream); + throw t; + } + finally { + STATS.connectionReleased(); + stream.close(); + } + }); + } + catch (Exception e) { + throw propagate(e); + } + } + @Override public void seek(long pos) + throws IOException { - checkState(!closed, "already closed"); - checkArgument(pos >= 0, "position is negative: %s", pos); + checkClosed(); + if (pos < 0) { + throw new EOFException(NEGATIVE_SEEK); + } // this allows a seek beyond the end of the stream but the next read will fail nextReadPosition = pos; @@ -850,6 +931,7 @@ public int read() public int read(byte[] buffer, int offset, int length) throws IOException { + checkClosed(); try { int bytesRead = retry() .maxAttempts(maxAttempts) @@ -874,14 +956,8 @@ public int read(byte[] buffer, int offset, int length) } return bytesRead; } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } catch (Exception e) { - throwIfInstanceOf(e, IOException.class); - throwIfUnchecked(e); - throw new RuntimeException(e); + throw propagate(e); } } @@ -950,12 +1026,12 @@ private InputStream openStream(Path path, long start) STATS.newGetObjectError(); if (e instanceof AmazonS3Exception) { switch (((AmazonS3Exception) e).getStatusCode()) { - case SC_REQUESTED_RANGE_NOT_SATISFIABLE: + case HTTP_RANGE_NOT_SATISFIABLE: // ignore request for start past end of object return new ByteArrayInputStream(new byte[0]); - case SC_FORBIDDEN: - case SC_NOT_FOUND: - case SC_BAD_REQUEST: + case HTTP_FORBIDDEN: + case HTTP_NOT_FOUND: + case HTTP_BAD_REQUEST: throw new UnrecoverableS3OperationException(path, e); } } @@ -963,35 +1039,54 @@ private InputStream openStream(Path path, long start) } }); } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } catch (Exception e) { - throwIfInstanceOf(e, IOException.class); - throwIfUnchecked(e); - throw new RuntimeException(e); + throw propagate(e); } } private void closeStream() { if (in != null) { - try { - if (in instanceof S3ObjectInputStream) { - ((S3ObjectInputStream) in).abort(); - } - else { - in.close(); - } - } - catch (IOException | AbortedException ignored) { - // thrown if the current thread is in the interrupted state - } + abortStream(in); in = null; STATS.connectionReleased(); } } + + private void checkClosed() + throws IOException + { + if (closed.get()) { + throw new IOException(STREAM_IS_CLOSED); + } + } + + private static void abortStream(InputStream in) + { + try { + if (in instanceof S3ObjectInputStream) { + ((S3ObjectInputStream) in).abort(); + } + else { + in.close(); + } + } + catch (IOException | AbortedException ignored) { + // thrown if the current thread is in the interrupted state + } + } + + private static RuntimeException propagate(Exception e) + throws IOException + { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + throw new InterruptedIOException(); + } + throwIfInstanceOf(e, IOException.class); + throwIfUnchecked(e); + throw new IOException(e); + } } private static class PrestoS3OutputStream diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystemStats.java b/presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystemStats.java index 45b58e086b5ea..5b6d0af285687 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystemStats.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystemStats.java @@ -14,8 +14,8 @@ package com.facebook.presto.hive.s3; import com.amazonaws.AbortedException; -import io.airlift.stats.CounterStat; -import io.airlift.stats.TimeStat; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.TimeStat; import io.airlift.units.Duration; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -251,15 +251,15 @@ public void newListObjectsCall() listObjectsCalls.update(1); } - public void newReadError(Exception e) + public void newReadError(Throwable t) { - if (e instanceof SocketException) { + if (t instanceof SocketException) { socketExceptions.update(1); } - else if (e instanceof SocketTimeoutException) { + else if (t instanceof SocketTimeoutException) { socketTimeoutExceptions.update(1); } - else if (e instanceof AbortedException) { + else if (t instanceof AbortedException) { awsAbortedExceptions.update(1); } else { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/s3/S3FileSystemType.java b/presto-hive/src/main/java/com/facebook/presto/hive/s3/S3FileSystemType.java index 2f3c56876b586..648b6e0b866de 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/s3/S3FileSystemType.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/s3/S3FileSystemType.java @@ -17,4 +17,5 @@ public enum S3FileSystemType { PRESTO, EMRFS, + HADOOP_DEFAULT, } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/security/HiveSecurityModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/security/HiveSecurityModule.java index bab5f43e3661a..17824d6f22ddd 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/security/HiveSecurityModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/security/HiveSecurityModule.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.hive.security; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.facebook.presto.plugin.base.security.FileBasedAccessControlModule; import com.facebook.presto.plugin.base.security.ReadOnlySecurityModule; import com.google.inject.Binder; import com.google.inject.Module; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import static io.airlift.configuration.ConditionalModule.installModuleIf; +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; public class HiveSecurityModule extends AbstractConfigurationAwareModule diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacyAccessControl.java b/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacyAccessControl.java index 0e98a5f18baed..6778e019aaf68 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacyAccessControl.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacyAccessControl.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive.security; -import com.facebook.presto.hive.HiveTransactionHandle; -import com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore; +import com.facebook.presto.hive.HiveTransactionManager; +import com.facebook.presto.hive.TransactionalMetadata; import com.facebook.presto.hive.metastore.Table; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.connector.ConnectorAccessControl; @@ -27,7 +27,6 @@ import java.util.Optional; import java.util.Set; -import java.util.function.Function; import static com.facebook.presto.spi.security.AccessDeniedException.denyAddColumn; import static com.facebook.presto.spi.security.AccessDeniedException.denyDropColumn; @@ -39,7 +38,7 @@ public class LegacyAccessControl implements ConnectorAccessControl { - private final Function metastoreProvider; + private final HiveTransactionManager hiveTransactionManager; private final boolean allowDropTable; private final boolean allowRenameTable; private final boolean allowAddColumn; @@ -48,10 +47,10 @@ public class LegacyAccessControl @Inject public LegacyAccessControl( - Function metastoreProvider, + HiveTransactionManager hiveTransactionManager, LegacySecurityConfig securityConfig) { - this.metastoreProvider = requireNonNull(metastoreProvider, "metastoreProvider is null"); + this.hiveTransactionManager = requireNonNull(hiveTransactionManager, "hiveTransactionManager is null"); requireNonNull(securityConfig, "securityConfig is null"); allowDropTable = securityConfig.getAllowDropTable(); @@ -99,7 +98,8 @@ public void checkCanDropTable(ConnectorTransactionHandle transaction, ConnectorI denyDropTable(tableName.toString()); } - Optional
target = metastoreProvider.apply(((HiveTransactionHandle) transaction)).getTable(tableName.getSchemaName(), tableName.getTableName()); + TransactionalMetadata metadata = hiveTransactionManager.get(transaction); + Optional
target = metadata.getMetastore().getTable(tableName.getSchemaName(), tableName.getTableName()); if (!target.isPresent()) { denyDropTable(tableName.toString(), "Table not found"); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacySecurityConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacySecurityConfig.java index c1573809dedad..0aaf363052509 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacySecurityConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacySecurityConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive.security; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; public class LegacySecurityConfig { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacySecurityModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacySecurityModule.java index 9bbae24c34572..f5384c87ebde4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacySecurityModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacySecurityModule.java @@ -18,7 +18,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; public class LegacySecurityModule implements Module diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/security/SecurityConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/security/SecurityConfig.java index fb7692204d379..9e6ed10b97f61 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/security/SecurityConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/security/SecurityConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive.security; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import javax.validation.constraints.NotNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/security/SqlStandardAccessControl.java b/presto-hive/src/main/java/com/facebook/presto/hive/security/SqlStandardAccessControl.java index 1bd32165a94b9..318a49a5ca20c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/security/SqlStandardAccessControl.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/security/SqlStandardAccessControl.java @@ -14,7 +14,8 @@ package com.facebook.presto.hive.security; import com.facebook.presto.hive.HiveConnectorId; -import com.facebook.presto.hive.HiveTransactionHandle; +import com.facebook.presto.hive.HiveTransactionManager; +import com.facebook.presto.hive.TransactionalMetadata; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore; import com.facebook.presto.spi.SchemaTableName; @@ -30,7 +31,6 @@ import java.util.Optional; import java.util.Set; -import java.util.function.Function; import static com.facebook.presto.hive.metastore.Database.DEFAULT_DATABASE_NAME; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege; @@ -81,15 +81,15 @@ public class SqlStandardAccessControl private static final SchemaTableName ROLES = new SchemaTableName(INFORMATION_SCHEMA_NAME, "roles"); private final String connectorId; - private final Function metastoreProvider; + private final HiveTransactionManager hiveTransactionManager; @Inject public SqlStandardAccessControl( HiveConnectorId connectorId, - Function metastoreProvider) + HiveTransactionManager hiveTransactionManager) { this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); - this.metastoreProvider = requireNonNull(metastoreProvider, "metastoreProvider is null"); + this.hiveTransactionManager = requireNonNull(hiveTransactionManager, "hiveTransactionManager is null"); } @Override @@ -317,7 +317,7 @@ public void checkCanRevokeRoles(ConnectorTransactionHandle transactionHandle, Co @Override public void checkCanSetRole(ConnectorTransactionHandle transaction, ConnectorIdentity identity, String role, String catalogName) { - SemiTransactionalHiveMetastore metastore = metastoreProvider.apply(((HiveTransactionHandle) transaction)); + SemiTransactionalHiveMetastore metastore = getMetastore(transaction); if (!isRoleApplicable(metastore, new PrestoPrincipal(USER, identity.getUser()), role)) { denySetRole(role); } @@ -343,7 +343,7 @@ public void checkCanShowRoleGrants(ConnectorTransactionHandle transactionHandle, private boolean isAdmin(ConnectorTransactionHandle transaction, ConnectorIdentity identity) { - SemiTransactionalHiveMetastore metastore = metastoreProvider.apply(((HiveTransactionHandle) transaction)); + SemiTransactionalHiveMetastore metastore = getMetastore(transaction); return isRoleEnabled(identity, metastore::listRoleGrants, ADMIN_ROLE_NAME); } @@ -358,7 +358,7 @@ private boolean isDatabaseOwner(ConnectorTransactionHandle transaction, Connecto return true; } - SemiTransactionalHiveMetastore metastore = metastoreProvider.apply(((HiveTransactionHandle) transaction)); + SemiTransactionalHiveMetastore metastore = getMetastore(transaction); Optional databaseMetadata = metastore.getDatabase(databaseName); if (!databaseMetadata.isPresent()) { return false; @@ -400,7 +400,7 @@ private boolean checkTablePermission( return true; } - SemiTransactionalHiveMetastore metastore = metastoreProvider.apply(((HiveTransactionHandle) transaction)); + SemiTransactionalHiveMetastore metastore = getMetastore(transaction); return listEnabledTablePrivileges(metastore, tableName.getSchemaName(), tableName.getTableName(), identity) .filter(privilegeInfo -> !grantOptionRequired || privilegeInfo.isGrantOption()) .anyMatch(privilegeInfo -> privilegeInfo.getHivePrivilege().equals(requiredPrivilege)); @@ -412,7 +412,7 @@ private boolean hasGrantOptionForPrivilege(ConnectorTransactionHandle transactio return true; } - SemiTransactionalHiveMetastore metastore = metastoreProvider.apply(((HiveTransactionHandle) transaction)); + SemiTransactionalHiveMetastore metastore = getMetastore(transaction); return listApplicableTablePrivileges( metastore, tableName.getSchemaName(), @@ -427,11 +427,17 @@ private boolean hasAdminOptionForRoles(ConnectorTransactionHandle transaction, C return true; } - SemiTransactionalHiveMetastore metastore = metastoreProvider.apply(((HiveTransactionHandle) transaction)); + SemiTransactionalHiveMetastore metastore = getMetastore(transaction); Set rolesWithGrantOption = listApplicableRoles(new PrestoPrincipal(USER, identity.getUser()), metastore::listRoleGrants) .filter(RoleGrant::isGrantable) .map(RoleGrant::getRoleName) .collect(toSet()); return rolesWithGrantOption.containsAll(roles); } + + private SemiTransactionalHiveMetastore getMetastore(ConnectorTransactionHandle transaction) + { + TransactionalMetadata metadata = hiveTransactionManager.get(transaction); + return metadata.getMetastore(); + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/statistics/MetastoreHiveStatisticsProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/statistics/MetastoreHiveStatisticsProvider.java index 924ca2fd5002c..82faec216a5af 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/statistics/MetastoreHiveStatisticsProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/statistics/MetastoreHiveStatisticsProvider.java @@ -14,15 +14,16 @@ package com.facebook.presto.hive.statistics; +import com.facebook.airlift.log.Logger; import com.facebook.presto.hive.HiveBasicStatistics; import com.facebook.presto.hive.HiveColumnHandle; import com.facebook.presto.hive.HivePartition; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.metastore.DateStatistics; import com.facebook.presto.hive.metastore.DecimalStatistics; import com.facebook.presto.hive.metastore.DoubleStatistics; import com.facebook.presto.hive.metastore.HiveColumnStatistics; import com.facebook.presto.hive.metastore.IntegerStatistics; +import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorSession; @@ -43,7 +44,6 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Shorts; import com.google.common.primitives.SignedBytes; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import java.math.BigDecimal; @@ -59,11 +59,11 @@ import java.util.OptionalLong; import java.util.Set; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_CORRUPTED_COLUMN_STATISTICS; import static com.facebook.presto.hive.HivePartition.UNPARTITIONED_ID; import static com.facebook.presto.hive.HiveSessionProperties.getPartitionStatisticsSampleSize; import static com.facebook.presto.hive.HiveSessionProperties.isIgnoreCorruptedStatistics; import static com.facebook.presto.hive.HiveSessionProperties.isStatisticsEnabled; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_CORRUPTED_COLUMN_STATISTICS; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.Chars.isCharType; import static com.facebook.presto.spi.type.DateType.DATE; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/AsyncQueue.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/AsyncQueue.java index aca08b128ad6a..2c779305e8550 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/AsyncQueue.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/AsyncQueue.java @@ -39,7 +39,7 @@ public class AsyncQueue private final int targetQueueSize; @GuardedBy("this") - private final Queue elements; + private Queue elements; // This future is completed when the queue transitions from full to not. But it will be replaced by a new instance of future immediately. @GuardedBy("this") private SettableFuture notFullSignal = SettableFuture.create(); @@ -84,6 +84,8 @@ private synchronized void signalIfFinishing() { if (finishing && borrowerCount == 0) { if (elements.size() == 0) { + // Reset elements queue after finishing to avoid holding on to the full sized empty array inside + elements = new ArrayDeque<>(0); completeAsync(executor, notEmptySignal); notEmptySignal = SettableFuture.create(); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java index 4595b720c77ad..f80308ef6904f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java @@ -13,13 +13,11 @@ */ package com.facebook.presto.hive.util; -import com.facebook.presto.hive.DirectoryLister; +import com.facebook.airlift.stats.TimeStat; +import com.facebook.presto.hive.HiveFileInfo; import com.facebook.presto.hive.NamenodeStats; import com.facebook.presto.spi.PrestoException; import com.google.common.collect.AbstractIterator; -import io.airlift.stats.TimeStat; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; @@ -30,12 +28,12 @@ import java.util.Deque; import java.util.Iterator; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILE_NOT_FOUND; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static java.util.Objects.requireNonNull; public class HiveFileIterator - extends AbstractIterator + extends AbstractIterator { public enum NestedDirectoryPolicy { @@ -45,53 +43,50 @@ public enum NestedDirectoryPolicy } private final Deque paths = new ArrayDeque<>(); - private final FileSystem fileSystem; - private final DirectoryLister directoryLister; + private final ListDirectoryOperation listDirectoryOperation; private final NamenodeStats namenodeStats; private final NestedDirectoryPolicy nestedDirectoryPolicy; - private Iterator remoteIterator = Collections.emptyIterator(); + private Iterator remoteIterator = Collections.emptyIterator(); public HiveFileIterator( Path path, - FileSystem fileSystem, - DirectoryLister directoryLister, + ListDirectoryOperation listDirectoryOperation, NamenodeStats namenodeStats, NestedDirectoryPolicy nestedDirectoryPolicy) { paths.addLast(requireNonNull(path, "path is null")); - this.fileSystem = requireNonNull(fileSystem, "fileSystem is null"); - this.directoryLister = requireNonNull(directoryLister, "directoryLister is null"); + this.listDirectoryOperation = requireNonNull(listDirectoryOperation, "listDirectoryOperation is null"); this.namenodeStats = requireNonNull(namenodeStats, "namenodeStats is null"); this.nestedDirectoryPolicy = requireNonNull(nestedDirectoryPolicy, "nestedDirectoryPolicy is null"); } @Override - protected LocatedFileStatus computeNext() + protected HiveFileInfo computeNext() { while (true) { while (remoteIterator.hasNext()) { - LocatedFileStatus status = getLocatedFileStatus(remoteIterator); + HiveFileInfo fileInfo = getLocatedFileStatus(remoteIterator); // Ignore hidden files and directories. Hive ignores files starting with _ and . as well. - String fileName = status.getPath().getName(); + String fileName = fileInfo.getPath().getName(); if (fileName.startsWith("_") || fileName.startsWith(".")) { continue; } - if (status.isDirectory()) { + if (fileInfo.isDirectory()) { switch (nestedDirectoryPolicy) { case IGNORED: continue; case RECURSE: - paths.add(status.getPath()); + paths.add(fileInfo.getPath()); continue; case FAIL: throw new NestedDirectoryNotAllowedException(); } } - return status; + return fileInfo; } if (paths.isEmpty()) { @@ -101,14 +96,14 @@ protected LocatedFileStatus computeNext() } } - private Iterator getLocatedFileStatusRemoteIterator(Path path) + private Iterator getLocatedFileStatusRemoteIterator(Path path) { try (TimeStat.BlockTimer ignored = namenodeStats.getListLocatedStatus().time()) { - return new FileStatusIterator(path, fileSystem, directoryLister, namenodeStats); + return new FileStatusIterator(path, listDirectoryOperation, namenodeStats); } } - private LocatedFileStatus getLocatedFileStatus(Iterator iterator) + private HiveFileInfo getLocatedFileStatus(Iterator iterator) { try (TimeStat.BlockTimer ignored = namenodeStats.getRemoteIteratorNext().time()) { return iterator.next(); @@ -116,18 +111,18 @@ private LocatedFileStatus getLocatedFileStatus(Iterator itera } private static class FileStatusIterator - implements Iterator + implements Iterator { private final Path path; private final NamenodeStats namenodeStats; - private final RemoteIterator fileStatusIterator; + private final RemoteIterator fileStatusIterator; - private FileStatusIterator(Path path, FileSystem fileSystem, DirectoryLister directoryLister, NamenodeStats namenodeStats) + private FileStatusIterator(Path path, ListDirectoryOperation listDirectoryOperation, NamenodeStats namenodeStats) { this.path = path; this.namenodeStats = namenodeStats; try { - this.fileStatusIterator = directoryLister.list(fileSystem, path); + this.fileStatusIterator = listDirectoryOperation.list(path); } catch (IOException e) { throw processException(e); @@ -146,7 +141,7 @@ public boolean hasNext() } @Override - public LocatedFileStatus next() + public HiveFileInfo next() { try { return fileStatusIterator.next(); @@ -174,4 +169,10 @@ public NestedDirectoryNotAllowedException() super("Nested sub-directories are not allowed"); } } + + public interface ListDirectoryOperation + { + RemoteIterator list(Path path) + throws IOException; + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java index 01acb85b1da84..4a108a492932c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive.util; +import com.facebook.presto.hive.HiveFileInfo; import com.facebook.presto.hive.HiveSplitPartitionInfo; import com.facebook.presto.hive.InternalHiveSplit; import com.facebook.presto.hive.InternalHiveSplit.InternalHiveBlock; @@ -23,7 +24,6 @@ import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputFormat; @@ -70,28 +70,29 @@ public InternalHiveSplitFactory( this.schedulerUsesHostAddresses = schedulerUsesHostAddresses; } - public Optional createInternalHiveSplit(LocatedFileStatus status, boolean splittable) + public Optional createInternalHiveSplit(HiveFileInfo fileInfo, boolean splittable) { - return createInternalHiveSplit(status, OptionalInt.empty(), OptionalInt.empty(), splittable); + return createInternalHiveSplit(fileInfo, OptionalInt.empty(), OptionalInt.empty(), splittable); } - public Optional createInternalHiveSplit(LocatedFileStatus status, int readBucketNumber, int tableBucketNumber, boolean splittable) + public Optional createInternalHiveSplit(HiveFileInfo fileInfo, int readBucketNumber, int tableBucketNumber, boolean splittable) { - return createInternalHiveSplit(status, OptionalInt.of(readBucketNumber), OptionalInt.of(tableBucketNumber), splittable); + return createInternalHiveSplit(fileInfo, OptionalInt.of(readBucketNumber), OptionalInt.of(tableBucketNumber), splittable); } - private Optional createInternalHiveSplit(LocatedFileStatus status, OptionalInt readBucketNumber, OptionalInt tableBucketNumber, boolean splittable) + private Optional createInternalHiveSplit(HiveFileInfo fileInfo, OptionalInt readBucketNumber, OptionalInt tableBucketNumber, boolean splittable) { - splittable = splittable && isSplittable(inputFormat, fileSystem, status.getPath()); + splittable = splittable && isSplittable(inputFormat, fileSystem, fileInfo.getPath()); return createInternalHiveSplit( - status.getPath(), - status.getBlockLocations(), + fileInfo.getPath(), + fileInfo.getBlockLocations(), 0, - status.getLen(), - status.getLen(), + fileInfo.getLength(), + fileInfo.getLength(), readBucketNumber, tableBucketNumber, - splittable); + splittable, + fileInfo.getExtraFileInfo()); } public Optional createInternalHiveSplit(FileSplit split) @@ -106,7 +107,8 @@ public Optional createInternalHiveSplit(FileSplit split) file.getLen(), OptionalInt.empty(), OptionalInt.empty(), - false); + false, + Optional.empty()); } private Optional createInternalHiveSplit( @@ -117,7 +119,8 @@ private Optional createInternalHiveSplit( long fileSize, OptionalInt readBucketNumber, OptionalInt tableBucketNumber, - boolean splittable) + boolean splittable, + Optional extraFileInfo) { String pathString = path.toString(); if (!pathMatchesPredicate(pathDomain, pathString)) { @@ -179,7 +182,8 @@ private Optional createInternalHiveSplit( splittable, forceLocalScheduling && allBlocksHaveRealAddress(blocks), s3SelectPushdownEnabled && S3SelectPushdown.isCompressionCodecSupported(inputFormat, path), - partitionInfo)); + partitionInfo, + extraFileInfo)); } private boolean needsHostAddresses(boolean forceLocalScheduling, List addresses) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/ResumableTasks.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/ResumableTasks.java index 216047793d10a..6b8e21e77b787 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/ResumableTasks.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/ResumableTasks.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive.util; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import java.util.concurrent.Executor; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/TempFileReader.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/TempFileReader.java index fb3a3ffc775c1..ebb220559ef5f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/TempFileReader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/TempFileReader.java @@ -17,12 +17,14 @@ import com.facebook.presto.orc.OrcDataSource; import com.facebook.presto.orc.OrcPredicate; import com.facebook.presto.orc.OrcReader; +import com.facebook.presto.orc.OrcReaderOptions; +import com.facebook.presto.orc.StorageStripeMetadataSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.Type; import com.google.common.collect.AbstractIterator; -import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; import java.io.IOException; @@ -42,21 +44,25 @@ public class TempFileReader extends AbstractIterator { - private final List types; + private final int columnCount; private final OrcBatchRecordReader reader; public TempFileReader(List types, OrcDataSource dataSource) { - this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); + requireNonNull(types, "types is null"); + this.columnCount = types.size(); try { OrcReader orcReader = new OrcReader( dataSource, ORC, - new DataSize(1, MEGABYTE), - new DataSize(8, MEGABYTE), - new DataSize(8, MEGABYTE), - new DataSize(16, MEGABYTE)); + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + new OrcReaderOptions( + new DataSize(1, MEGABYTE), + new DataSize(8, MEGABYTE), + new DataSize(16, MEGABYTE), + false)); Map includedColumns = new HashMap<>(); for (int i = 0; i < types.size(); i++) { @@ -88,9 +94,9 @@ protected Page computeNext() return endOfData(); } - Block[] blocks = new Block[types.size()]; - for (int i = 0; i < types.size(); i++) { - blocks[i] = reader.readBlock(types.get(i), i).getLoadedBlock(); + Block[] blocks = new Block[columnCount]; + for (int i = 0; i < columnCount; i++) { + blocks[i] = reader.readBlock(i).getLoadedBlock(); } return new Page(batchSize, blocks); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index c0d4693a6bf4c..7fc96a7e0140b 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.GroupByHashPageIndexerFactory; import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; import com.facebook.presto.hive.LocationService.WriteInfo; @@ -24,6 +27,7 @@ import com.facebook.presto.hive.metastore.HivePrivilegeInfo; import com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege; import com.facebook.presto.hive.metastore.Partition; +import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.hive.metastore.PartitionWithStatistics; import com.facebook.presto.hive.metastore.PrestoTableType; import com.facebook.presto.hive.metastore.PrincipalPrivileges; @@ -36,15 +40,18 @@ import com.facebook.presto.hive.metastore.thrift.TestingHiveCluster; import com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore; import com.facebook.presto.hive.orc.OrcBatchPageSource; +import com.facebook.presto.hive.orc.OrcSelectivePageSource; import com.facebook.presto.hive.parquet.ParquetPageSource; import com.facebook.presto.hive.rcfile.RcFilePageSource; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorInsertTableHandle; import com.facebook.presto.spi.ConnectorOutputTableHandle; import com.facebook.presto.spi.ConnectorPageSink; import com.facebook.presto.spi.ConnectorPageSource; +import com.facebook.presto.spi.ConnectorPushdownFilterResult; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.ConnectorSplitSource; @@ -64,6 +71,7 @@ import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.SchemaTablePrefix; import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.TableNotFoundException; import com.facebook.presto.spi.ViewNotFoundException; import com.facebook.presto.spi.block.Block; @@ -74,11 +82,15 @@ import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingContext; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.NullableValue; import com.facebook.presto.spi.predicate.Range; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.predicate.ValueSet; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.security.ConnectorIdentity; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.statistics.ColumnStatistics; import com.facebook.presto.spi.statistics.TableStatistics; @@ -91,22 +103,24 @@ import com.facebook.presto.spi.type.SqlTimestamp; import com.facebook.presto.spi.type.SqlVarbinary; import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.TimeZoneKey; import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.JoinCompiler; +import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.MaterializedRow; import com.facebook.presto.testing.TestingConnectorSession; import com.facebook.presto.testing.TestingNodeManager; +import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.net.HostAndPort; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import io.airlift.slice.Slice; -import io.airlift.stats.CounterStat; +import io.airlift.slice.Slices; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.apache.hadoop.fs.FileStatus; @@ -114,6 +128,7 @@ import org.apache.hadoop.fs.Path; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.testng.SkipException; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -126,7 +141,9 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalInt; @@ -140,6 +157,15 @@ import java.util.stream.IntStream; import java.util.stream.LongStream; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; +import static com.facebook.airlift.testing.Assertions.assertLessThanOrEqual; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; import static com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.COMMIT; import static com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_APPEND_PAGE; import static com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_BEGIN_INSERT; @@ -152,12 +178,13 @@ import static com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; +import static com.facebook.presto.hive.HiveColumnHandle.MAX_PARTITION_KEY_COLUMN_INDEX; import static com.facebook.presto.hive.HiveColumnHandle.bucketColumnHandle; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH; -import static com.facebook.presto.hive.HiveMetadata.PRESTO_QUERY_ID_NAME; import static com.facebook.presto.hive.HiveMetadata.PRESTO_VERSION_NAME; import static com.facebook.presto.hive.HiveMetadata.convertToPredicate; +import static com.facebook.presto.hive.HiveSessionProperties.OFFLINE_DATA_DEBUG_MODE_ENABLED; +import static com.facebook.presto.hive.HiveSessionProperties.SORTED_WRITE_TO_TEMP_PATH_ENABLED; import static com.facebook.presto.hive.HiveStorageFormat.AVRO; import static com.facebook.presto.hive.HiveStorageFormat.DWRF; import static com.facebook.presto.hive.HiveStorageFormat.JSON; @@ -172,27 +199,34 @@ import static com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY; import static com.facebook.presto.hive.HiveTableProperties.SORTED_BY_PROPERTY; import static com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY; +import static com.facebook.presto.hive.HiveTestUtils.FILTER_STATS_CALCULATOR_SERVICE; import static com.facebook.presto.hive.HiveTestUtils.FUNCTION_RESOLUTION; +import static com.facebook.presto.hive.HiveTestUtils.METADATA; import static com.facebook.presto.hive.HiveTestUtils.PAGE_SORTER; import static com.facebook.presto.hive.HiveTestUtils.ROW_EXPRESSION_SERVICE; import static com.facebook.presto.hive.HiveTestUtils.SESSION; import static com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER; import static com.facebook.presto.hive.HiveTestUtils.arrayType; -import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveDataStreamFactories; +import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveBatchPageSourceFactories; import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories; import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider; +import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveSelectivePageSourceFactories; import static com.facebook.presto.hive.HiveTestUtils.getDefaultOrcFileWriterFactory; import static com.facebook.presto.hive.HiveTestUtils.getTypes; import static com.facebook.presto.hive.HiveTestUtils.mapType; import static com.facebook.presto.hive.HiveTestUtils.rowType; +import static com.facebook.presto.hive.HiveType.HIVE_BOOLEAN; +import static com.facebook.presto.hive.HiveType.HIVE_BYTE; +import static com.facebook.presto.hive.HiveType.HIVE_DOUBLE; +import static com.facebook.presto.hive.HiveType.HIVE_FLOAT; import static com.facebook.presto.hive.HiveType.HIVE_INT; import static com.facebook.presto.hive.HiveType.HIVE_LONG; +import static com.facebook.presto.hive.HiveType.HIVE_SHORT; import static com.facebook.presto.hive.HiveType.HIVE_STRING; import static com.facebook.presto.hive.HiveType.toHiveType; import static com.facebook.presto.hive.HiveUtil.columnExtraInfo; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; -import static com.facebook.presto.hive.HiveWriteUtils.createDirectory; import static com.facebook.presto.hive.LocationHandle.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_PARTITION_VALUE; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createDateColumnStatistics; @@ -200,13 +234,16 @@ import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createDoubleColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createStringColumnStatistics; +import static com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME; +import static com.facebook.presto.hive.metastore.MetastoreUtil.createDirectory; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.StandardErrorCode.TRANSACTION_CONFLICT; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING; import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.spi.predicate.TupleDomain.withColumnDomains; import static com.facebook.presto.spi.security.PrincipalType.USER; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -228,6 +265,8 @@ import static com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType; import static com.facebook.presto.spi.type.VarcharType.createVarcharType; import static com.facebook.presto.spi.type.Varchars.isVarcharType; +import static com.facebook.presto.sql.planner.VariablesExtractor.extractUnique; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; import static com.facebook.presto.testing.DateTimeTestingUtils.sqlTimestampOf; import static com.facebook.presto.testing.MaterializedResult.materializeSourceDataStream; import static com.google.common.base.MoreObjects.toStringHelper; @@ -246,15 +285,7 @@ import static com.google.common.hash.Hashing.sha256; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.slice.Slices.utf8Slice; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertGreaterThanOrEqual; -import static io.airlift.testing.Assertions.assertInstanceOf; -import static io.airlift.testing.Assertions.assertLessThanOrEqual; import static io.airlift.units.DataSize.Unit.KILOBYTE; import static java.lang.Float.floatToRawIntBits; import static java.lang.Math.toIntExact; @@ -318,7 +349,7 @@ public abstract class AbstractTestHiveClient .row(3L, "bye", (byte) 46, (short) 346, 345, 456L, 754.2008f, 98.1, false, ImmutableList.of("ape", "bear"), ImmutableMap.of("three", 3L, "four", 4L), ImmutableList.of("false", 0L, false)) .build(); - private static final List CREATE_TABLE_COLUMNS_PARTITIONED = ImmutableList.builder() + protected static final List CREATE_TABLE_COLUMNS_PARTITIONED = ImmutableList.builder() .addAll(CREATE_TABLE_COLUMNS) .add(new ColumnMetadata("ds", createUnboundedVarcharType())) .build(); @@ -418,6 +449,7 @@ private static RowType toRowType(List columns) .add(new ColumnMetadata("struct_to_struct", MISMATCH_SCHEMA_ROW_TYPE_APPEND)) .add(new ColumnMetadata("list_to_list", arrayType(MISMATCH_SCHEMA_ROW_TYPE_APPEND))) .add(new ColumnMetadata("map_to_map", mapType(MISMATCH_SCHEMA_PRIMITIVE_COLUMN_AFTER.get(1).getType(), MISMATCH_SCHEMA_ROW_TYPE_DROP))) + .add(new ColumnMetadata("tinyint_append", TINYINT)) .add(new ColumnMetadata("ds", createUnboundedVarcharType())) .build(); @@ -441,11 +473,76 @@ private static RowType toRowType(List columns) result.add(appendFieldRowResult); result.add(Arrays.asList(appendFieldRowResult, null, appendFieldRowResult)); result.add(ImmutableMap.of(result.get(1), dropFieldRowResult)); + result.add(null); result.add(result.get(9)); return new MaterializedRow(materializedRow.getPrecision(), result); }).collect(toList())) .build(); + private static final SubfieldExtractor SUBFIELD_EXTRACTOR = new SubfieldExtractor(FUNCTION_RESOLUTION, ROW_EXPRESSION_SERVICE.getExpressionOptimizer(), SESSION); + + private static final TypeProvider TYPE_PROVIDER_AFTER = TypeProvider.copyOf(MISMATCH_SCHEMA_TABLE_AFTER.stream() + .collect(toImmutableMap(ColumnMetadata::getName, ColumnMetadata::getType))); + + private static final TestingRowExpressionTranslator ROW_EXPRESSION_TRANSLATOR = new TestingRowExpressionTranslator(METADATA); + + private static final List MISMATCH_SCHEMA_TABLE_AFTER_FILTERS = ImmutableList.of( + // integer_to_varchar + toRowExpression("integer_to_varchar", VARCHAR, Domain.singleValue(VARCHAR, Slices.utf8Slice("17"))), + toRowExpression("integer_to_varchar", VARCHAR, Domain.notNull(VARCHAR)), + toRowExpression("integer_to_varchar", VARCHAR, Domain.onlyNull(VARCHAR)), + // varchar_to_integer + toRowExpression("varchar_to_integer", INTEGER, Domain.singleValue(INTEGER, -923L)), + toRowExpression("varchar_to_integer", INTEGER, Domain.notNull(INTEGER)), + toRowExpression("varchar_to_integer", INTEGER, Domain.onlyNull(INTEGER)), + // tinyint_append + toRowExpression("tinyint_append", TINYINT, Domain.singleValue(TINYINT, 1L)), + toRowExpression("tinyint_append", TINYINT, Domain.onlyNull(TINYINT)), + toRowExpression("tinyint_append", TINYINT, Domain.notNull(TINYINT)), + // struct_to_struct + toRowExpression("struct_to_struct.f_integer_to_varchar", MISMATCH_SCHEMA_ROW_TYPE_APPEND, Domain.singleValue(VARCHAR, Slices.utf8Slice("-27"))), + toRowExpression("struct_to_struct.f_varchar_to_integer", MISMATCH_SCHEMA_ROW_TYPE_APPEND, Domain.singleValue(INTEGER, 2147483647L)), + toRowExpression("struct_to_struct.f_tinyint_to_smallint_append", MISMATCH_SCHEMA_ROW_TYPE_APPEND, Domain.singleValue(TINYINT, 1L)), + toRowExpression("struct_to_struct.f_tinyint_to_smallint_append", MISMATCH_SCHEMA_ROW_TYPE_APPEND, Domain.onlyNull(TINYINT)), + toRowExpression("struct_to_struct.f_tinyint_to_smallint_append", MISMATCH_SCHEMA_ROW_TYPE_APPEND, Domain.notNull(TINYINT)), + // filter functions + toRowExpression("tinyint_to_smallint + 1 > 0"), + toRowExpression("tinyint_to_smallint * 2 < 0")); + + private static RowExpression toRowExpression(String name, Type type, Domain domain) + { + RowExpression expression = SUBFIELD_EXTRACTOR.toRowExpression(new Subfield(name), type); + return ROW_EXPRESSION_SERVICE.getDomainTranslator().toPredicate(TupleDomain.withColumnDomains(ImmutableMap.of(expression, domain))); + } + + private static RowExpression toRowExpression(String sql) + { + return ROW_EXPRESSION_TRANSLATOR.translate(expression(sql), TYPE_PROVIDER_AFTER); + } + + private static final List> MISMATCH_SCHEMA_TABLE_AFTER_RESULT_PREDICATES = ImmutableList.of( + // integer_to_varchar + row -> Objects.equals(row.getField(6), "17"), + row -> row.getField(6) != null, + row -> row.getField(6) == null, + // varchar_to_integer + row -> Objects.equals(row.getField(7), -923), + row -> row.getField(7) != null, + row -> row.getField(7) == null, + // tinyint_append + row -> false, + row -> true, + row -> false, + // struct_to_struct + row -> Objects.equals(row.getField(6), "-27"), + row -> Objects.equals(row.getField(7), 2147483647), + row -> false, + row -> true, + row -> false, + // filter functions + row -> row.getField(0) != null && (short) row.getField(0) + 1 > 0, + row -> row.getField(0) != null && (short) row.getField(0) + 1 < 0); + protected Set createTableFormats = difference(ImmutableSet.copyOf(HiveStorageFormat.values()), ImmutableSet.of(AVRO)); private static final JoinCompiler JOIN_COMPILER = new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig()); @@ -656,21 +753,26 @@ protected void setupHive(String connectorId, String databaseName, String timeZon invalidTableLayoutHandle = new HiveTableLayoutHandle( invalidTable, ImmutableList.of(), + ImmutableList.of(), + ImmutableMap.of(), ImmutableList.of(new HivePartition(invalidTable, "unknown", ImmutableMap.of())), TupleDomain.all(), TRUE_CONSTANT, ImmutableMap.of(), TupleDomain.all(), Optional.empty(), - Optional.empty()); + Optional.empty(), + false, + "layout"); - dsColumn = new HiveColumnHandle("ds", HIVE_STRING, parseTypeSignature(StandardTypes.VARCHAR), -1, PARTITION_KEY, Optional.empty()); - fileFormatColumn = new HiveColumnHandle("file_format", HIVE_STRING, parseTypeSignature(StandardTypes.VARCHAR), -1, PARTITION_KEY, Optional.empty()); - dummyColumn = new HiveColumnHandle("dummy", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), -1, PARTITION_KEY, Optional.empty()); - intColumn = new HiveColumnHandle("t_int", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), -1, PARTITION_KEY, Optional.empty()); + int partitionColumnIndex = MAX_PARTITION_KEY_COLUMN_INDEX; + dsColumn = new HiveColumnHandle("ds", HIVE_STRING, parseTypeSignature(StandardTypes.VARCHAR), partitionColumnIndex--, PARTITION_KEY, Optional.empty()); + fileFormatColumn = new HiveColumnHandle("file_format", HIVE_STRING, parseTypeSignature(StandardTypes.VARCHAR), partitionColumnIndex--, PARTITION_KEY, Optional.empty()); + dummyColumn = new HiveColumnHandle("dummy", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), partitionColumnIndex--, PARTITION_KEY, Optional.empty()); + intColumn = new HiveColumnHandle("t_int", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), partitionColumnIndex--, PARTITION_KEY, Optional.empty()); invalidColumnHandle = new HiveColumnHandle(INVALID_COLUMN, HIVE_STRING, parseTypeSignature(StandardTypes.VARCHAR), 0, REGULAR, Optional.empty()); - List partitionColumns = ImmutableList.of(dsColumn, fileFormatColumn, dummyColumn); + List partitionColumns = ImmutableList.of(dsColumn, fileFormatColumn, dummyColumn); List partitions = ImmutableList.builder() .add(new HivePartition(tablePartitionFormat, "ds=2012-12-29/file_format=textfile/dummy=1", @@ -706,47 +808,84 @@ protected void setupHive(String connectorId, String databaseName, String timeZon TupleDomain domainPredicate = tupleDomain.transform(HiveColumnHandle.class::cast) .transform(column -> new Subfield(column.getName(), ImmutableList.of())); tableLayout = new ConnectorTableLayout( - new HiveTableLayoutHandle(tablePartitionFormat, partitionColumns, partitions, domainPredicate, TRUE_CONSTANT, ImmutableMap.of(dsColumn.getName(), dsColumn), tupleDomain, Optional.empty(), Optional.empty()), + new HiveTableLayoutHandle( + tablePartitionFormat, + partitionColumns, + ImmutableList.of( + new Column("t_string", HIVE_STRING, Optional.empty()), + new Column("t_tinyint", HIVE_BYTE, Optional.empty()), + new Column("t_smallint", HIVE_SHORT, Optional.empty()), + new Column("t_int", HIVE_INT, Optional.empty()), + new Column("t_bigint", HIVE_LONG, Optional.empty()), + new Column("t_float", HIVE_FLOAT, Optional.empty()), + new Column("t_double", HIVE_DOUBLE, Optional.empty()), + new Column("t_boolean", HIVE_BOOLEAN, Optional.empty())), + ImmutableMap.of(), + partitions, + domainPredicate, + TRUE_CONSTANT, + ImmutableMap.of(dsColumn.getName(), dsColumn), + tupleDomain, + Optional.empty(), + Optional.empty(), + false, + "layout"), Optional.empty(), - TupleDomain.withColumnDomains(ImmutableMap.of( + withColumnDomains(ImmutableMap.of( dsColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("2012-12-29"))), false), fileFormatColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("textfile")), Range.equal(createUnboundedVarcharType(), utf8Slice("sequencefile")), Range.equal(createUnboundedVarcharType(), utf8Slice("rctext")), Range.equal(createUnboundedVarcharType(), utf8Slice("rcbinary"))), false), dummyColumn, Domain.create(ValueSet.ofRanges(Range.equal(INTEGER, 1L), Range.equal(INTEGER, 2L), Range.equal(INTEGER, 3L), Range.equal(INTEGER, 4L)), false))), Optional.empty(), Optional.empty(), - Optional.of(new DiscretePredicates(partitionColumns, ImmutableList.of( - TupleDomain.withColumnDomains(ImmutableMap.of( + Optional.of(new DiscretePredicates(ImmutableList.copyOf(partitionColumns), ImmutableList.of( + withColumnDomains(ImmutableMap.of( dsColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("2012-12-29"))), false), fileFormatColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("textfile"))), false), dummyColumn, Domain.create(ValueSet.ofRanges(Range.equal(INTEGER, 1L)), false))), - TupleDomain.withColumnDomains(ImmutableMap.of( + withColumnDomains(ImmutableMap.of( dsColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("2012-12-29"))), false), fileFormatColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("sequencefile"))), false), dummyColumn, Domain.create(ValueSet.ofRanges(Range.equal(INTEGER, 2L)), false))), - TupleDomain.withColumnDomains(ImmutableMap.of( + withColumnDomains(ImmutableMap.of( dsColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("2012-12-29"))), false), fileFormatColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("rctext"))), false), dummyColumn, Domain.create(ValueSet.ofRanges(Range.equal(INTEGER, 3L)), false))), - TupleDomain.withColumnDomains(ImmutableMap.of( + withColumnDomains(ImmutableMap.of( dsColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("2012-12-29"))), false), fileFormatColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("rcbinary"))), false), dummyColumn, Domain.create(ValueSet.ofRanges(Range.equal(INTEGER, 4L)), false)))))), ImmutableList.of()); List unpartitionedPartitions = ImmutableList.of(new HivePartition(tableUnpartitioned)); - unpartitionedTableLayout = new ConnectorTableLayout(new HiveTableLayoutHandle(tableUnpartitioned, ImmutableList.of(), unpartitionedPartitions, TupleDomain.all(), TRUE_CONSTANT, ImmutableMap.of(), TupleDomain.all(), Optional.empty(), Optional.empty())); + unpartitionedTableLayout = new ConnectorTableLayout(new HiveTableLayoutHandle( + tableUnpartitioned, + ImmutableList.of(), + ImmutableList.of( + new Column("t_string", HIVE_STRING, Optional.empty()), + new Column("t_tinyint", HIVE_BYTE, Optional.empty())), + ImmutableMap.of(), + unpartitionedPartitions, + TupleDomain.all(), + TRUE_CONSTANT, + ImmutableMap.of(), + TupleDomain.all(), + Optional.empty(), + Optional.empty(), + false, + "layout")); timeZone = DateTimeZone.forTimeZone(TimeZone.getTimeZone(timeZoneId)); } protected final void setup(String host, int port, String databaseName, String timeZone) { HiveClientConfig hiveClientConfig = getHiveClientConfig(); + MetastoreClientConfig metastoreClientConfig = getMetastoreClientConfig(); hiveClientConfig.setTimeZone(timeZone); String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy"); if (proxy != null) { - hiveClientConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy)); + metastoreClientConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy)); } - HiveCluster hiveCluster = new TestingHiveCluster(hiveClientConfig, host, port); + HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, host, port); ExtendedHiveMetastore metastore = new CachingHiveMetastore( new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster)), executor, @@ -754,18 +893,18 @@ protected final void setup(String host, int port, String databaseName, String ti Duration.valueOf("15s"), 10000); - setup(databaseName, hiveClientConfig, metastore); + setup(databaseName, hiveClientConfig, metastoreClientConfig, metastore); } - protected final void setup(String databaseName, HiveClientConfig hiveClientConfig, ExtendedHiveMetastore hiveMetastore) + protected final void setup(String databaseName, HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig, ExtendedHiveMetastore hiveMetastore) { HiveConnectorId connectorId = new HiveConnectorId("hive-test"); setupHive(connectorId.toString(), databaseName, hiveClientConfig.getTimeZone()); metastoreClient = hiveMetastore; - HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationUpdater(hiveClientConfig)); - hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication()); + HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig, metastoreClientConfig), ImmutableSet.of()); + hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); locationService = new HiveLocationService(hdfsEnvironment); metadataFactory = new HiveMetadataFactory( metastoreClient, @@ -781,17 +920,20 @@ protected final void setup(String databaseName, HiveClientConfig hiveClientConfi TYPE_MANAGER, locationService, FUNCTION_RESOLUTION, + METADATA.getFunctionManager(), ROW_EXPRESSION_SERVICE, + FILTER_STATS_CALCULATOR_SERVICE, new TableParameterCodec(), PARTITION_UPDATE_CODEC, listeningDecorator(executor), new HiveTypeTranslator(), new HiveStagingFileCommitter(hdfsEnvironment, listeningDecorator(executor)), new HiveZeroRowFileCreator(hdfsEnvironment, listeningDecorator(executor)), - TEST_SERVER_VERSION); + TEST_SERVER_VERSION, + new HivePartitionObjectBuilder()); transactionManager = new HiveTransactionManager(); splitManager = new HiveSplitManager( - transactionHandle -> ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore(), + transactionManager, new NamenodeStats(), hdfsEnvironment, new HadoopDirectoryLister(), @@ -806,21 +948,22 @@ protected final void setup(String databaseName, HiveClientConfig hiveClientConfi hiveClientConfig.getSplitLoaderConcurrency(), false); pageSinkProvider = new HivePageSinkProvider( - getDefaultHiveFileWriterFactories(hiveClientConfig), + getDefaultHiveFileWriterFactories(hiveClientConfig, metastoreClientConfig), hdfsEnvironment, PAGE_SORTER, metastoreClient, new GroupByHashPageIndexerFactory(JOIN_COMPILER), TYPE_MANAGER, getHiveClientConfig(), + getMetastoreClientConfig(), locationService, PARTITION_UPDATE_CODEC, new TestingNodeManager("fake-environment"), new HiveEventClient(), new HiveSessionProperties(hiveClientConfig, new OrcFileWriterConfig(), new ParquetFileWriterConfig()), new HiveWriterStats(), - getDefaultOrcFileWriterFactory(hiveClientConfig)); - pageSourceProvider = new HivePageSourceProvider(hiveClientConfig, hdfsEnvironment, getDefaultHiveRecordCursorProvider(hiveClientConfig), getDefaultHiveDataStreamFactories(hiveClientConfig), ImmutableSet.of(), TYPE_MANAGER, ROW_EXPRESSION_SERVICE); + getDefaultOrcFileWriterFactory(hiveClientConfig, metastoreClientConfig)); + pageSourceProvider = new HivePageSourceProvider(hiveClientConfig, hdfsEnvironment, getDefaultHiveRecordCursorProvider(hiveClientConfig, metastoreClientConfig), getDefaultHiveBatchPageSourceFactories(hiveClientConfig, metastoreClientConfig), getDefaultHiveSelectivePageSourceFactories(hiveClientConfig, metastoreClientConfig), TYPE_MANAGER, ROW_EXPRESSION_SERVICE); } /** @@ -834,11 +977,93 @@ protected HiveClientConfig getHiveClientConfig() .setTemporaryTableSchema(database); } + protected MetastoreClientConfig getMetastoreClientConfig() + { + return new MetastoreClientConfig(); + } + protected ConnectorSession newSession() { return new TestingConnectorSession(new HiveSessionProperties(getHiveClientConfig(), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties()); } + protected ConnectorSession newSession(Map extraProperties) + { + ConnectorSession session = newSession(); + return new ConnectorSession() { + @Override + public String getQueryId() + { + return session.getQueryId(); + } + + @Override + public Optional getSource() + { + return session.getSource(); + } + + @Override + public ConnectorIdentity getIdentity() + { + return session.getIdentity(); + } + + @Override + public TimeZoneKey getTimeZoneKey() + { + return session.getTimeZoneKey(); + } + + @Override + public Locale getLocale() + { + return session.getLocale(); + } + + @Override + public Optional getTraceToken() + { + return session.getTraceToken(); + } + + @Override + public Optional getClientInfo() + { + return session.getClientInfo(); + } + + @Override + public long getStartTime() + { + return session.getStartTime(); + } + + @Override + public boolean isLegacyTimestamp() + { + return session.isLegacyTimestamp(); + } + + @Override + public SqlFunctionProperties getSqlFunctionProperties() + { + return session.getSqlFunctionProperties(); + } + + @Override + public T getProperty(String name, Class type) + { + Object value = extraProperties.get(name); + if (value != null) { + return type.cast(value); + } + + return session.getProperty(name, type); + } + }; + } + protected Transaction newTransaction() { return new HiveTransaction(transactionManager, metadataFactory.get()); @@ -890,7 +1115,7 @@ public SemiTransactionalHiveMetastore getMetastore(String schema) private SemiTransactionalHiveMetastore getMetastore() { - return ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore(); + return transactionManager.get(transactionHandle).getMetastore(); } @Override @@ -904,7 +1129,7 @@ public void commit() { checkState(!closed); closed = true; - HiveMetadata metadata = (HiveMetadata) transactionManager.remove(transactionHandle); + TransactionalMetadata metadata = transactionManager.remove(transactionHandle); checkArgument(metadata != null, "no such transaction: %s", transactionHandle); metadata.commit(); } @@ -914,7 +1139,7 @@ public void rollback() { checkState(!closed); closed = true; - HiveMetadata metadata = (HiveMetadata) transactionManager.remove(transactionHandle); + TransactionalMetadata metadata = transactionManager.remove(transactionHandle); checkArgument(metadata != null, "no such transaction: %s", transactionHandle); metadata.rollback(); } @@ -1003,6 +1228,7 @@ public void testListUnknownSchema() @Test public void testGetPartitions() + throws Exception { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); @@ -1047,6 +1273,8 @@ public void testGetPartitionNames() public void testMismatchSchemaTable() throws Exception { + boolean pushdownFilterEnabled = getHiveClientConfig().isPushdownFilterEnabled(); + for (HiveStorageFormat storageFormat : createTableFormats) { // TODO: fix coercion for JSON if (storageFormat == JSON) { @@ -1060,7 +1288,9 @@ public void testMismatchSchemaTable() MISMATCH_SCHEMA_TABLE_BEFORE, MISMATCH_SCHEMA_TABLE_DATA_BEFORE, MISMATCH_SCHEMA_TABLE_AFTER, - MISMATCH_SCHEMA_TABLE_DATA_AFTER); + MISMATCH_SCHEMA_TABLE_DATA_AFTER, + pushdownFilterEnabled ? MISMATCH_SCHEMA_TABLE_AFTER_FILTERS : ImmutableList.of(), + pushdownFilterEnabled ? MISMATCH_SCHEMA_TABLE_AFTER_RESULT_PREDICATES : ImmutableList.of()); } finally { dropTable(temporaryMismatchSchemaTable); @@ -1074,7 +1304,9 @@ protected void doTestMismatchSchemaTable( List tableBefore, MaterializedResult dataBefore, List tableAfter, - MaterializedResult dataAfter) + MaterializedResult dataAfter, + List afterFilters, + List> afterResultPredicates) throws Exception { String schemaName = schemaTableName.getSchemaName(); @@ -1143,6 +1375,30 @@ protected void doTestMismatchSchemaTable( MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.empty()); assertEqualsIgnoreOrder(result.getMaterializedRows(), dataAfter.getMaterializedRows()); + int filterCount = afterFilters.size(); + for (int i = 0; i < filterCount; i++) { + RowExpression predicate = afterFilters.get(i); + ConnectorTableLayoutHandle layoutHandle = metadata.pushdownFilter(session, tableHandle, predicate, Optional.empty()).getLayout().getHandle(); + + // Read all columns with a filter + MaterializedResult filteredResult = readTable(transaction, tableHandle, layoutHandle, columnHandles, session, OptionalInt.empty(), Optional.empty()); + + Predicate rowPredicate = afterResultPredicates.get(i); + List expectedRows = dataAfter.getMaterializedRows().stream().filter(rowPredicate::apply).collect(toList()); + + assertEqualsIgnoreOrder(filteredResult.getMaterializedRows(), expectedRows); + + // Read all columns except the ones used in the filter + Set filterColumnNames = extractUnique(predicate).stream().map(VariableReferenceExpression::getName).collect(toImmutableSet()); + + List nonFilterColumns = columnHandles.stream() + .filter(column -> !filterColumnNames.contains(((HiveColumnHandle) column).getName())) + .collect(toList()); + + int resultCount = readTable(transaction, tableHandle, layoutHandle, nonFilterColumns, session, OptionalInt.empty(), Optional.empty()).getRowCount(); + assertEquals(resultCount, expectedRows.size()); + } + transaction.commit(); } @@ -1350,7 +1606,8 @@ private void assertTableStatsComputed( ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName); - TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, Constraint.alwaysTrue()); + List allColumnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); + TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, Optional.empty(), allColumnHandles, Constraint.alwaysTrue()); assertFalse(tableStatistics.getRowCount().isUnknown(), "row count is unknown"); @@ -1459,8 +1716,8 @@ public void testGetPartitionSplitsTableOfflinePartition() assertNotNull(dsColumn); Domain domain = Domain.singleValue(createUnboundedVarcharType(), utf8Slice("2012-12-30")); - TupleDomain tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of(dsColumn, domain)); - List tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); + TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of(dsColumn, domain)); + ConnectorTableLayout tableLayout = getTableLayout(session, metadata, tableHandle, new Constraint<>(tupleDomain)); try { getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), SPLIT_SCHEDULING_CONTEXT)); fail("Expected PartitionOfflineException"); @@ -1470,6 +1727,22 @@ public void testGetPartitionSplitsTableOfflinePartition() assertEquals(e.getPartition(), "ds=2012-12-30"); } } + + try (Transaction transaction = newTransaction()) { + ConnectorMetadata metadata = transaction.getMetadata(); + ConnectorSession session = newSession(ImmutableMap.of(OFFLINE_DATA_DEBUG_MODE_ENABLED, true)); + + ConnectorTableHandle tableHandle = getTableHandle(metadata, tableOfflinePartition); + assertNotNull(tableHandle); + + ColumnHandle dsColumn = metadata.getColumnHandles(session, tableHandle).get("ds"); + assertNotNull(dsColumn); + + Domain domain = Domain.singleValue(createUnboundedVarcharType(), utf8Slice("2012-12-30")); + TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of(dsColumn, domain)); + ConnectorTableLayout tableLayout = getTableLayout(session, metadata, tableHandle, new Constraint<>(tupleDomain)); + getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, tableLayout.getHandle(), SPLIT_SCHEDULING_CONTEXT)); + } } @Test @@ -1496,6 +1769,20 @@ public void testGetPartitionSplitsTableNotReadablePartition() assertEquals(e.getPartition(), Optional.empty()); } } + + try (Transaction transaction = newTransaction()) { + ConnectorMetadata metadata = transaction.getMetadata(); + ConnectorSession session = newSession(ImmutableMap.of(OFFLINE_DATA_DEBUG_MODE_ENABLED, true)); + + ConnectorTableHandle tableHandle = getTableHandle(metadata, tableNotReadable); + assertNotNull(tableHandle); + + ColumnHandle dsColumn = metadata.getColumnHandles(session, tableHandle).get("ds"); + assertNotNull(dsColumn); + + List tableLayoutResults = metadata.getTableLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); + getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), SPLIT_SCHEDULING_CONTEXT)); + } } @Test @@ -1645,16 +1932,16 @@ private void doTestBucketedTableEvolutionWithDifferentReadCount(HiveStorageForma ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); - ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName); + ConnectorTableHandle hiveTableHandle = getTableHandle(metadata, tableName); // read entire table List columnHandles = ImmutableList.builder() - .addAll(metadata.getColumnHandles(session, tableHandle).values()) + .addAll(metadata.getColumnHandles(session, hiveTableHandle).values()) .build(); List tableLayoutResults = transaction.getMetadata().getTableLayouts( session, - tableHandle, + hiveTableHandle, new Constraint<>(TupleDomain.all()), Optional.empty()); @@ -1664,20 +1951,26 @@ private void doTestBucketedTableEvolutionWithDifferentReadCount(HiveStorageForma HiveTableLayoutHandle modifiedReadBucketCountLayoutHandle = new HiveTableLayoutHandle( layoutHandle.getSchemaTableName(), layoutHandle.getPartitionColumns(), + layoutHandle.getDataColumns(), + layoutHandle.getTableParameters(), layoutHandle.getPartitions().get(), layoutHandle.getDomainPredicate(), layoutHandle.getRemainingPredicate(), layoutHandle.getPredicateColumns(), layoutHandle.getPartitionColumnPredicate(), Optional.of(new HiveBucketHandle(bucketHandle.getColumns(), bucketHandle.getTableBucketCount(), 2)), - layoutHandle.getBucketFilter()); + layoutHandle.getBucketFilter(), + false, + "layout"); - List splits = getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, modifiedReadBucketCountLayoutHandle, SPLIT_SCHEDULING_CONTEXT)); + List splits = getAllSplits(session, transaction, modifiedReadBucketCountLayoutHandle); assertEquals(splits.size(), 16); + TableHandle tableHandle = toTableHandle(transaction, hiveTableHandle, modifiedReadBucketCountLayoutHandle); + ImmutableList.Builder allRows = ImmutableList.builder(); for (ConnectorSplit split : splits) { - try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) { + try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, tableHandle.getLayout().get(), columnHandles)) { MaterializedResult intermediateResult = materializeSourceDataStream(session, pageSource, getTypes(columnHandles)); allRows.addAll(intermediateResult.getMaterializedRows()); } @@ -1749,7 +2042,6 @@ private void doTestBucketedTableEvolution(HiveStorageFormat storageFormat, Schem ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName); - // read entire table List columnHandles = ImmutableList.builder() .addAll(metadata.getColumnHandles(session, tableHandle).values()) @@ -1765,12 +2057,25 @@ private void doTestBucketedTableEvolution(HiveStorageFormat storageFormat, Schem assertBucketTableEvolutionResult(result, columnHandles, ImmutableSet.of(0, 1, 2, 3, 4, 5, 6, 7), rowCount); // read single bucket (table/logical bucket) + + NullableValue singleBucket = NullableValue.of(INTEGER, 6L); + ConnectorTableLayoutHandle layoutHandle; + if (HiveSessionProperties.isPushdownFilterEnabled(session)) { + TupleDomain bucketDomain = TupleDomain.fromFixedValues(ImmutableMap.of(new VariableReferenceExpression(BUCKET_COLUMN_NAME, BIGINT), singleBucket)); + + RowExpression predicate = ROW_EXPRESSION_SERVICE.getDomainTranslator().toPredicate(bucketDomain); + layoutHandle = metadata.pushdownFilter(session, tableHandle, predicate, Optional.empty()).getLayout().getHandle(); + } + else { + layoutHandle = getOnlyElement(metadata.getTableLayouts(session, tableHandle, new Constraint<>(TupleDomain.fromFixedValues(ImmutableMap.of(bucketColumnHandle(), singleBucket))), Optional.empty())).getTableLayout().getHandle(); + } + result = readTable( transaction, tableHandle, + layoutHandle, columnHandles, session, - TupleDomain.fromFixedValues(ImmutableMap.of(bucketColumnHandle(), NullableValue.of(INTEGER, 6L))), OptionalInt.empty(), Optional.empty()); assertBucketTableEvolutionResult(result, columnHandles, ImmutableSet.of(6), rowCount); @@ -1784,9 +2089,9 @@ private void doTestBucketedTableEvolution(HiveStorageFormat storageFormat, Schem result = readTable( transaction, tableHandle, + layoutHandle, columnHandles, session, - TupleDomain.fromFixedValues(ImmutableMap.of(bucketColumnHandle(), NullableValue.of(INTEGER, 6L))), OptionalInt.empty(), Optional.empty()); assertBucketTableEvolutionResult(result, columnHandles, ImmutableSet.of(6), rowCount); @@ -1847,13 +2152,15 @@ public void testGetRecords() ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); - ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); - ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle); - List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); + ConnectorTableHandle hiveTableHandle = getTableHandle(metadata, tablePartitionFormat); + ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session, hiveTableHandle); + List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, hiveTableHandle).values()); Map columnIndex = indexColumns(columnHandles); - List splits = getAllSplits(transaction, tableHandle, TupleDomain.all()); + ConnectorTableLayoutHandle layoutHandle = getLayout(session, transaction, hiveTableHandle, TupleDomain.all()); + List splits = getAllSplits(session, transaction, layoutHandle); assertEquals(splits.size(), partitionCount); + for (ConnectorSplit split : splits) { HiveSplit hiveSplit = (HiveSplit) split; @@ -1865,7 +2172,7 @@ public void testGetRecords() long rowNumber = 0; long completedBytes = 0; - try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, hiveSplit, columnHandles)) { + try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, hiveSplit, layoutHandle, columnHandles)) { MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles)); assertPageSourceType(pageSource, fileType); @@ -1938,12 +2245,14 @@ public void testGetPartialRecords() ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); - ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); - List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); + ConnectorTableHandle hiveTableHandle = getTableHandle(metadata, tablePartitionFormat); + List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, hiveTableHandle).values()); Map columnIndex = indexColumns(columnHandles); - List splits = getAllSplits(transaction, tableHandle, TupleDomain.all()); + ConnectorTableLayoutHandle layoutHandle = getLayout(session, transaction, hiveTableHandle, TupleDomain.all()); + List splits = getAllSplits(session, transaction, layoutHandle); assertEquals(splits.size(), partitionCount); + for (ConnectorSplit split : splits) { HiveSplit hiveSplit = (HiveSplit) split; @@ -1954,7 +2263,7 @@ public void testGetPartialRecords() int dummyPartition = Integer.parseInt(partitionKeys.get(2).getValue()); long rowNumber = 0; - try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, hiveSplit, columnHandles)) { + try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, hiveSplit, layoutHandle, columnHandles)) { assertPageSourceType(pageSource, fileType); MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles)); for (MaterializedRow row : result) { @@ -1979,11 +2288,12 @@ public void testGetRecordsUnpartitioned() ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); - ConnectorTableHandle tableHandle = getTableHandle(metadata, tableUnpartitioned); - List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); + ConnectorTableHandle hiveTableHandle = getTableHandle(metadata, tableUnpartitioned); + List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, hiveTableHandle).values()); Map columnIndex = indexColumns(columnHandles); - List splits = getAllSplits(transaction, tableHandle, TupleDomain.all()); + ConnectorTableLayoutHandle layoutHandle = getLayout(session, transaction, hiveTableHandle, TupleDomain.all()); + List splits = getAllSplits(session, transaction, layoutHandle); assertEquals(splits.size(), 1); for (ConnectorSplit split : splits) { @@ -1992,7 +2302,7 @@ public void testGetRecordsUnpartitioned() assertEquals(hiveSplit.getPartitionKeys(), ImmutableList.of()); long rowNumber = 0; - try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) { + try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, layoutHandle, columnHandles)) { assertPageSourceType(pageSource, TEXTFILE); MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles)); @@ -2051,8 +2361,7 @@ public void testPartitionSchemaNonCanonical() ConnectorTableHandle table = getTableHandle(metadata, tablePartitionSchemaChangeNonCanonical); ColumnHandle column = metadata.getColumnHandles(session, table).get("t_boolean"); assertNotNull(column); - List tableLayoutResults = metadata.getTableLayouts(session, table, new Constraint<>(TupleDomain.fromFixedValues(ImmutableMap.of(column, NullableValue.of(BOOLEAN, false)))), Optional.empty()); - ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); + ConnectorTableLayoutHandle layoutHandle = getTableLayout(session, metadata, table, new Constraint<>(withColumnDomains(ImmutableMap.of(intColumn, Domain.singleValue(BIGINT, 5L))))).getHandle(); assertEquals(getAllPartitions(layoutHandle).size(), 1); assertEquals(getPartitionId(getAllPartitions(layoutHandle).get(0)), "t_boolean=0"); @@ -2060,7 +2369,7 @@ public void testPartitionSchemaNonCanonical() ConnectorSplit split = getOnlyElement(getAllSplits(splitSource)); ImmutableList columnHandles = ImmutableList.of(column); - try (ConnectorPageSource ignored = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) { + try (ConnectorPageSource ignored = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, layoutHandle, columnHandles)) { fail("expected exception"); } catch (PrestoException e) { @@ -2069,6 +2378,19 @@ public void testPartitionSchemaNonCanonical() } } + private static ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorMetadata metadata, ConnectorTableHandle tableHandle, Constraint constraint) + { + if (HiveSessionProperties.isPushdownFilterEnabled(session)) { + assertTrue(constraint.getSummary().isAll()); + + ConnectorPushdownFilterResult pushdownFilterResult = metadata.pushdownFilter(session, tableHandle, TRUE_CONSTANT, Optional.empty()); + return pushdownFilterResult.getLayout(); + } + + List tableLayoutResults = metadata.getTableLayouts(session, tableHandle, constraint, Optional.empty()); + return getOnlyElement(tableLayoutResults).getTableLayout(); + } + @Test public void testTypesTextFile() throws Exception @@ -2115,13 +2437,22 @@ public void testTypesParquet() public void testEmptyTextFile() throws Exception { + checkSupportedStorageFormat(TEXTFILE); assertEmptyFile(TEXTFILE); } + private void checkSupportedStorageFormat(HiveStorageFormat storageFormat) + { + if (!createTableFormats.contains(storageFormat)) { + throw new SkipException(storageFormat + " format is not supported"); + } + } + @Test(expectedExceptions = PrestoException.class, expectedExceptionsMessageRegExp = "Error opening Hive split .*SequenceFile.*EOFException") public void testEmptySequenceFile() throws Exception { + checkSupportedStorageFormat(SEQUENCEFILE); assertEmptyFile(SEQUENCEFILE); } @@ -2129,6 +2460,7 @@ public void testEmptySequenceFile() public void testEmptyRcTextFile() throws Exception { + checkSupportedStorageFormat(RCTEXT); assertEmptyFile(RCTEXT); } @@ -2136,6 +2468,7 @@ public void testEmptyRcTextFile() public void testEmptyRcBinaryFile() throws Exception { + checkSupportedStorageFormat(RCBINARY); assertEmptyFile(RCBINARY); } @@ -2143,6 +2476,7 @@ public void testEmptyRcBinaryFile() public void testEmptyOrcFile() throws Exception { + checkSupportedStorageFormat(ORC); assertEmptyFile(ORC); } @@ -2150,6 +2484,7 @@ public void testEmptyOrcFile() public void testEmptyDwrfFile() throws Exception { + checkSupportedStorageFormat(DWRF); assertEmptyFile(DWRF); } @@ -2326,7 +2661,7 @@ public void testTableCreationIgnoreExisting() try { try (Transaction transaction = newTransaction()) { LocationService locationService = getLocationService(); - LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(schemaName), session, schemaName, tableName); + LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(schemaName), session, schemaName, tableName, false); targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath(); Table table = createSimpleTable(schemaTableName, columns, session, targetPath, "q1"); transaction.getMetastore(schemaName) @@ -2401,23 +2736,26 @@ public void testBucketSortedTables() throws Exception { SchemaTableName table = temporaryTable("create_sorted"); + SchemaTableName tableWithTempPath = temporaryTable("create_sorted_with_temp_path"); try { - doTestBucketSortedTables(table); + doTestBucketSortedTables(table, false, ORC); + doTestBucketSortedTables(tableWithTempPath, true, ORC); } finally { dropTable(table); + dropTable(tableWithTempPath); } } - private void doTestBucketSortedTables(SchemaTableName table) + private void doTestBucketSortedTables(SchemaTableName table, boolean useTempPath, HiveStorageFormat storageFormat) throws IOException { int bucketCount = 3; int expectedRowCount = 0; try (Transaction transaction = newTransaction()) { - ConnectorSession session = newSession(); ConnectorMetadata metadata = transaction.getMetadata(); + ConnectorSession session = newSession(ImmutableMap.of(SORTED_WRITE_TO_TEMP_PATH_ENABLED, useTempPath)); // begin creating the table ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata( @@ -2429,7 +2767,7 @@ private void doTestBucketSortedTables(SchemaTableName table) .add(new ColumnMetadata("ds", VARCHAR)) .build(), ImmutableMap.builder() - .put(STORAGE_FORMAT_PROPERTY, RCBINARY) + .put(STORAGE_FORMAT_PROPERTY, storageFormat) .put(PARTITIONED_BY_PROPERTY, ImmutableList.of("ds")) .put(BUCKETED_BY_PROPERTY, ImmutableList.of("id")) .put(BUCKET_COUNT_PROPERTY, bucketCount) @@ -2461,17 +2799,18 @@ private void doTestBucketSortedTables(SchemaTableName table) } // verify we have enough temporary files per bucket to require multiple passes - Path stagingPathRoot = getStagingPathRoot(outputHandle); + Path path = useTempPath ? getTempFilePathRoot(outputHandle).get() : getStagingPathRoot(outputHandle); HdfsContext context = new HdfsContext(session, table.getSchemaName(), table.getTableName()); - assertThat(listAllDataFiles(context, stagingPathRoot)) - .filteredOn(file -> file.contains(".tmp-sort.")) + Set files = listAllDataFiles(context, path); + assertThat(listAllDataFiles(context, path)) + .filteredOn(file -> file.contains(".tmp-sort")) .size().isGreaterThan(bucketCount * getHiveClientConfig().getMaxOpenSortFiles() * 2); // finish the write Collection fragments = getFutureValue(sink.finish()); // verify there are no temporary files - for (String file : listAllDataFiles(context, stagingPathRoot)) { + for (String file : listAllDataFiles(context, path)) { assertThat(file).doesNotContain(".tmp-sort."); } @@ -2484,17 +2823,18 @@ private void doTestBucketSortedTables(SchemaTableName table) // verify that bucket files are sorted try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); - ConnectorSession session = newSession(); + ConnectorSession session = newSession(ImmutableMap.of(SORTED_WRITE_TO_TEMP_PATH_ENABLED, useTempPath)); - ConnectorTableHandle tableHandle = getTableHandle(metadata, table); - List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); + ConnectorTableHandle hiveTableHandle = getTableHandle(metadata, table); + List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, hiveTableHandle).values()); - List splits = getAllSplits(transaction, tableHandle, TupleDomain.all()); + ConnectorTableLayoutHandle layoutHandle = getLayout(session, transaction, hiveTableHandle, TupleDomain.all()); + List splits = getAllSplits(session, transaction, layoutHandle); assertThat(splits).hasSize(bucketCount); int actualRowCount = 0; for (ConnectorSplit split : splits) { - try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) { + try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, layoutHandle, columnHandles)) { String lastValueAsc = null; long lastValueDesc = -1; @@ -2533,6 +2873,15 @@ private void doTestBucketSortedTables(SchemaTableName table) } } + private TableHandle toTableHandle(Transaction transaction, ConnectorTableHandle connectorTableHandle, ConnectorTableLayoutHandle connectorLayoutHandle) + { + return new TableHandle( + new ConnectorId(clientId), + connectorTableHandle, + transaction.getTransactionHandle(), + Optional.of(connectorLayoutHandle)); + } + @Test public void testInsert() throws Exception @@ -2954,7 +3303,7 @@ public void testIllegalStorageFormatDuringTableScan() String tableOwner = session.getUser(); String schemaName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); - LocationHandle locationHandle = getLocationService().forNewTable(transaction.getMetastore(schemaName), session, schemaName, tableName); + LocationHandle locationHandle = getLocationService().forNewTable(transaction.getMetastore(schemaName), session, schemaName, tableName, false); Path targetPath = getLocationService().getQueryWriteInfo(locationHandle).getTargetPath(); //create table whose storage format is null Table.Builder tableBuilder = Table.builder() @@ -3196,7 +3545,7 @@ protected String partitionTargetPath(SchemaTableName schemaTableName, String par SemiTransactionalHiveMetastore metastore = transaction.getMetastore(schemaTableName.getSchemaName()); LocationService locationService = getLocationService(); Table table = metastore.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName()).get(); - LocationHandle handle = locationService.forExistingTable(metastore, session, table); + LocationHandle handle = locationService.forExistingTable(metastore, session, table, false); return locationService.getPartitionWriteInfo(handle, Optional.empty(), partitionName).getTargetPath().toString(); } } @@ -3228,8 +3577,9 @@ protected void testPartitionStatisticsSampling(List columns, Par ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = metadata.getTableHandle(session, tableName); - TableStatistics unsampledStatistics = metadata.getTableStatistics(sampleSize(2), tableHandle, Constraint.alwaysTrue()); - TableStatistics sampledStatistics = metadata.getTableStatistics(sampleSize(1), tableHandle, Constraint.alwaysTrue()); + List allColumnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); + TableStatistics unsampledStatistics = metadata.getTableStatistics(sampleSize(2), tableHandle, Optional.empty(), allColumnHandles, Constraint.alwaysTrue()); + TableStatistics sampledStatistics = metadata.getTableStatistics(sampleSize(1), tableHandle, Optional.empty(), allColumnHandles, Constraint.alwaysTrue()); assertEquals(sampledStatistics, unsampledStatistics); } } @@ -3329,7 +3679,7 @@ protected void doCreateTable(SchemaTableName tableName, HiveStorageFormat storag if (pageSinkProperties.isPartitionCommitRequired()) { assertValidPartitionCommitFragments(fragments); - metadata.commitPartition(session, outputHandle, fragments); + metadata.commitPartitionAsync(session, outputHandle, fragments).get(); } // verify all new files start with the unique prefix @@ -3498,7 +3848,7 @@ private void doInsert(HiveStorageFormat storageFormat, SchemaTableName tableName Collection fragments = getFutureValue(sink.finish()); if (pageSinkProperties.isPartitionCommitRequired()) { assertValidPartitionCommitFragments(fragments); - metadata.commitPartition(session, insertTableHandle, fragments); + metadata.commitPartitionAsync(session, insertTableHandle, fragments).get(); } metadata.finishInsert(session, insertTableHandle, fragments, ImmutableList.of()); @@ -3593,6 +3943,14 @@ protected Path getTargetPathRoot(ConnectorInsertTableHandle insertTableHandle) .getTargetPath(); } + protected Optional getTempFilePathRoot(ConnectorOutputTableHandle outputTableHandle) + { + HiveOutputTableHandle handle = (HiveOutputTableHandle) outputTableHandle; + return getLocationService() + .getQueryWriteInfo(handle.getLocationHandle()) + .getTempPath(); + } + protected Set listAllDataFiles(Transaction transaction, String schemaName, String tableName) throws IOException { @@ -3713,7 +4071,7 @@ private void doInsertIntoNewPartition(HiveStorageFormat storageFormat, SchemaTab Collection fragments = getFutureValue(sink.finish()); if (pageSinkProperties.isPartitionCommitRequired()) { assertValidPartitionCommitFragments(fragments); - metadata.commitPartition(session, insertTableHandle, fragments); + metadata.commitPartitionAsync(session, insertTableHandle, fragments).get(); } metadata.finishInsert(session, insertTableHandle, fragments, ImmutableList.of()); @@ -3832,7 +4190,7 @@ private void doInsertIntoExistingPartition(HiveStorageFormat storageFormat, Sche Collection fragments = getFutureValue(sink.finish()); if (pageSinkProperties.isPartitionCommitRequired()) { assertValidPartitionCommitFragments(fragments); - metadata.commitPartition(session, insertTableHandle, fragments); + metadata.commitPartitionAsync(session, insertTableHandle, fragments).get(); } metadata.finishInsert(session, insertTableHandle, fragments, ImmutableList.of()); @@ -4079,7 +4437,7 @@ private void doTestMetadataDelete(HiveStorageFormat storageFormat, SchemaTableNa // delete ds=2015-07-01 and 2015-07-02 session = newSession(); - TupleDomain tupleDomain2 = TupleDomain.withColumnDomains( + TupleDomain tupleDomain2 = withColumnDomains( ImmutableMap.of(dsColumnHandle, Domain.create(ValueSet.ofRanges(Range.range(createUnboundedVarcharType(), utf8Slice("2015-07-01"), true, utf8Slice("2015-07-02"), true)), false))); Constraint constraint2 = new Constraint<>(tupleDomain2, convertToPredicate(tupleDomain2)); List tableLayoutResults2 = metadata.getTableLayouts(session, tableHandle, constraint2, Optional.empty()); @@ -4126,22 +4484,20 @@ protected void assertGetRecords(String tableName, HiveStorageFormat hiveStorageF ConnectorTableHandle tableHandle = getTableHandle(metadata, new SchemaTableName(database, tableName)); ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle); - HiveSplit hiveSplit = getHiveSplit(transaction, tableHandle); + + ConnectorTableLayoutHandle layoutHandle = getLayout(session, transaction, tableHandle, TupleDomain.all()); + List splits = getAllSplits(session, transaction, layoutHandle); + assertEquals(splits.size(), 1); + + HiveSplit hiveSplit = (HiveSplit) getOnlyElement(splits); List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); - ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, hiveSplit, columnHandles); + ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, hiveSplit, layoutHandle, columnHandles); assertGetRecords(hiveStorageFormat, tableMetadata, hiveSplit, pageSource, columnHandles); } } - protected HiveSplit getHiveSplit(Transaction transaction, ConnectorTableHandle tableHandle) - { - List splits = getAllSplits(transaction, tableHandle, TupleDomain.all()); - assertEquals(splits.size(), 1); - return (HiveSplit) getOnlyElement(splits); - } - protected void assertGetRecords( HiveStorageFormat hiveStorageFormat, ConnectorTableMetadata tableMetadata, @@ -4394,7 +4750,7 @@ protected ConnectorTableHandle getTableHandle(ConnectorMetadata metadata, Schema private MaterializedResult readTable( Transaction transaction, - ConnectorTableHandle tableHandle, + ConnectorTableHandle hiveTableHandle, List columnHandles, ConnectorSession session, TupleDomain tupleDomain, @@ -4402,20 +4758,30 @@ private MaterializedResult readTable( Optional expectedStorageFormat) throws Exception { - List tableLayoutResults = transaction.getMetadata().getTableLayouts( - session, - tableHandle, - new Constraint<>(tupleDomain), - Optional.empty()); - ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); - List splits = getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, SPLIT_SCHEDULING_CONTEXT)); + ConnectorTableLayoutHandle layoutHandle = getTableLayout(session, transaction.getMetadata(), hiveTableHandle, new Constraint<>(tupleDomain)).getHandle(); + return readTable(transaction, hiveTableHandle, layoutHandle, columnHandles, session, expectedSplitCount, expectedStorageFormat); + } + + private MaterializedResult readTable( + Transaction transaction, + ConnectorTableHandle hiveTableHandle, + ConnectorTableLayoutHandle hiveTableLayoutHandle, + List columnHandles, + ConnectorSession session, + OptionalInt expectedSplitCount, + Optional expectedStorageFormat) + throws Exception + { + List splits = getAllSplits(session, transaction, hiveTableLayoutHandle); if (expectedSplitCount.isPresent()) { assertEquals(splits.size(), expectedSplitCount.getAsInt()); } + TableHandle tableHandle = toTableHandle(transaction, hiveTableHandle, hiveTableLayoutHandle); + ImmutableList.Builder allRows = ImmutableList.builder(); for (ConnectorSplit split : splits) { - try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) { + try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, tableHandle.getLayout().get(), columnHandles)) { expectedStorageFormat.ifPresent(format -> assertPageSourceType(pageSource, format)); MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles)); allRows.addAll(result.getMaterializedRows()); @@ -4446,10 +4812,20 @@ protected static int getSplitCount(ConnectorSplitSource splitSource) private List getAllSplits(Transaction transaction, ConnectorTableHandle tableHandle, TupleDomain tupleDomain) { ConnectorSession session = newSession(); + ConnectorTableLayoutHandle layoutHandle = getLayout(session, transaction, tableHandle, tupleDomain); + return getAllSplits(session, transaction, layoutHandle); + } + + private List getAllSplits(ConnectorSession session, Transaction transaction, ConnectorTableLayoutHandle layoutHandle) + { + return getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, SPLIT_SCHEDULING_CONTEXT)); + } + + private static ConnectorTableLayoutHandle getLayout(ConnectorSession session, Transaction transaction, ConnectorTableHandle tableHandle, TupleDomain tupleDomain) + { ConnectorMetadata metadata = transaction.getMetadata(); List tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); - ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); - return getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, SPLIT_SCHEDULING_CONTEXT)); + return getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); } protected static List getAllSplits(ConnectorSplitSource splitSource) @@ -4474,7 +4850,10 @@ protected String getPartitionId(Object partition) protected static void assertPageSourceType(ConnectorPageSource pageSource, HiveStorageFormat hiveStorageFormat) { - if (pageSource instanceof RecordPageSource) { + if (pageSource instanceof OrcSelectivePageSource) { + assertTrue(hiveStorageFormat == ORC || hiveStorageFormat == DWRF); + } + else if (pageSource instanceof RecordPageSource) { RecordCursor hiveRecordCursor = ((RecordPageSource) pageSource).getCursor(); hiveRecordCursor = ((HiveRecordCursor) hiveRecordCursor).getRegularColumnRecordCursor(); if (hiveRecordCursor instanceof HiveCoercionRecordCursor) { @@ -4660,7 +5039,7 @@ protected Table createEmptyTable(SchemaTableName schemaTableName, HiveStorageFor String tableName = schemaTableName.getTableName(); LocationService locationService = getLocationService(); - LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(schemaName), session, schemaName, tableName); + LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(schemaName), session, schemaName, tableName, false); targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath(); Table.Builder tableBuilder = Table.builder() @@ -4870,7 +5249,7 @@ private void doTestTransactionDeleteInsert( // Query 1: delete session = newSession(); HiveColumnHandle dsColumnHandle = (HiveColumnHandle) metadata.getColumnHandles(session, tableHandle).get("pk2"); - TupleDomain tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of( + TupleDomain tupleDomain = withColumnDomains(ImmutableMap.of( dsColumnHandle, domainToDrop)); Constraint constraint = new Constraint<>(tupleDomain, convertToPredicate(tupleDomain)); List tableLayoutResults = metadata.getTableLayouts(session, tableHandle, constraint, Optional.empty()); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientLocal.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientLocal.java index 75e5bb506ab8f..41af472bd8926 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientLocal.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientLocal.java @@ -66,8 +66,9 @@ public void initialize() HiveClientConfig hiveConfig = new HiveClientConfig() .setTimeZone("America/Los_Angeles"); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); - setup(testDbName, hiveConfig, metastore); + setup(testDbName, hiveConfig, metastoreClientConfig, metastore); } @AfterClass(alwaysRun = true) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileFormats.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileFormats.java index 1078ec5b4a998..9b23250fb926a 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileFormats.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileFormats.java @@ -88,11 +88,11 @@ import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; -import static com.facebook.presto.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; import static com.facebook.presto.hive.HiveTestUtils.SESSION; import static com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER; import static com.facebook.presto.hive.HiveTestUtils.mapType; import static com.facebook.presto.hive.HiveUtil.isStructuralType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.HIVE_DEFAULT_DYNAMIC_PARTITION; import static com.facebook.presto.hive.util.ConfigurationUtils.configureCompression; import static com.facebook.presto.hive.util.SerDeUtils.serializeObject; import static com.facebook.presto.spi.type.BigintType.BIGINT; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java index 6e51f2b2f0239..99873487e77cb 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.GroupByHashPageIndexerFactory; import com.facebook.presto.hive.AbstractTestHiveClient.HiveTransaction; import com.facebook.presto.hive.AbstractTestHiveClient.Transaction; @@ -30,6 +33,7 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorOutputTableHandle; import com.facebook.presto.spi.ConnectorPageSink; import com.facebook.presto.spi.ConnectorPageSource; @@ -42,6 +46,7 @@ import com.facebook.presto.spi.Constraint; import com.facebook.presto.spi.PageSinkProperties; import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.TableNotFoundException; import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.connector.ConnectorPageSinkProvider; @@ -55,16 +60,11 @@ import com.facebook.presto.testing.MaterializedRow; import com.facebook.presto.testing.TestingConnectorSession; import com.facebook.presto.testing.TestingNodeManager; -import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.ImmutableSet; import com.google.common.net.HostAndPort; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; -import io.airlift.stats.CounterStat; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.testng.annotations.AfterClass; @@ -79,18 +79,25 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutorService; +import java.util.function.BiFunction; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static com.facebook.presto.hive.AbstractTestHiveClient.createTableProperties; import static com.facebook.presto.hive.AbstractTestHiveClient.filterNonHiddenColumnHandles; import static com.facebook.presto.hive.AbstractTestHiveClient.filterNonHiddenColumnMetadata; import static com.facebook.presto.hive.AbstractTestHiveClient.getAllSplits; +import static com.facebook.presto.hive.HiveTestUtils.FILTER_STATS_CALCULATOR_SERVICE; import static com.facebook.presto.hive.HiveTestUtils.FUNCTION_RESOLUTION; +import static com.facebook.presto.hive.HiveTestUtils.METADATA; import static com.facebook.presto.hive.HiveTestUtils.PAGE_SORTER; import static com.facebook.presto.hive.HiveTestUtils.ROW_EXPRESSION_SERVICE; import static com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER; -import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveDataStreamFactories; +import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveBatchPageSourceFactories; import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories; import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider; +import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveSelectivePageSourceFactories; import static com.facebook.presto.hive.HiveTestUtils.getDefaultOrcFileWriterFactory; import static com.facebook.presto.hive.HiveTestUtils.getTypes; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING; @@ -100,9 +107,6 @@ import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static java.util.Locale.ENGLISH; import static java.util.concurrent.Executors.newCachedThreadPool; import static org.testng.Assert.assertEquals; @@ -129,6 +133,7 @@ public abstract class AbstractTestHiveFileSystem private ExecutorService executor; private HiveClientConfig config; + private MetastoreClientConfig metastoreClientConfig; @BeforeClass public void setUp() @@ -147,7 +152,7 @@ public void tearDown() protected abstract Path getBasePath(); - protected void setup(String host, int port, String databaseName, Function hdfsConfigurationProvider, boolean s3SelectPushdownEnabled) + protected void setup(String host, int port, String databaseName, BiFunction hdfsConfigurationProvider, boolean s3SelectPushdownEnabled) { database = databaseName; table = new SchemaTableName(database, "presto_test_external_fs"); @@ -156,29 +161,31 @@ protected void setup(String host, int port, String databaseName, Function partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class); metadataFactory = new HiveMetadataFactory( config, + metastoreClientConfig, metastoreClient, hdfsEnvironment, hivePartitionManager, @@ -186,16 +193,19 @@ protected void setup(String host, int port, String databaseName, Function ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore(), + transactionManager, new NamenodeStats(), hdfsEnvironment, new HadoopDirectoryLister(), @@ -210,21 +220,22 @@ protected void setup(String host, int port, String databaseName, Function columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values()); + ConnectorTableHandle hiveTableHandle = getTableHandle(metadata, tableName); + List columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, hiveTableHandle).values()); // verify the metadata ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session, getTableHandle(metadata, tableName)); assertEquals(filterNonHiddenColumnMetadata(tableMetadata.getColumns()), columns); // verify the data - List tableLayoutResults = metadata.getTableLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); + List tableLayoutResults = metadata.getTableLayouts(session, hiveTableHandle, Constraint.alwaysTrue(), Optional.empty()); HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); assertEquals(layoutHandle.getPartitions().get().size(), 1); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, SPLIT_SCHEDULING_CONTEXT); ConnectorSplit split = getOnlyElement(getAllSplits(splitSource)); - try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) { + TableHandle tableHandle = new TableHandle(new ConnectorId("hive"), hiveTableHandle, transaction.getTransactionHandle(), Optional.of(layoutHandle)); + + try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, tableHandle.getLayout().get(), columnHandles)) { MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles)); assertEqualsIgnoreOrder(result.getMaterializedRows(), data.getMaterializedRows()); } @@ -462,9 +477,9 @@ private static class TestingHiveMetastore private final Path basePath; private final HdfsEnvironment hdfsEnvironment; - public TestingHiveMetastore(ExtendedHiveMetastore delegate, ExecutorService executor, HiveClientConfig hiveClientConfig, Path basePath, HdfsEnvironment hdfsEnvironment) + public TestingHiveMetastore(ExtendedHiveMetastore delegate, ExecutorService executor, MetastoreClientConfig metastoreClientConfig, Path basePath, HdfsEnvironment hdfsEnvironment) { - super(delegate, executor, hiveClientConfig); + super(delegate, executor, metastoreClientConfig); this.basePath = basePath; this.hdfsEnvironment = hdfsEnvironment; } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/HiveBenchmarkQueryRunner.java b/presto-hive/src/test/java/com/facebook/presto/hive/HiveBenchmarkQueryRunner.java index 41b4eb7b5d8e0..dbe6669a224bd 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/HiveBenchmarkQueryRunner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/HiveBenchmarkQueryRunner.java @@ -28,7 +28,7 @@ import java.util.Map; import java.util.Optional; -import static com.facebook.presto.hive.metastore.file.FileHiveMetastore.createTestingFileHiveMetastore; +import static com.facebook.presto.hive.TestHiveUtil.createTestingFileHiveMetastore; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java b/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java index c83b0c8a16033..aadce8b94d649 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.log.Logging; import com.facebook.presto.Session; import com.facebook.presto.execution.QueryManagerConfig.ExchangeMaterializationStrategy; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; @@ -27,8 +29,7 @@ import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import io.airlift.log.Logging; +import com.google.common.collect.ImmutableSet; import io.airlift.tpch.TpchTable; import org.intellij.lang.annotations.Language; import org.joda.time.DateTimeZone; @@ -38,6 +39,8 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.airlift.log.Level.ERROR; +import static com.facebook.airlift.log.Level.WARN; import static com.facebook.presto.SystemSessionProperties.COLOCATED_JOIN; import static com.facebook.presto.SystemSessionProperties.EXCHANGE_MATERIALIZATION_STRATEGY; import static com.facebook.presto.SystemSessionProperties.GROUPED_EXECUTION_FOR_AGGREGATION; @@ -47,8 +50,6 @@ import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tests.QueryAssertions.copyTpchTables; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static io.airlift.log.Level.ERROR; -import static io.airlift.log.Level.WARN; import static io.airlift.units.Duration.nanosSince; import static java.lang.String.format; import static java.util.Locale.ENGLISH; @@ -82,22 +83,56 @@ public static DistributedQueryRunner createQueryRunner(Iterable> ta return createQueryRunner(tables, ImmutableMap.of(), Optional.empty()); } + public static DistributedQueryRunner createQueryRunner( + Iterable> tables, + Map extraProperties, + Map extraCoordinatorProperties, + Optional baseDataDir) + throws Exception + { + return createQueryRunner(tables, extraProperties, extraCoordinatorProperties, "sql-standard", ImmutableMap.of(), baseDataDir); + } + public static DistributedQueryRunner createQueryRunner(Iterable> tables, Map extraProperties, Optional baseDataDir) throws Exception { - return createQueryRunner(tables, extraProperties, "sql-standard", ImmutableMap.of(), baseDataDir); + return createQueryRunner(tables, extraProperties, ImmutableMap.of(), "sql-standard", ImmutableMap.of(), baseDataDir); + } + + public static DistributedQueryRunner createQueryRunner( + Iterable> tables, + Map extraProperties, + String security, + Map extraHiveProperties, + Optional baseDataDir) + throws Exception + { + return createQueryRunner(tables, extraProperties, ImmutableMap.of(), security, extraHiveProperties, baseDataDir); } - public static DistributedQueryRunner createQueryRunner(Iterable> tables, Map extraProperties, String security, Map extraHiveProperties, Optional baseDataDir) + public static DistributedQueryRunner createQueryRunner( + Iterable> tables, + Map extraProperties, + Map extraCoordinatorProperties, + String security, + Map extraHiveProperties, + Optional baseDataDir) throws Exception { assertEquals(DateTimeZone.getDefault(), TIME_ZONE, "Timezone not configured correctly. Add -Duser.timezone=America/Bahia_Banderas to your JVM arguments"); setupLogging(); + Map systemProperties = ImmutableMap.builder() + .put("task.writer-count", "2") + .put("task.partitioned-writer-count", "4") + .putAll(extraProperties) + .build(); + DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(createSession(Optional.of(new SelectedRole(ROLE, Optional.of("admin"))))) .setNodeCount(4) - .setExtraProperties(extraProperties) + .setExtraProperties(systemProperties) + .setCoordinatorProperties(extraCoordinatorProperties) .setBaseDataDir(baseDataDir) .build(); try { @@ -107,8 +142,9 @@ public static DistributedQueryRunner createQueryRunner(Iterable> ta File baseDir = queryRunner.getCoordinator().getBaseDataDir().resolve("hive_data").toFile(); HiveClientConfig hiveClientConfig = new HiveClientConfig(); - HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationUpdater(hiveClientConfig)); - HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication()); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig, metastoreClientConfig), ImmutableSet.of()); + HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); FileHiveMetastore metastore = new FileHiveMetastore(hdfsEnvironment, baseDir.toURI().toString(), "test"); queryRunner.installPlugin(new HivePlugin(HIVE_CATALOG, Optional.of(metastore))); @@ -122,12 +158,19 @@ public static DistributedQueryRunner createQueryRunner(Iterable> ta .put("hive.collect-column-statistics-on-write", "true") .put("hive.temporary-table-schema", TEMPORARY_TABLE_SCHEMA) .build(); + + Map storageProperties = extraHiveProperties.containsKey("hive.storage-format") ? + ImmutableMap.copyOf(hiveProperties) : + ImmutableMap.builder() + .putAll(hiveProperties) + .put("hive.storage-format", "TEXTFILE") + .put("hive.compression-codec", "NONE") + .build(); + Map hiveBucketedProperties = ImmutableMap.builder() - .putAll(hiveProperties) + .putAll(storageProperties) .put("hive.max-initial-split-size", "10kB") // so that each bucket has multiple splits .put("hive.max-split-size", "10kB") // so that each bucket has multiple splits - .put("hive.storage-format", "TEXTFILE") // so that there's no minimum split size for the file - .put("hive.compression-codec", "NONE") // so that the file is splittable .build(); queryRunner.createCatalog(HIVE_CATALOG, HIVE_CATALOG, hiveProperties); queryRunner.createCatalog(HIVE_BUCKETED_CATALOG, HIVE_CATALOG, hiveBucketedProperties); @@ -171,9 +214,10 @@ public static DistributedQueryRunner createMaterializingQueryRunner(Iterable role) "hive", Optional.empty(), role.map(selectedRole -> ImmutableMap.of(HIVE_CATALOG, selectedRole)) - .orElse(ImmutableMap.of()))) + .orElse(ImmutableMap.of()), + ImmutableMap.of())) .setCatalog(HIVE_CATALOG) .setSchema(TPCH_SCHEMA) .build(); @@ -210,7 +255,8 @@ public static Session createBucketedSession(Optional role) "hive", Optional.empty(), role.map(selectedRole -> ImmutableMap.of(HIVE_BUCKETED_CATALOG, selectedRole)) - .orElse(ImmutableMap.of()))) + .orElse(ImmutableMap.of()), + ImmutableMap.of())) .setCatalog(HIVE_BUCKETED_CATALOG) .setSchema(TPCH_BUCKETED_SCHEMA) .build(); @@ -223,7 +269,8 @@ public static Session createMaterializeExchangesSession(Optional r "hive", Optional.empty(), role.map(selectedRole -> ImmutableMap.of("hive", selectedRole)) - .orElse(ImmutableMap.of()))) + .orElse(ImmutableMap.of()), + ImmutableMap.of())) .setSystemProperty(PARTITIONING_PROVIDER_CATALOG, HIVE_CATALOG) .setSystemProperty(EXCHANGE_MATERIALIZATION_STRATEGY, ExchangeMaterializationStrategy.ALL.toString()) .setSystemProperty(HASH_PARTITION_COUNT, "13") diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java b/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java index 0760675645ca9..9d65f6d74f048 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java @@ -15,9 +15,17 @@ import com.facebook.presto.PagesIndexPageSorter; import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.cost.ConnectorFilterStatsCalculatorService; +import com.facebook.presto.cost.FilterStatsCalculator; +import com.facebook.presto.cost.ScalarStatsCalculator; +import com.facebook.presto.cost.StatsNormalizer; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; +import com.facebook.presto.hive.gcs.HiveGcsConfig; +import com.facebook.presto.hive.gcs.HiveGcsConfigurationInitializer; import com.facebook.presto.hive.orc.DwrfBatchPageSourceFactory; +import com.facebook.presto.hive.orc.DwrfSelectivePageSourceFactory; import com.facebook.presto.hive.orc.OrcBatchPageSourceFactory; +import com.facebook.presto.hive.orc.OrcSelectivePageSourceFactory; import com.facebook.presto.hive.parquet.ParquetPageSourceFactory; import com.facebook.presto.hive.rcfile.RcFilePageSourceFactory; import com.facebook.presto.hive.s3.HiveS3Config; @@ -25,14 +33,18 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.operator.PagesIndex; +import com.facebook.presto.orc.StorageStripeMetadataSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.FilterStatsCalculatorService; import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.PredicateCompiler; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.MapType; @@ -43,6 +55,7 @@ import com.facebook.presto.spi.type.TypeSignatureParameter; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.RowExpressionPredicateCompiler; +import com.facebook.presto.sql.planner.planPrinter.RowExpressionFormatter; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; @@ -75,7 +88,8 @@ private HiveTestUtils() public static final StandardFunctionResolution FUNCTION_RESOLUTION = new FunctionResolution(METADATA.getFunctionManager()); - public static final RowExpressionService ROW_EXPRESSION_SERVICE = new RowExpressionService() { + public static final RowExpressionService ROW_EXPRESSION_SERVICE = new RowExpressionService() + { @Override public DomainTranslator getDomainTranslator() { @@ -99,51 +113,71 @@ public DeterminismEvaluator getDeterminismEvaluator() { return new RowExpressionDeterminismEvaluator(METADATA); } + + @Override + public String formatRowExpression(ConnectorSession session, RowExpression expression) + { + return new RowExpressionFormatter(METADATA.getFunctionManager()).formatRowExpression(session, expression); + } }; + public static final FilterStatsCalculatorService FILTER_STATS_CALCULATOR_SERVICE = new ConnectorFilterStatsCalculatorService( + new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA), new StatsNormalizer())); + static { // associate TYPE_MANAGER with a function manager new FunctionManager(TYPE_MANAGER, new BlockEncodingManager(TYPE_MANAGER), new FeaturesConfig()); } public static final HiveClientConfig HIVE_CLIENT_CONFIG = new HiveClientConfig(); + public static final MetastoreClientConfig METASTORE_CLIENT_CONFIG = new MetastoreClientConfig(); - public static final HdfsEnvironment HDFS_ENVIRONMENT = createTestHdfsEnvironment(HIVE_CLIENT_CONFIG); + public static final HdfsEnvironment HDFS_ENVIRONMENT = createTestHdfsEnvironment(HIVE_CLIENT_CONFIG, METASTORE_CLIENT_CONFIG); public static final PageSorter PAGE_SORTER = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)); - public static Set getDefaultHiveDataStreamFactories(HiveClientConfig hiveClientConfig) + public static Set getDefaultHiveBatchPageSourceFactories(HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig) { FileFormatDataSourceStats stats = new FileFormatDataSourceStats(); - HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); + HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig, metastoreClientConfig); return ImmutableSet.builder() - .add(new RcFilePageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats)) - .add(new OrcBatchPageSourceFactory(TYPE_MANAGER, hiveClientConfig, testHdfsEnvironment, stats)) - .add(new DwrfBatchPageSourceFactory(TYPE_MANAGER, hiveClientConfig, testHdfsEnvironment, stats)) - .add(new ParquetPageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats)) + .add(new RcFilePageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats, new HadoopFileOpener())) + .add(new OrcBatchPageSourceFactory(TYPE_MANAGER, hiveClientConfig, testHdfsEnvironment, stats, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())) + .add(new DwrfBatchPageSourceFactory(TYPE_MANAGER, hiveClientConfig, testHdfsEnvironment, stats, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())) + .add(new ParquetPageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats, new HadoopFileOpener())) + .build(); + } + + public static Set getDefaultHiveSelectivePageSourceFactories(HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig) + { + FileFormatDataSourceStats stats = new FileFormatDataSourceStats(); + HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig, metastoreClientConfig); + return ImmutableSet.builder() + .add(new OrcSelectivePageSourceFactory(TYPE_MANAGER, FUNCTION_RESOLUTION, ROW_EXPRESSION_SERVICE, hiveClientConfig, testHdfsEnvironment, stats, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())) + .add(new DwrfSelectivePageSourceFactory(TYPE_MANAGER, FUNCTION_RESOLUTION, ROW_EXPRESSION_SERVICE, hiveClientConfig, testHdfsEnvironment, stats, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())) .build(); } - public static Set getDefaultHiveRecordCursorProvider(HiveClientConfig hiveClientConfig) + public static Set getDefaultHiveRecordCursorProvider(HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig) { - HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); + HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig, metastoreClientConfig); return ImmutableSet.builder() .add(new GenericHiveRecordCursorProvider(testHdfsEnvironment)) .build(); } - public static Set getDefaultHiveFileWriterFactories(HiveClientConfig hiveClientConfig) + public static Set getDefaultHiveFileWriterFactories(HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig) { - HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); + HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig, metastoreClientConfig); return ImmutableSet.builder() .add(new RcFileFileWriterFactory(testHdfsEnvironment, TYPE_MANAGER, new NodeVersion("test_version"), hiveClientConfig, new FileFormatDataSourceStats())) - .add(getDefaultOrcFileWriterFactory(hiveClientConfig)) + .add(getDefaultOrcFileWriterFactory(hiveClientConfig, metastoreClientConfig)) .build(); } - public static OrcFileWriterFactory getDefaultOrcFileWriterFactory(HiveClientConfig hiveClientConfig) + public static OrcFileWriterFactory getDefaultOrcFileWriterFactory(HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig) { - HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); + HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig, metastoreClientConfig); return new OrcFileWriterFactory( testHdfsEnvironment, TYPE_MANAGER, @@ -162,10 +196,16 @@ public static List getTypes(List columnHandles) return types.build(); } - public static HdfsEnvironment createTestHdfsEnvironment(HiveClientConfig config) + public static HdfsEnvironment createTestHdfsEnvironment(HiveClientConfig config, MetastoreClientConfig metastoreClientConfig) { - HdfsConfiguration hdfsConfig = new HiveHdfsConfiguration(new HdfsConfigurationUpdater(config, new PrestoS3ConfigurationUpdater(new HiveS3Config()))); - return new HdfsEnvironment(hdfsConfig, config, new NoHdfsAuthentication()); + HdfsConfiguration hdfsConfig = new HiveHdfsConfiguration( + new HdfsConfigurationInitializer( + config, + metastoreClientConfig, + new PrestoS3ConfigurationUpdater(new HiveS3Config()), + new HiveGcsConfigurationInitializer(new HiveGcsConfig())), + ImmutableSet.of()); + return new HdfsEnvironment(hdfsConfig, metastoreClientConfig, new NoHdfsAuthentication()); } public static MapType mapType(Type keyType, Type valueType) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestBackgroundHiveSplitLoader.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestBackgroundHiveSplitLoader.java index 482e6e736af39..d97763c558f38 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestBackgroundHiveSplitLoader.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestBackgroundHiveSplitLoader.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.hive.HiveBucketing.HiveBucketFilter; import com.facebook.presto.hive.HiveColumnHandle.ColumnType; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; @@ -21,15 +22,12 @@ import com.facebook.presto.hive.metastore.Table; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.SchemaTableName; -import com.facebook.presto.spi.Subfield; import com.facebook.presto.spi.predicate.Domain; -import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.testing.TestingConnectorSession; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.stats.CounterStat; import io.airlift.units.DataSize; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; @@ -39,7 +37,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; import org.testng.annotations.Test; @@ -47,26 +44,23 @@ import java.net.URI; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.concurrent.Executor; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.hive.BackgroundHiveSplitLoader.BucketSplitInfo.createBucketSplitInfo; -import static com.facebook.presto.hive.HiveColumnHandle.PATH_COLUMN_NAME; -import static com.facebook.presto.hive.HiveColumnHandle.pathColumnHandle; +import static com.facebook.presto.hive.HiveFileInfo.createHiveFileInfo; import static com.facebook.presto.hive.HiveTestUtils.SESSION; import static com.facebook.presto.hive.HiveType.HIVE_INT; import static com.facebook.presto.hive.HiveType.HIVE_STRING; import static com.facebook.presto.hive.HiveUtil.getRegularColumnHandles; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; +import static com.facebook.presto.hive.util.HiveFileIterator.NestedDirectoryPolicy; import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; -import static com.facebook.presto.spi.predicate.TupleDomain.withColumnDomains; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; @@ -88,11 +82,9 @@ public class TestBackgroundHiveSplitLoader private static final Domain RETURNED_PATH_DOMAIN = Domain.singleValue(VARCHAR, utf8Slice(RETURNED_PATH.toString())); - private static final TupleDomain RETURNED_PATH_TUPLE_DOMAIN = withColumnDomains(ImmutableMap.of(new Subfield(PATH_COLUMN_NAME, ImmutableList.of()), RETURNED_PATH_DOMAIN)); - - private static final List TEST_FILES = ImmutableList.of( - locatedFileStatus(RETURNED_PATH), - locatedFileStatus(FILTERED_PATH)); + private static final List TEST_FILES = ImmutableList.of( + createHiveFileInfo(locatedFileStatus(RETURNED_PATH), Optional.empty()), + createHiveFileInfo(locatedFileStatus(FILTERED_PATH), Optional.empty())); private static final List PARTITION_COLUMNS = ImmutableList.of( new Column("partitionColumn", HIVE_INT, Optional.empty())); @@ -113,7 +105,7 @@ public void testNoPathFilter() TEST_FILES, Optional.empty()); - HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, TupleDomain.all(), ImmutableMap.of()); + HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader); backgroundHiveSplitLoader.start(hiveSplitSource); assertEquals(drain(hiveSplitSource).size(), 2); @@ -127,7 +119,7 @@ public void testPathFilter() TEST_FILES, Optional.of(RETURNED_PATH_DOMAIN)); - HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, RETURNED_PATH_TUPLE_DOMAIN, ImmutableMap.of(PATH_COLUMN_NAME, pathColumnHandle())); + HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader); backgroundHiveSplitLoader.start(hiveSplitSource); List paths = drain(hiveSplitSource); assertEquals(paths.size(), 1); @@ -145,7 +137,7 @@ public void testPathFilterOneBucketMatchPartitionedTable() PARTITIONED_TABLE, Optional.of(new HiveBucketHandle(BUCKET_COLUMN_HANDLES, BUCKET_COUNT, BUCKET_COUNT))); - HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, RETURNED_PATH_TUPLE_DOMAIN, ImmutableMap.of(PATH_COLUMN_NAME, pathColumnHandle())); + HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader); backgroundHiveSplitLoader.start(hiveSplitSource); List paths = drain(hiveSplitSource); assertEquals(paths.size(), 1); @@ -167,7 +159,7 @@ public void testPathFilterBucketedPartitionedTable() BUCKET_COUNT, BUCKET_COUNT))); - HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, RETURNED_PATH_TUPLE_DOMAIN, ImmutableMap.of(PATH_COLUMN_NAME, pathColumnHandle())); + HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader); backgroundHiveSplitLoader.start(hiveSplitSource); List paths = drain(hiveSplitSource); assertEquals(paths.size(), 1); @@ -179,10 +171,10 @@ public void testEmptyFileWithNoBlocks() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( - ImmutableList.of(locatedFileStatusWithNoBlocks(RETURNED_PATH)), + ImmutableList.of(createHiveFileInfo(locatedFileStatusWithNoBlocks(RETURNED_PATH), Optional.empty())), Optional.empty()); - HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, TupleDomain.none(), ImmutableMap.of()); + HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader); backgroundHiveSplitLoader.start(hiveSplitSource); List splits = drainSplits(hiveSplitSource); @@ -196,7 +188,7 @@ public void testNoHangIfPartitionIsOffline() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoaderOfflinePartitions(); - HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, TupleDomain.all(), ImmutableMap.of()); + HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader); backgroundHiveSplitLoader.start(hiveSplitSource); assertThrows(RuntimeException.class, () -> drain(hiveSplitSource)); @@ -225,7 +217,7 @@ private static List drainSplits(HiveSplitSource source) } private static BackgroundHiveSplitLoader backgroundHiveSplitLoader( - List files, + List files, Optional pathDomain) { return backgroundHiveSplitLoader( @@ -237,7 +229,7 @@ private static BackgroundHiveSplitLoader backgroundHiveSplitLoader( } private static BackgroundHiveSplitLoader backgroundHiveSplitLoader( - List files, + List files, Optional pathDomain, Optional hiveBucketFilter, Table table, @@ -316,18 +308,12 @@ protected HivePartitionMetadata computeNext() }; } - private static HiveSplitSource hiveSplitSource( - BackgroundHiveSplitLoader backgroundHiveSplitLoader, - TupleDomain effectivePredicate, - Map predicateColumns) + private static HiveSplitSource hiveSplitSource(BackgroundHiveSplitLoader backgroundHiveSplitLoader) { return HiveSplitSource.allAtOnce( SESSION, SIMPLE_TABLE.getDatabaseName(), SIMPLE_TABLE.getTableName(), - effectivePredicate, - TRUE_CONSTANT, - predicateColumns, 1, 1, new DataSize(32, MEGABYTE), @@ -399,32 +385,17 @@ private static LocatedFileStatus locatedFileStatusWithNoBlocks(Path path) private static class TestingDirectoryLister implements DirectoryLister { - private final List files; + private final List files; - public TestingDirectoryLister(List files) + public TestingDirectoryLister(List files) { this.files = files; } @Override - public RemoteIterator list(FileSystem fs, Path path) + public Iterator list(FileSystem fs, Path path, NamenodeStats namenodeStats, NestedDirectoryPolicy nestedDirectoryPolicy) { - return new RemoteIterator() - { - private final Iterator iterator = files.iterator(); - - @Override - public boolean hasNext() - { - return iterator.hasNext(); - } - - @Override - public LocatedFileStatus next() - { - return iterator.next(); - } - }; + return files.iterator(); } } @@ -434,8 +405,8 @@ private static class TestingHdfsEnvironment public TestingHdfsEnvironment() { super( - new HiveHdfsConfiguration(new HdfsConfigurationUpdater(new HiveClientConfig())), - new HiveClientConfig(), + new HiveHdfsConfiguration(new HdfsConfigurationInitializer(new HiveClientConfig(), new MetastoreClientConfig()), ImmutableSet.of()), + new MetastoreClientConfig(), new NoHdfsAuthentication()); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestCoercingFilters.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestCoercingFilters.java new file mode 100644 index 0000000000000..1f4db0122256c --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestCoercingFilters.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.hive.HiveCoercer.IntegerNumberToVarcharCoercer; +import com.facebook.presto.hive.HiveCoercer.VarcharToIntegerNumberCoercer; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.TupleDomainFilter.BigintRange; +import com.facebook.presto.orc.TupleDomainFilter.BytesRange; +import com.facebook.presto.spi.Subfield; +import org.testng.annotations.Test; + +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestCoercingFilters +{ + @Test + public void testIntegerToVarchar() + { + TupleDomainFilter filter = BytesRange.of("10".getBytes(), false, "10".getBytes(), false, false); + + HiveCoercer coercer = new IntegerNumberToVarcharCoercer(INTEGER, VARCHAR); + + TupleDomainFilter coercingFilter = coercer.toCoercingFilter(filter, new Subfield("c")); + + assertTrue(coercingFilter.testLong(10)); + assertFalse(coercingFilter.testLong(25)); + assertFalse(coercingFilter.testNull()); + } + + @Test + public void testVarcharToInteger() + { + TupleDomainFilter filter = BigintRange.of(100, Integer.MAX_VALUE, false); + + HiveCoercer coercer = new VarcharToIntegerNumberCoercer(VARCHAR, INTEGER); + + TupleDomainFilter coercingFilter = coercer.toCoercingFilter(filter, new Subfield("c")); + + assertTrue(coercingFilter.testLength(1)); + assertTrue(coercingFilter.testLength(2)); + assertTrue(coercingFilter.testLength(3)); + + assertTrue(coercingFilter.testBytes("100".getBytes(), 0, 3)); + assertTrue(coercingFilter.testBytes("145".getBytes(), 0, 3)); + assertTrue(coercingFilter.testBytes("2147483647".getBytes(), 0, 10)); + + assertFalse(coercingFilter.testBytes("50".getBytes(), 0, 2)); + assertFalse(coercingFilter.testBytes("-50".getBytes(), 0, 3)); + + // parsing error + assertFalse(coercingFilter.testBytes("abc".getBytes(), 0, 3)); + + // out of range + assertFalse(coercingFilter.testBytes("2147483648".getBytes(), 0, 10)); + + assertFalse(coercingFilter.testNull()); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestDomainTranslator.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestDomainTranslator.java index 9156e22a37244..0491f2da8b45a 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestDomainTranslator.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestDomainTranslator.java @@ -14,15 +14,18 @@ package com.facebook.presto.hive; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Subfield; -import com.facebook.presto.spi.block.TestingSession; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.Range; +import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.predicate.ValueSet; import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.DomainTranslator.ExtractionResult; +import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -32,6 +35,7 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; +import com.facebook.presto.testing.TestingConnectorSession; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.slice.Slices; @@ -40,14 +44,20 @@ import java.util.List; import java.util.Map; +import java.util.function.Function; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.block.BlockAssertions.createArrayBigintBlock; +import static com.facebook.presto.block.BlockAssertions.createMapBlock; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.hive.HiveTestUtils.mapType; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.spi.function.OperatorType.SUBSCRIPT; import static com.facebook.presto.spi.predicate.TupleDomain.withColumnDomains; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.spi.relation.ConstantExpression.createConstantExpression; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.DEREFERENCE; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IN; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IS_NULL; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.IntegerType.INTEGER; @@ -56,7 +66,6 @@ import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.specialForm; -import static com.facebook.presto.util.Reflection.methodHandle; import static com.google.common.collect.ImmutableList.toImmutableList; import static org.testng.Assert.assertEquals; @@ -73,9 +82,24 @@ public class TestDomainTranslator RowType.field("d", mapType(BIGINT, BIGINT)), RowType.field("e", mapType(VARCHAR, BIGINT))))); + private static final ExpressionOptimizer TEST_EXPRESSION_OPTIMIZER = new ExpressionOptimizer() + { + @Override + public RowExpression optimize(RowExpression rowExpression, Level level, ConnectorSession session) + { + return rowExpression; + } + + @Override + public Object optimize(RowExpression expression, Level level, ConnectorSession session, Function variableResolver) + { + throw new UnsupportedOperationException(); + } + }; + private Metadata metadata; private RowExpressionDomainTranslator domainTranslator; - private SubfieldExtractor columnExtractor; + private DomainTranslator.ColumnExtractor columnExtractor; @BeforeClass public void setup() @@ -84,8 +108,12 @@ public void setup() domainTranslator = new RowExpressionDomainTranslator(metadata); columnExtractor = new SubfieldExtractor( new FunctionResolution(metadata.getFunctionManager()), - (rowExpression, level, session) -> rowExpression, - TestingSession.SESSION); + TEST_EXPRESSION_OPTIMIZER, + new TestingConnectorSession( + new HiveSessionProperties( + new HiveClientConfig().setRangeFiltersOnSubscriptsEnabled(true), + new OrcFileWriterConfig(), + new ParquetFileWriterConfig()).getSessionProperties())).toColumnExtractor(); } @Test @@ -94,12 +122,12 @@ public void testSubfields() Map expressions = ImmutableMap.builder() .put("c_bigint", C_BIGINT) .put("c_bigint_array[5]", arraySubscript(C_BIGINT_ARRAY, 5)) - .put("c_bigint_to_bigint_map[5]", mapSubscript(C_BIGINT_TO_BIGINT_MAP, constant(5, BIGINT))) + .put("c_bigint_to_bigint_map[5]", mapSubscript(C_BIGINT_TO_BIGINT_MAP, constant(5L, BIGINT))) .put("c_varchar_to_bigint_map[\"foo\"]", mapSubscript(C_VARCHAR_TO_BIGINT_MAP, constant(Slices.utf8Slice("foo"), VARCHAR))) .put("c_struct.a", dereference(C_STRUCT, 0)) .put("c_struct.b.x", dereference(dereference(C_STRUCT, 1), 0)) .put("c_struct.c[5]", arraySubscript(dereference(C_STRUCT, 2), 5)) - .put("c_struct.d[5]", mapSubscript(dereference(C_STRUCT, 3), constant(5, BIGINT))) + .put("c_struct.d[5]", mapSubscript(dereference(C_STRUCT, 3), constant(5L, BIGINT))) .put("c_struct.e[\"foo\"]", mapSubscript(dereference(C_STRUCT, 4), constant(Slices.utf8Slice("foo"), VARCHAR))) .build(); @@ -117,12 +145,32 @@ public void testSubfields() assertPredicateTranslates(bigintIn(expression, ImmutableList.of(1L, 2L)), subfield, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L), Range.equal(BIGINT, 2L)), false)); } + + Type arrayType = C_BIGINT_ARRAY.getType(); + assertPredicateTranslates(isNull(C_BIGINT_ARRAY), C_BIGINT_ARRAY.getName(), Domain.create(ValueSet.none(arrayType), true)); + assertPredicateTranslates(not(isNull(C_BIGINT_ARRAY)), C_BIGINT_ARRAY.getName(), Domain.create(ValueSet.all(arrayType), false)); + assertPredicateDoesNotTranslate(equal(C_BIGINT_ARRAY, createConstantExpression(createArrayBigintBlock(ImmutableList.of(ImmutableList.of(1L, 2L, 3L))), arrayType))); + + MapType mapType = (MapType) C_BIGINT_TO_BIGINT_MAP.getType(); + assertPredicateTranslates(isNull(C_BIGINT_TO_BIGINT_MAP), C_BIGINT_TO_BIGINT_MAP.getName(), Domain.create(ValueSet.none(mapType), true)); + assertPredicateTranslates(not(isNull(C_BIGINT_TO_BIGINT_MAP)), C_BIGINT_TO_BIGINT_MAP.getName(), Domain.create(ValueSet.all(mapType), false)); + assertPredicateDoesNotTranslate(equal(C_BIGINT_TO_BIGINT_MAP, createConstantExpression(createMapBlock(mapType, ImmutableMap.of(1, 100)), mapType))); } private RowExpression dereference(RowExpression base, int field) { Type fieldType = base.getType().getTypeParameters().get(field); - return specialForm(DEREFERENCE, fieldType, ImmutableList.of(base, new ConstantExpression(field, INTEGER))); + return specialForm(DEREFERENCE, fieldType, ImmutableList.of(base, new ConstantExpression((long) field, INTEGER))); + } + + private RowExpression isNull(RowExpression expression) + { + return specialForm(IS_NULL, BOOLEAN, expression); + } + + private RowExpression not(RowExpression expression) + { + return call("not", new FunctionResolution(metadata.getFunctionManager()).notFunction(), BOOLEAN, expression); } private RowExpression arraySubscript(RowExpression arrayExpression, int index) @@ -132,7 +180,7 @@ private RowExpression arraySubscript(RowExpression arrayExpression, int index) return call(SUBSCRIPT.name(), operator(SUBSCRIPT, arrayType, elementType), elementType, - ImmutableList.of(arrayExpression, constant(index, INTEGER))); + ImmutableList.of(arrayExpression, constant((long) index, INTEGER))); } private RowExpression mapSubscript(RowExpression mapExpression, RowExpression keyExpression) @@ -144,22 +192,6 @@ private RowExpression mapSubscript(RowExpression mapExpression, RowExpression ke ImmutableList.of(mapExpression, keyExpression)); } - private static MapType mapType(Type keyType, Type valueType) - { - return new MapType( - keyType, - valueType, - methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException"), - methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException"), - methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException"), - methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException")); - } - - public static void throwUnsupportedOperationException() - { - throw new UnsupportedOperationException(); - } - private FunctionHandle operator(OperatorType operatorType, Type... types) { return metadata.getFunctionManager().resolveOperator(operatorType, fromTypes(types)); @@ -172,6 +204,13 @@ private void assertPredicateTranslates(RowExpression predicate, String subfield, assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(new Subfield(subfield), domain))); } + private void assertPredicateDoesNotTranslate(RowExpression predicate) + { + ExtractionResult result = domainTranslator.fromPredicate(TEST_SESSION.toConnectorSession(), predicate, columnExtractor); + assertEquals(result.getTupleDomain(), TupleDomain.all()); + assertEquals(result.getRemainingExpression(), predicate); + } + private RowExpression greaterThan(RowExpression left, RowExpression right) { return binaryOperator(OperatorType.GREATER_THAN, left, right); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestFileSystemCache.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestFileSystemCache.java index 177d72503a24e..db7362fcbe5f6 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestFileSystemCache.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestFileSystemCache.java @@ -15,6 +15,7 @@ import com.facebook.presto.hive.authentication.ImpersonatingHdfsAuthentication; import com.facebook.presto.hive.authentication.SimpleHadoopAuthentication; +import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -34,8 +35,8 @@ public void testFileSystemCache() ImpersonatingHdfsAuthentication auth = new ImpersonatingHdfsAuthentication(new SimpleHadoopAuthentication()); HdfsEnvironment environment = new HdfsEnvironment( - new HiveHdfsConfiguration(new HdfsConfigurationUpdater(new HiveClientConfig())), - new HiveClientConfig(), + new HiveHdfsConfiguration(new HdfsConfigurationInitializer(new HiveClientConfig(), new MetastoreClientConfig()), ImmutableSet.of()), + new MetastoreClientConfig(), auth); FileSystem fs1 = getFileSystem(environment, "user"); FileSystem fs2 = getFileSystem(environment, "user"); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java index e57c96fdb7e72..ef1b04f862d9d 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java @@ -13,14 +13,13 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.facebook.presto.hive.HiveClientConfig.HdfsAuthenticationType; import com.facebook.presto.hive.HiveClientConfig.HiveMetastoreAuthenticationType; import com.facebook.presto.hive.s3.S3FileSystemType; import com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.net.HostAndPort; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import io.airlift.units.Duration; @@ -49,13 +48,6 @@ public void testDefaults() .setMaxOutstandingSplitsSize(new DataSize(256, Unit.MEGABYTE)) .setMaxSplitIteratorThreads(1_000) .setAllowCorruptWritesForTesting(false) - .setMetastoreCacheTtl(new Duration(0, TimeUnit.SECONDS)) - .setMetastoreRefreshInterval(new Duration(0, TimeUnit.SECONDS)) - .setMetastoreCacheMaximumSize(10000) - .setPerTransactionMetastoreCacheMaximumSize(1000) - .setMaxMetastoreRefreshThreads(100) - .setMetastoreSocksProxy(null) - .setMetastoreTimeout(new Duration(10, TimeUnit.SECONDS)) .setMinPartitionBatchSize(10) .setMaxPartitionBatchSize(100) .setMaxInitialSplits(200) @@ -71,7 +63,6 @@ public void testDefaults() .setIpcPingInterval(new Duration(10, TimeUnit.SECONDS)) .setDfsConnectTimeout(new Duration(500, TimeUnit.MILLISECONDS)) .setDfsConnectMaxRetries(5) - .setVerifyChecksum(true) .setDomainSocketPath(null) .setS3FileSystemType(S3FileSystemType.PRESTO) .setResourceConfigFiles("") @@ -86,6 +77,7 @@ public void testDefaults() .setTextMaxLineLength(new DataSize(100, Unit.MEGABYTE)) .setUseParquetColumnNames(false) .setFailOnCorruptedParquetStatistics(true) + .setParquetMaxReadBlockSize(new DataSize(16, Unit.MEGABYTE)) .setUseOrcColumnNames(false) .setAssumeCanonicalPartitionKeys(false) .setOrcBloomFiltersEnabled(false) @@ -109,6 +101,8 @@ public void testDefaults() .setBucketExecutionEnabled(true) .setIgnoreTableBucketing(false) .setMaxBucketsForGroupedExecution(1_000_000) + .setSortedWriteToTempPathEnabled(false) + .setSortedWriteTempPathSubdirectoryCount(10) .setFileSystemMaxCacheSize(1000) .setTableStatisticsEnabled(true) .setOptimizeMismatchedBucketCount(false) @@ -117,9 +111,6 @@ public void testDefaults() .setHdfsWireEncryptionEnabled(false) .setPartitionStatisticsSampleSize(100) .setIgnoreCorruptedStatistics(false) - .setRecordingPath(null) - .setRecordingDuration(new Duration(0, TimeUnit.MINUTES)) - .setReplay(false) .setCollectColumnStatisticsOnWrite(false) .setCollectColumnStatisticsOnWrite(false) .setS3SelectPushdownEnabled(false) @@ -129,7 +120,9 @@ public void testDefaults() .setTemporaryTableSchema("default") .setTemporaryTableStorageFormat(ORC) .setTemporaryTableCompressionCodec(SNAPPY) - .setPushdownFilterEnabled(false)); + .setPushdownFilterEnabled(false) + .setZstdJniDecompressionEnabled(false) + .setRangeFiltersOnSubscriptsEnabled(false)); } @Test @@ -143,20 +136,12 @@ public void testExplicitPropertyMappings() .put("hive.max-outstanding-splits-size", "32MB") .put("hive.max-split-iterator-threads", "10") .put("hive.allow-corrupt-writes-for-testing", "true") - .put("hive.metastore-cache-ttl", "2h") - .put("hive.metastore-refresh-interval", "30m") - .put("hive.metastore-cache-maximum-size", "5000") - .put("hive.per-transaction-metastore-cache-maximum-size", "500") - .put("hive.metastore-refresh-max-threads", "2500") - .put("hive.metastore.thrift.client.socks-proxy", "localhost:1080") - .put("hive.metastore-timeout", "20s") .put("hive.metastore.partition-batch-size.min", "1") .put("hive.metastore.partition-batch-size.max", "1000") .put("hive.dfs.ipc-ping-interval", "34s") .put("hive.dfs-timeout", "33s") .put("hive.dfs.connect.timeout", "20s") .put("hive.dfs.connect.max-retries", "10") - .put("hive.dfs.verify-checksum", "false") .put("hive.dfs.domain-socket-path", "/foo") .put("hive.s3-file-system-type", "EMRFS") .put("hive.config.resources", "/foo.xml,/bar.xml") @@ -180,6 +165,7 @@ public void testExplicitPropertyMappings() .put("hive.text.max-line-length", "13MB") .put("hive.parquet.use-column-names", "true") .put("hive.parquet.fail-on-corrupted-statistics", "false") + .put("hive.parquet.max-read-block-size", "66kB") .put("hive.orc.use-column-names", "true") .put("hive.orc.bloom-filters.enabled", "true") .put("hive.orc.default-bloom-filter-fpp", "0.96") @@ -203,6 +189,8 @@ public void testExplicitPropertyMappings() .put("hive.sorted-writing", "false") .put("hive.ignore-table-bucketing", "true") .put("hive.max-buckets-for-grouped-execution", "100") + .put("hive.sorted-write-to-temp-path-enabled", "true") + .put("hive.sorted-write-temp-path-subdirectory-count", "50") .put("hive.fs.cache.max-size", "1010") .put("hive.table-statistics-enabled", "false") .put("hive.optimize-mismatched-bucket-count", "true") @@ -211,9 +199,6 @@ public void testExplicitPropertyMappings() .put("hive.hdfs.wire-encryption.enabled", "true") .put("hive.partition-statistics-sample-size", "1234") .put("hive.ignore-corrupted-statistics", "true") - .put("hive.metastore-recording-path", "/foo/bar") - .put("hive.metastore-recoding-duration", "42s") - .put("hive.replay-metastore-recording", "true") .put("hive.collect-column-statistics-on-write", "true") .put("hive.s3select-pushdown.enabled", "true") .put("hive.s3select-pushdown.max-connections", "1234") @@ -223,6 +208,8 @@ public void testExplicitPropertyMappings() .put("hive.temporary-table-storage-format", "DWRF") .put("hive.temporary-table-compression-codec", "NONE") .put("hive.pushdown-filter-enabled", "true") + .put("hive.range-filters-on-subscripts-enabled", "true") + .put("hive.zstd-jni-decompression-enabled", "true") .build(); HiveClientConfig expected = new HiveClientConfig() @@ -233,13 +220,6 @@ public void testExplicitPropertyMappings() .setMaxOutstandingSplitsSize(new DataSize(32, Unit.MEGABYTE)) .setMaxSplitIteratorThreads(10) .setAllowCorruptWritesForTesting(true) - .setMetastoreCacheTtl(new Duration(2, TimeUnit.HOURS)) - .setMetastoreRefreshInterval(new Duration(30, TimeUnit.MINUTES)) - .setMetastoreCacheMaximumSize(5000) - .setPerTransactionMetastoreCacheMaximumSize(500) - .setMaxMetastoreRefreshThreads(2500) - .setMetastoreSocksProxy(HostAndPort.fromParts("localhost", 1080)) - .setMetastoreTimeout(new Duration(20, TimeUnit.SECONDS)) .setMinPartitionBatchSize(1) .setMaxPartitionBatchSize(1000) .setMaxInitialSplits(10) @@ -255,7 +235,6 @@ public void testExplicitPropertyMappings() .setDfsTimeout(new Duration(33, TimeUnit.SECONDS)) .setDfsConnectTimeout(new Duration(20, TimeUnit.SECONDS)) .setDfsConnectMaxRetries(10) - .setVerifyChecksum(false) .setResourceConfigFiles(ImmutableList.of("/foo.xml", "/bar.xml")) .setHiveStorageFormat(HiveStorageFormat.SEQUENCEFILE) .setCompressionCodec(HiveCompressionCodec.NONE) @@ -269,6 +248,7 @@ public void testExplicitPropertyMappings() .setTextMaxLineLength(new DataSize(13, Unit.MEGABYTE)) .setUseParquetColumnNames(true) .setFailOnCorruptedParquetStatistics(false) + .setParquetMaxReadBlockSize(new DataSize(66, Unit.KILOBYTE)) .setUseOrcColumnNames(true) .setAssumeCanonicalPartitionKeys(true) .setOrcBloomFiltersEnabled(true) @@ -293,6 +273,8 @@ public void testExplicitPropertyMappings() .setSortedWritingEnabled(false) .setIgnoreTableBucketing(true) .setMaxBucketsForGroupedExecution(100) + .setSortedWriteToTempPathEnabled(true) + .setSortedWriteTempPathSubdirectoryCount(50) .setFileSystemMaxCacheSize(1010) .setTableStatisticsEnabled(false) .setOptimizeMismatchedBucketCount(true) @@ -301,9 +283,6 @@ public void testExplicitPropertyMappings() .setHdfsWireEncryptionEnabled(true) .setPartitionStatisticsSampleSize(1234) .setIgnoreCorruptedStatistics(true) - .setRecordingPath("/foo/bar") - .setRecordingDuration(new Duration(42, TimeUnit.SECONDS)) - .setReplay(true) .setCollectColumnStatisticsOnWrite(true) .setCollectColumnStatisticsOnWrite(true) .setS3SelectPushdownEnabled(true) @@ -313,7 +292,9 @@ public void testExplicitPropertyMappings() .setTemporaryTableSchema("other") .setTemporaryTableStorageFormat(DWRF) .setTemporaryTableCompressionCodec(NONE) - .setPushdownFilterEnabled(true); + .setPushdownFilterEnabled(true) + .setZstdJniDecompressionEnabled(true) + .setRangeFiltersOnSubscriptsEnabled(true); ConfigAssertions.assertFullMapping(properties, expected); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientFileMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientFileMetastore.java index e280b74b0b284..db19750b6961b 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientFileMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientFileMetastore.java @@ -16,6 +16,7 @@ import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.file.FileHiveMetastore; +import com.google.common.collect.ImmutableSet; import org.testng.SkipException; import java.io.File; @@ -28,9 +29,10 @@ protected ExtendedHiveMetastore createMetastore(File tempDir) { File baseDir = new File(tempDir, "metastore"); HiveClientConfig hiveConfig = new HiveClientConfig(); - HdfsConfigurationUpdater updator = new HdfsConfigurationUpdater(hiveConfig); - HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(updator); - HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveConfig, new NoHdfsAuthentication()); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + HdfsConfigurationInitializer updator = new HdfsConfigurationInitializer(hiveConfig, metastoreClientConfig); + HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(updator, ImmutableSet.of()); + HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); return new FileHiveMetastore(hdfsEnvironment, baseDir.toURI().toString(), "test"); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientInMemoryMetastoreWithFilterPushdown.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientInMemoryMetastoreWithFilterPushdown.java new file mode 100644 index 0000000000000..609b93b642ef9 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientInMemoryMetastoreWithFilterPushdown.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; +import com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore; +import com.facebook.presto.hive.metastore.thrift.InMemoryHiveMetastore; +import com.google.common.collect.ImmutableSet; + +import java.io.File; + +import static com.facebook.presto.hive.HiveStorageFormat.DWRF; +import static com.facebook.presto.hive.HiveStorageFormat.ORC; + +public class TestHiveClientInMemoryMetastoreWithFilterPushdown + extends AbstractTestHiveClientLocal +{ + private TestHiveClientInMemoryMetastoreWithFilterPushdown() + { + createTableFormats = ImmutableSet.of(ORC, DWRF); + } + + @Override + protected HiveClientConfig getHiveClientConfig() + { + return super.getHiveClientConfig().setPushdownFilterEnabled(true); + } + + @Override + protected ExtendedHiveMetastore createMetastore(File tempDir) + { + File baseDir = new File(tempDir, "metastore"); + InMemoryHiveMetastore hiveMetastore = new InMemoryHiveMetastore(baseDir); + return new BridgingHiveMetastore(hiveMetastore); + } + + @Override + public void testMetadataDelete() + { + // InMemoryHiveMetastore ignores "removeData" flag in dropPartition + } + + @Override + public void testTransactionDeleteInsert() + { + // InMemoryHiveMetastore does not check whether partition exist in createPartition and dropPartition + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveColumnHandle.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveColumnHandle.java index f9f001aeed720..c94cc113dcf75 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveColumnHandle.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveColumnHandle.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.type.StandardTypes; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; import java.util.Optional; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveConnectorFactory.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveConnectorFactory.java index d300e25bf85f5..620a2b8ac0f32 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveConnectorFactory.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveConnectorFactory.java @@ -25,9 +25,9 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.airlift.testing.Assertions.assertContains; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.spi.transaction.IsolationLevel.READ_UNCOMMITTED; -import static io.airlift.testing.Assertions.assertContains; -import static io.airlift.testing.Assertions.assertInstanceOf; import static org.testng.Assert.fail; public class TestHiveConnectorFactory diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java index 5b9475e076e6e..12d3f7dbd4533 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java @@ -56,12 +56,6 @@ public void testIntersect() // decimal type is not supported by the Hive hash code function } - @Override - public void testNullOnLhsOfInPredicateAllowed() - { - // unknown type is not supported by the Hive hash code function - } - @Override public void testQuantifiedComparison() { @@ -74,12 +68,6 @@ public void testSemiJoin() // decimal type is not supported by the Hive hash code function } - @Override - public void testUnion() - { - // unknown type is not supported by the Hive hash code function - } - @Override public void testUnionRequiringCoercion() { @@ -92,12 +80,6 @@ public void testValues() // decimal type is not supported by the Hive hash code function } - @Override - public void testAntiJoinNullHandling() - { - // Unsupported Hive type: unknown - } - @Test public void testExchangeMaterializationWithConstantFolding() { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithOptimizedRepartitioning.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithOptimizedRepartitioning.java new file mode 100644 index 0000000000000..abcd10852346a --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithOptimizedRepartitioning.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.tests.AbstractTestDistributedQueries; +import com.google.common.collect.ImmutableMap; + +import java.util.Optional; + +import static io.airlift.tpch.TpchTable.getTables; + +public class TestHiveDistributedQueriesWithOptimizedRepartitioning + extends AbstractTestDistributedQueries +{ + public TestHiveDistributedQueriesWithOptimizedRepartitioning() + { + super(() -> HiveQueryRunner.createQueryRunner( + getTables(), + ImmutableMap.of( + "experimental.optimized-repartitioning", "true", + // Use small SerialiedPages to force flushing + "driver.max-page-partitioning-buffer-size", "10000B"), + Optional.empty())); + } + + @Override + protected boolean supportsNotNullColumns() + { + return false; + } + + @Override + public void testDelete() + { + // Hive connector currently does not support row-by-row delete + } + + // Hive specific tests should normally go in TestHiveIntegrationSmokeTest +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java index 7b58c9aebbcbc..53010865b6703 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java @@ -13,19 +13,27 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.metastore.Column; +import com.facebook.presto.hive.metastore.Storage; +import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.hive.orc.DwrfBatchPageSourceFactory; import com.facebook.presto.hive.orc.OrcBatchPageSourceFactory; import com.facebook.presto.hive.parquet.ParquetPageSourceFactory; import com.facebook.presto.hive.rcfile.RcFilePageSourceFactory; import com.facebook.presto.orc.OrcWriterOptions; +import com.facebook.presto.orc.StorageStripeMetadataSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.RecordCursor; import com.facebook.presto.spi.RecordPageSource; +import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.RowType; import com.facebook.presto.testing.TestingConnectorSession; -import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -53,10 +61,11 @@ import java.util.List; import java.util.Optional; import java.util.OptionalInt; -import java.util.Properties; import java.util.TimeZone; import java.util.stream.Collectors; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH; import static com.facebook.presto.hive.HiveStorageFormat.AVRO; import static com.facebook.presto.hive.HiveStorageFormat.DWRF; import static com.facebook.presto.hive.HiveStorageFormat.JSON; @@ -68,19 +77,35 @@ import static com.facebook.presto.hive.HiveStorageFormat.TEXTFILE; import static com.facebook.presto.hive.HiveTestUtils.HDFS_ENVIRONMENT; import static com.facebook.presto.hive.HiveTestUtils.HIVE_CLIENT_CONFIG; +import static com.facebook.presto.hive.HiveTestUtils.ROW_EXPRESSION_SERVICE; import static com.facebook.presto.hive.HiveTestUtils.SESSION; import static com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER; import static com.facebook.presto.hive.HiveTestUtils.getTypes; -import static com.google.common.base.Predicates.not; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_PARTITION_VALUE; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType; +import static com.facebook.presto.tests.StructuralTestUtil.arrayBlockOf; +import static com.facebook.presto.tests.StructuralTestUtil.mapBlockOf; +import static com.facebook.presto.tests.StructuralTestUtil.rowBlockOf; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.filter; -import static com.google.common.collect.Iterables.transform; import static io.airlift.slice.Slices.utf8Slice; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT; -import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB; +import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardListObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector; import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaBooleanObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaDoubleObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaFloatObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaIntObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaLongObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector; +import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaTimestampObjectInspector; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; @@ -177,7 +202,7 @@ public void testRcTextPageSource(int rowCount) assertThatFileFormat(RCTEXT) .withColumns(TEST_COLUMNS) .withRowsCount(rowCount) - .isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS)); + .isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener())); } @Test(dataProvider = "rowCount") @@ -198,7 +223,7 @@ public void testRcTextOptimizedWriter(int rowCount) .withSession(session) .withFileWriterFactory(new RcFileFileWriterFactory(HDFS_ENVIRONMENT, TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE, STATS)) .isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)) - .isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS)); + .isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener())); } @Test(dataProvider = "rowCount") @@ -229,7 +254,7 @@ public void testRcBinaryPageSource(int rowCount) assertThatFileFormat(RCBINARY) .withColumns(testColumns) .withRowsCount(rowCount) - .isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS)); + .isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener())); } @Test(dataProvider = "rowCount") @@ -252,7 +277,7 @@ public void testRcBinaryOptimizedWriter(int rowCount) .withSession(session) .withFileWriterFactory(new RcFileFileWriterFactory(HDFS_ENVIRONMENT, TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE, STATS)) .isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)) - .isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS)); + .isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener())); } @Test(dataProvider = "rowCount") @@ -262,7 +287,7 @@ public void testOrc(int rowCount) assertThatFileFormat(ORC) .withColumns(TEST_COLUMNS) .withRowsCount(rowCount) - .isReadableByPageSource(new OrcBatchPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT, STATS, 100)); + .isReadableByPageSource(new OrcBatchPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT, STATS, 100, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())); } @Test(dataProvider = "rowCount") @@ -288,7 +313,7 @@ public void testOrcOptimizedWriter(int rowCount) .withSession(session) .withFileWriterFactory(new OrcFileWriterFactory(HDFS_ENVIRONMENT, TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE, STATS, new OrcWriterOptions())) .isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)) - .isReadableByPageSource(new OrcBatchPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT, STATS, 100)); + .isReadableByPageSource(new OrcBatchPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT, STATS, 100, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())); } @Test(dataProvider = "rowCount") @@ -302,7 +327,7 @@ public void testOrcUseColumnNames(int rowCount) .withRowsCount(rowCount) .withReadColumns(Lists.reverse(TEST_COLUMNS)) .withSession(session) - .isReadableByPageSource(new OrcBatchPageSourceFactory(TYPE_MANAGER, true, HDFS_ENVIRONMENT, STATS, 100)); + .isReadableByPageSource(new OrcBatchPageSourceFactory(TYPE_MANAGER, true, HDFS_ENVIRONMENT, STATS, 100, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())); } @Test(dataProvider = "rowCount") @@ -334,7 +359,7 @@ public void testParquetPageSource(int rowCount) .withColumns(testColumns) .withSession(parquetPageSourceSession) .withRowsCount(rowCount) - .isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS)); + .isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener())); } @Test(dataProvider = "rowCount") @@ -357,7 +382,7 @@ public void testParquetPageSourceSchemaEvolution(int rowCount) .withReadColumns(readColumns) .withSession(parquetPageSourceSession) .withRowsCount(rowCount) - .isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS)); + .isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener())); // test name-based access readColumns = Lists.reverse(writeColumns); @@ -365,7 +390,7 @@ public void testParquetPageSourceSchemaEvolution(int rowCount) .withWriteColumns(writeColumns) .withReadColumns(readColumns) .withSession(parquetPageSourceSessionUseName) - .isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS)); + .isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener())); } private static List getTestColumnsSupportedByParquet() @@ -393,7 +418,7 @@ public void testDwrf(int rowCount) assertThatFileFormat(DWRF) .withColumns(testColumns) .withRowsCount(rowCount) - .isReadableByPageSource(new DwrfBatchPageSourceFactory(TYPE_MANAGER, HIVE_CLIENT_CONFIG, HDFS_ENVIRONMENT, STATS)); + .isReadableByPageSource(new DwrfBatchPageSourceFactory(TYPE_MANAGER, HIVE_CLIENT_CONFIG, HDFS_ENVIRONMENT, STATS, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())); } @Test(dataProvider = "rowCount") @@ -421,7 +446,7 @@ public void testDwrfOptimizedWriter(int rowCount) .withSession(session) .withFileWriterFactory(new OrcFileWriterFactory(HDFS_ENVIRONMENT, TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE, STATS, new OrcWriterOptions())) .isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)) - .isReadableByPageSource(new DwrfBatchPageSourceFactory(TYPE_MANAGER, HIVE_CLIENT_CONFIG, HDFS_ENVIRONMENT, STATS)); + .isReadableByPageSource(new DwrfBatchPageSourceFactory(TYPE_MANAGER, HIVE_CLIENT_CONFIG, HDFS_ENVIRONMENT, STATS, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())); } @Test @@ -434,25 +459,25 @@ public void testTruncateVarcharColumn() assertThatFileFormat(RCTEXT) .withWriteColumns(ImmutableList.of(writeColumn)) .withReadColumns(ImmutableList.of(readColumn)) - .isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS)) + .isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener())) .isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)); assertThatFileFormat(RCBINARY) .withWriteColumns(ImmutableList.of(writeColumn)) .withReadColumns(ImmutableList.of(readColumn)) - .isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS)) + .isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener())) .isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)); assertThatFileFormat(ORC) .withWriteColumns(ImmutableList.of(writeColumn)) .withReadColumns(ImmutableList.of(readColumn)) - .isReadableByPageSource(new OrcBatchPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT, STATS, 100)); + .isReadableByPageSource(new OrcBatchPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT, STATS, 100, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())); assertThatFileFormat(PARQUET) .withWriteColumns(ImmutableList.of(writeColumn)) .withReadColumns(ImmutableList.of(readColumn)) .withSession(parquetPageSourceSession) - .isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS)); + .isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener())); assertThatFileFormat(AVRO) .withWriteColumns(ImmutableList.of(writeColumn)) @@ -479,27 +504,27 @@ public void testFailForLongVarcharPartitionColumn() List columns = ImmutableList.of(partitionColumn, varcharColumn); - HiveErrorCode expectedErrorCode = HiveErrorCode.HIVE_INVALID_PARTITION_VALUE; + MetastoreErrorCode expectedErrorCode = HIVE_INVALID_PARTITION_VALUE; String expectedMessage = "Invalid partition value 'test' for varchar(3) partition key: partition_column"; assertThatFileFormat(RCTEXT) .withColumns(columns) - .isFailingForPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS), expectedErrorCode, expectedMessage) + .isFailingForPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessage) .isFailingForRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage); assertThatFileFormat(RCBINARY) .withColumns(columns) - .isFailingForPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS), expectedErrorCode, expectedMessage) + .isFailingForPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessage) .isFailingForRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage); assertThatFileFormat(ORC) .withColumns(columns) - .isFailingForPageSource(new OrcBatchPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT, STATS, 100), expectedErrorCode, expectedMessage); + .isFailingForPageSource(new OrcBatchPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT, STATS, 100, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener()), expectedErrorCode, expectedMessage); assertThatFileFormat(PARQUET) .withColumns(columns) .withSession(parquetPageSourceSession) - .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS), expectedErrorCode, expectedMessage); + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessage); assertThatFileFormat(SEQUENCEFILE) .withColumns(columns) @@ -510,6 +535,179 @@ public void testFailForLongVarcharPartitionColumn() .isFailingForRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage); } + @Test + public void testSchemaMismatch() + throws Exception + { + TestColumn floatColumn = new TestColumn("column_name", javaFloatObjectInspector, 5.1f, 5.1f); + TestColumn doubleColumn = new TestColumn("column_name", javaDoubleObjectInspector, 5.1, 5.1); + TestColumn booleanColumn = new TestColumn("column_name", javaBooleanObjectInspector, true, true); + TestColumn stringColumn = new TestColumn("column_name", javaStringObjectInspector, "test", utf8Slice("test")); + TestColumn intColumn = new TestColumn("column_name", javaIntObjectInspector, 3, 3); + TestColumn longColumn = new TestColumn("column_name", javaLongObjectInspector, 4L, 4L); + TestColumn timestampColumn = new TestColumn("column_name", javaTimestampObjectInspector, 4L, 4L); + TestColumn mapLongColumn = new TestColumn("column_name", + getStandardMapObjectInspector(javaLongObjectInspector, javaLongObjectInspector), + ImmutableMap.of(4L, 4L), + mapBlockOf(BIGINT, BIGINT, 4L, 4L)); + TestColumn mapDoubleColumn = new TestColumn("column_name", + getStandardMapObjectInspector(javaDoubleObjectInspector, javaDoubleObjectInspector), + ImmutableMap.of(5.1, 5.2), + mapBlockOf(DOUBLE, DOUBLE, 5.1, 5.2)); + TestColumn arrayStringColumn = new TestColumn("column_name", + getStandardListObjectInspector(javaStringObjectInspector), + ImmutableList.of("test"), + arrayBlockOf(createUnboundedVarcharType(), "test")); + TestColumn arrayBooleanColumn = new TestColumn("column_name", + getStandardListObjectInspector(javaBooleanObjectInspector), + ImmutableList.of(true), + arrayBlockOf(BOOLEAN, true)); + TestColumn rowLongColumn = new TestColumn("column_name", + getStandardStructObjectInspector(ImmutableList.of("s_bigint"), ImmutableList.of(javaLongObjectInspector)), + new Long[] {1L}, + rowBlockOf(ImmutableList.of(BIGINT), 1)); + TestColumn nestColumn = new TestColumn("column_name", + getStandardMapObjectInspector( + javaStringObjectInspector, + getStandardListObjectInspector( + getStandardStructObjectInspector( + ImmutableList.of("s_int"), + ImmutableList.of(javaIntObjectInspector)))), + ImmutableMap.of("test", ImmutableList.of(new Integer[] {1})), + mapBlockOf(createUnboundedVarcharType(), new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER))), + "test", arrayBlockOf(RowType.anonymous(ImmutableList.of(INTEGER)), rowBlockOf(ImmutableList.of(INTEGER), 1L)))); + + HiveErrorCode expectedErrorCode = HIVE_PARTITION_SCHEMA_MISMATCH; + String expectedMessageFloatDouble = "The column column_name is declared as type double, but the Parquet file declares the column as type FLOAT"; + + // Make sure INT64 is still readable as Timestamp see https://github.com/prestodb/presto/issues/13855 + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(longColumn)) + .withReadColumns(ImmutableList.of(timestampColumn)) + .withSession(parquetPageSourceSession) + .isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener())); + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(floatColumn)) + .withReadColumns(ImmutableList.of(doubleColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageFloatDouble); + + String expectedMessageDoubleLong = "The column column_name is declared as type bigint, but the Parquet file declares the column as type DOUBLE"; + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(doubleColumn)) + .withReadColumns(ImmutableList.of(longColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageDoubleLong); + + String expectedMessageFloatInt = "The column column_name is declared as type int, but the Parquet file declares the column as type FLOAT"; + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(floatColumn)) + .withReadColumns(ImmutableList.of(intColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageFloatInt); + + String expectedMessageIntBoolean = "The column column_name is declared as type boolean, but the Parquet file declares the column as type INT32"; + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(intColumn)) + .withReadColumns(ImmutableList.of(booleanColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageIntBoolean); + + String expectedMessageStringLong = "The column column_name is declared as type string, but the Parquet file declares the column as type INT64"; + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(longColumn)) + .withReadColumns(ImmutableList.of(stringColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageStringLong); + + String expectedMessageIntString = "The column column_name is declared as type int, but the Parquet file declares the column as type BINARY"; + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(stringColumn)) + .withReadColumns(ImmutableList.of(intColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageIntString); + + String expectedMessageMapLongLong = "The column column_name is declared as type map, but the Parquet file declares the column as type INT64"; + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(longColumn)) + .withReadColumns(ImmutableList.of(mapLongColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageMapLongLong); + + String expectedMessageMapLongMapDouble = "The column column_name is declared as type map, but the Parquet file declares the column as type optional group column_name (MAP) {\n" + + " repeated group map (MAP_KEY_VALUE) {\n" + + " required double key;\n" + + " optional double value;\n" + + " }\n" + + "}"; + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(mapDoubleColumn)) + .withReadColumns(ImmutableList.of(mapLongColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageMapLongMapDouble); + + String expectedMessageArrayStringArrayBoolean = "The column column_name is declared as type array, but the Parquet file declares the column as type optional group column_name (LIST) {\n" + + " repeated group bag {\n" + + " optional boolean array_element;\n" + + " }\n" + + "}"; + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(arrayBooleanColumn)) + .withReadColumns(ImmutableList.of(arrayStringColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageArrayStringArrayBoolean); + + String expectedMessageBooleanArrayBoolean = "The column column_name is declared as type array, but the Parquet file declares the column as type BOOLEAN"; + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(booleanColumn)) + .withReadColumns(ImmutableList.of(arrayBooleanColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageBooleanArrayBoolean); + + String expectedMessageRowLongLong = "The column column_name is declared as type bigint, but the Parquet file declares the column as type optional group column_name {\n" + + " optional int64 s_bigint;\n" + + "}"; + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(rowLongColumn)) + .withReadColumns(ImmutableList.of(longColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageRowLongLong); + + String expectedMessageMapLongRowLong = "The column column_name is declared as type struct, but the Parquet file declares the column as type optional group column_name (MAP) {\n" + + " repeated group map (MAP_KEY_VALUE) {\n" + + " required int64 key;\n" + + " optional int64 value;\n" + + " }\n" + + "}"; + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(mapLongColumn)) + .withReadColumns(ImmutableList.of(rowLongColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageMapLongRowLong); + + String expectedMessageRowLongNest = "The column column_name is declared as type map>>, but the Parquet file declares the column as type optional group column_name {\n" + + " optional int64 s_bigint;\n" + + "}"; + + assertThatFileFormat(PARQUET) + .withWriteColumns(ImmutableList.of(rowLongColumn)) + .withReadColumns(ImmutableList.of(nestColumn)) + .withSession(parquetPageSourceSession) + .isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HadoopFileOpener()), expectedErrorCode, expectedMessageRowLongNest); + } + private void testCursorProvider(HiveRecordCursorProvider cursorProvider, FileSplit split, HiveStorageFormat storageFormat, @@ -517,17 +715,17 @@ private void testCursorProvider(HiveRecordCursorProvider cursorProvider, ConnectorSession session, int rowCount) { - Properties splitProperties = new Properties(); - splitProperties.setProperty(FILE_INPUT_FORMAT, storageFormat.getInputFormat()); - splitProperties.setProperty(SERIALIZATION_LIB, storageFormat.getSerDe()); - splitProperties.setProperty("columns", Joiner.on(',').join(transform(filter(testColumns, not(TestColumn::isPartitionKey)), TestColumn::getName))); - splitProperties.setProperty("columns.types", Joiner.on(',').join(transform(filter(testColumns, not(TestColumn::isPartitionKey)), TestColumn::getType))); - List partitionKeys = testColumns.stream() .filter(TestColumn::isPartitionKey) .map(input -> new HivePartitionKey(input.getName(), (String) input.getWriteValue())) .collect(toList()); + List partitionKeyColumnHandles = getColumnHandles(testColumns.stream().filter(TestColumn::isPartitionKey).collect(toImmutableList())); + List tableDataColumns = testColumns.stream() + .filter(column -> !column.isPartitionKey()) + .map(column -> new Column(column.getName(), HiveType.valueOf(column.getType()), Optional.empty())) + .collect(toImmutableList()); + Configuration configuration = new Configuration(); configuration.set("io.compression.codecs", LzoCodec.class.getName() + "," + LzopCodec.class.getName()); Optional pageSource = HivePageSourceProvider.createHivePageSource( @@ -540,15 +738,30 @@ private void testCursorProvider(HiveRecordCursorProvider cursorProvider, split.getStart(), split.getLength(), split.getLength(), - splitProperties, + new Storage( + StorageFormat.create(storageFormat.getSerDe(), storageFormat.getInputFormat(), storageFormat.getOutputFormat()), + "location", + Optional.empty(), + false, + ImmutableMap.of()), TupleDomain.all(), getColumnHandles(testColumns), + ImmutableMap.of(), partitionKeys, DateTimeZone.getDefault(), TYPE_MANAGER, + new SchemaTableName("schema", "table"), + partitionKeyColumnHandles, + tableDataColumns, ImmutableMap.of(), + tableDataColumns.size(), + ImmutableMap.of(), + Optional.empty(), + false, Optional.empty(), - false); + TRUE_CONSTANT, + false, + ROW_EXPRESSION_SERVICE); RecordCursor cursor = ((RecordPageSource) pageSource.get()).getCursor(); @@ -563,17 +776,17 @@ private void testPageSourceFactory(HiveBatchPageSourceFactory sourceFactory, int rowCount) throws IOException { - Properties splitProperties = new Properties(); - splitProperties.setProperty(FILE_INPUT_FORMAT, storageFormat.getInputFormat()); - splitProperties.setProperty(SERIALIZATION_LIB, storageFormat.getSerDe()); - splitProperties.setProperty("columns", Joiner.on(',').join(transform(filter(testColumns, not(TestColumn::isPartitionKey)), TestColumn::getName))); - splitProperties.setProperty("columns.types", Joiner.on(',').join(transform(filter(testColumns, not(TestColumn::isPartitionKey)), TestColumn::getType))); - List partitionKeys = testColumns.stream() .filter(TestColumn::isPartitionKey) .map(input -> new HivePartitionKey(input.getName(), (String) input.getWriteValue())) .collect(toList()); + List partitionKeyColumnHandles = getColumnHandles(testColumns.stream().filter(TestColumn::isPartitionKey).collect(toImmutableList())); + List tableDataColumns = testColumns.stream() + .filter(column -> !column.isPartitionKey()) + .map(column -> new Column(column.getName(), HiveType.valueOf(column.getType()), Optional.empty())) + .collect(toImmutableList()); + List columnHandles = getColumnHandles(testColumns); Optional pageSource = HivePageSourceProvider.createHivePageSource( @@ -586,15 +799,30 @@ private void testPageSourceFactory(HiveBatchPageSourceFactory sourceFactory, split.getStart(), split.getLength(), split.getLength(), - splitProperties, + new Storage( + StorageFormat.create(storageFormat.getSerDe(), storageFormat.getInputFormat(), storageFormat.getOutputFormat()), + "location", + Optional.empty(), + false, + ImmutableMap.of()), TupleDomain.all(), columnHandles, + ImmutableMap.of(), partitionKeys, DateTimeZone.getDefault(), TYPE_MANAGER, + new SchemaTableName("schema", "table"), + partitionKeyColumnHandles, + tableDataColumns, ImmutableMap.of(), + tableDataColumns.size(), + ImmutableMap.of(), + Optional.empty(), + false, Optional.empty(), - false); + TRUE_CONSTANT, + false, + ROW_EXPRESSION_SERVICE); assertTrue(pageSource.isPresent()); @@ -733,14 +961,14 @@ public FileFormatAssertion isReadableByRecordCursor(HiveRecordCursorProvider cur return this; } - public FileFormatAssertion isFailingForPageSource(HiveBatchPageSourceFactory pageSourceFactory, HiveErrorCode expectedErrorCode, String expectedMessage) + public FileFormatAssertion isFailingForPageSource(HiveBatchPageSourceFactory pageSourceFactory, ErrorCodeSupplier expectedErrorCode, String expectedMessage) throws Exception { assertFailure(Optional.of(pageSourceFactory), Optional.empty(), expectedErrorCode, expectedMessage); return this; } - public FileFormatAssertion isFailingForRecordCursor(HiveRecordCursorProvider cursorProvider, HiveErrorCode expectedErrorCode, String expectedMessage) + public FileFormatAssertion isFailingForRecordCursor(HiveRecordCursorProvider cursorProvider, ErrorCodeSupplier expectedErrorCode, String expectedMessage) throws Exception { assertFailure(Optional.empty(), Optional.of(cursorProvider), expectedErrorCode, expectedMessage); @@ -793,7 +1021,7 @@ private void assertRead(Optional pageSourceFactory, private void assertFailure( Optional pageSourceFactory, Optional cursorProvider, - HiveErrorCode expectedErrorCode, + ErrorCodeSupplier expectedErrorCode, String expectedMessage) throws Exception { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java index 8ac2f83dc3f3e..10704e14b8849 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java @@ -36,6 +36,7 @@ import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.planner.planPrinter.IOPlanPrinter.ColumnConstraint; @@ -46,6 +47,7 @@ import com.facebook.presto.sql.planner.planPrinter.IOPlanPrinter.IOPlan.TableColumnInfo; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.MaterializedRow; +import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestIntegrationSmokeTest; import com.facebook.presto.tests.DistributedQueryRunner; import com.google.common.collect.ImmutableList; @@ -69,6 +71,7 @@ import java.util.function.Function; import java.util.stream.LongStream; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.SystemSessionProperties.COLOCATED_JOIN; import static com.facebook.presto.SystemSessionProperties.CONCURRENT_LIFESPANS_PER_NODE; import static com.facebook.presto.SystemSessionProperties.DYNAMIC_SCHEDULE_FOR_GROUPED_EXECUTION; @@ -87,13 +90,17 @@ import static com.facebook.presto.hive.HiveQueryRunner.createQueryRunner; import static com.facebook.presto.hive.HiveSessionProperties.PUSHDOWN_FILTER_ENABLED; import static com.facebook.presto.hive.HiveSessionProperties.RCFILE_OPTIMIZED_WRITER_ENABLED; +import static com.facebook.presto.hive.HiveSessionProperties.SORTED_WRITE_TEMP_PATH_SUBDIRECTORY_COUNT; +import static com.facebook.presto.hive.HiveSessionProperties.SORTED_WRITE_TO_TEMP_PATH_ENABLED; import static com.facebook.presto.hive.HiveSessionProperties.getInsertExistingPartitionsBehavior; +import static com.facebook.presto.hive.HiveSessionProperties.isPushdownFilterEnabled; import static com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY; import static com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY; import static com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY; import static com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY; import static com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER; import static com.facebook.presto.hive.HiveUtil.columnExtraInfo; +import static com.facebook.presto.spi.predicate.Marker.Bound.ABOVE; import static com.facebook.presto.spi.predicate.Marker.Bound.EXACTLY; import static com.facebook.presto.spi.security.SelectedRole.Type.ROLE; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -123,7 +130,6 @@ import static com.google.common.io.Files.createTempDir; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.tpch.TpchTable.CUSTOMER; import static io.airlift.tpch.TpchTable.LINE_ITEM; import static io.airlift.tpch.TpchTable.ORDERS; @@ -183,7 +189,7 @@ private List getPartitions(HiveTableLayoutHandle tableLayoutHandle) public void testSchemaOperations() { Session admin = Session.builder(getQueryRunner().getDefaultSession()) - .setIdentity(new Identity("hive", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ROLE, Optional.of("admin"))))) + .setIdentity(new Identity("hive", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ROLE, Optional.of("admin"))), ImmutableMap.of())) .build(); assertUpdate(admin, "CREATE SCHEMA new_schema"); @@ -204,56 +210,81 @@ public void testIOExplain() computeActual("CREATE TABLE test_orders WITH (partitioned_by = ARRAY['orderkey', 'processing']) AS SELECT custkey, orderkey, orderstatus = 'P' processing FROM orders WHERE orderkey < 3"); MaterializedResult result = computeActual("EXPLAIN (TYPE IO, FORMAT JSON) INSERT INTO test_orders SELECT custkey, orderkey, processing FROM test_orders where custkey <= 10"); + ImmutableSet.Builder expectedConstraints = ImmutableSet.builder(); + expectedConstraints.add(new ColumnConstraint( + "orderkey", + BIGINT.getTypeSignature(), + new FormattedDomain( + false, + ImmutableSet.of( + new FormattedRange( + new FormattedMarker(Optional.of("1"), EXACTLY), + new FormattedMarker(Optional.of("1"), EXACTLY)), + new FormattedRange( + new FormattedMarker(Optional.of("2"), EXACTLY), + new FormattedMarker(Optional.of("2"), EXACTLY)))))); + expectedConstraints.add(new ColumnConstraint( + "processing", + BOOLEAN.getTypeSignature(), + new FormattedDomain( + false, + ImmutableSet.of( + new FormattedRange( + new FormattedMarker(Optional.of("false"), EXACTLY), + new FormattedMarker(Optional.of("false"), EXACTLY)))))); + if (isPushdownFilterEnabled(getConnectorSession(getSession()))) { + expectedConstraints.add(new ColumnConstraint( + "custkey", + BIGINT.getTypeSignature(), + new FormattedDomain( + false, + ImmutableSet.of( + new FormattedRange( + new FormattedMarker(Optional.empty(), ABOVE), + new FormattedMarker(Optional.of("10"), EXACTLY)))))); + } assertEquals( jsonCodec(IOPlan.class).fromJson((String) getOnlyElement(result.getOnlyColumnAsSet())), new IOPlan( ImmutableSet.of(new TableColumnInfo( new CatalogSchemaTableName(catalog, "tpch", "test_orders"), - ImmutableSet.of( - new ColumnConstraint( - "orderkey", - BIGINT.getTypeSignature(), - new FormattedDomain( - false, - ImmutableSet.of( - new FormattedRange( - new FormattedMarker(Optional.of("1"), EXACTLY), - new FormattedMarker(Optional.of("1"), EXACTLY)), - new FormattedRange( - new FormattedMarker(Optional.of("2"), EXACTLY), - new FormattedMarker(Optional.of("2"), EXACTLY))))), - new ColumnConstraint( - "processing", - BOOLEAN.getTypeSignature(), - new FormattedDomain( - false, - ImmutableSet.of( - new FormattedRange( - new FormattedMarker(Optional.of("false"), EXACTLY), - new FormattedMarker(Optional.of("false"), EXACTLY)))))))), + expectedConstraints.build())), Optional.of(new CatalogSchemaTableName(catalog, "tpch", "test_orders")))); assertUpdate("DROP TABLE test_orders"); - // Test IO explain with large number of discrete components where Domain::simpify comes into play. + // Test IO explain with large number of discrete components where Domain::simplify comes into play. computeActual("CREATE TABLE test_orders WITH (partitioned_by = ARRAY['orderkey']) AS select custkey, orderkey FROM orders where orderkey < 200"); + expectedConstraints = ImmutableSet.builder(); + expectedConstraints.add(new ColumnConstraint( + "orderkey", + BIGINT.getTypeSignature(), + new FormattedDomain( + false, + ImmutableSet.of( + new FormattedRange( + new FormattedMarker(Optional.of("1"), EXACTLY), + new FormattedMarker(Optional.of("199"), EXACTLY)))))); + if (isPushdownFilterEnabled(getConnectorSession(getSession()))) { + expectedConstraints.add(new ColumnConstraint( + "custkey", + BIGINT.getTypeSignature(), + new FormattedDomain( + false, + ImmutableSet.of( + new FormattedRange( + new FormattedMarker(Optional.empty(), ABOVE), + new FormattedMarker(Optional.of("10"), EXACTLY)))))); + } + result = computeActual("EXPLAIN (TYPE IO, FORMAT JSON) INSERT INTO test_orders SELECT custkey, orderkey + 10 FROM test_orders where custkey <= 10"); assertEquals( jsonCodec(IOPlan.class).fromJson((String) getOnlyElement(result.getOnlyColumnAsSet())), new IOPlan( ImmutableSet.of(new TableColumnInfo( new CatalogSchemaTableName(catalog, "tpch", "test_orders"), - ImmutableSet.of( - new ColumnConstraint( - "orderkey", - BIGINT.getTypeSignature(), - new FormattedDomain( - false, - ImmutableSet.of( - new FormattedRange( - new FormattedMarker(Optional.of("1"), EXACTLY), - new FormattedMarker(Optional.of("199"), EXACTLY)))))))), + expectedConstraints.build())), Optional.of(new CatalogSchemaTableName(catalog, "tpch", "test_orders")))); assertUpdate("DROP TABLE test_orders"); @@ -1674,6 +1705,15 @@ public void testMetadataDelete() assertQuery("SELECT * from test_metadata_delete", "SELECT orderkey, linenumber, linestatus FROM lineitem WHERE linestatus<>'F' or linenumber<>3"); + // TODO This use case can be supported + try { + getQueryRunner().execute("DELETE FROM test_metadata_delete WHERE lower(LINE_STATUS)='f' and LINE_NUMBER=CAST(4 AS INTEGER)"); + fail("expected exception"); + } + catch (RuntimeException e) { + assertEquals(e.getMessage(), "This connector only supports delete where one or more partitions are deleted entirely"); + } + assertUpdate("DELETE FROM test_metadata_delete WHERE LINE_STATUS='O'"); assertQuery("SELECT * from test_metadata_delete", "SELECT orderkey, linenumber, linestatus FROM lineitem WHERE linestatus<>'O' and linenumber<>3"); @@ -1911,6 +1951,113 @@ private void testBucketedExecution(Session session) "SELECT custkey, COUNT(*) FROM orders GROUP BY custkey"); } + @Test + public void testBucketPruning() + { + Session session = getSession(); + QueryRunner queryRunner = getQueryRunner(); + queryRunner.execute("CREATE TABLE orders_bucketed WITH (bucket_count = 11, bucketed_by = ARRAY['orderkey']) AS " + + "SELECT * FROM orders"); + + try { + assertQuery(session, "SELECT * FROM orders_bucketed WHERE orderkey = 100", "SELECT * FROM orders WHERE orderkey = 100"); + + assertQuery(session, "SELECT * FROM orders_bucketed WHERE orderkey = 100 OR orderkey = 101", "SELECT * FROM orders WHERE orderkey = 100 OR orderkey = 101"); + + assertQuery(session, "SELECT * FROM orders_bucketed WHERE orderkey IN (100, 101, 133)", "SELECT * FROM orders WHERE orderkey IN (100, 101, 133)"); + + assertQuery(session, "SELECT * FROM orders_bucketed", "SELECT * FROM orders"); + + assertQuery(session, "SELECT * FROM orders_bucketed WHERE orderkey > 100", "SELECT * FROM orders WHERE orderkey > 100"); + + assertQuery(session, "SELECT * FROM orders_bucketed WHERE orderkey != 100", "SELECT * FROM orders WHERE orderkey != 100"); + } + finally { + queryRunner.execute("DROP TABLE orders_bucketed"); + } + + queryRunner.execute("CREATE TABLE orders_bucketed WITH (bucket_count = 11, bucketed_by = ARRAY['orderkey', 'custkey']) AS " + + "SELECT * FROM orders"); + + try { + assertQuery(session, "SELECT * FROM orders_bucketed WHERE orderkey = 101 AND custkey = 280", "SELECT * FROM orders WHERE orderkey = 101 AND custkey = 280"); + + assertQuery(session, "SELECT * FROM orders_bucketed WHERE orderkey IN (101, 71) AND custkey = 280", "SELECT * FROM orders WHERE orderkey IN (101, 71) AND custkey = 280"); + + assertQuery(session, "SELECT * FROM orders_bucketed WHERE orderkey IN (101, 71) AND custkey IN (280, 34)", "SELECT * FROM orders WHERE orderkey IN (101, 71) AND custkey IN (280, 34)"); + + assertQuery(session, "SELECT * FROM orders_bucketed WHERE orderkey = 101 AND custkey = 280 AND orderstatus <> '0'", "SELECT * FROM orders WHERE orderkey = 101 AND custkey = 280 AND orderstatus <> '0'"); + + assertQuery(session, "SELECT * FROM orders_bucketed WHERE orderkey = 101", "SELECT * FROM orders WHERE orderkey = 101"); + + assertQuery(session, "SELECT * FROM orders_bucketed WHERE custkey = 280", "SELECT * FROM orders WHERE custkey = 280"); + + assertQuery(session, "SELECT * FROM orders_bucketed WHERE orderkey = 101 AND custkey > 280", "SELECT * FROM orders WHERE orderkey = 101 AND custkey > 280"); + } + finally { + queryRunner.execute("DROP TABLE orders_bucketed"); + } + } + + @Test + public void testWriteSortedTable() + { + testWriteSortedTable(getSession()); + testWriteSortedTable(Session.builder(getSession()) + .setCatalogSessionProperty(catalog, SORTED_WRITE_TO_TEMP_PATH_ENABLED, "true") + .setCatalogSessionProperty(catalog, SORTED_WRITE_TEMP_PATH_SUBDIRECTORY_COUNT, "10") + .build()); + } + + private void testWriteSortedTable(Session session) + { + try { + // create table + assertUpdate( + session, + "CREATE TABLE create_partitioned_sorted_table (orderkey, custkey, totalprice, orderdate, orderpriority, clerk, shippriority, comment, orderstatus)\n" + + "WITH (partitioned_by = ARRAY['orderstatus'], bucketed_by = ARRAY['custkey'], bucket_count = 11, sorted_by = ARRAY['orderkey']) AS\n" + + "SELECT orderkey, custkey, totalprice, orderdate, orderpriority, clerk, shippriority, comment, orderstatus FROM tpch.sf1.orders", + (long) computeActual("SELECT count(*) FROM tpch.sf1.orders").getOnlyValue()); + assertQuery( + session, + "SELECT count(*) FROM create_partitioned_sorted_table", + "SELECT count(*) * 100 FROM orders"); + + assertUpdate( + session, + "CREATE TABLE create_unpartitioned_sorted_table (orderkey, custkey, totalprice, orderdate, orderpriority, clerk, shippriority, comment, orderstatus)\n" + + "WITH (bucketed_by = ARRAY['custkey'], bucket_count = 11, sorted_by = ARRAY['orderkey']) AS\n" + + "SELECT orderkey, custkey, totalprice, orderdate, orderpriority, clerk, shippriority, comment, orderstatus FROM tpch.sf1.orders", + (long) computeActual("SELECT count(*) FROM tpch.sf1.orders").getOnlyValue()); + assertQuery( + session, + "SELECT count(*) FROM create_unpartitioned_sorted_table", + "SELECT count(*) * 100 FROM orders"); + + // insert + assertUpdate( + session, + "CREATE TABLE insert_partitioned_sorted_table (LIKE create_partitioned_sorted_table) " + + "WITH (partitioned_by = ARRAY['orderstatus'], bucketed_by = ARRAY['custkey'], bucket_count = 11, sorted_by = ARRAY['orderkey'])"); + + assertUpdate( + session, + "INSERT INTO insert_partitioned_sorted_table\n" + + "SELECT orderkey, custkey, totalprice, orderdate, orderpriority, clerk, shippriority, comment, orderstatus FROM tpch.sf1.orders", + (long) computeActual("SELECT count(*) FROM tpch.sf1.orders").getOnlyValue()); + assertQuery( + session, + "SELECT count(*) FROM insert_partitioned_sorted_table", + "SELECT count(*) * 100 FROM orders"); + } + finally { + assertUpdate(session, "DROP TABLE IF EXISTS create_partitioned_sorted_table"); + assertUpdate(session, "DROP TABLE IF EXISTS create_unpartitioned_sorted_table"); + assertUpdate(session, "DROP TABLE IF EXISTS insert_partitioned_sorted_table"); + } + } + @Test public void testScaleWriters() { @@ -1920,6 +2067,7 @@ public void testScaleWriters() Session.builder(getSession()) .setSystemProperty("scale_writers", "true") .setSystemProperty("writer_min_size", "32MB") + .setSystemProperty("task_writer_count", "1") .build(), "CREATE TABLE scale_writers_small AS SELECT * FROM tpch.tiny.orders", (long) computeActual("SELECT count(*) FROM tpch.tiny.orders").getOnlyValue()); @@ -1931,6 +2079,7 @@ public void testScaleWriters() Session.builder(getSession()) .setSystemProperty("scale_writers", "true") .setSystemProperty("writer_min_size", "1MB") + .setSystemProperty("task_writer_count", "1") .build(), "CREATE TABLE scale_writers_large WITH (format = 'RCBINARY') AS SELECT * FROM tpch.sf1.orders", (long) computeActual("SELECT count(*) FROM tpch.sf1.orders").getOnlyValue()); @@ -2642,7 +2791,7 @@ public void testMaterializedPartitioning() assertQueryFails( materializeAllWrongPartitioningProvider, "SELECT orderkey, COUNT(*) lines FROM lineitem GROUP BY orderkey", - "Catalog \"tpch\" does not support custom partitioning and cannot be used as a partitioning provider"); + "Catalog \"tpch\" cannot be used as a partitioning provider: This connector does not support custom partitioning"); // make sure that bucketing is not ignored for temporary tables Session bucketingIgnored = Session.builder(materializeExchangesSession) @@ -3417,6 +3566,9 @@ private void testGroupedExecution(Session session) assertQuery(notColocated, tableScan, expectedTableScan); assertQuery(eligibleTableScans, tableScan, expectedTableScan); + + // Input connector returns non-empty TablePartitioning for unpartitioned table. + assertQuery(eligibleTableScans, "SELECT orderkey FROM tpch.tiny.orders", "SELECT orderkey FROM orders"); } finally { assertUpdate(session, "DROP TABLE IF EXISTS test_grouped_join1"); @@ -3536,18 +3688,18 @@ public void testCollectColumnStatisticsOnCreateTable() " partitioned_by = ARRAY['p_varchar'] " + ") " + "AS " + - "SELECT c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, p_varchar " + + "SELECT c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, c_array, p_varchar " + "FROM ( " + " VALUES " + - " (null, null, null, null, null, null, 'p1'), " + - " (null, null, null, null, null, null, 'p1'), " + - " (true, BIGINT '1', DOUBLE '2.2', TIMESTAMP '2012-08-08 01:00', CAST('abc1' AS VARCHAR), CAST('bcd1' AS VARBINARY), 'p1')," + - " (false, BIGINT '0', DOUBLE '1.2', TIMESTAMP '2012-08-08 00:00', CAST('abc2' AS VARCHAR), CAST('bcd2' AS VARBINARY), 'p1')," + - " (null, null, null, null, null, null, 'p2'), " + - " (null, null, null, null, null, null, 'p2'), " + - " (true, BIGINT '2', DOUBLE '3.3', TIMESTAMP '2012-09-09 01:00', CAST('cba1' AS VARCHAR), CAST('dcb1' AS VARBINARY), 'p2'), " + - " (false, BIGINT '1', DOUBLE '2.3', TIMESTAMP '2012-09-09 00:00', CAST('cba2' AS VARCHAR), CAST('dcb2' AS VARBINARY), 'p2') " + - ") AS x (c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, p_varchar)", tableName), 8); + " (null, null, null, null, null, null, null, 'p1'), " + + " (null, null, null, null, null, null, null, 'p1'), " + + " (true, BIGINT '1', DOUBLE '2.2', TIMESTAMP '2012-08-08 01:00', CAST('abc1' AS VARCHAR), CAST('bcd1' AS VARBINARY), sequence(0, 10), 'p1')," + + " (false, BIGINT '0', DOUBLE '1.2', TIMESTAMP '2012-08-08 00:00', CAST('abc2' AS VARCHAR), CAST('bcd2' AS VARBINARY), sequence(10, 20), 'p1')," + + " (null, null, null, null, null, null, null, 'p2'), " + + " (null, null, null, null, null, null, null, 'p2'), " + + " (true, BIGINT '2', DOUBLE '3.3', TIMESTAMP '2012-09-09 01:00', CAST('cba1' AS VARCHAR), CAST('dcb1' AS VARBINARY), sequence(20, 25), 'p2'), " + + " (false, BIGINT '1', DOUBLE '2.3', TIMESTAMP '2012-09-09 00:00', CAST('cba2' AS VARCHAR), CAST('dcb2' AS VARBINARY), sequence(30, 35), 'p2') " + + ") AS x (c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, c_array, p_varchar)", tableName), 8); assertQuery(format("SHOW STATS FOR (SELECT * FROM %s WHERE p_varchar = 'p1')", tableName), "SELECT * FROM VALUES " + @@ -3557,6 +3709,7 @@ public void testCollectColumnStatisticsOnCreateTable() "('c_timestamp', null, 2.0E0, 0.5E0, null, null, null), " + "('c_varchar', 8.0E0, 2.0E0, 0.5E0, null, null, null), " + "('c_varbinary', 8.0E0, null, 0.5E0, null, null, null), " + + "('c_array', 176.0E0, null, 0.5, null, null, null), " + "('p_varchar', 8.0E0, 1.0E0, 0.0E0, null, null, null), " + "(null, null, null, null, 4.0E0, null, null)"); assertQuery(format("SHOW STATS FOR (SELECT * FROM %s WHERE p_varchar = 'p2')", tableName), @@ -3567,6 +3720,7 @@ public void testCollectColumnStatisticsOnCreateTable() "('c_timestamp', null, 2.0E0, 0.5E0, null, null, null), " + "('c_varchar', 8.0E0, 2.0E0, 0.5E0, null, null, null), " + "('c_varbinary', 8.0E0, null, 0.5E0, null, null, null), " + + "('c_array', 96.0E0, null, 0.5, null, null, null), " + "('p_varchar', 8.0E0, 1.0E0, 0.0E0, null, null, null), " + "(null, null, null, null, 4.0E0, null, null)"); @@ -3579,6 +3733,7 @@ public void testCollectColumnStatisticsOnCreateTable() "('c_timestamp', null, 0E0, 0E0, null, null, null), " + "('c_varchar', 0E0, 0E0, 0E0, null, null, null), " + "('c_varbinary', null, 0E0, 0E0, null, null, null), " + + "('c_array', null, 0E0, 0E0, null, null, null), " + "('p_varchar', 0E0, 0E0, 0E0, null, null, null), " + "(null, null, null, null, 0E0, null, null)"); @@ -3597,6 +3752,7 @@ public void testCollectColumnStatisticsOnInsert() " c_timestamp TIMESTAMP, " + " c_varchar VARCHAR, " + " c_varbinary VARBINARY, " + + " c_array ARRAY(BIGINT), " + " p_varchar VARCHAR " + ") " + "WITH ( " + @@ -3605,18 +3761,18 @@ public void testCollectColumnStatisticsOnInsert() assertUpdate(format("" + "INSERT INTO %s " + - "SELECT c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, p_varchar " + + "SELECT c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, c_array, p_varchar " + "FROM ( " + " VALUES " + - " (null, null, null, null, null, null, 'p1'), " + - " (null, null, null, null, null, null, 'p1'), " + - " (true, BIGINT '1', DOUBLE '2.2', TIMESTAMP '2012-08-08 01:00', CAST('abc1' AS VARCHAR), CAST('bcd1' AS VARBINARY), 'p1')," + - " (false, BIGINT '0', DOUBLE '1.2', TIMESTAMP '2012-08-08 00:00', CAST('abc2' AS VARCHAR), CAST('bcd2' AS VARBINARY), 'p1')," + - " (null, null, null, null, null, null, 'p2'), " + - " (null, null, null, null, null, null, 'p2'), " + - " (true, BIGINT '2', DOUBLE '3.3', TIMESTAMP '2012-09-09 01:00', CAST('cba1' AS VARCHAR), CAST('dcb1' AS VARBINARY), 'p2'), " + - " (false, BIGINT '1', DOUBLE '2.3', TIMESTAMP '2012-09-09 00:00', CAST('cba2' AS VARCHAR), CAST('dcb2' AS VARBINARY), 'p2') " + - ") AS x (c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, p_varchar)", tableName), 8); + " (null, null, null, null, null, null, null, 'p1'), " + + " (null, null, null, null, null, null, null, 'p1'), " + + " (true, BIGINT '1', DOUBLE '2.2', TIMESTAMP '2012-08-08 01:00', CAST('abc1' AS VARCHAR), CAST('bcd1' AS VARBINARY), sequence(0, 10), 'p1')," + + " (false, BIGINT '0', DOUBLE '1.2', TIMESTAMP '2012-08-08 00:00', CAST('abc2' AS VARCHAR), CAST('bcd2' AS VARBINARY), sequence(10, 20), 'p1')," + + " (null, null, null, null, null, null, null, 'p2'), " + + " (null, null, null, null, null, null, null, 'p2'), " + + " (true, BIGINT '2', DOUBLE '3.3', TIMESTAMP '2012-09-09 01:00', CAST('cba1' AS VARCHAR), CAST('dcb1' AS VARBINARY), sequence(20, 25), 'p2'), " + + " (false, BIGINT '1', DOUBLE '2.3', TIMESTAMP '2012-09-09 00:00', CAST('cba2' AS VARCHAR), CAST('dcb2' AS VARBINARY), sequence(30, 35), 'p2') " + + ") AS x (c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, c_array, p_varchar)", tableName), 8); assertQuery(format("SHOW STATS FOR (SELECT * FROM %s WHERE p_varchar = 'p1')", tableName), "SELECT * FROM VALUES " + @@ -3626,6 +3782,7 @@ public void testCollectColumnStatisticsOnInsert() "('c_timestamp', null, 2.0E0, 0.5E0, null, null, null), " + "('c_varchar', 8.0E0, 2.0E0, 0.5E0, null, null, null), " + "('c_varbinary', 8.0E0, null, 0.5E0, null, null, null), " + + "('c_array', 176.0E0, null, 0.5E0, null, null, null), " + "('p_varchar', 8.0E0, 1.0E0, 0.0E0, null, null, null), " + "(null, null, null, null, 4.0E0, null, null)"); assertQuery(format("SHOW STATS FOR (SELECT * FROM %s WHERE p_varchar = 'p2')", tableName), @@ -3636,6 +3793,7 @@ public void testCollectColumnStatisticsOnInsert() "('c_timestamp', null, 2.0E0, 0.5E0, null, null, null), " + "('c_varchar', 8.0E0, 2.0E0, 0.5E0, null, null, null), " + "('c_varbinary', 8.0E0, null, 0.5E0, null, null, null), " + + "('c_array', 96.0E0, null, 0.5E0, null, null, null), " + "('p_varchar', 8.0E0, 1.0E0, 0.0E0, null, null, null), " + "(null, null, null, null, 4.0E0, null, null)"); @@ -3648,6 +3806,7 @@ public void testCollectColumnStatisticsOnInsert() "('c_timestamp', null, 0E0, 0E0, null, null, null), " + "('c_varchar', 0E0, 0E0, 0E0, null, null, null), " + "('c_varbinary', null, 0E0, 0E0, null, null, null), " + + "('c_array', null, 0E0, 0E0, null, null, null), " + "('p_varchar', 0E0, 0E0, 0E0, null, null, null), " + "(null, null, null, null, 0E0, null, null)"); @@ -3729,6 +3888,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, null, null, null, null, null), " + "('c_varchar', null, null, null, null, null, null), " + "('c_varbinary', null, null, null, null, null, null), " + + "('c_array', null, null, null, null, null, null), " + "('p_varchar', 24.0, 3.0, 0.25, null, null, null), " + "('p_bigint', null, 2.0, 0.25, null, '7', '8'), " + "(null, null, null, null, 16.0, null, null)"); @@ -3743,6 +3903,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, null, null, null, null, null), " + "('c_varchar', null, null, null, null, null, null), " + "('c_varbinary', null, null, null, null, null, null), " + + "('c_array', null, null, null, null, null, null), " + "('p_varchar', 24.0, 3.0, 0.25, null, null, null), " + "('p_bigint', null, 2.0, 0.25, null, '7', '8'), " + "(null, null, null, null, 16.0, null, null)"); @@ -3758,6 +3919,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, 2.0, 0.5, null, null, null), " + "('c_varchar', 8.0, 2.0, 0.5, null, null, null), " + "('c_varbinary', 4.0, null, 0.5, null, null, null), " + + "('c_array', 176.0, null, 0.5, null, null, null), " + "('p_varchar', 8.0, 1.0, 0.0, null, null, null), " + "('p_bigint', null, 1.0, 0.0, null, '7', '7'), " + "(null, null, null, null, 4.0, null, null)"); @@ -3769,6 +3931,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, 2.0, 0.5, null, null, null), " + "('c_varchar', 8.0, 2.0, 0.5, null, null, null), " + "('c_varbinary', 4.0, null, 0.5, null, null, null), " + + "('c_array', 96.0, null, 0.5, null, null, null), " + "('p_varchar', 8.0, 1.0, 0.0, null, null, null), " + "('p_bigint', null, 1.0, 0.0, null, '7', '7'), " + "(null, null, null, null, 4.0, null, null)"); @@ -3780,6 +3943,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, 4.0, 0.0, null, null, null), " + "('c_varchar', 16.0, 4.0, 0.0, null, null, null), " + "('c_varbinary', 8.0, null, 0.0, null, null, null), " + + "('c_array', 192.0, null, 0.0, null, null, null), " + "('p_varchar', 0.0, 0.0, 1.0, null, null, null), " + "('p_bigint', null, 0.0, 1.0, null, null, null), " + "(null, null, null, null, 4.0, null, null)"); @@ -3793,6 +3957,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, null, null, null, null, null), " + "('c_varchar', null, null, null, null, null, null), " + "('c_varbinary', null, null, null, null, null, null), " + + "('c_array', null, null, null, null, null, null), " + "('p_varchar', 8.0, 1.0, 0.0, null, null, null), " + "('p_bigint', null, 1.0, 0.0, null, '8', '8'), " + "(null, null, null, null, 4.0, null, null)"); @@ -3804,6 +3969,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, null, null, null, null, null), " + "('c_varchar', null, null, null, null, null, null), " + "('c_varbinary', null, null, null, null, null, null), " + + "('c_array', null, null, null, null, null, null), " + "('p_varchar', 0.0, 0.0, 0.0, null, null, null), " + "('p_bigint', null, 0.0, 0.0, null, null, null), " + "(null, null, null, null, 0.0, null, null)"); @@ -3815,6 +3981,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, null, null, null, null, null), " + "('c_varchar', null, null, null, null, null, null), " + "('c_varbinary', null, null, null, null, null, null), " + + "('c_array', null, null, null, null, null, null), " + "('p_varchar', 0.0, 0.0, 0.0, null, null, null), " + "('p_bigint', null, 0.0, 0.0, null, null, null), " + "(null, null, null, null, 0.0, null, null)"); @@ -3831,6 +3998,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, 2.0, 0.5, null, null, null), " + "('c_varchar', 8.0, 2.0, 0.5, null, null, null), " + "('c_varbinary', 4.0, null, 0.5, null, null, null), " + + "('c_array', 176.0, null, 0.5, null, null, null), " + "('p_varchar', 8.0, 1.0, 0.0, null, null, null), " + "('p_bigint', null, 1.0, 0.0, null, '7', '7'), " + "(null, null, null, null, 4.0, null, null)"); @@ -3842,6 +4010,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, 2.0, 0.5, null, null, null), " + "('c_varchar', 8.0, 2.0, 0.5, null, null, null), " + "('c_varbinary', 4.0, null, 0.5, null, null, null), " + + "('c_array', 96.0, null, 0.5, null, null, null), " + "('p_varchar', 8.0, 1.0, 0.0, null, null, null), " + "('p_bigint', null, 1.0, 0.0, null, '7', '7'), " + "(null, null, null, null, 4.0, null, null)"); @@ -3853,6 +4022,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, 4.0, 0.0, null, null, null), " + "('c_varchar', 16.0, 4.0, 0.0, null, null, null), " + "('c_varbinary', 8.0, null, 0.0, null, null, null), " + + "('c_array', 192.0, null, 0.0, null, null, null), " + "('p_varchar', 0.0, 0.0, 1.0, null, null, null), " + "('p_bigint', null, 0.0, 1.0, null, null, null), " + "(null, null, null, null, 4.0, null, null)"); @@ -3864,6 +4034,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, 2.0, 0.5, null, null, null), " + "('c_varchar', 8.0, 2.0, 0.5, null, null, null), " + "('c_varbinary', 4.0, null, 0.5, null, null, null), " + + "('c_array', 96.0, null, 0.5, null, null, null), " + "('p_varchar', 8.0, 1.0, 0.0, null, null, null), " + "('p_bigint', null, 1.0, 0.0, null, '8', '8'), " + "(null, null, null, null, 4.0, null, null)"); @@ -3875,6 +4046,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, 0.0, 0.0, null, null, null), " + "('c_varchar', 0.0, 0.0, 0.0, null, null, null), " + "('c_varbinary', 0.0, null, 0.0, null, null, null), " + + "('c_array', 0.0, null, 0.0, null, null, null), " + "('p_varchar', 0.0, 0.0, 0.0, null, null, null), " + "('p_bigint', null, 0.0, 0.0, null, null, null), " + "(null, null, null, null, 0.0, null, null)"); @@ -3886,6 +4058,7 @@ public void testAnalyzePartitionedTable() "('c_timestamp', null, 0.0, 0.0, null, null, null), " + "('c_varchar', 0.0, 0.0, 0.0, null, null, null), " + "('c_varbinary', 0.0, null, 0.0, null, null, null), " + + "('c_array', 0.0, null, 0.0, null, null, null), " + "('p_varchar', 0.0, 0.0, 0.0, null, null, null), " + "('p_bigint', null, 0.0, 0.0, null, null, null), " + "(null, null, null, null, 0.0, null, null)"); @@ -3909,6 +4082,7 @@ public void testAnalyzeUnpartitionedTable() "('c_timestamp', null, null, null, null, null, null), " + "('c_varchar', null, null, null, null, null, null), " + "('c_varbinary', null, null, null, null, null, null), " + + "('c_array', null, null, null, null, null, null), " + "('p_varchar', null, null, null, null, null, null), " + "('p_bigint', null, null, null, null, null, null), " + "(null, null, null, null, 16.0, null, null)"); @@ -3924,6 +4098,7 @@ public void testAnalyzeUnpartitionedTable() "('c_timestamp', null, 10.0, 0.375, null, null, null), " + "('c_varchar', 40.0, 10.0, 0.375, null, null, null), " + "('c_varbinary', 20.0, null, 0.375, null, null, null), " + + "('c_array', 560.0, null, 0.375, null, null, null), " + "('p_varchar', 24.0, 3.0, 0.25, null, null, null), " + "('p_bigint', null, 2.0, 0.25, null, '7', '8'), " + "(null, null, null, null, 16.0, null, null)"); @@ -3958,30 +4133,30 @@ private void createTableForAnalyzeTest(String tableName, boolean partitioned) tableName + (partitioned ? " WITH (partitioned_by = ARRAY['p_varchar', 'p_bigint'])\n" : " ") + "AS " + - "SELECT c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, p_varchar, p_bigint " + + "SELECT c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, c_array, p_varchar, p_bigint " + "FROM ( " + " VALUES " + // p_varchar = 'p1', p_bigint = BIGINT '7' - " (null, null, null, null, null, null, 'p1', BIGINT '7'), " + - " (null, null, null, null, null, null, 'p1', BIGINT '7'), " + - " (true, BIGINT '1', DOUBLE '2.2', TIMESTAMP '2012-08-08 01:00', 'abc1', X'bcd1', 'p1', BIGINT '7'), " + - " (false, BIGINT '0', DOUBLE '1.2', TIMESTAMP '2012-08-08 00:00', 'abc2', X'bcd2', 'p1', BIGINT '7'), " + + " (null, null, null, null, null, null, null, 'p1', BIGINT '7'), " + + " (null, null, null, null, null, null, null, 'p1', BIGINT '7'), " + + " (true, BIGINT '1', DOUBLE '2.2', TIMESTAMP '2012-08-08 01:00', 'abc1', X'bcd1', sequence(0, 10), 'p1', BIGINT '7'), " + + " (false, BIGINT '0', DOUBLE '1.2', TIMESTAMP '2012-08-08 00:00', 'abc2', X'bcd2', sequence(10, 20), 'p1', BIGINT '7'), " + // p_varchar = 'p2', p_bigint = BIGINT '7' - " (null, null, null, null, null, null, 'p2', BIGINT '7'), " + - " (null, null, null, null, null, null, 'p2', BIGINT '7'), " + - " (true, BIGINT '2', DOUBLE '3.3', TIMESTAMP '2012-09-09 01:00', 'cba1', X'dcb1', 'p2', BIGINT '7'), " + - " (false, BIGINT '1', DOUBLE '2.3', TIMESTAMP '2012-09-09 00:00', 'cba2', X'dcb2', 'p2', BIGINT '7'), " + + " (null, null, null, null, null, null, null, 'p2', BIGINT '7'), " + + " (null, null, null, null, null, null, null, 'p2', BIGINT '7'), " + + " (true, BIGINT '2', DOUBLE '3.3', TIMESTAMP '2012-09-09 01:00', 'cba1', X'dcb1', sequence(20, 25), 'p2', BIGINT '7'), " + + " (false, BIGINT '1', DOUBLE '2.3', TIMESTAMP '2012-09-09 00:00', 'cba2', X'dcb2', sequence(30, 35), 'p2', BIGINT '7'), " + // p_varchar = 'p3', p_bigint = BIGINT '8' - " (null, null, null, null, null, null, 'p3', BIGINT '8'), " + - " (null, null, null, null, null, null, 'p3', BIGINT '8'), " + - " (true, BIGINT '3', DOUBLE '4.4', TIMESTAMP '2012-10-10 01:00', 'bca1', X'cdb1', 'p3', BIGINT '8'), " + - " (false, BIGINT '2', DOUBLE '3.4', TIMESTAMP '2012-10-10 00:00', 'bca2', X'cdb2', 'p3', BIGINT '8'), " + + " (null, null, null, null, null, null, null, 'p3', BIGINT '8'), " + + " (null, null, null, null, null, null, null, 'p3', BIGINT '8'), " + + " (true, BIGINT '3', DOUBLE '4.4', TIMESTAMP '2012-10-10 01:00', 'bca1', X'cdb1', sequence(40, 45), 'p3', BIGINT '8'), " + + " (false, BIGINT '2', DOUBLE '3.4', TIMESTAMP '2012-10-10 00:00', 'bca2', X'cdb2', sequence(50, 55), 'p3', BIGINT '8'), " + // p_varchar = NULL, p_bigint = NULL - " (false, BIGINT '7', DOUBLE '7.7', TIMESTAMP '1977-07-07 07:07', 'efa1', X'efa1', NULL, NULL), " + - " (false, BIGINT '6', DOUBLE '6.7', TIMESTAMP '1977-07-07 07:06', 'efa2', X'efa2', NULL, NULL), " + - " (false, BIGINT '5', DOUBLE '5.7', TIMESTAMP '1977-07-07 07:05', 'efa3', X'efa3', NULL, NULL), " + - " (false, BIGINT '4', DOUBLE '4.7', TIMESTAMP '1977-07-07 07:04', 'efa4', X'efa4', NULL, NULL) " + - ") AS x (c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, p_varchar, p_bigint)", 16); + " (false, BIGINT '7', DOUBLE '7.7', TIMESTAMP '1977-07-07 07:07', 'efa1', X'efa1', sequence(60, 65), NULL, NULL), " + + " (false, BIGINT '6', DOUBLE '6.7', TIMESTAMP '1977-07-07 07:06', 'efa2', X'efa2', sequence(70, 75), NULL, NULL), " + + " (false, BIGINT '5', DOUBLE '5.7', TIMESTAMP '1977-07-07 07:05', 'efa3', X'efa3', sequence(80, 85), NULL, NULL), " + + " (false, BIGINT '4', DOUBLE '4.7', TIMESTAMP '1977-07-07 07:04', 'efa4', X'efa4', sequence(90, 95), NULL, NULL) " + + ") AS x (c_boolean, c_bigint, c_double, c_timestamp, c_varchar, c_varbinary, c_array, p_varchar, p_bigint)", 16); if (partitioned) { // Create empty partitions @@ -4154,6 +4329,16 @@ public void testCtasFailsWithAvroSchemaUrl() assertQueryFails(ctasSql, "CREATE TABLE AS not supported when Avro schema url is set"); } + @Test + public void testBucketedTablesFailWithTooManyBuckets() + throws Exception + { + @Language("SQL") String createSql = "CREATE TABLE t (dummy VARCHAR)\n" + + "WITH (bucket_count = 1000001, bucketed_by=ARRAY['dummy'])"; + + assertQueryFails(createSql, "bucket_count should be no more than 1000000"); + } + @Test public void testBucketedTablesFailWithAvroSchemaUrl() throws Exception @@ -4221,7 +4406,7 @@ public void testLikeSerializesWithPushdownFilter() Session pushdownFilterEnabled = Session.builder(getQueryRunner().getDefaultSession()) .setCatalogSessionProperty(catalog, PUSHDOWN_FILTER_ENABLED, "true") .build(); - assertQueryFails(pushdownFilterEnabled, "SELECT comment FROM lineitem WHERE comment LIKE 'abc%'", "Error opening Hive split.*Unsupported type: VARCHAR"); + assertQuerySucceeds(pushdownFilterEnabled, "SELECT comment FROM lineitem WHERE comment LIKE 'abc%'"); } @Test @@ -4251,6 +4436,24 @@ public void testIgnoreTableBucketing() assertQueryFails(ignoreBucketingSession, query, "Table bucketing is ignored\\. The virtual \"\\$bucket\" column cannot be referenced\\."); } + @Test + public void testTableWriterMergeNodeIsPresent() + { + assertUpdate( + getSession(), + "CREATE TABLE test_table_writer_merge_operator AS SELECT orderkey FROM orders", + 15000, + assertTableWriterMergeNodeIsPresent()); + } + + private static Consumer assertTableWriterMergeNodeIsPresent() + { + return plan -> assertTrue(searchFrom(plan.getRoot()) + .where(node -> node instanceof TableWriterMergeNode) + .findFirst() + .isPresent()); + } + private HiveInsertTableHandle getHiveInsertTableHandle(Session session, String tableName) { Metadata metadata = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getMetadata(); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java index 3c0c282be2568..a45b3eeabffb8 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java @@ -15,10 +15,10 @@ import com.facebook.presto.Session; import com.facebook.presto.cost.StatsProvider; -import com.facebook.presto.metadata.CastType; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.Subfield; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.TableScanNode; @@ -28,6 +28,8 @@ import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.assertions.MatchResult; import com.facebook.presto.sql.planner.assertions.Matcher; @@ -36,37 +38,49 @@ import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; import com.facebook.presto.tests.DistributedQueryRunner; import com.google.common.base.Functions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import io.airlift.slice.Slice; import io.airlift.slice.Slices; import org.testng.annotations.Test; import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; +import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG; import static com.facebook.presto.hive.HiveQueryRunner.createQueryRunner; +import static com.facebook.presto.hive.HiveSessionProperties.COLLECT_COLUMN_STATISTICS_ON_WRITE; import static com.facebook.presto.hive.HiveSessionProperties.PUSHDOWN_FILTER_ENABLED; +import static com.facebook.presto.hive.HiveSessionProperties.RANGE_FILTERS_ON_SUBSCRIPTS_ENABLED; import static com.facebook.presto.hive.TestHiveIntegrationSmokeTest.assertRemoteExchangesCount; -import static com.facebook.presto.spi.function.OperatorType.CAST; import static com.facebook.presto.spi.function.OperatorType.EQUAL; +import static com.facebook.presto.spi.predicate.Domain.multipleValues; +import static com.facebook.presto.spi.predicate.Domain.notNull; +import static com.facebook.presto.spi.predicate.Domain.singleValue; import static com.facebook.presto.spi.predicate.TupleDomain.withColumnDomains; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.spi.type.VarcharType.createVarcharType; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.planner.assertions.MatchResult.NO_MATCH; import static com.facebook.presto.sql.planner.assertions.MatchResult.match; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.exchange; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.output; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; @@ -74,13 +88,18 @@ import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; +import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.tpch.TpchTable.LINE_ITEM; +import static io.airlift.tpch.TpchTable.ORDERS; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; public class TestHiveLogicalPlanner @@ -88,7 +107,52 @@ public class TestHiveLogicalPlanner { public TestHiveLogicalPlanner() { - super(() -> createQueryRunner(ImmutableList.of(LINE_ITEM), ImmutableMap.of("experimental.pushdown-subfields-enabled", "true"), Optional.empty())); + super(() -> createQueryRunner(ImmutableList.of(ORDERS, LINE_ITEM), ImmutableMap.of("experimental.pushdown-subfields-enabled", "true"), Optional.empty())); + } + + @Test + public void testRepeatedFilterPushdown() + { + QueryRunner queryRunner = getQueryRunner(); + + try { + queryRunner.execute("CREATE TABLE orders_partitioned WITH (partitioned_by = ARRAY['ds']) AS " + + "SELECT orderkey, orderpriority, '2019-11-01' as ds FROM orders WHERE orderkey < 1000 " + + "UNION ALL " + + "SELECT orderkey, orderpriority, '2019-11-02' as ds FROM orders WHERE orderkey < 1000"); + + queryRunner.execute("CREATE TABLE lineitem_unpartitioned AS " + + "SELECT orderkey, linenumber, shipmode, '2019-11-01' as ds FROM lineitem WHERE orderkey < 1000 " + + "UNION ALL " + + "SELECT orderkey, linenumber, shipmode, '2019-11-02' as ds FROM lineitem WHERE orderkey < 1000 "); + + TupleDomain ordersDomain = withColumnDomains(ImmutableMap.of( + "orderpriority", singleValue(createVarcharType(15), utf8Slice("1-URGENT")))); + + TupleDomain lineitemDomain = withColumnDomains(ImmutableMap.of( + "shipmode", singleValue(createVarcharType(10), utf8Slice("MAIL")), + "ds", singleValue(createVarcharType(10), utf8Slice("2019-11-02")))); + + assertPlan(pushdownFilterEnabled(), + "WITH a AS (\n" + + " SELECT ds, orderkey\n" + + " FROM orders_partitioned\n" + + " WHERE orderpriority = '1-URGENT' AND ds > '2019-11-01'\n" + + "),\n" + + "b AS (\n" + + " SELECT ds, orderkey, linenumber\n" + + " FROM lineitem_unpartitioned\n" + + " WHERE shipmode = 'MAIL'\n" + + ")\n" + + "SELECT * FROM a LEFT JOIN b ON a.ds = b.ds", + anyTree(node(JoinNode.class, + anyTree(tableScan("orders_partitioned", ordersDomain, TRUE_CONSTANT, ImmutableSet.of("orderpriority"))), + anyTree(tableScan("lineitem_unpartitioned", lineitemDomain, TRUE_CONSTANT, ImmutableSet.of("shipmode", "ds")))))); + } + finally { + queryRunner.execute("DROP TABLE IF EXISTS orders_partitioned"); + queryRunner.execute("DROP TABLE IF EXISTS lineitem_unpartitioned"); + } } @Test @@ -105,12 +169,12 @@ public void testPushdownFilter() assertPlan(pushdownFilterEnabled, "SELECT linenumber FROM lineitem WHERE partkey = 10", output(exchange( strictTableScan("lineitem", identityMap("linenumber")))), - plan -> assertTableLayout(plan, "lineitem", withColumnDomains(ImmutableMap.of(new Subfield("partkey", ImmutableList.of()), Domain.singleValue(BIGINT, 10L))), TRUE_CONSTANT, ImmutableSet.of("partkey"))); + plan -> assertTableLayout(plan, "lineitem", withColumnDomains(ImmutableMap.of(new Subfield("partkey", ImmutableList.of()), singleValue(BIGINT, 10L))), TRUE_CONSTANT, ImmutableSet.of("partkey"))); assertPlan(pushdownFilterEnabled, "SELECT partkey, linenumber FROM lineitem WHERE partkey = 10", output(exchange( strictTableScan("lineitem", identityMap("partkey", "linenumber")))), - plan -> assertTableLayout(plan, "lineitem", withColumnDomains(ImmutableMap.of(new Subfield("partkey", ImmutableList.of()), Domain.singleValue(BIGINT, 10L))), TRUE_CONSTANT, ImmutableSet.of("partkey"))); + plan -> assertTableLayout(plan, "lineitem", withColumnDomains(ImmutableMap.of(new Subfield("partkey", ImmutableList.of()), singleValue(BIGINT, 10L))), TRUE_CONSTANT, ImmutableSet.of("partkey"))); // Only remaining predicate assertPlan("SELECT linenumber FROM lineitem WHERE mod(orderkey, 2) = 1", @@ -151,12 +215,75 @@ public void testPushdownFilter() assertPlan(pushdownFilterEnabled, "SELECT linenumber FROM lineitem WHERE partkey = 10 AND mod(orderkey, 2) = 1", output(exchange( strictTableScan("lineitem", identityMap("linenumber")))), - plan -> assertTableLayout(plan, "lineitem", withColumnDomains(ImmutableMap.of(new Subfield("partkey", ImmutableList.of()), Domain.singleValue(BIGINT, 10L))), remainingPredicate, ImmutableSet.of("partkey", "orderkey"))); + plan -> assertTableLayout(plan, "lineitem", withColumnDomains(ImmutableMap.of(new Subfield("partkey", ImmutableList.of()), singleValue(BIGINT, 10L))), remainingPredicate, ImmutableSet.of("partkey", "orderkey"))); assertPlan(pushdownFilterEnabled, "SELECT partkey, orderkey, linenumber FROM lineitem WHERE partkey = 10 AND mod(orderkey, 2) = 1", output(exchange( strictTableScan("lineitem", identityMap("partkey", "orderkey", "linenumber")))), - plan -> assertTableLayout(plan, "lineitem", withColumnDomains(ImmutableMap.of(new Subfield("partkey", ImmutableList.of()), Domain.singleValue(BIGINT, 10L))), remainingPredicate, ImmutableSet.of("partkey", "orderkey"))); + plan -> assertTableLayout(plan, "lineitem", withColumnDomains(ImmutableMap.of(new Subfield("partkey", ImmutableList.of()), singleValue(BIGINT, 10L))), remainingPredicate, ImmutableSet.of("partkey", "orderkey"))); + } + + @Test + public void testPartitionPruning() + { + QueryRunner queryRunner = getQueryRunner(); + queryRunner.execute("CREATE TABLE test_partition_pruning WITH (partitioned_by = ARRAY['ds']) AS " + + "SELECT orderkey, CAST(to_iso8601(date_add('DAY', orderkey % 7, date('2019-11-01'))) AS VARCHAR) AS ds FROM orders WHERE orderkey < 1000"); + + Session pushdownFilterEnabled = pushdownFilterEnabled(); + try { + assertPlan(pushdownFilterEnabled, "SELECT * FROM test_partition_pruning WHERE ds = '2019-11-01'", + anyTree(tableScanWithConstraint("test_partition_pruning", ImmutableMap.of("ds", singleValue(VARCHAR, utf8Slice("2019-11-01")))))); + + assertPlan(pushdownFilterEnabled, "SELECT * FROM test_partition_pruning WHERE date(ds) = date('2019-11-01')", + anyTree(tableScanWithConstraint("test_partition_pruning", ImmutableMap.of("ds", singleValue(VARCHAR, utf8Slice("2019-11-01")))))); + + assertPlan(pushdownFilterEnabled, "SELECT * FROM test_partition_pruning WHERE date(ds) BETWEEN date('2019-11-02') AND date('2019-11-04')", + anyTree(tableScanWithConstraint("test_partition_pruning", ImmutableMap.of("ds", multipleValues(VARCHAR, utf8Slices("2019-11-02", "2019-11-03", "2019-11-04")))))); + + assertPlan(pushdownFilterEnabled, "SELECT * FROM test_partition_pruning WHERE ds < '2019-11-05'", + anyTree(tableScanWithConstraint("test_partition_pruning", ImmutableMap.of("ds", multipleValues(VARCHAR, utf8Slices("2019-11-01", "2019-11-02", "2019-11-03", "2019-11-04")))))); + + assertPlan(pushdownFilterEnabled, "SELECT * FROM test_partition_pruning WHERE date(ds) > date('2019-11-02')", + anyTree(tableScanWithConstraint("test_partition_pruning", ImmutableMap.of("ds", multipleValues(VARCHAR, utf8Slices("2019-11-03", "2019-11-04", "2019-11-05", "2019-11-06", "2019-11-07")))))); + + assertPlan(pushdownFilterEnabled, "SELECT * FROM test_partition_pruning WHERE ds < '2019-11-05' AND date(ds) > date('2019-11-02')", + anyTree(tableScanWithConstraint("test_partition_pruning", ImmutableMap.of("ds", multipleValues(VARCHAR, utf8Slices("2019-11-03", "2019-11-04")))))); + } + finally { + queryRunner.execute("DROP TABLE test_partition_pruning"); + } + } + + private static List utf8Slices(String... values) + { + return Arrays.stream(values).map(Slices::utf8Slice).collect(toImmutableList()); + } + + private static PlanMatchPattern tableScanWithConstraint(String tableName, Map expectedConstraint) + { + return PlanMatchPattern.tableScan(tableName).with(new Matcher() { + @Override + public boolean shapeMatches(PlanNode node) + { + return node instanceof TableScanNode; + } + + @Override + public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) + { + TableScanNode tableScan = (TableScanNode) node; + TupleDomain constraint = tableScan.getCurrentConstraint() + .transform(HiveColumnHandle.class::cast) + .transform(HiveColumnHandle::getName); + + if (!expectedConstraint.equals(constraint.getDomains().get())) { + return NO_MATCH; + } + + return match(); + } + }); } @Test @@ -174,34 +301,37 @@ public void testPushdownFilterOnSubfields() "e map(varchar, bigint)))"); assertPushdownFilterOnSubfields("SELECT * FROM test_pushdown_filter_on_subfields WHERE a[1] = 1", - ImmutableMap.of(new Subfield("a[1]"), Domain.singleValue(BIGINT, 1L))); + ImmutableMap.of(new Subfield("a[1]"), singleValue(BIGINT, 1L))); assertPushdownFilterOnSubfields("SELECT * FROM test_pushdown_filter_on_subfields where a[1 + 1] = 1", - ImmutableMap.of(new Subfield("a[2]"), Domain.singleValue(BIGINT, 1L))); + ImmutableMap.of(new Subfield("a[2]"), singleValue(BIGINT, 1L))); assertPushdownFilterOnSubfields("SELECT * FROM test_pushdown_filter_on_subfields WHERE b['foo'] = 1", - ImmutableMap.of(new Subfield("b[\"foo\"]"), Domain.singleValue(BIGINT, 1L))); + ImmutableMap.of(new Subfield("b[\"foo\"]"), singleValue(BIGINT, 1L))); assertPushdownFilterOnSubfields("SELECT * FROM test_pushdown_filter_on_subfields WHERE b[concat('f','o', 'o')] = 1", - ImmutableMap.of(new Subfield("b[\"foo\"]"), Domain.singleValue(BIGINT, 1L))); + ImmutableMap.of(new Subfield("b[\"foo\"]"), singleValue(BIGINT, 1L))); assertPushdownFilterOnSubfields("SELECT * FROM test_pushdown_filter_on_subfields WHERE c.a = 1", - ImmutableMap.of(new Subfield("c.a"), Domain.singleValue(BIGINT, 1L))); + ImmutableMap.of(new Subfield("c.a"), singleValue(BIGINT, 1L))); assertPushdownFilterOnSubfields("SELECT * FROM test_pushdown_filter_on_subfields WHERE c.b.x = 1", - ImmutableMap.of(new Subfield("c.b.x"), Domain.singleValue(BIGINT, 1L))); + ImmutableMap.of(new Subfield("c.b.x"), singleValue(BIGINT, 1L))); assertPushdownFilterOnSubfields("SELECT * FROM test_pushdown_filter_on_subfields WHERE c.c[5] = 1", - ImmutableMap.of(new Subfield("c.c[5]"), Domain.singleValue(BIGINT, 1L))); + ImmutableMap.of(new Subfield("c.c[5]"), singleValue(BIGINT, 1L))); assertPushdownFilterOnSubfields("SELECT * FROM test_pushdown_filter_on_subfields WHERE c.d[5] = 1", - ImmutableMap.of(new Subfield("c.d[5]"), Domain.singleValue(BIGINT, 1L))); + ImmutableMap.of(new Subfield("c.d[5]"), singleValue(BIGINT, 1L))); assertPushdownFilterOnSubfields("SELECT * FROM test_pushdown_filter_on_subfields WHERE c.e[concat('f', 'o', 'o')] = 1", - ImmutableMap.of(new Subfield("c.e[\"foo\"]"), Domain.singleValue(BIGINT, 1L))); + ImmutableMap.of(new Subfield("c.e[\"foo\"]"), singleValue(BIGINT, 1L))); assertPushdownFilterOnSubfields("SELECT * FROM test_pushdown_filter_on_subfields WHERE c.e['foo'] = 1", - ImmutableMap.of(new Subfield("c.e[\"foo\"]"), Domain.singleValue(BIGINT, 1L))); + ImmutableMap.of(new Subfield("c.e[\"foo\"]"), singleValue(BIGINT, 1L))); + + assertPushdownFilterOnSubfields("SELECT * FROM test_pushdown_filter_on_subfields WHERE c.a IS NOT NULL AND c.c IS NOT NULL", + ImmutableMap.of(new Subfield("c.a"), notNull(BIGINT), new Subfield("c.c"), notNull(new ArrayType(BIGINT)))); assertUpdate("DROP TABLE test_pushdown_filter_on_subfields"); } @@ -209,7 +339,11 @@ public void testPushdownFilterOnSubfields() @Test public void testPushdownArraySubscripts() { - assertUpdate("CREATE TABLE test_pushdown_array_subscripts(id bigint, a array(bigint), b array(array(varchar)))"); + assertUpdate("CREATE TABLE test_pushdown_array_subscripts(id bigint, " + + "a array(bigint), " + + "b array(array(varchar)), " + + "y array(row(a bigint, b varchar, c double, d row(d1 bigint, d2 double))), " + + "z array(array(row(p bigint, e row(e1 bigint, e2 varchar)))))"); assertPushdownSubscripts("test_pushdown_array_subscripts"); @@ -242,7 +376,12 @@ public void testPushdownArraySubscripts() @Test public void testPushdownMapSubscripts() { - assertUpdate("CREATE TABLE test_pushdown_map_subscripts(id bigint, a map(bigint, bigint), b map(bigint, map(bigint, varchar)), c map(varchar, bigint))"); + assertUpdate("CREATE TABLE test_pushdown_map_subscripts(id bigint, " + + "a map(bigint, bigint), " + + "b map(bigint, map(bigint, varchar)), " + + "c map(varchar, bigint), \n" + + "y map(bigint, row(a bigint, b varchar, c double, d row(d1 bigint, d2 double)))," + + "z map(bigint, map(bigint, row(p bigint, e row(e1 bigint, e2 varchar)))))"); assertPushdownSubscripts("test_pushdown_map_subscripts"); @@ -351,6 +490,24 @@ private void assertPushdownSubscripts(String tableName) assertPushdownSubfields(format("SELECT min(a[1]) FROM %s GROUP BY id", tableName), tableName, ImmutableMap.of("a", toSubfields("a[1]"))); + assertPushdownSubfields(format("SELECT arbitrary(y[1]).a FROM %s GROUP BY id", tableName), tableName, + ImmutableMap.of("y", toSubfields("y[1].a"))); + + assertPushdownSubfields(format("SELECT arbitrary(y[1]).d.d1 FROM %s GROUP BY id", tableName), tableName, + ImmutableMap.of("y", toSubfields("y[1].d.d1"))); + + assertPushdownSubfields(format("SELECT arbitrary(y[2].d).d1 FROM %s GROUP BY id", tableName), tableName, + ImmutableMap.of("y", toSubfields("y[2].d.d1"))); + + assertPushdownSubfields(format("SELECT arbitrary(y[3].d.d1) FROM %s GROUP BY id", tableName), tableName, + ImmutableMap.of("y", toSubfields("y[3].d.d1"))); + + assertPushdownSubfields(format("SELECT arbitrary(z[1][2]).e.e1 FROM %s GROUP BY id", tableName), tableName, + ImmutableMap.of("z", toSubfields("z[1][2].e.e1"))); + + assertPushdownSubfields(format("SELECT arbitrary(z[2][3].e).e2 FROM %s GROUP BY id", tableName), tableName, + ImmutableMap.of("z", toSubfields("z[2][3].e.e2"))); + // Union assertPlan(format("SELECT a[1] FROM %s UNION ALL SELECT a[2] FROM %s", tableName, tableName), anyTree(exchange( @@ -392,6 +549,14 @@ private void assertPushdownSubscripts(String tableName) assertPushdownSubfields(format("SELECT a[1] FROM (SELECT DISTINCT * FROM %s) LIMIT 10", tableName), tableName, ImmutableMap.of()); + + // No pass through subfield pruning + assertPushdownSubfields(format("SELECT id, min(y[1]).a FROM %s GROUP BY 1", tableName), tableName, + ImmutableMap.of("y", toSubfields("y[1]"))); + assertPushdownSubfields(format("SELECT id, min(y[1]).a, min(y[1].d).d1 FROM %s GROUP BY 1", tableName), tableName, + ImmutableMap.of("y", toSubfields("y[1]"))); + assertPushdownSubfields(format("SELECT id, min(z[1][2]).e.e1 FROM %s GROUP BY 1", tableName), tableName, + ImmutableMap.of("z", toSubfields("z[1][2]"))); } @Test @@ -441,6 +606,21 @@ public void testPushdownSubfields() assertPushdownSubfields("SELECT id, min(x.a + length(y[2].b)) * avg(x.d.d1) FROM test_pushdown_struct_subfields GROUP BY 1", "test_pushdown_struct_subfields", ImmutableMap.of("x", toSubfields("x.a", "x.d.d1"), "y", toSubfields("y[2].b"))); + assertPushdownSubfields("SELECT id, arbitrary(x.a) FROM test_pushdown_struct_subfields GROUP BY 1", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.a"))); + + assertPushdownSubfields("SELECT id, arbitrary(x).a FROM test_pushdown_struct_subfields GROUP BY 1", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.a"))); + + assertPushdownSubfields("SELECT id, arbitrary(x).d.d1 FROM test_pushdown_struct_subfields GROUP BY 1", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.d.d1"))); + + assertPushdownSubfields("SELECT id, arbitrary(x.d).d1 FROM test_pushdown_struct_subfields GROUP BY 1", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.d.d1"))); + + assertPushdownSubfields("SELECT id, arbitrary(x.d.d2) FROM test_pushdown_struct_subfields GROUP BY 1", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.d.d2"))); + // Unnest assertPushdownSubfields("SELECT t.a, t.d.d1, x.a FROM test_pushdown_struct_subfields CROSS JOIN UNNEST(y) as t(a, b, c, d)", "test_pushdown_struct_subfields", ImmutableMap.of("x", toSubfields("x.a"), "y", toSubfields("y[*].a", "y[*].d.d1"))); @@ -451,6 +631,28 @@ public void testPushdownSubfields() assertPushdownSubfields("SELECT id, x.a FROM test_pushdown_struct_subfields CROSS JOIN UNNEST(y) as t(a, b, c, d)", "test_pushdown_struct_subfields", ImmutableMap.of("x", toSubfields("x.a"))); + // Legacy unnest + Session legacyUnnest = Session.builder(getSession()).setSystemProperty("legacy_unnest", "true").build(); + assertPushdownSubfields(legacyUnnest, "SELECT t.y.a, t.y.d.d1, x.a FROM test_pushdown_struct_subfields CROSS JOIN UNNEST(y) as t(y)", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.a"), "y", toSubfields("y[*].a", "y[*].d.d1"))); + + assertPushdownSubfields(legacyUnnest, "SELECT t.*, x.a FROM test_pushdown_struct_subfields CROSS JOIN UNNEST(y) as t(y)", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.a"))); + + assertPushdownSubfields(legacyUnnest, "SELECT id, x.a FROM test_pushdown_struct_subfields CROSS JOIN UNNEST(y) as t(y)", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.a"))); + + // Case sensitivity + assertPushdownSubfields("SELECT x.a, x.b, x.A + 2 FROM test_pushdown_struct_subfields WHERE x.B LIKE 'abc%'", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.a", "x.b"))); + + // No pass-through subfield pruning + assertPushdownSubfields("SELECT id, min(x.d).d1 FROM test_pushdown_struct_subfields GROUP BY 1", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.d"))); + + assertPushdownSubfields("SELECT id, min(x.d).d1, min(x.d.d2) FROM test_pushdown_struct_subfields GROUP BY 1", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.d"))); + assertUpdate("DROP TABLE test_pushdown_struct_subfields"); } @@ -462,7 +664,11 @@ public void testPushdownSubfieldsAssorted() "a array(bigint), " + "b map(bigint, bigint), " + "c map(varchar, bigint), " + - "d row(d1 bigint, d2 array(bigint), d3 map(bigint, bigint), d4 row(x double, y double)))"); + "d row(d1 bigint, d2 array(bigint), d3 map(bigint, bigint), d4 row(x double, y double)), " + + "w array(array(row(p bigint, e row(e1 bigint, e2 varchar)))), " + + "x row(a bigint, b varchar, c double, d row(d1 bigint, d2 double)), " + + "y array(row(a bigint, b varchar, c double, d row(d1 bigint, d2 double))), " + + "z row(a bigint, b varchar, c double))"); assertPushdownSubfields("SELECT id, a[1], mod(a[2], 3), b[10], c['cat'] + c['dog'], d.d1 * d.d2[5] / d.d3[2], d.d4.x FROM test_pushdown_subfields", "test_pushdown_subfields", ImmutableMap.of( @@ -483,6 +689,44 @@ public void testPushdownSubfieldsAssorted() "a", toSubfields("a[1]"), "d", toSubfields("d.d3[5]"))); + // Subfield pruning should pass-through arbitrary() function + assertPushdownSubfields("SELECT id, " + + "arbitrary(x.a), " + + "arbitrary(x).a, " + + "arbitrary(x).d.d1, " + + "arbitrary(x.d).d1, " + + "arbitrary(x.d.d2), " + + "arbitrary(y[1]).a, " + + "arbitrary(y[1]).d.d1, " + + "arbitrary(y[2]).d.d1, " + + "arbitrary(y[3].d.d1), " + + "arbitrary(z).c, " + + "arbitrary(w[1][2]).e.e1, " + + "arbitrary(w[2][3].e.e2) " + + "FROM test_pushdown_subfields " + + "GROUP BY 1", "test_pushdown_subfields", + ImmutableMap.of("x", toSubfields("x.a", "x.d.d1", "x.d.d2"), + "y", toSubfields("y[1].a", "y[1].d.d1", "y[2].d.d1", "y[3].d.d1"), + "z", toSubfields("z.c"), + "w", toSubfields("w[1][2].e.e1", "w[2][3].e.e2"))); + + // Subfield pruning should not pass-through other aggregate functions e.g. min() function + assertPushdownSubfields("SELECT id, " + + "min(x.d).d1, " + + "min(x.d.d2), " + + "min(z).c, " + + "min(z.b), " + + "min(y[1]).a, " + + "min(y[1]).d.d1, " + + "min(y[2].d.d1), " + + "min(w[1][2]).e.e1, " + + "min(w[2][3].e.e2) " + + "FROM test_pushdown_subfields " + + "GROUP BY 1", "test_pushdown_subfields", + ImmutableMap.of("x", toSubfields("x.d"), + "y", toSubfields("y[1]", "y[2].d.d1"), + "w", toSubfields("w[1][2]", "w[2][3].e.e2"))); + assertUpdate("DROP TABLE test_pushdown_subfields"); } @@ -526,6 +770,145 @@ public void testVirtualBucketing() } } + // Make sure subfield pruning doesn't interfere with cost-based optimizer + @Test + public void testPushdownSubfieldsAndJoinReordering() + { + Session collectStatistics = Session.builder(getSession()) + .setCatalogSessionProperty(HIVE_CATALOG, COLLECT_COLUMN_STATISTICS_ON_WRITE, "true") + .build(); + + getQueryRunner().execute(collectStatistics, "CREATE TABLE orders_ex AS SELECT orderkey, custkey, array[custkey] as keys FROM orders"); + + try { + Session joinReorderingOn = Session.builder(pushdownFilterEnabled()) + .setSystemProperty(JOIN_REORDERING_STRATEGY, FeaturesConfig.JoinReorderingStrategy.AUTOMATIC.name()) + .build(); + + Session joinReorderingOff = Session.builder(pushdownFilterEnabled()) + .setSystemProperty(JOIN_REORDERING_STRATEGY, FeaturesConfig.JoinReorderingStrategy.ELIMINATE_CROSS_JOINS.name()) + .build(); + + assertPlan(joinReorderingOff, "SELECT sum(custkey) FROM orders_ex o, lineitem l WHERE o.orderkey = l.orderkey", + anyTree(join(INNER, ImmutableList.of(equiJoinClause("o_orderkey", "l_orderkey")), + anyTree(PlanMatchPattern.tableScan("orders_ex", ImmutableMap.of("o_orderkey", "orderkey"))), + anyTree(PlanMatchPattern.tableScan("lineitem", ImmutableMap.of("l_orderkey", "orderkey")))))); + + assertPlan(joinReorderingOff, "SELECT sum(keys[1]) FROM orders_ex o, lineitem l WHERE o.orderkey = l.orderkey", + anyTree(join(INNER, ImmutableList.of(equiJoinClause("o_orderkey", "l_orderkey")), + anyTree(PlanMatchPattern.tableScan("orders_ex", ImmutableMap.of("o_orderkey", "orderkey"))), + anyTree(PlanMatchPattern.tableScan("lineitem", ImmutableMap.of("l_orderkey", "orderkey")))))); + + assertPlan(joinReorderingOn, "SELECT sum(custkey) FROM orders_ex o, lineitem l WHERE o.orderkey = l.orderkey", + anyTree(join(INNER, ImmutableList.of(equiJoinClause("l_orderkey", "o_orderkey")), + anyTree(PlanMatchPattern.tableScan("lineitem", ImmutableMap.of("l_orderkey", "orderkey"))), + anyTree(PlanMatchPattern.tableScan("orders_ex", ImmutableMap.of("o_orderkey", "orderkey")))))); + + assertPlan(joinReorderingOn, "SELECT sum(keys[1]) FROM orders_ex o, lineitem l WHERE o.orderkey = l.orderkey", + anyTree(join(INNER, ImmutableList.of(equiJoinClause("l_orderkey", "o_orderkey")), + anyTree(PlanMatchPattern.tableScan("lineitem", ImmutableMap.of("l_orderkey", "orderkey"))), + anyTree(PlanMatchPattern.tableScan("orders_ex", ImmutableMap.of("o_orderkey", "orderkey")))))); + + assertPlan(joinReorderingOff, "SELECT l.discount, l.orderkey, o.totalprice FROM lineitem l, orders o WHERE l.orderkey = o.orderkey AND l.quantity < 2 AND o.totalprice BETWEEN 0 AND 200000", + anyTree( + node(JoinNode.class, + anyTree(tableScan("lineitem", ImmutableMap.of())), + anyTree(tableScan("orders", ImmutableMap.of()))))); + + assertPlan(joinReorderingOn, "SELECT l.discount, l.orderkey, o.totalprice FROM lineitem l, orders o WHERE l.orderkey = o.orderkey AND l.quantity < 2 AND o.totalprice BETWEEN 0 AND 200000", + anyTree( + node(JoinNode.class, + anyTree(tableScan("orders", ImmutableMap.of())), + anyTree(tableScan("lineitem", ImmutableMap.of()))))); + + assertPlan(joinReorderingOff, "SELECT keys[1] FROM orders_ex o, lineitem l WHERE o.orderkey = l.orderkey AND o.keys[1] > 0", + anyTree(join(INNER, ImmutableList.of(equiJoinClause("o_orderkey", "l_orderkey")), + anyTree(PlanMatchPattern.tableScan("orders_ex", ImmutableMap.of("o_orderkey", "orderkey"))), + anyTree(PlanMatchPattern.tableScan("lineitem", ImmutableMap.of("l_orderkey", "orderkey")))))); + + assertPlan(joinReorderingOn, "SELECT keys[1] FROM orders_ex o, lineitem l WHERE o.orderkey = l.orderkey AND o.keys[1] > 0", + anyTree(join(INNER, ImmutableList.of(equiJoinClause("l_orderkey", "o_orderkey")), + anyTree(PlanMatchPattern.tableScan("lineitem", ImmutableMap.of("l_orderkey", "orderkey"))), + anyTree(PlanMatchPattern.tableScan("orders_ex", ImmutableMap.of("o_orderkey", "orderkey")))))); + } + finally { + assertUpdate("DROP TABLE orders_ex"); + } + } + + @Test + public void testBucketPruning() + { + QueryRunner queryRunner = getQueryRunner(); + queryRunner.execute("CREATE TABLE orders_bucketed WITH (bucket_count = 11, bucketed_by = ARRAY['orderkey']) AS " + + "SELECT * FROM orders"); + + try { + assertPlan(getSession(), "SELECT * FROM orders_bucketed WHERE orderkey = 100", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertBucketFilter(plan, "orders_bucketed", 11, ImmutableSet.of(1))); + + assertPlan(getSession(), "SELECT * FROM orders_bucketed WHERE orderkey = 100 OR orderkey = 101", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertBucketFilter(plan, "orders_bucketed", 11, ImmutableSet.of(1, 2))); + + assertPlan(getSession(), "SELECT * FROM orders_bucketed WHERE orderkey IN (100, 101, 133)", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertBucketFilter(plan, "orders_bucketed", 11, ImmutableSet.of(1, 2))); + + assertPlan(getSession(), "SELECT * FROM orders_bucketed", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertNoBucketFilter(plan, "orders_bucketed", 11)); + + assertPlan(getSession(), "SELECT * FROM orders_bucketed WHERE orderkey > 100", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertNoBucketFilter(plan, "orders_bucketed", 11)); + + assertPlan(getSession(), "SELECT * FROM orders_bucketed WHERE orderkey != 100", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertNoBucketFilter(plan, "orders_bucketed", 11)); + } + finally { + queryRunner.execute("DROP TABLE orders_bucketed"); + } + + queryRunner.execute("CREATE TABLE orders_bucketed WITH (bucket_count = 11, bucketed_by = ARRAY['orderkey', 'custkey']) AS " + + "SELECT * FROM orders"); + + try { + assertPlan(getSession(), "SELECT * FROM orders_bucketed WHERE orderkey = 101 AND custkey = 280", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertBucketFilter(plan, "orders_bucketed", 11, ImmutableSet.of(1))); + + assertPlan(getSession(), "SELECT * FROM orders_bucketed WHERE orderkey IN (101, 71) AND custkey = 280", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertBucketFilter(plan, "orders_bucketed", 11, ImmutableSet.of(1, 6))); + + assertPlan(getSession(), "SELECT * FROM orders_bucketed WHERE orderkey IN (101, 71) AND custkey IN (280, 34)", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertBucketFilter(plan, "orders_bucketed", 11, ImmutableSet.of(1, 2, 6, 8))); + + assertPlan(getSession(), "SELECT * FROM orders_bucketed WHERE orderkey = 101 AND custkey = 280 AND orderstatus <> '0'", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertBucketFilter(plan, "orders_bucketed", 11, ImmutableSet.of(1))); + + assertPlan(getSession(), "SELECT * FROM orders_bucketed WHERE orderkey = 101", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertNoBucketFilter(plan, "orders_bucketed", 11)); + + assertPlan(getSession(), "SELECT * FROM orders_bucketed WHERE custkey = 280", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertNoBucketFilter(plan, "orders_bucketed", 11)); + + assertPlan(getSession(), "SELECT * FROM orders_bucketed WHERE orderkey = 101 AND custkey > 280", + anyTree(PlanMatchPattern.tableScan("orders_bucketed")), + plan -> assertNoBucketFilter(plan, "orders_bucketed", 11)); + } + finally { + queryRunner.execute("DROP TABLE orders_bucketed"); + } + } + private static Set toSubfields(String... subfieldPaths) { return Arrays.stream(subfieldPaths) @@ -556,7 +939,7 @@ private static boolean isTableScanNode(PlanNode node, String tableName) private void assertPushdownFilterOnSubfields(String query, Map predicateDomains) { String tableName = "test_pushdown_filter_on_subfields"; - assertPlan(pushdownFilterEnabled(), query, + assertPlan(pushdownFilterAndNestedColumnFilterEnabled(), query, output(exchange(PlanMatchPattern.tableScan(tableName))), plan -> assertTableLayout( plan, @@ -573,12 +956,17 @@ private Session pushdownFilterEnabled() .build(); } + private Session pushdownFilterAndNestedColumnFilterEnabled() + { + return Session.builder(getQueryRunner().getDefaultSession()) + .setCatalogSessionProperty(HIVE_CATALOG, PUSHDOWN_FILTER_ENABLED, "true") + .setCatalogSessionProperty(HIVE_CATALOG, RANGE_FILTERS_ON_SUBSCRIPTS_ENABLED, "true") + .build(); + } + private RowExpression constant(long value) { - return new CallExpression(CAST.name(), - getQueryRunner().getMetadata().getFunctionManager().lookupCast(CastType.CAST, VARCHAR.getTypeSignature(), BIGINT.getTypeSignature()), - BIGINT, - ImmutableList.of(new ConstantExpression(Slices.utf8Slice(String.valueOf(value)), VARCHAR))); + return new ConstantExpression(value, BIGINT); } private static Map identityMap(String...values) @@ -598,6 +986,68 @@ private void assertTableLayout(Plan plan, String tableName, TupleDomain bucketsToKeep) + { + TableScanNode tableScan = searchFrom(plan.getRoot()) + .where(node -> isTableScanNode(node, tableName)) + .findOnlyElement(); + + assertTrue(tableScan.getTable().getLayout().isPresent()); + HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) tableScan.getTable().getLayout().get(); + + assertTrue(layoutHandle.getBucketHandle().isPresent()); + assertTrue(layoutHandle.getBucketFilter().isPresent()); + assertEquals(layoutHandle.getBucketHandle().get().getReadBucketCount(), readBucketCount); + assertEquals(layoutHandle.getBucketFilter().get().getBucketsToKeep(), bucketsToKeep); + } + + private void assertNoBucketFilter(Plan plan, String tableName, int readBucketCount) + { + TableScanNode tableScan = searchFrom(plan.getRoot()) + .where(node -> isTableScanNode(node, tableName)) + .findOnlyElement(); + + assertTrue(tableScan.getTable().getLayout().isPresent()); + HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) tableScan.getTable().getLayout().get(); + + assertEquals(layoutHandle.getBucketHandle().get().getReadBucketCount(), readBucketCount); + assertFalse(layoutHandle.getBucketFilter().isPresent()); + } + + private static PlanMatchPattern tableScan(String tableName, TupleDomain domainPredicate, RowExpression remainingPredicate, Set predicateColumnNames) + { + return PlanMatchPattern.tableScan(tableName).with(new Matcher() { + @Override + public boolean shapeMatches(PlanNode node) + { + return node instanceof TableScanNode; + } + + @Override + public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) + { + TableScanNode tableScan = (TableScanNode) node; + + Optional layout = tableScan.getTable().getLayout(); + + if (!layout.isPresent()) { + return NO_MATCH; + } + + HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) layout.get(); + + if (!Objects.equals(layoutHandle.getPredicateColumns().keySet(), predicateColumnNames) || + !Objects.equals(layoutHandle.getDomainPredicate(), domainPredicate.transform(Subfield::new)) || + !Objects.equals(layoutHandle.getRemainingPredicate(), remainingPredicate)) { + return NO_MATCH; + } + + return match(); + } + }); } private static final class HiveTableScanMatcher diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java index e5e1b085166ea..e640b1a84e320 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java @@ -13,10 +13,15 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.GroupByHashPageIndexerFactory; +import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HivePageSinkMetadata; +import com.facebook.presto.hive.metastore.Storage; +import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorPageSink; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; @@ -24,6 +29,7 @@ import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.PageSinkProperties; import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.Type; @@ -32,12 +38,9 @@ import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.TestingConnectorSession; import com.facebook.presto.testing.TestingNodeManager; -import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.io.Files; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slices; import io.airlift.tpch.LineItem; import io.airlift.tpch.LineItemColumn; @@ -52,18 +55,22 @@ import java.util.List; import java.util.Optional; import java.util.OptionalInt; -import java.util.Properties; import java.util.stream.Stream; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.HiveCompressionCodec.NONE; +import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG; import static com.facebook.presto.hive.HiveTestUtils.PAGE_SORTER; import static com.facebook.presto.hive.HiveTestUtils.ROW_EXPRESSION_SERVICE; import static com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER; import static com.facebook.presto.hive.HiveTestUtils.createTestHdfsEnvironment; -import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveDataStreamFactories; +import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveBatchPageSourceFactories; import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories; import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider; +import static com.facebook.presto.hive.HiveTestUtils.getDefaultHiveSelectivePageSourceFactories; import static com.facebook.presto.hive.HiveTestUtils.getDefaultOrcFileWriterFactory; import static com.facebook.presto.hive.HiveType.HIVE_DATE; import static com.facebook.presto.hive.HiveType.HIVE_DOUBLE; @@ -72,23 +79,19 @@ import static com.facebook.presto.hive.HiveType.HIVE_STRING; import static com.facebook.presto.hive.LocationHandle.TableType.NEW; import static com.facebook.presto.hive.LocationHandle.WriteMode.DIRECT_TO_TARGET_NEW_DIRECTORY; -import static com.facebook.presto.hive.metastore.file.FileHiveMetastore.createTestingFileHiveMetastore; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.hive.TestHiveUtil.createTestingFileHiveMetastore; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.DateType.DATE; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType; import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.testing.Assertions.assertGreaterThan; import static java.lang.String.format; import static java.util.stream.Collectors.toList; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT; -import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB; import static org.testng.Assert.assertTrue; public class TestHivePageSink @@ -103,13 +106,14 @@ public void testAllFormats() throws Exception { HiveClientConfig config = new HiveClientConfig(); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); File tempDir = Files.createTempDir(); try { ExtendedHiveMetastore metastore = createTestingFileHiveMetastore(new File(tempDir, "metastore")); for (HiveStorageFormat format : HiveStorageFormat.values()) { config.setHiveStorageFormat(format); config.setCompressionCodec(NONE); - long uncompressedLength = writeTestFile(config, metastore, makeFileName(tempDir, config)); + long uncompressedLength = writeTestFile(config, metastoreClientConfig, metastore, makeFileName(tempDir, config)); assertGreaterThan(uncompressedLength, 0L); for (HiveCompressionCodec codec : HiveCompressionCodec.values()) { @@ -117,7 +121,7 @@ public void testAllFormats() continue; } config.setCompressionCodec(codec); - long length = writeTestFile(config, metastore, makeFileName(tempDir, config)); + long length = writeTestFile(config, metastoreClientConfig, metastore, makeFileName(tempDir, config)); assertTrue(uncompressedLength > length, format("%s with %s compressed to %s which is not less than %s", format, codec, length, uncompressedLength)); } } @@ -132,11 +136,11 @@ private static String makeFileName(File tempDir, HiveClientConfig config) return tempDir.getAbsolutePath() + "/" + config.getHiveStorageFormat().name() + "." + config.getCompressionCodec().name(); } - private static long writeTestFile(HiveClientConfig config, ExtendedHiveMetastore metastore, String outputPath) + private static long writeTestFile(HiveClientConfig config, MetastoreClientConfig metastoreClientConfig, ExtendedHiveMetastore metastore, String outputPath) { HiveTransactionHandle transaction = new HiveTransactionHandle(); HiveWriterStats stats = new HiveWriterStats(); - ConnectorPageSink pageSink = createPageSink(transaction, config, metastore, new Path("file:///" + outputPath), stats); + ConnectorPageSink pageSink = createPageSink(transaction, config, metastoreClientConfig, metastore, new Path("file:///" + outputPath), stats); List columns = getTestColumns(); List columnTypes = columns.stream() .map(LineItemColumn::getType) @@ -185,7 +189,7 @@ private static long writeTestFile(HiveClientConfig config, ExtendedHiveMetastore File outputFile = getOnlyElement(files); long length = outputFile.length(); - ConnectorPageSource pageSource = createPageSource(transaction, config, outputFile); + ConnectorPageSource pageSource = createPageSource(transaction, config, metastoreClientConfig, outputFile); List pages = new ArrayList<>(); while (!pageSource.isFinished()) { @@ -211,13 +215,8 @@ public static MaterializedResult toMaterializedResult(ConnectorSession session, return resultBuilder.build(); } - private static ConnectorPageSource createPageSource(HiveTransactionHandle transaction, HiveClientConfig config, File outputFile) + private static ConnectorPageSource createPageSource(HiveTransactionHandle transaction, HiveClientConfig config, MetastoreClientConfig metastoreClientConfig, File outputFile) { - Properties splitProperties = new Properties(); - splitProperties.setProperty(FILE_INPUT_FORMAT, config.getHiveStorageFormat().getInputFormat()); - splitProperties.setProperty(SERIALIZATION_LIB, config.getHiveStorageFormat().getSerDe()); - splitProperties.setProperty("columns", Joiner.on(',').join(getColumnHandles().stream().map(HiveColumnHandle::getName).collect(toList()))); - splitProperties.setProperty("columns.types", Joiner.on(',').join(getColumnHandles().stream().map(HiveColumnHandle::getHiveType).map(hiveType -> hiveType.getHiveTypeName().toString()).collect(toList()))); HiveSplit split = new HiveSplit( SCHEMA_NAME, TABLE_NAME, @@ -226,25 +225,48 @@ private static ConnectorPageSource createPageSource(HiveTransactionHandle transa 0, outputFile.length(), outputFile.length(), - splitProperties, + new Storage( + StorageFormat.create(config.getHiveStorageFormat().getSerDe(), config.getHiveStorageFormat().getInputFormat(), config.getHiveStorageFormat().getOutputFormat()), + "location", + Optional.empty(), + false, + ImmutableMap.of()), ImmutableList.of(), ImmutableList.of(), OptionalInt.empty(), OptionalInt.empty(), false, - TupleDomain.all(), - TRUE_CONSTANT, - ImmutableMap.of(), + getColumnHandles().size(), ImmutableMap.of(), Optional.empty(), - false); - HivePageSourceProvider provider = new HivePageSourceProvider(config, createTestHdfsEnvironment(config), getDefaultHiveRecordCursorProvider(config), getDefaultHiveDataStreamFactories(config), ImmutableSet.of(), TYPE_MANAGER, ROW_EXPRESSION_SERVICE); - return provider.createPageSource(transaction, getSession(config), split, ImmutableList.copyOf(getColumnHandles())); + false, + Optional.empty()); + TableHandle tableHandle = new TableHandle( + new ConnectorId(HIVE_CATALOG), + new HiveTableHandle(SCHEMA_NAME, TABLE_NAME), + transaction, + Optional.of(new HiveTableLayoutHandle( + new SchemaTableName(SCHEMA_NAME, TABLE_NAME), + ImmutableList.of(), + getColumnHandles().stream() + .map(column -> new Column(column.getName(), column.getHiveType(), Optional.empty())) + .collect(toImmutableList()), + ImmutableMap.of(), + TupleDomain.all(), + TRUE_CONSTANT, + ImmutableMap.of(), + TupleDomain.all(), + Optional.empty(), + Optional.empty(), + false, + "layout"))); + HivePageSourceProvider provider = new HivePageSourceProvider(config, createTestHdfsEnvironment(config, metastoreClientConfig), getDefaultHiveRecordCursorProvider(config, metastoreClientConfig), getDefaultHiveBatchPageSourceFactories(config, metastoreClientConfig), getDefaultHiveSelectivePageSourceFactories(config, metastoreClientConfig), TYPE_MANAGER, ROW_EXPRESSION_SERVICE); + return provider.createPageSource(transaction, getSession(config), split, tableHandle.getLayout().get(), ImmutableList.copyOf(getColumnHandles())); } - private static ConnectorPageSink createPageSink(HiveTransactionHandle transaction, HiveClientConfig config, ExtendedHiveMetastore metastore, Path outputPath, HiveWriterStats stats) + private static ConnectorPageSink createPageSink(HiveTransactionHandle transaction, HiveClientConfig config, MetastoreClientConfig metastoreClientConfig, ExtendedHiveMetastore metastore, Path outputPath, HiveWriterStats stats) { - LocationHandle locationHandle = new LocationHandle(outputPath, outputPath, NEW, DIRECT_TO_TARGET_NEW_DIRECTORY); + LocationHandle locationHandle = new LocationHandle(outputPath, outputPath, Optional.empty(), NEW, DIRECT_TO_TARGET_NEW_DIRECTORY); HiveOutputTableHandle handle = new HiveOutputTableHandle( SCHEMA_NAME, TABLE_NAME, @@ -260,22 +282,23 @@ private static ConnectorPageSink createPageSink(HiveTransactionHandle transactio "test", ImmutableMap.of()); JsonCodec partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class); - HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(config); + HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(config, metastoreClientConfig); HivePageSinkProvider provider = new HivePageSinkProvider( - getDefaultHiveFileWriterFactories(config), + getDefaultHiveFileWriterFactories(config, metastoreClientConfig), hdfsEnvironment, PAGE_SORTER, metastore, new GroupByHashPageIndexerFactory(new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig())), TYPE_MANAGER, config, + metastoreClientConfig, new HiveLocationService(hdfsEnvironment), partitionUpdateCodec, new TestingNodeManager("fake-environment"), new HiveEventClient(), new HiveSessionProperties(config, new OrcFileWriterConfig(), new ParquetFileWriterConfig()), stats, - getDefaultOrcFileWriterFactory(config)); + getDefaultOrcFileWriterFactory(config, metastoreClientConfig)); return provider.createPageSink(transaction, getSession(config), handle, PageSinkProperties.defaultProperties()); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePartitionManager.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePartitionManager.java new file mode 100644 index 0000000000000..98da5794877e2 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePartitionManager.java @@ -0,0 +1,185 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.hive; + +import com.facebook.presto.hive.metastore.Column; +import com.facebook.presto.hive.metastore.PrestoTableType; +import com.facebook.presto.hive.metastore.Storage; +import com.facebook.presto.hive.metastore.Table; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.Constraint; +import com.facebook.presto.spi.predicate.Domain; +import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.TestingTypeManager; +import com.facebook.presto.testing.TestingConnectorSession; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; +import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; +import static com.facebook.presto.hive.HiveColumnHandle.MAX_PARTITION_KEY_COLUMN_INDEX; +import static com.facebook.presto.hive.HiveColumnHandle.bucketColumnHandle; +import static com.facebook.presto.hive.HiveStorageFormat.ORC; +import static com.facebook.presto.hive.HiveType.HIVE_INT; +import static com.facebook.presto.hive.HiveType.HIVE_STRING; +import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static io.airlift.slice.Slices.utf8Slice; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestHivePartitionManager +{ + private static final String SCHEMA_NAME = "schema"; + private static final String TABLE_NAME = "table"; + private static final String USER_NAME = "user"; + private static final String LOCATION = "somewhere/over/the/rainbow"; + private static final Column PARTITION_COLUMN = new Column("ds", HIVE_STRING, Optional.empty()); + private static final Column BUCKET_COLUMN = new Column("c1", HIVE_INT, Optional.empty()); + private static final Table TABLE = new Table( + SCHEMA_NAME, + TABLE_NAME, + USER_NAME, + PrestoTableType.MANAGED_TABLE, + new Storage(fromHiveStorageFormat(ORC), LOCATION, Optional.of(new HiveBucketProperty(ImmutableList.of(BUCKET_COLUMN.getName()), 100, ImmutableList.of())), false, ImmutableMap.of()), + ImmutableList.of(BUCKET_COLUMN), + ImmutableList.of(PARTITION_COLUMN), + ImmutableMap.of(), + Optional.empty(), + Optional.empty()); + + private static final List PARTITIONS = ImmutableList.of("ds=2019-07-23", "ds=2019-08-23"); + + private HivePartitionManager hivePartitionManager = new HivePartitionManager(new TestingTypeManager(), new HiveClientConfig()); + private final TestingSemiTransactionalHiveMetastore metastore = TestingSemiTransactionalHiveMetastore.create(); + + @BeforeClass + public void setUp() + { + metastore.addTable(SCHEMA_NAME, TABLE_NAME, TABLE, PARTITIONS); + } + + @Test + public void testUsesBucketingIfSmallEnough() + { + HiveTableHandle tableHandle = new HiveTableHandle(SCHEMA_NAME, TABLE_NAME); + HivePartitionResult result = hivePartitionManager.getPartitions( + metastore, + tableHandle, + Constraint.alwaysTrue(), + new TestingConnectorSession(new HiveSessionProperties(new HiveClientConfig(), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties())); + assertTrue(result.getBucketHandle().isPresent(), "bucketHandle is not present"); + assertFalse(result.getBucketFilter().isPresent(), "bucketFilter is present"); + } + + @Test + public void testIgnoresBucketingWhenTooManyBuckets() + { + ConnectorSession session = new TestingConnectorSession( + new HiveSessionProperties( + new HiveClientConfig().setMaxBucketsForGroupedExecution(100), + new OrcFileWriterConfig(), + new ParquetFileWriterConfig()) + .getSessionProperties()); + HivePartitionResult result = hivePartitionManager.getPartitions(metastore, new HiveTableHandle(SCHEMA_NAME, TABLE_NAME), Constraint.alwaysTrue(), session); + assertFalse(result.getBucketHandle().isPresent(), "bucketHandle is present"); + assertFalse(result.getBucketFilter().isPresent(), "bucketFilter is present"); + } + + @Test + public void testUsesBucketingWithPartitionFilters() + { + ConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveClientConfig().setMaxBucketsForGroupedExecution(100), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties()); + HiveTableHandle tableHandle = new HiveTableHandle(SCHEMA_NAME, TABLE_NAME); + HivePartitionResult result = hivePartitionManager.getPartitions( + metastore, + tableHandle, + new Constraint<>(TupleDomain.withColumnDomains( + ImmutableMap.of( + new HiveColumnHandle( + PARTITION_COLUMN.getName(), + PARTITION_COLUMN.getType(), + parseTypeSignature(StandardTypes.VARCHAR), + MAX_PARTITION_KEY_COLUMN_INDEX, + PARTITION_KEY, + Optional.empty()), + Domain.singleValue(VARCHAR, utf8Slice("2019-07-23"))))), + session); + assertTrue(result.getBucketHandle().isPresent(), "bucketHandle is not present"); + assertFalse(result.getBucketFilter().isPresent(), "bucketFilter is present"); + } + + @Test + public void testUsesBucketingWithBucketFilters() + { + ConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveClientConfig().setMaxBucketsForGroupedExecution(100), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties()); + HiveTableHandle tableHandle = new HiveTableHandle(SCHEMA_NAME, TABLE_NAME); + HivePartitionResult result = hivePartitionManager.getPartitions( + metastore, + tableHandle, + new Constraint<>(TupleDomain.withColumnDomains( + ImmutableMap.of( + new HiveColumnHandle( + BUCKET_COLUMN.getName(), + BUCKET_COLUMN.getType(), + parseTypeSignature(StandardTypes.VARCHAR), + 0, + REGULAR, + Optional.empty()), + Domain.singleValue(INTEGER, 1L)))), + session); + assertTrue(result.getBucketHandle().isPresent(), "bucketHandle is not present"); + assertTrue(result.getBucketFilter().isPresent(), "bucketFilter is present"); + } + + @Test + public void testUsesBucketingWithBucketColumn() + { + ConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveClientConfig().setMaxBucketsForGroupedExecution(1), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties()); + HiveTableHandle tableHandle = new HiveTableHandle(SCHEMA_NAME, TABLE_NAME); + HivePartitionResult result = hivePartitionManager.getPartitions( + metastore, + tableHandle, + new Constraint<>(TupleDomain.withColumnDomains( + ImmutableMap.of( + bucketColumnHandle(), + Domain.singleValue(INTEGER, 1L)))), + session); + assertTrue(result.getBucketHandle().isPresent(), "bucketHandle is not present"); + assertTrue(result.getBucketFilter().isPresent(), "bucketFilter is present"); + } + + @Test + public void testIgnoresBucketingWhenConfigured() + { + ConnectorSession session = new TestingConnectorSession( + new HiveSessionProperties( + new HiveClientConfig().setIgnoreTableBucketing(true), + new OrcFileWriterConfig(), + new ParquetFileWriterConfig()) + .getSessionProperties()); + HivePartitionResult result = hivePartitionManager.getPartitions(metastore, new HiveTableHandle(SCHEMA_NAME, TABLE_NAME), Constraint.alwaysTrue(), session); + assertFalse(result.getBucketHandle().isPresent(), "bucketHandle is present"); + assertFalse(result.getBucketFilter().isPresent(), "bucketFilter is present"); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownDistributedQueries.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownDistributedQueries.java new file mode 100644 index 0000000000000..50619dd85bf0a --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownDistributedQueries.java @@ -0,0 +1,56 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.tests.AbstractTestDistributedQueries; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.hive.HiveQueryRunner.createQueryRunner; +import static com.facebook.presto.sql.tree.ExplainType.Type.LOGICAL; +import static com.google.common.collect.Iterables.getOnlyElement; +import static io.airlift.tpch.TpchTable.getTables; +import static org.testng.Assert.assertEquals; + +public class TestHivePushdownDistributedQueries + extends AbstractTestDistributedQueries +{ + public TestHivePushdownDistributedQueries() + { + super(() -> createQueryRunner(getTables(), ImmutableMap.of("experimental.pushdown-subfields-enabled", "true"), "sql-standard", ImmutableMap.of("hive.pushdown-filter-enabled", "true"), Optional.empty())); + } + + @Override + protected boolean supportsNotNullColumns() + { + return false; + } + + @Override + public void testDelete() + { + // Hive connector currently does not support row-by-row delete + } + + @Test + public void testExplainOfCreateTableAs() + { + String query = "CREATE TABLE copy_orders AS SELECT * FROM orders"; + MaterializedResult result = computeActual("EXPLAIN " + query); + assertEquals(getOnlyElement(result.getOnlyColumnAsSet()), getExplainPlan(query, LOGICAL)); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownFilterQueries.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownFilterQueries.java index c8bca5da192f1..fe7f264b81a94 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownFilterQueries.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownFilterQueries.java @@ -17,23 +17,81 @@ import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; import com.facebook.presto.tests.DistributedQueryRunner; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; import java.util.Optional; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG; import static com.facebook.presto.hive.HiveSessionProperties.PUSHDOWN_FILTER_ENABLED; +import static com.facebook.presto.hive.HiveStorageFormat.RCBINARY; +import static com.facebook.presto.hive.HiveStorageFormat.RCTEXT; +import static com.facebook.presto.hive.HiveStorageFormat.TEXTFILE; +import static com.facebook.presto.spi.type.StandardTypes.BIGINT; +import static com.facebook.presto.spi.type.StandardTypes.BOOLEAN; +import static com.facebook.presto.spi.type.StandardTypes.DATE; +import static com.facebook.presto.spi.type.StandardTypes.DOUBLE; +import static com.facebook.presto.spi.type.StandardTypes.INTEGER; +import static com.facebook.presto.spi.type.StandardTypes.REAL; +import static com.facebook.presto.spi.type.StandardTypes.SMALLINT; +import static com.facebook.presto.spi.type.StandardTypes.TINYINT; +import static com.facebook.presto.spi.type.StandardTypes.VARCHAR; +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.airlift.tpch.TpchTable.getTables; +import static java.lang.String.format; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.util.stream.Collectors.joining; public class TestHivePushdownFilterQueries extends AbstractTestQueryFramework { - private static final String WITH_LINEITEM_EX = "WITH lineitem_ex AS (" + - "SELECT linenumber, orderkey, " + - " CASE WHEN linenumber % 7 = 0 THEN null ELSE shipmode = 'AIR' END AS ship_by_air, " + - " CASE WHEN linenumber % 5 = 0 THEN null ELSE returnflag = 'R' END AS is_returned " + - "FROM lineitem)"; + private static final Pattern ARRAY_SUBSCRIPT_PATTERN = Pattern.compile("([a-z_+]+)((\\[[0-9]+\\])+)"); + + private static final String WITH_LINEITEM_EX = "WITH lineitem_ex AS (\n" + + "SELECT linenumber, orderkey, partkey, suppkey, quantity, extendedprice, tax, shipinstruct, shipmode, \n" + + " CASE WHEN linenumber % 5 = 0 THEN null ELSE shipmode = 'AIR' END AS ship_by_air, \n" + + " CASE WHEN linenumber % 7 = 0 THEN null ELSE returnflag = 'R' END AS is_returned, \n" + + " CASE WHEN linenumber % 4 = 0 THEN null ELSE CAST(day(shipdate) AS TINYINT) END AS ship_day, " + + " CASE WHEN linenumber % 6 = 0 THEN null ELSE CAST(month(shipdate) AS TINYINT) END AS ship_month, " + + " CASE WHEN linenumber % 3 = 0 THEN null ELSE CAST(shipdate AS TIMESTAMP) END AS ship_timestamp, \n" + + " CASE WHEN orderkey % 3 = 0 THEN null ELSE CAST(commitdate AS TIMESTAMP) END AS commit_timestamp, \n" + + " CASE WHEN orderkey % 5 = 0 THEN null ELSE CAST(discount AS REAL) END AS discount_real, \n" + + " CASE WHEN orderkey % 43 = 0 THEN null ELSE discount END as discount, \n" + + " CASE WHEN orderkey % 7 = 0 THEN null ELSE CAST(tax AS REAL) END AS tax_real, \n" + + " CASE WHEN linenumber % 2 = 0 THEN null ELSE (CAST(day(shipdate) AS TINYINT) , CAST(month(shipdate) AS TINYINT)) END AS ship_day_month, " + + " CASE WHEN orderkey % 37 = 0 THEN null ELSE CAST(discount AS DECIMAL(20, 8)) END AS discount_long_decimal, " + + " CASE WHEN orderkey % 41 = 0 THEN null ELSE CAST(tax AS DECIMAL(3, 2)) END AS tax_short_decimal, " + + " CASE WHEN orderkey % 43 = 0 THEN null ELSE (CAST(discount AS DECIMAL(20, 8)), CAST(tax AS DECIMAL(20, 8))) END AS long_decimals, " + + " CASE WHEN orderkey % 11 = 0 THEN null ELSE (orderkey, partkey, suppkey) END AS keys, \n" + + " CASE WHEN orderkey % 41 = 0 THEN null ELSE (extendedprice, discount, tax) END AS doubles, \n" + + " CASE WHEN orderkey % 13 = 0 THEN null ELSE ((orderkey, partkey), (suppkey,), CASE WHEN orderkey % 17 = 0 THEN null ELSE (orderkey, partkey) END) END AS nested_keys, \n" + + " CASE WHEN orderkey % 17 = 0 THEN null ELSE (shipmode = 'AIR', returnflag = 'R') END as flags, \n" + + " CASE WHEN orderkey % 19 = 0 THEN null ELSE (CAST(discount AS REAL), CAST(tax AS REAL)) END as reals, \n" + + " CASE WHEN orderkey % 23 = 0 THEN null ELSE (orderkey, linenumber, (CAST(day(shipdate) as TINYINT), CAST(month(shipdate) AS TINYINT), CAST(year(shipdate) AS INTEGER))) END AS info, \n" + + " CASE WHEN orderkey % 31 = 0 THEN null ELSE (" + + " (CAST(day(shipdate) AS TINYINT), CAST(month(shipdate) AS TINYINT), CAST(year(shipdate) AS INTEGER)), " + + " (CAST(day(commitdate) AS TINYINT), CAST(month(commitdate) AS TINYINT), CAST(year(commitdate) AS INTEGER)), " + + " (CAST(day(receiptdate) AS TINYINT), CAST(month(receiptdate) AS TINYINT), CAST(year(receiptdate) AS INTEGER))) END AS dates, \n" + + " CASE WHEN orderkey % 37 = 0 THEN null ELSE (CAST(shipdate AS TIMESTAMP), CAST(commitdate AS TIMESTAMP)) END AS timestamps, \n" + + " CASE WHEN orderkey % 43 = 0 THEN null ELSE comment END AS comment, \n" + + " CASE WHEN orderkey % 43 = 0 THEN null ELSE upper(comment) END AS uppercase_comment, \n" + + " CAST('' as VARBINARY) AS empty_comment, \n" + + " CASE WHEN orderkey % 47 = 0 THEN null ELSE CAST(comment AS CHAR(5)) END AS fixed_comment, \n" + + " CASE WHEN orderkey % 49 = 0 THEN null ELSE (CAST(comment AS CHAR(4)), CAST(comment AS CHAR(3)), CAST(SUBSTR(comment,length(comment) - 4) AS CHAR(4))) END AS char_array, \n" + + " CASE WHEN orderkey % 49 = 0 THEN null ELSE (comment, comment) END AS varchar_array \n" + + + "FROM lineitem)\n"; protected TestHivePushdownFilterQueries() { @@ -44,22 +102,89 @@ private static QueryRunner createQueryRunner() throws Exception { DistributedQueryRunner queryRunner = HiveQueryRunner.createQueryRunner(getTables(), - ImmutableMap.of(), + ImmutableMap.of("experimental.pushdown-subfields-enabled", "true"), "sql-standard", ImmutableMap.of("hive.pushdown-filter-enabled", "true"), Optional.empty()); queryRunner.execute(noPushdownFilter(queryRunner.getDefaultSession()), - "CREATE TABLE lineitem_ex (linenumber, orderkey, ship_by_air, is_returned) AS " + - "SELECT linenumber, " + - " orderkey, " + - " IF (linenumber % 7 = 0, null, shipmode = 'AIR') AS ship_by_air, " + - " IF (linenumber % 5 = 0, null, returnflag = 'R') AS is_returned " + + "CREATE TABLE lineitem_ex (linenumber, orderkey, partkey, suppkey, quantity, extendedprice, tax, shipinstruct, shipmode, ship_by_air, is_returned, ship_day, ship_month, ship_timestamp, commit_timestamp, discount_real, discount, tax_real, ship_day_month, discount_long_decimal, tax_short_decimal, long_decimals, keys, doubles, nested_keys, flags, reals, info, dates, timestamps, comment, uppercase_comment, empty_comment, fixed_comment, char_array, varchar_array) AS " + + "SELECT linenumber, orderkey, partkey, suppkey, quantity, extendedprice, tax, shipinstruct, shipmode, " + + " IF (linenumber % 5 = 0, null, shipmode = 'AIR') AS ship_by_air, " + + " IF (linenumber % 7 = 0, null, returnflag = 'R') AS is_returned, " + + " IF (linenumber % 4 = 0, null, CAST(day(shipdate) AS TINYINT)) AS ship_day, " + + " IF (linenumber % 6 = 0, null, CAST(month(shipdate) AS TINYINT)) AS ship_month, " + + " IF (linenumber % 3 = 0, null, CAST(shipdate AS TIMESTAMP)) AS ship_timestamp, " + + " IF (orderkey % 3 = 0, null, CAST(commitdate AS TIMESTAMP)) AS commit_timestamp, " + + " IF (orderkey % 5 = 0, null, CAST(discount AS REAL)) AS discount_real, " + + " IF (orderkey % 43 = 0, null, discount) AS discount, " + + " IF (orderkey % 7 = 0, null, CAST(tax AS REAL)) AS tax_real, " + + " IF (linenumber % 2 = 0, null, ARRAY[CAST(day(shipdate) AS TINYINT), CAST(month(shipdate) AS TINYINT)]) AS ship_day_month, " + + " IF (orderkey % 37 = 0, null, CAST(discount AS DECIMAL(20, 8))) AS discount_long_decimal, " + + " IF (orderkey % 41 = 0, null, CAST(tax AS DECIMAL(3, 2))) AS tax_short_decimal, " + + " IF (orderkey % 43 = 0, null, ARRAY[CAST(discount AS DECIMAL(20, 8)), CAST(tax AS DECIMAL(20, 8))]) AS long_decimals, " + + " IF (orderkey % 11 = 0, null, ARRAY[orderkey, partkey, suppkey]) AS keys, " + + " IF (orderkey % 41 = 0, null, ARRAY[extendedprice, discount, tax]) AS doubles, " + + " IF (orderkey % 13 = 0, null, ARRAY[ARRAY[orderkey, partkey], ARRAY[suppkey], IF (orderkey % 17 = 0, null, ARRAY[orderkey, partkey])]) AS nested_keys, " + + " IF (orderkey % 17 = 0, null, ARRAY[shipmode = 'AIR', returnflag = 'R']) AS flags, " + + " IF (orderkey % 19 = 0, null, ARRAY[CAST(discount AS REAL), CAST(tax AS REAL)]), " + + " IF (orderkey % 23 = 0, null, CAST(ROW(orderkey, linenumber, ROW(day(shipdate), month(shipdate), year(shipdate))) AS ROW(orderkey BIGINT, linenumber INTEGER, shipdate ROW(ship_day TINYINT, ship_month TINYINT, ship_year INTEGER)))), " + + " IF (orderkey % 31 = 0, NULL, ARRAY[" + + " CAST(ROW(day(shipdate), month(shipdate), year(shipdate)) AS ROW(day TINYINT, month TINYINT, year INTEGER)), " + + " CAST(ROW(day(commitdate), month(commitdate), year(commitdate)) AS ROW(day TINYINT, month TINYINT, year INTEGER)), " + + " CAST(ROW(day(receiptdate), month(receiptdate), year(receiptdate)) AS ROW(day TINYINT, month TINYINT, year INTEGER))]), " + + " IF (orderkey % 37 = 0, NULL, ARRAY[CAST(shipdate AS TIMESTAMP), CAST(commitdate AS TIMESTAMP)]) AS timestamps, " + + " IF (orderkey % 43 = 0, NULL, comment) AS comment, " + + " IF (orderkey % 43 = 0, NULL, upper(comment)) AS uppercase_comment, " + + " CAST('' as VARBINARY) AS empty_comment, \n" + + " IF (orderkey % 47 = 0, NULL, CAST(comment AS CHAR(5))) AS fixed_comment, " + + " IF (orderkey % 49 = 0, NULL, ARRAY[CAST(comment AS CHAR(4)), CAST(comment AS CHAR(3)), CAST(SUBSTR(comment,length(comment) - 4) AS CHAR(4))]) AS char_array, " + + " IF (orderkey % 49 = 0, NULL, ARRAY[comment, comment]) AS varchar_array " + "FROM lineitem"); - return queryRunner; } + @Test + public void testTableSampling() + { + assertQuerySucceeds("SELECT * FROM lineitem TABLESAMPLE BERNOULLI (1)"); + + assertQuerySucceeds("SELECT * FROM lineitem TABLESAMPLE BERNOULLI (1) WHERE orderkey > 1000"); + + assertQuerySucceeds("SELECT * FROM lineitem TABLESAMPLE BERNOULLI (1) WHERE orderkey % 2 = 0"); + + assertQueryReturnsEmptyResult("SELECT * FROM lineitem WHERE rand() > 1"); + + // error in filter function with no inputs + assertQueryFails("SELECT * FROM lineitem WHERE array[1, 2, 3][cast(floor(rand()) as integer)] > 0", "SQL array indices start at 1"); + + // error in filter function with no inputs is masked by another filter + assertQuerySucceeds("SELECT * FROM lineitem WHERE array[1, 2, 3][cast(floor(rand()) as integer)] > 0 AND linenumber < 0"); + assertQuerySucceeds("SELECT * FROM lineitem WHERE array[1, 2, 3][cast(floor(rand()) as integer)] > 0 AND linenumber % 2 < 0"); + + // error in filter function with no inputs is masked by an error in another filter + assertQueryFails("SELECT * FROM lineitem WHERE array[1, 2, 3][cast(floor(rand()) as integer)] > 0 AND array[1, 2, 3][cast(floor(rand() * linenumber) as integer) - linenumber] > 0", "Array subscript is negative"); + } + + @Test + public void testLegacyUnnest() + { + Session legacyUnnest = Session.builder(getSession()).setSystemProperty("legacy_unnest", "true").build(); + + assertQuery(legacyUnnest, "SELECT orderkey, date.day FROM lineitem_ex CROSS JOIN UNNEST(dates) t(date)", + "SELECT orderkey, day(shipdate) FROM lineitem WHERE orderkey % 31 <> 0 UNION ALL " + + "SELECT orderkey, day(commitdate) FROM lineitem WHERE orderkey % 31 <> 0 UNION ALL " + + "SELECT orderkey, day(receiptdate) FROM lineitem WHERE orderkey % 31 <> 0"); + } + + @Test + public void testPushdownWithDisjointFilters() + { + assertQueryUsingH2Cte("SELECT * FROM lineitem_ex where orderkey = 1 and orderkey = 2"); + + assertQueryUsingH2Cte("SELECT count(*) FROM orders WHERE orderkey = 100 and orderkey = 101"); + } + @Test public void testBooleans() { @@ -84,6 +209,34 @@ public void testBooleans() assertQueryUsingH2Cte("SELECT COUNT(*) FROM lineitem_ex WHERE ship_by_air is not null AND is_returned = true"); } + @Test + public void testBytes() + { + // Single tinyint column + assertQueryUsingH2Cte("SELECT ship_day FROM lineitem_ex"); + + assertQueryUsingH2Cte("SELECT ship_day FROM lineitem_ex WHERE ship_day < 15"); + + assertQueryUsingH2Cte("SELECT count(*) FROM lineitem_ex WHERE ship_day > 15"); + + assertQueryUsingH2Cte("SELECT count(*) FROM lineitem_ex WHERE ship_day is null"); + + // Two tinyint columns + assertQueryUsingH2Cte("SELECT ship_day, ship_month FROM lineitem_ex"); + + assertQueryUsingH2Cte("SELECT ship_day, ship_month FROM lineitem_ex WHERE ship_month = 1"); + + assertQueryUsingH2Cte("SELECT ship_day, ship_month FROM lineitem_ex WHERE ship_day = 1 AND ship_month = 1"); + + assertQueryUsingH2Cte("SELECT COUNT(*) FROM lineitem_ex WHERE ship_month is null"); + + assertQueryUsingH2Cte("SELECT COUNT(*) FROM lineitem_ex WHERE ship_day is not null AND ship_month = 1"); + + assertQueryUsingH2Cte("SELECT ship_day, ship_month FROM lineitem_ex WHERE ship_day > 15 AND ship_month < 5 AND (ship_day + ship_month) < 20"); + + assertQueryUsingH2Cte("SELECT count(*) FROM lineitem_ex WHERE ship_day_month[2] = 12"); + } + @Test public void testNumeric() { @@ -102,6 +255,448 @@ public void testNumeric() assertQueryUsingH2Cte("SELECT linenumber, ship_by_air, is_returned FROM lineitem_ex WHERE orderkey < 30000 AND ship_by_air = true"); } + @Test + public void testTimestamps() + { + // Single timestamp column + assertQueryUsingH2Cte("SELECT ship_timestamp FROM lineitem_ex"); + + assertQueryUsingH2Cte("SELECT ship_timestamp FROM lineitem_ex WHERE ship_timestamp < TIMESTAMP '1993-01-01 01:00:00'"); + + assertQueryUsingH2Cte("SELECT count(*) FROM lineitem_ex WHERE ship_timestamp IS NOT NULL"); + + assertQueryUsingH2Cte("SELECT count(*) FROM lineitem_ex WHERE ship_timestamp = TIMESTAMP '2012-08-08 01:00:00'"); + + // Two timestamp columns + assertQueryUsingH2Cte("SELECT commit_timestamp, ship_timestamp FROM lineitem_ex"); + + assertQueryUsingH2Cte("SELECT commit_timestamp, ship_timestamp FROM lineitem_ex WHERE ship_timestamp > TIMESTAMP '1993-08-08 01:00:00' AND commit_timestamp < TIMESTAMP '1993-08-08 01:00:00'"); + + assertQueryReturnsEmptyResult("SELECT commit_timestamp, ship_timestamp FROM lineitem_ex WHERE year(ship_timestamp) - year(commit_timestamp) > 1"); + + assertQueryUsingH2Cte("SELECT commit_timestamp, ship_timestamp, orderkey FROM lineitem_ex WHERE year(commit_timestamp) > 1993 and year(ship_timestamp) > 1993 and year(ship_timestamp) - year(commit_timestamp) = 1"); + + assertQueryUsingH2Cte("SELECT count(*) from lineitem_ex where timestamps[1] > TIMESTAMP '1993-08-08 01:00:00'"); + + assertQueryUsingH2Cte("SELECT count(*) from lineitem_ex where year(timestamps[1]) != year(timestamps[2])"); + } + + @Test + public void testDouble() + { + assertQuery("SELECT quantity, extendedprice, discount, tax FROM lineitem"); + + assertQueryUsingH2Cte("SELECT count(discount) FROM lineitem_ex"); + + assertFilterProject("discount IS NULL", "count(*)"); + + assertFilterProject("discount IS NOT NULL", "sum(quantity), sum(discount)"); + + assertFilterProject("is_returned = true", "quantity, extendedprice, discount"); + + assertFilterProject("quantity = 4", "orderkey, tax"); + + assertFilterProject("quantity = 4 AND discount = 0", "extendedprice, discount"); + + assertFilterProject("quantity = 4 AND discount = 0 AND tax = .05", "orderkey"); + + assertFilterProject("(discount + tax) < (quantity / 10)", "tax"); + + assertFilterProject("doubles[1] > 0.01", "count(*)"); + + // Compact values + assertFilterProject("discount + tax > .05 AND discount > .01 AND tax > .01", "tax"); + + // SucceedingPositionsToFail > 0 in readWithFilter + assertFilterProject("is_returned AND doubles[2] = .01 AND doubles[1] + discount > 0.10", "count(*)"); + } + + @Test + public void testFloats() + { + assertQueryUsingH2Cte("SELECT discount_real, tax_real FROM lineitem_ex"); + + assertFilterProject("tax_real IS NOT NULL", "count(*)"); + + assertFilterProject("tax_real IS NULL", "count(*)"); + + assertFilterProject("tax_real > 0.1", "count(*)"); + + assertFilterProject("tax_real < 0.03", "discount_real, tax_real"); + + assertFilterProject("tax_real < 0.05 AND discount_real > 0.05", "discount_real"); + + assertFilterProject("tax_real = discount_real", "discount_real"); + + assertFilterProject("discount_real > 0.01 AND tax_real > 0.01 AND (discount_real + tax_real) < 0.08", "discount_real"); + + assertFilterProject("reals[1] > 0.01", "count(*)"); + } + + @Test + public void testMaps() + { + getQueryRunner().execute("CREATE TABLE test_maps AS " + + "SELECT orderkey, " + + " linenumber, " + + " IF (keys IS NULL, null, MAP(ARRAY[1, 2, 3], keys)) AS map_keys, " + + " IF (flags IS NULL, null, MAP(ARRAY[1, 2], flags)) AS map_flags " + + "FROM lineitem_ex"); + + Function rewriter = query -> query.replaceAll("map_keys", "keys") + .replaceAll("map_flags", "flags") + .replaceAll("test_maps", "lineitem_ex") + .replaceAll("cardinality", "array_length"); + try { + //filter on nested columns + assertQueryUsingH2Cte("SELECT * FROM test_maps WHERE map_keys[1] > 10 and map_keys[1] < 20", rewriter); + + assertQueryUsingH2Cte("SELECT map_keys[1] FROM test_maps WHERE linenumber < 3", rewriter); + + assertQueryUsingH2Cte("SELECT cardinality(map_keys) FROM test_maps", rewriter); + assertQueryUsingH2Cte("SELECT cardinality(map_keys) FROM test_maps WHERE map_keys[1] % 2 = 0", rewriter); + + assertQueryUsingH2Cte("SELECT map_keys[1] FROM test_maps", rewriter); + assertQueryUsingH2Cte("SELECT map_keys[2] FROM test_maps", rewriter); + assertQueryUsingH2Cte("SELECT map_keys[1], map_keys[3] FROM test_maps", rewriter); + + assertQueryUsingH2Cte("SELECT map_keys[1] FROM test_maps WHERE map_keys[1] % 2 = 0", rewriter); + + assertQueryUsingH2Cte("SELECT map_keys[2] FROM test_maps WHERE map_keys[1] % 2 = 0", rewriter); + assertQueryUsingH2Cte("SELECT map_keys[1], map_keys[3] FROM test_maps WHERE map_keys[1] % 2 = 0", rewriter); + + assertQueryUsingH2Cte("SELECT map_keys[1], map_flags[2] FROM test_maps WHERE map_keys IS NOT NULL AND map_flags IS NOT NULL AND map_keys[1] % 2 = 0", rewriter); + + // filter-only map + assertQueryUsingH2Cte("SELECT linenumber FROM test_maps WHERE map_keys IS NOT NULL", rewriter); + + // equality filter + assertQuery("SELECT orderkey FROM test_maps WHERE map_flags = MAP(ARRAY[1, 2], ARRAY[true, true])", "SELECT orderkey FROM lineitem WHERE orderkey % 17 <> 0 AND shipmode = 'AIR' AND returnflag = 'R'"); + + assertQueryFails("SELECT map_keys[5] FROM test_maps WHERE map_keys[1] % 2 = 0", "Key not present in map: 5"); + assertQueryFails("SELECT map_keys[5] FROM test_maps WHERE map_keys[4] % 2 = 0", "Key not present in map: 4"); + } + finally { + getQueryRunner().execute("DROP TABLE test_maps"); + } + } + + @Test + public void testDecimals() + { + assertQueryUsingH2Cte("SELECT discount_long_decimal, tax_short_decimal FROM lineitem_ex"); + + assertFilterProject("discount_long_decimal IS NOT NULL", "count(*)"); + + assertFilterProject("discount_long_decimal IS NULL", "discount_long_decimal"); + + assertFilterProject("discount_long_decimal > 0.05", "discount_long_decimal"); + + assertFilterProject("tax_short_decimal < 0.03", "tax_short_decimal"); + + assertFilterProject("tax_short_decimal < 0.05 AND discount_long_decimal > 0.05", "discount_long_decimal"); + + assertFilterProject("tax_short_decimal < discount_long_decimal", "discount_long_decimal"); + + assertFilterProject("discount_long_decimal > 0.01 AND tax_short_decimal > 0.01 AND (discount_long_decimal + tax_short_decimal) < 0.03", "discount_long_decimal"); + + assertFilterProject("long_decimals[1] > 0.01", "count(*)"); + + assertFilterProject("tax_real > 0.01 and tax_short_decimal > 0.02 and (discount_long_decimal + tax_short_decimal) < 0.05", "tax_short_decimal, discount_long_decimal"); + } + + @Test + public void testStrings() + { + assertFilterProject("comment < 'a' OR comment BETWEEN 'c' AND 'd'", "empty_comment"); + //char + assertFilterProject("orderkey = 8480", "char_array"); + assertFilterProject("orderkey < 1000", "fixed_comment"); + + //varchar/char direct + assertFilterProject("comment is not NULL and linenumber=1 and orderkey<10", "comment"); + assertFilterProject("comment is NULL", "count(*)"); + assertFilterProject("length(comment) > 14 and orderkey < 150 and linenumber=2", "count(*)"); + assertFilterProject("comment like '%fluf%'", "comment"); + assertFilterProject("orderkey = 8480", "comment, fixed_comment"); + + //varchar/char dictionary + assertFilterProject("orderkey < 5000", "shipinstruct"); + assertFilterProject("shipinstruct IN ('NONE')", "comment, fixed_comment, char_array"); + assertFilterProject("trim(char_array[1]) = char_array[2]", "count(*)"); + assertFilterProject("char_array[1] IN ('along') and shipinstruct IN ('NONE')", "char_array"); + assertFilterProject("length(varchar_array[1]) > 10", "varchar_array"); + assertFilterProject("shipmode in ('AIR', 'MAIL', 'RAIL')\n" + + "AND shipinstruct in ('TAKE BACK RETURN', 'DELIVER IN PERSON')\n" + + "AND substr(shipinstruct, 2, 1) = substr(shipmode, 2, 1)\n" + + "AND shipmode = if(linenumber % 2 = 0, 'RAIL', 'MAIL')", "orderkey"); + + assertFilterProject("varchar_array[1] BETWEEN 'd' AND 'f'", "orderkey"); + assertFilterProject("comment between 'd' and 'f' AND uppercase_comment between 'D' and 'E' and length(comment) % 2 = linenumber % 2 and length(uppercase_comment) % 2 = linenumber % 2", "orderkey"); + + assertQueryUsingH2Cte("select shipmode from lineitem_ex where shipmode in ('AIR', 'MAIL', 'RAIL') and orderkey < 1000 and linenumber < 5 order by orderkey limit 20"); + + assertQueryUsingH2Cte("SELECT comment, varchar_array FROM lineitem_ex " + + "WHERE comment between 'a' and 'd' AND (length(comment) + orderkey + length(varchar_array[1])) % 2 = 0"); + } + + @Test + public void testArrays() + { + // read all positions + assertQueryUsingH2Cte("SELECT * FROM lineitem_ex"); + + // top-level IS [NOT] NULL filters + assertFilterProject("keys IS NULL", "orderkey, flags"); + assertFilterProject("nested_keys IS NULL", "keys, flags"); + + assertFilterProject("flags IS NOT NULL", "keys, orderkey"); + assertFilterProject("nested_keys IS NOT NULL", "keys, flags"); + + // mid-level IS [NOR] NULL filters + assertFilterProject("nested_keys[3] IS NULL", "keys, flags"); + assertFilterProject("nested_keys[3] IS NOT NULL", "keys, flags"); + + // read selected positions + assertQueryUsingH2Cte("SELECT * FROM lineitem_ex WHERE orderkey = 1"); + + // read all positions; extract selected positions + assertQueryUsingH2Cte("SELECT * FROM lineitem_ex WHERE orderkey % 3 = 1"); + + // filter + assertFilterProject("keys[2] = 1", "orderkey, flags"); + assertFilterProject("nested_keys[1][2] = 1", "orderkey, flags"); + + // filter function + assertFilterProject("keys[2] % 3 = 1", "orderkey, flags"); + assertFilterProject("nested_keys[1][2] % 3 = 1", "orderkey, flags"); + + // less selective filter + assertFilterProject("keys[1] < 1000", "orderkey, flags"); + assertFilterProject("nested_keys[1][1] < 1000", "orderkey, flags"); + + // filter plus filter function + assertFilterProject("keys[1] < 1000 AND keys[2] % 3 = 1", "orderkey, flags"); + assertFilterProject("nested_keys[1][1] < 1000 AND nested_keys[1][2] % 3 = 1", "orderkey, flags"); + + // filter function on multiple columns + assertFilterProject("keys[1] % 3 = 1 AND (orderkey + keys[2]) % 5 = 1", "orderkey, flags"); + assertFilterProject("nested_keys[1][1] % 3 = 1 AND (orderkey + nested_keys[1][2]) % 5 = 1", "orderkey, flags"); + + // filter on multiple columns, plus filter function + assertFilterProject("keys[1] < 1000 AND flags[2] = true AND keys[2] % 2 = if(flags[1], 0, 1)", "orderkey, flags"); + assertFilterProject("nested_keys[1][1] < 1000 AND flags[2] = true AND nested_keys[1][2] % 2 = if(flags[1], 0, 1)", "orderkey, flags"); + + // filters at different levels + assertFilterProject("nested_keys IS NOT NULL AND nested_keys[1][1] > 0", "keys"); + assertFilterProject("nested_keys[3] IS NULL AND nested_keys[2][1] > 10", "keys, flags"); + assertFilterProject("nested_keys[3] IS NOT NULL AND nested_keys[1][2] > 10", "keys, flags"); + assertFilterProject("nested_keys IS NOT NULL AND nested_keys[3] IS NOT NULL AND nested_keys[1][1] > 0", "keys"); + assertFilterProject("nested_keys IS NOT NULL AND nested_keys[3] IS NULL AND nested_keys[1][1] > 0", "keys"); + + // equality filter + assertQuery("SELECT orderkey FROM lineitem_ex WHERE keys = ARRAY[1, 22, 48]", "SELECT orderkey FROM lineitem WHERE orderkey = 1 AND partkey = 22 AND suppkey = 48"); + + // subfield pruning + assertQueryUsingH2Cte("SELECT nested_keys[2][1], nested_keys[1] FROM lineitem_ex"); + + assertFilterProjectFails("keys[5] > 0", "orderkey", "Array subscript out of bounds"); + assertFilterProjectFails("nested_keys[5][1] > 0", "orderkey", "Array subscript out of bounds"); + assertFilterProjectFails("nested_keys[1][5] > 0", "orderkey", "Array subscript out of bounds"); + assertFilterProjectFails("nested_keys[2][5] > 0", "orderkey", "Array subscript out of bounds"); + } + + @Test + public void testArraySubfieldPruning() + { + Function rewriter = query -> query.replaceAll("cardinality", "array_length"); + + // filter uses full column; that column is projected in full, partially or not at all + + // always-false filter + assertQueryUsingH2Cte("SELECT keys FROM lineitem_ex WHERE cardinality(keys) = 1", rewriter); + + assertQueryUsingH2Cte("SELECT keys[1] FROM lineitem_ex WHERE cardinality(keys) = 1", rewriter); + + assertQueryUsingH2Cte("SELECT orderkey FROM lineitem_ex WHERE cardinality(keys) = 1", rewriter); + + // some rows pass the filter + assertQueryUsingH2Cte("SELECT keys FROM lineitem_ex WHERE cardinality(keys) = 3", rewriter); + + assertQueryUsingH2Cte("SELECT keys[1] FROM lineitem_ex WHERE cardinality(keys) = 3", rewriter); + + assertQueryUsingH2Cte("SELECT orderkey FROM lineitem_ex WHERE cardinality(keys) = 3", rewriter); + + assertQueryUsingH2Cte("SELECT keys FROM lineitem_ex WHERE cardinality(keys) = 3 AND keys[1] < 1000", rewriter); + + assertQueryUsingH2Cte("SELECT keys[1] FROM lineitem_ex WHERE cardinality(keys) = 3 AND keys[1] < 1000", rewriter); + + assertQueryUsingH2Cte("SELECT orderkey FROM lineitem_ex WHERE cardinality(keys) = 3 AND keys[1] < 1000", rewriter); + + // range filter + assertQueryUsingH2Cte("SELECT keys FROM lineitem_ex WHERE keys is NOT NULL"); + + assertQueryUsingH2Cte("SELECT keys[1] FROM lineitem_ex WHERE keys is NOT NULL"); + + assertQueryUsingH2Cte("SELECT linenumber FROM lineitem_ex WHERE keys is NOT NULL"); + + // filter uses partial column; that column is projected in full, partially or not at all + assertQueryUsingH2Cte("SELECT keys FROM lineitem_ex WHERE keys[1] < 1000"); + assertQueryUsingH2Cte("SELECT keys FROM lineitem_ex WHERE keys[1] % 2 = 0"); + assertQueryUsingH2Cte("SELECT keys FROM lineitem_ex WHERE keys[1] < 1000 AND keys[2] % 2 = 0"); + + assertQueryUsingH2Cte("SELECT keys[1] FROM lineitem_ex WHERE keys[1] < 1000"); + assertQueryUsingH2Cte("SELECT keys[1] FROM lineitem_ex WHERE keys[1] % 2 = 0"); + assertQueryUsingH2Cte("SELECT keys[1] FROM lineitem_ex WHERE keys[1] < 1000 AND keys[2] % 2 = 0"); + + assertQueryUsingH2Cte("SELECT keys[2] FROM lineitem_ex WHERE keys[1] < 1000"); + assertQueryUsingH2Cte("SELECT keys[2] FROM lineitem_ex WHERE keys[1] % 2 = 0"); + assertQueryUsingH2Cte("SELECT keys[2] FROM lineitem_ex WHERE keys[1] < 1000 AND keys[2] % 2 = 0"); + + assertQueryUsingH2Cte("SELECT keys[1], keys[2] FROM lineitem_ex WHERE keys[1] < 1000"); + assertQueryUsingH2Cte("SELECT keys[1], keys[2] FROM lineitem_ex WHERE keys[1] % 2 = 0"); + assertQueryUsingH2Cte("SELECT keys[1], keys[2] FROM lineitem_ex WHERE keys[1] < 1000 AND keys[2] % 2 = 0"); + + assertQueryUsingH2Cte("SELECT linenumber FROM lineitem_ex WHERE keys[1] < 1000"); + assertQueryUsingH2Cte("SELECT linenumber FROM lineitem_ex WHERE keys[1] % 2 = 0"); + assertQueryUsingH2Cte("SELECT linenumber FROM lineitem_ex WHERE keys[1] < 1000 AND keys[2] % 2 = 0"); + + // no filter on array column; column is projected in full or partially + assertQueryUsingH2Cte("SELECT keys FROM lineitem_ex WHERE orderkey < 1000"); + + assertQueryUsingH2Cte("SELECT keys[1] FROM lineitem_ex WHERE orderkey < 1000"); + + assertQueryUsingH2Cte("SELECT keys[2], keys[3] FROM lineitem_ex WHERE orderkey < 1000"); + } + + @Test + public void testArrayOfMaps() + { + getQueryRunner().execute("CREATE TABLE test_arrays_of_maps AS\n" + + "SELECT orderkey, ARRAY[MAP(ARRAY[1, 2, 3], ARRAY[orderkey, partkey, suppkey]), MAP(ARRAY[1, 2, 3], ARRAY[orderkey + 1, partkey + 1, suppkey + 1])] as array_of_maps\n" + + "FROM lineitem"); + + try { + assertQuery("SELECT t.maps[1] FROM test_arrays_of_maps CROSS JOIN UNNEST(array_of_maps) AS t(maps)", "SELECT orderkey FROM lineitem UNION ALL SELECT orderkey + 1 FROM lineitem"); + + assertQuery("SELECT cardinality(array_of_maps[1]) > 0, t.maps[1] FROM test_arrays_of_maps CROSS JOIN UNNEST(array_of_maps) AS t(maps)", "SELECT true, orderkey FROM lineitem UNION ALL SELECT true, orderkey + 1 FROM lineitem"); + } + finally { + getQueryRunner().execute("DROP TABLE test_arrays_of_maps"); + } + } + + @Test + public void testMapsOfArrays() + { + getQueryRunner().execute("CREATE TABLE test_maps_of_arrays AS\n" + + "SELECT orderkey, map_from_entries(array_agg(row(linenumber, array[quantity, discount, tax]))) items\n" + + "FROM lineitem\n" + + "GROUP BY 1"); + + try { + assertQuery("SELECT t.doubles[2] FROM test_maps_of_arrays CROSS JOIN UNNEST(items) AS t(linenumber, doubles)", "SELECT discount FROM lineitem"); + + assertQuery("SELECT t.linenumber, t.doubles[2] FROM test_maps_of_arrays CROSS JOIN UNNEST(items) AS t(linenumber, doubles)", "SELECT linenumber, discount FROM lineitem"); + + assertQuery("SELECT cardinality(items[1]) > 0, t.doubles[2] FROM test_maps_of_arrays CROSS JOIN UNNEST(items) AS t(linenumber, doubles)", "SELECT true, discount FROM lineitem"); + } + finally { + getQueryRunner().execute("DROP TABLE test_maps_of_arrays"); + } + } + + @Test + public void testStructs() + { + assertQueryUsingH2Cte("SELECT orderkey, info, dates FROM lineitem_ex"); + + Function rewriter = query -> query.replaceAll("info.orderkey", "info[1]") + .replaceAll("info.linenumber", "info[2]") + .replaceAll("info.shipdate.ship_day", "info[3][1]") + .replaceAll("info.shipdate.ship_year", "info[3][3]") + .replaceAll("info.shipdate", "info[3]") + .replaceAll("dates\\[1\\].day", "dates[1][1]"); + + assertQueryUsingH2Cte("SELECT info.orderkey FROM lineitem_ex", rewriter); + assertQueryUsingH2Cte("SELECT info.orderkey, info.linenumber FROM lineitem_ex", rewriter); + + assertQueryUsingH2Cte("SELECT info.linenumber, info.shipdate.ship_year FROM lineitem_ex WHERE orderkey < 1000", rewriter); + + assertQueryUsingH2Cte("SELECT info.orderkey FROM lineitem_ex WHERE info IS NULL", rewriter); + assertQueryUsingH2Cte("SELECT info.orderkey FROM lineitem_ex WHERE info IS NOT NULL", rewriter); + + assertQueryUsingH2Cte("SELECT info, dates FROM lineitem_ex WHERE info.orderkey % 7 = 0", rewriter); + assertQueryUsingH2Cte("SELECT info.orderkey, dates FROM lineitem_ex WHERE info.orderkey % 7 = 0", rewriter); + assertQueryUsingH2Cte("SELECT info.linenumber, dates FROM lineitem_ex WHERE info.orderkey % 7 = 0", rewriter); + + assertQueryUsingH2Cte("SELECT dates FROM lineitem_ex WHERE info.orderkey % 7 = 0", rewriter); + assertQueryUsingH2Cte("SELECT info.orderkey FROM lineitem_ex WHERE info IS NOT NULL", rewriter); + + assertQueryUsingH2Cte("SELECT info.orderkey, info.shipdate FROM lineitem_ex WHERE info.orderkey % 7 = 0", rewriter); + + assertQueryUsingH2Cte("SELECT dates FROM lineitem_ex WHERE dates[1].day % 2 = 0", rewriter); + + assertQueryUsingH2Cte("SELECT info.orderkey, dates FROM lineitem_ex WHERE info IS NOT NULL AND dates IS NOT NULL AND info.orderkey % 7 = 0", rewriter); + + // filter-only struct + assertQueryUsingH2Cte("SELECT orderkey FROM lineitem_ex WHERE info IS NOT NULL"); + + // filters on subfields + assertQueryUsingH2Cte("SELECT info.orderkey, info.linenumber FROM lineitem_ex WHERE info.linenumber = 2", rewriter); + assertQueryUsingH2Cte("SELECT linenumber FROM lineitem_ex WHERE info.linenumber = 2", rewriter); + assertQueryUsingH2Cte("SELECT linenumber FROM lineitem_ex WHERE info IS NULL OR info.linenumber = 2", rewriter); + + assertQueryUsingH2Cte("SELECT info.shipdate.ship_day FROM lineitem_ex WHERE info.shipdate.ship_day < 15", rewriter); + assertQueryUsingH2Cte("SELECT info.linenumber FROM lineitem_ex WHERE info.shipdate.ship_day < 15", rewriter); + + // case sensitivity + assertQuery("SELECT INFO.orderkey FROM lineitem_ex", "SELECT CASE WHEN orderkey % 23 = 0 THEN null ELSE orderkey END FROM lineitem"); + assertQuery("SELECT INFO.ORDERKEY FROM lineitem_ex", "SELECT CASE WHEN orderkey % 23 = 0 THEN null ELSE orderkey END FROM lineitem"); + assertQuery("SELECT iNfO.oRdErKeY FROM lineitem_ex", "SELECT CASE WHEN orderkey % 23 = 0 THEN null ELSE orderkey END FROM lineitem"); + } + + @Test + public void testAllNullsInStruct() + { + List types = ImmutableList.of(BOOLEAN, TINYINT, SMALLINT, INTEGER, BIGINT, DOUBLE, REAL, VARCHAR, DATE); + + String query = String.format("SELECT orderkey, CAST(ROW(%s, 1) AS ROW(%s, a INTEGER)) as struct FROM orders", + types.stream() + .map(type -> "null") + .collect(joining(", ")), + types.stream() + .map(type -> String.format("null_%s %s", type.toLowerCase(getSession().getLocale()), type.toUpperCase())) + .collect(joining(", "))); + + getQueryRunner().execute("CREATE TABLE test_all_nulls_in_struct AS " + query); + + try { + for (String type : types) { + assertQuery( + format( + "SELECT struct.a, struct.null_%s FROM test_all_nulls_in_struct WHERE struct IS NOT NULL AND orderkey %% 2 = 0", + type.toLowerCase(getSession().getLocale())), + "SELECT 1, null FROM orders WHERE orderkey % 2 = 0"); + } + } + finally { + getQueryRunner().execute("DROP TABLE test_all_nulls_in_struct"); + } + } + + private void assertFilterProject(String filter, String projections) + { + assertQueryUsingH2Cte(format("SELECT * FROM lineitem_ex WHERE %s", filter)); + assertQueryUsingH2Cte(format("SELECT %s FROM lineitem_ex WHERE %s", projections, filter)); + } + + private void assertFilterProjectFails(String filter, String projections, String expectedMessageRegExp) + { + assertQueryFails(format("SELECT * FROM lineitem_ex WHERE %s", filter), expectedMessageRegExp); + assertQueryFails(format("SELECT %s FROM lineitem_ex WHERE %s", projections, filter), expectedMessageRegExp); + } + @Test public void testFilterFunctions() { @@ -124,28 +719,72 @@ public void testFilterFunctions() assertQueryFails("SELECT custkey, orderdate FROM orders WHERE array[1, 2, 3][orderkey % 5 + custkey % 7 + 1] > 0", "Array subscript out of bounds"); // filter function with "recoverable" error - assertQuery("SELECT custkey, orderdate FROM orders WHERE array[1, 2, 3][orderkey % 5 + custkey %7 + 1] > 0 AND orderkey % 5 = 1 AND custkey % 7 = 0", "SELECT custkey, orderdate FROM orders WHERE orderkey % 5 = 1 AND custkey % 7 = 0"); + assertQuery("SELECT custkey, orderdate FROM orders WHERE array[1, 2, 3][orderkey % 5 + custkey % 7 + 1] > 0 AND orderkey % 5 = 1 AND custkey % 7 = 0", "SELECT custkey, orderdate FROM orders WHERE orderkey % 5 = 1 AND custkey % 7 = 0"); + + // filter function on numeric and boolean columns + assertFilterProject("if(is_returned, linenumber, orderkey) % 5 = 0", "linenumber"); + + // filter functions on array columns + assertFilterProject("keys[1] % 5 = 0", "orderkey"); + assertFilterProject("nested_keys[1][1] % 5 = 0", "orderkey"); + + assertFilterProject("keys[1] % 5 = 0 AND keys[2] > 100", "orderkey"); + assertFilterProject("keys[1] % 5 = 0 AND nested_keys[1][2] > 100", "orderkey"); - // filter function on numeric and boolean columnss - assertQuery("SELECT linenumber FROM lineitem_ex WHERE if(is_returned, linenumber, orderkey) % 5 = 0", WITH_LINEITEM_EX + "SELECT linenumber FROM lineitem_ex WHERE CASE WHEN is_returned THEN linenumber ELSE orderkey END % 5 = 0"); + assertFilterProject("keys[1] % 5 = 0 AND keys[2] % 7 = 0", "orderkey"); + assertFilterProject("keys[1] % 5 = 0 AND nested_keys[1][2] % 7 = 0", "orderkey"); + + assertFilterProject("(cast(keys[1] as integer) + keys[3]) % 5 = 0", "orderkey"); + assertFilterProject("(cast(keys[1] as integer) + nested_keys[1][2]) % 5 = 0", "orderkey"); + + // subscript out of bounds + assertQueryFails("SELECT orderkey FROM lineitem_ex WHERE keys[5] % 7 = 0", "Array subscript out of bounds"); + assertQueryFails("SELECT orderkey FROM lineitem_ex WHERE nested_keys[1][5] % 7 = 0", "Array subscript out of bounds"); + + assertQueryFails("SELECT * FROM lineitem_ex WHERE nested_keys[1][5] > 0", "Array subscript out of bounds"); + assertQueryFails("SELECT orderkey FROM lineitem_ex WHERE nested_keys[1][5] > 0", "Array subscript out of bounds"); + assertQueryFails("SELECT * FROM lineitem_ex WHERE nested_keys[1][5] > 0 AND orderkey % 5 = 0", "Array subscript out of bounds"); + + assertFilterProject("nested_keys[1][5] > 0 AND orderkey % 5 > 10", "keys"); + } + + @Test + public void testPushdownComposition() + { + // Tests composing two pushdowns each with a range filter and filter function. + assertQuery( + "WITH data AS (" + + " SELECT l.suppkey, l.linenumber, l.shipmode, MAX(o.orderdate)" + + " FROM lineitem l, orders o WHERE" + + " o.orderkey = l.orderkey AND linenumber IN (2, 3, 4, 6) AND shipmode LIKE '%AIR%'" + + " GROUP BY l.suppkey, l.linenumber, l.shipmode)" + + "SELECT COUNT(*) FROM data WHERE suppkey BETWEEN 10 AND 30 AND shipmode LIKE '%REG%'"); } @Test public void testPartitionColumns() { - assertUpdate("CREATE TABLE test_partition_columns WITH (partitioned_by = ARRAY['p']) AS\n" + - "SELECT * FROM (VALUES (1, 'abc'), (2, 'abc')) as t(x, p)", 2); + assertUpdate("CREATE TABLE test_partition_columns WITH (partitioned_by = ARRAY['p', 'q']) AS\n" + + "SELECT * FROM (VALUES (1, 'abc', 'cba'), (2, 'abc', 'def')) as t(x, p, q)", 2); + + assertQuery("SELECT * FROM test_partition_columns", "SELECT 1, 'abc', 'cba' UNION ALL SELECT 2, 'abc', 'def'"); - assertQuery("SELECT * FROM test_partition_columns", "SELECT 1, 'abc' UNION ALL SELECT 2, 'abc'"); + assertQuery("SELECT x FROM test_partition_columns", "SELECT 1 UNION ALL SELECT 2"); - assertQuery("SELECT * FROM test_partition_columns WHERE p = 'abc'", "SELECT 1, 'abc' UNION ALL SELECT 2, 'abc'"); + assertQuery("SELECT * FROM test_partition_columns WHERE p = 'abc'", "SELECT 1, 'abc', 'cba' UNION ALL SELECT 2, 'abc', 'def'"); - assertQuery("SELECT * FROM test_partition_columns WHERE p LIKE 'a%'", "SELECT 1, 'abc' UNION ALL SELECT 2, 'abc'"); + assertQuery("SELECT * FROM test_partition_columns WHERE p LIKE 'a%'", "SELECT 1, 'abc', 'cba' UNION ALL SELECT 2, 'abc', 'def'"); - assertQuery("SELECT * FROM test_partition_columns WHERE substr(p, x, 1) = 'a'", "SELECT 1, 'abc'"); + assertQuery("SELECT * FROM test_partition_columns WHERE substr(p, x, 1) = 'a' and substr(q, 1, 1) = 'c'", "SELECT 1, 'abc', 'cba'"); assertQueryReturnsEmptyResult("SELECT * FROM test_partition_columns WHERE p = 'xxx'"); + assertQueryReturnsEmptyResult("SELECT * FROM test_partition_columns WHERE p = 'abc' and p='def'"); + + assertUpdate("INSERT into test_partition_columns values (3, 'abc', NULL)", 1); + + assertQuerySucceeds(getSession(), "select * from test_partition_columns"); + assertUpdate("DROP TABLE test_partition_columns"); } @@ -169,9 +808,262 @@ public void testPathColumn() assertQuerySucceeds(session, "SELECT linenumber, \"$path\" FROM lineitem WHERE length(\"$path\") % 2 = linenumber % 2"); } + @Test + public void testTextfileFormatWithPushdown() + { + assertUpdate("CREATE TABLE textfile (id BIGINT) WITH (format = 'TEXTFILE')"); + assertUpdate("INSERT INTO textfile VALUES (1), (2), (3)", 3); + assertQuery("SELECT id FROM textfile WHERE id = 1", "SELECT 1"); + assertQuery("SELECT id FROM textfile", "SELECT 1 UNION SELECT 2 UNION SELECT 3 "); + assertUpdate("DROP TABLE textfile"); + } + + @Test + public void testSchemaEvolution() + { + assertUpdate("CREATE TABLE test_schema_evolution WITH (partitioned_by = ARRAY['regionkey']) AS SELECT nationkey, regionkey FROM nation", 25); + assertUpdate("ALTER TABLE test_schema_evolution ADD COLUMN nation_plus_region BIGINT"); + + // constant filter function errors + assertQueryFails("SELECT * FROM test_schema_evolution WHERE coalesce(nation_plus_region, fail('constant filter error')) is not null", "constant filter error"); + assertQuerySucceeds("SELECT * FROM test_schema_evolution WHERE nationkey < 0 AND coalesce(nation_plus_region, fail('constant filter error')) is not null"); + assertQueryFails("SELECT * FROM test_schema_evolution WHERE nationkey % 2 = 0 AND coalesce(nation_plus_region, fail('constant filter error')) is not null", "constant filter error"); + + // non-deterministic filter function with constant inputs + assertQueryReturnsEmptyResult("SELECT * FROM test_schema_evolution WHERE nation_plus_region * rand() < 0"); + assertQuery("SELECT nationkey FROM test_schema_evolution WHERE nation_plus_region * rand() IS NULL", "SELECT nationkey FROM nation"); + assertQuerySucceeds("SELECT nationkey FROM test_schema_evolution WHERE coalesce(nation_plus_region, 1) * rand() < 0.5"); + + assertUpdate("INSERT INTO test_schema_evolution SELECT nationkey, nationkey + regionkey, regionkey FROM nation", 25); + assertUpdate("ALTER TABLE test_schema_evolution ADD COLUMN nation_minus_region BIGINT"); + assertUpdate("INSERT INTO test_schema_evolution SELECT nationkey, nationkey + regionkey, nationkey - regionkey, regionkey FROM nation", 25); + + String cte = "WITH test_schema_evolution AS (" + + "SELECT nationkey, null AS nation_plus_region, null AS nation_minus_region, regionkey FROM nation " + + "UNION ALL SELECT nationkey, nationkey + regionkey, null, regionkey FROM nation " + + "UNION ALL SELECT nationkey, nationkey + regionkey, nationkey - regionkey, regionkey FROM nation)"; + + assertQueryUsingH2Cte("SELECT * FROM test_schema_evolution", cte); + assertQueryUsingH2Cte("SELECT * FROM test_schema_evolution WHERE nation_plus_region IS NULL", cte); + assertQueryUsingH2Cte("SELECT * FROM test_schema_evolution WHERE nation_plus_region > 10", cte); + assertQueryUsingH2Cte("SELECT * FROM test_schema_evolution WHERE nation_plus_region + 1 > 10", cte); + assertQueryUsingH2Cte("SELECT * FROM test_schema_evolution WHERE nation_plus_region + nation_minus_region > 20", cte); + assertQueryUsingH2Cte("select * from test_schema_evolution where nation_plus_region = regionkey", cte); + assertUpdate("DROP TABLE test_schema_evolution"); + } + + @Test + public void testRcAndTextFormats() + throws IOException + { + getQueryRunner().execute("CREATE TABLE lineitem_ex_partitioned WITH (format = 'ORC', partitioned_by = ARRAY['ds']) AS\n" + + "SELECT\n" + + " linenumber,\n" + + " orderkey,\n" + + " partkey,\n" + + " suppkey,\n" + + " quantity,\n" + + " extendedprice,\n" + + " tax,\n" + + " shipinstruct,\n" + + " shipmode,\n" + + " ship_by_air,\n" + + " is_returned,\n" + + " ship_day,\n" + + " ship_month,\n" + + " ship_timestamp,\n" + + " commit_timestamp,\n" + + " discount_real,\n" + + " discount,\n" + + " tax_real,\n" + + " ship_day_month,\n" + + " discount_long_decimal,\n" + + " tax_short_decimal,\n" + + " long_decimals,\n" + + " keys,\n" + + " doubles,\n" + + " nested_keys,\n" + + " flags,\n" + + " reals,\n" + + " info,\n" + + " dates,\n" + + " timestamps,\n" + + " comment,\n" + + " uppercase_comment,\n" + + " fixed_comment,\n" + + " char_array,\n" + + " varchar_array,\n" + + " '2019-11-01' AS ds\n" + + "FROM lineitem_ex"); + try { + for (HiveStorageFormat format : ImmutableList.of(RCBINARY, RCTEXT, TEXTFILE)) { + assertFileFormat(format); + } + } + finally { + assertUpdate("DROP TABLE lineitem_ex_partitioned"); + } + } + + private void assertFileFormat(HiveStorageFormat storageFormat) + throws IOException + { + // Make an ORC table backed by file of some other format + QueryRunner queryRunner = getQueryRunner(); + queryRunner.execute("CREATE TABLE test_file_format_orc WITH (format='ORC', partitioned_by=ARRAY['ds']) AS " + + "SELECT * FROM lineitem_ex_partitioned LIMIT 1"); + try { + queryRunner.execute(format("CREATE TABLE test_file_format WITH (format='%s', partitioned_by=ARRAY['ds']) AS " + + "SELECT * FROM lineitem_ex_partitioned", storageFormat)); + Path orcDirectory = getPartitionDirectory("test_file_format_orc", "ds='2019-11-01'"); + deleteRecursively(orcDirectory, ALLOW_INSECURE); + + Path otherDirectory = getPartitionDirectory("test_file_format", "ds='2019-11-01'"); + Files.move(otherDirectory, orcDirectory, REPLACE_EXISTING); + + String cte = WITH_LINEITEM_EX + ", test_file_format_orc AS " + + "(SELECT\n" + + " linenumber,\n" + + " orderkey,\n" + + " partkey,\n" + + " suppkey,\n" + + " quantity,\n" + + " extendedprice,\n" + + " tax,\n" + + " shipinstruct,\n" + + " shipmode,\n" + + " ship_by_air,\n" + + " is_returned,\n" + + " ship_day,\n" + + " ship_month,\n" + + " ship_timestamp,\n" + + " commit_timestamp,\n" + + " discount_real,\n" + + " discount,\n" + + " tax_real,\n" + + " ship_day_month,\n" + + " discount_long_decimal,\n" + + " tax_short_decimal,\n" + + " long_decimals,\n" + + " keys,\n" + + " doubles,\n" + + " nested_keys,\n" + + " flags,\n" + + " reals,\n" + + " info,\n" + + " dates,\n" + + " timestamps,\n" + + " comment,\n" + + " uppercase_comment,\n" + + " fixed_comment,\n" + + " char_array,\n" + + " varchar_array,\n" + + " '2019-11-01' AS ds\n" + + "FROM lineitem_ex)"; + + // no filter + assertQueryUsingH2Cte("SELECT * FROM test_file_format_orc", cte); + assertQueryUsingH2Cte("SELECT comment FROM test_file_format_orc", cte); + assertQueryUsingH2Cte("SELECT COUNT(*) FROM test_file_format_orc", cte); + + // filter on partition column + assertQueryUsingH2Cte("SELECT comment from test_file_format_orc WHERE ds='2019-11-01'", cte); + assertQueryReturnsEmptyResult("SELECT comment FROM test_file_format_orc WHERE ds='2019-11-02'"); + + // range filters and filter functions + assertQueryUsingH2Cte("SELECT orderkey from test_file_format_orc WHERE orderkey < 1000", cte); + assertQueryUsingH2Cte("SELECT orderkey, comment from test_file_format_orc WHERE orderkey < 1000 AND comment LIKE '%final%'", cte); + assertQueryUsingH2Cte("SELECT COUNT(*) from test_file_format_orc WHERE orderkey < 1000", cte); + + assertQueryUsingH2Cte("SELECT COUNT(*) FROM test_file_format_orc WHERE concat(ds,'*') = '2019-11-01*'", cte); + assertQueryUsingH2Cte("SELECT orderkey FROM test_file_format_orc WHERE comment LIKE '%final%'", cte); + + assertQueryUsingH2Cte("SELECT discount FROM test_file_format_orc WHERE discount > 0.01", cte); + assertQueryUsingH2Cte("SELECT * FROM test_file_format_orc WHERE discount > 0.01 and discount + tax > 0.03", cte); + assertQueryUsingH2Cte("SELECT COUNT(*) FROM test_file_format_orc WHERE discount = 0.0", cte); + + assertQueryUsingH2Cte("SELECT COUNT(*) FROM test_file_format_orc WHERE discount_real > 0.01", cte); + assertQueryUsingH2Cte("SELECT * FROM test_file_format_orc WHERE tax_real > 0.01 and discount_real > 0.01", cte); + + assertQueryUsingH2Cte("SELECT keys FROM test_file_format_orc WHERE keys IS NOT NULL", cte); + assertQueryUsingH2Cte("SELECT keys FROM test_file_format_orc WHERE keys IS NULL", cte); + assertQueryUsingH2Cte("SELECT linenumber FROM test_file_format_orc WHERE keys[1] % 5 = 0 AND keys[2] > 100", cte); + + assertQueryUsingH2Cte("SELECT * FROM test_file_format_orc WHERE is_returned=false", cte); + assertQueryUsingH2Cte("SELECT * FROM test_file_format_orc WHERE is_returned is NULL", cte); + + assertQueryUsingH2Cte("SELECT ship_day FROM test_file_format_orc WHERE ship_day > 2", cte); + + assertQueryUsingH2Cte("SELECT discount_long_decimal FROM test_file_format_orc WHERE discount_long_decimal > 0.05", cte); + assertQueryUsingH2Cte("SELECT tax_short_decimal FROM test_file_format_orc WHERE tax_short_decimal < 0.03", cte); + assertQueryUsingH2Cte("SELECT discount_long_decimal FROM test_file_format_orc WHERE discount_long_decimal > 0.01 AND tax_short_decimal > 0.01 AND (discount_long_decimal + tax_short_decimal) < 0.03", cte); + + Function rewriter = query -> query.replaceAll("info.orderkey", "info[1]") + .replaceAll("dates\\[1\\].day", "dates[1][1]"); + + assertQueryUsingH2Cte("SELECT dates FROM test_file_format_orc WHERE dates[1].day % 2 = 0", cte, rewriter); + assertQueryUsingH2Cte("SELECT info.orderkey, dates FROM test_file_format_orc WHERE info IS NOT NULL AND dates IS NOT NULL AND info.orderkey % 7 = 0", cte, rewriter); + + // empty result + assertQueryReturnsEmptyResult("SELECT comment FROM test_file_format_orc WHERE orderkey < 0"); + assertQueryReturnsEmptyResult("SELECT comment FROM test_file_format_orc WHERE comment LIKE '???'"); + } + finally { + assertUpdate("DROP TABLE IF EXISTS test_file_format"); + assertUpdate("DROP TABLE test_file_format_orc"); + } + } + + private Path getPartitionDirectory(String tableName, String partitionClause) + { + String filePath = ((String) computeActual(noPushdownFilter(getSession()), format("SELECT \"$path\" FROM %s WHERE %s LIMIT 1", tableName, partitionClause)).getOnlyValue()) + .replace("file:", ""); + return Paths.get(filePath).getParent(); + } + + private void assertQueryUsingH2Cte(String query, String cte) + { + assertQueryUsingH2Cte(query, cte, Function.identity()); + } + + private void assertQueryUsingH2Cte(String query, String cte, Function rewriter) + { + assertQuery(query, cte + toH2(rewriter.apply(query))); + } + private void assertQueryUsingH2Cte(String query) { - assertQuery(query, WITH_LINEITEM_EX + query); + assertQueryUsingH2Cte(query, Function.identity()); + } + + private void assertQueryUsingH2Cte(String query, Function rewriter) + { + assertQuery(query, WITH_LINEITEM_EX + toH2(rewriter.apply(query))); + } + + private static String toH2(String query) + { + return replaceArraySubscripts(query).replaceAll(" if\\(", " casewhen("); + } + + private static String replaceArraySubscripts(String query) + { + Matcher matcher = ARRAY_SUBSCRIPT_PATTERN.matcher(query); + + StringBuilder builder = new StringBuilder(); + int offset = 0; + while (matcher.find()) { + String expression = matcher.group(1); + List indices = Splitter.onPattern("[^0-9]").omitEmptyStrings().splitToList(matcher.group(2)); + for (int i = 0; i < indices.size(); i++) { + expression = format("array_get(%s, %s)", expression, indices.get(i)); + } + + builder.append(query, offset, matcher.start()).append(expression); + offset = matcher.end(); + } + builder.append(query.substring(offset)); + return builder.toString(); } private static Session noPushdownFilter(Session session) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownIntegrationSmokeTest.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownIntegrationSmokeTest.java new file mode 100644 index 0000000000000..42eb0625c841f --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownIntegrationSmokeTest.java @@ -0,0 +1,43 @@ +package com.facebook.presto.hive; +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.facebook.presto.spi.security.SelectedRole; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.Optional; + +import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG; +import static com.facebook.presto.hive.HiveQueryRunner.createBucketedSession; +import static com.facebook.presto.hive.HiveQueryRunner.createMaterializeExchangesSession; +import static com.facebook.presto.hive.HiveQueryRunner.createQueryRunner; +import static com.facebook.presto.spi.security.SelectedRole.Type.ROLE; +import static io.airlift.tpch.TpchTable.CUSTOMER; +import static io.airlift.tpch.TpchTable.LINE_ITEM; +import static io.airlift.tpch.TpchTable.ORDERS; +import static io.airlift.tpch.TpchTable.PART_SUPPLIER; + +public class TestHivePushdownIntegrationSmokeTest + extends TestHiveIntegrationSmokeTest +{ + public TestHivePushdownIntegrationSmokeTest() + { + super(() -> createQueryRunner(ImmutableList.of(ORDERS, CUSTOMER, LINE_ITEM, PART_SUPPLIER), ImmutableMap.of("experimental.pushdown-subfields-enabled", "true"), "sql-standard", ImmutableMap.of("hive.pushdown-filter-enabled", "true"), Optional.empty()), + createBucketedSession(Optional.of(new SelectedRole(ROLE, Optional.of("admin")))), + createMaterializeExchangesSession(Optional.of(new SelectedRole(ROLE, Optional.of("admin")))), + HIVE_CATALOG, + new HiveTypeTranslator()); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveRecoverableGroupedExecution.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveRecoverableGroupedExecution.java index c88950b175ebe..5d6e6ee703a55 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveRecoverableGroupedExecution.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveRecoverableGroupedExecution.java @@ -13,31 +13,27 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; -import com.facebook.presto.execution.QueryState; -import com.facebook.presto.execution.StageInfo; -import com.facebook.presto.execution.StageState; -import com.facebook.presto.server.BasicQueryInfo; import com.facebook.presto.server.testing.TestingPrestoServer; -import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.security.SelectedRole; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.tests.DistributedQueryRunner; +import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import org.intellij.lang.annotations.Language; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.util.List; import java.util.Optional; import java.util.OptionalLong; -import java.util.Set; import static com.facebook.presto.SystemSessionProperties.COLOCATED_JOIN; import static com.facebook.presto.SystemSessionProperties.CONCURRENT_LIFESPANS_PER_NODE; @@ -47,242 +43,297 @@ import static com.facebook.presto.SystemSessionProperties.RECOVERABLE_GROUPED_EXECUTION; import static com.facebook.presto.SystemSessionProperties.REDISTRIBUTE_WRITES; import static com.facebook.presto.SystemSessionProperties.SCALE_WRITERS; +import static com.facebook.presto.SystemSessionProperties.TASK_PARTITIONED_WRITER_COUNT; import static com.facebook.presto.SystemSessionProperties.TASK_WRITER_COUNT; +import static com.facebook.presto.execution.TaskState.RUNNING; import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG; import static com.facebook.presto.hive.HiveQueryRunner.TPCH_BUCKETED_SCHEMA; import static com.facebook.presto.hive.HiveQueryRunner.createQueryRunner; import static com.facebook.presto.hive.HiveSessionProperties.VIRTUAL_BUCKET_COUNT; import static com.facebook.presto.spi.security.SelectedRole.Type.ROLE; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static com.google.common.collect.MoreCollectors.toOptional; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; import static io.airlift.tpch.TpchTable.ORDERS; import static java.lang.Thread.sleep; -import static java.util.Objects.requireNonNull; +import static java.util.Collections.shuffle; import static java.util.concurrent.Executors.newCachedThreadPool; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.stream.Collectors.toList; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -@Test(singleThreaded = true, enabled = false) +@Test(singleThreaded = true) public class TestHiveRecoverableGroupedExecution { - private static final Set SPLIT_SCHEDULING_STARTED_STATES = ImmutableSet.of(StageState.SCHEDULING_SPLITS, StageState.SCHEDULED, StageState.RUNNING, StageState.FINISHED); + private static final Logger log = Logger.get(TestHiveRecoverableGroupedExecution.class); - private final Session recoverableSession; - private final DistributedQueryRunnerSupplier distributedQueryRunnerSupplier; - private ListeningExecutorService executor; - - @SuppressWarnings("unused") - public TestHiveRecoverableGroupedExecution() - { - this(() -> createQueryRunner( - ImmutableList.of(ORDERS), - ImmutableMap.of("query.remote-task.max-error-duration", "1s"), - Optional.empty()), - createRecoverableSession(Optional.of(new SelectedRole(ROLE, Optional.of("admin"))))); - } + private static final int TEST_TIMEOUT = 120_000; + private static final int INVOCATION_COUNT = 1; - protected TestHiveRecoverableGroupedExecution(DistributedQueryRunnerSupplier distributedQueryRunnerSupplier, Session recoverableSession) - { - this.distributedQueryRunnerSupplier = requireNonNull(distributedQueryRunnerSupplier, "distributedQueryRunnerSupplier is null"); - this.recoverableSession = requireNonNull(recoverableSession, "recoverableSession is null"); - } + private DistributedQueryRunner queryRunner; + private ListeningExecutorService executor; @BeforeClass public void setUp() + throws Exception { + queryRunner = createQueryRunner( + ImmutableList.of(ORDERS), + // extra properties + ImmutableMap.of( + // task get results timeout has to be significantly higher that the task status update timeout + "exchange.max-error-duration", "5m", + // set the timeout of a single HTTP request to 1s, to make sure that the task result requests are actually failing + // The default is 10s, that might not be sufficient to make sure that the request fails before the recoverable execution kicks in + "exchange.http-client.request-timeout", "1s"), + // extra coordinator properties + ImmutableMap.of( + // set the timeout of the task update requests to something low to improve overall test latency + "scheduler.http-client.request-timeout", "1s", + // this effectively disables the retries + "query.remote-task.max-error-duration", "1s", + // allow 2 out of 4 tasks to fail + "max-failed-task-percentage", "0.6"), + + Optional.empty()); executor = listeningDecorator(newCachedThreadPool()); } @AfterClass(alwaysRun = true) public void shutdown() { - executor.shutdownNow(); + if (executor != null) { + executor.shutdownNow(); + } + executor = null; + + if (queryRunner != null) { + queryRunner.close(); + } + queryRunner = null; + } + + @DataProvider(name = "writerConcurrency") + public static Object[][] writerConcurrency() + { + return new Object[][] {{1}, {2}}; } - @Test(timeOut = 60_000, enabled = false) - public void testCreateBucketedTable() + @Test(timeOut = TEST_TIMEOUT, dataProvider = "writerConcurrency", invocationCount = INVOCATION_COUNT) + public void testCreateBucketedTable(int writerConcurrency) throws Exception { testRecoverableGroupedExecution( + writerConcurrency, ImmutableList.of( - "CREATE TABLE test_table1\n" + + "CREATE TABLE create_bucketed_table_1\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key1']) AS\n" + "SELECT orderkey key1, comment value1 FROM orders", - "CREATE TABLE test_table2\n" + + "CREATE TABLE create_bucketed_table_2\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key2']) AS\n" + "SELECT orderkey key2, comment value2 FROM orders", - "CREATE TABLE test_table3\n" + + "CREATE TABLE create_bucketed_table_3\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key3']) AS\n" + "SELECT orderkey key3, comment value3 FROM orders"), - "CREATE TABLE test_success WITH (bucket_count = 13, bucketed_by = ARRAY['key1']) AS\n" + + "CREATE TABLE create_bucketed_table_success WITH (bucket_count = 13, bucketed_by = ARRAY['key1']) AS\n" + "SELECT key1, value1, key2, value2, key3, value3\n" + - "FROM test_table1\n" + - "JOIN test_table2\n" + + "FROM create_bucketed_table_1\n" + + "JOIN create_bucketed_table_2\n" + "ON key1 = key2\n" + - "JOIN test_table3\n" + + "JOIN create_bucketed_table_3\n" + "ON key2 = key3", - "CREATE TABLE test_failure WITH (bucket_count = 13, bucketed_by = ARRAY['key1']) AS\n" + + "CREATE TABLE create_bucketed_table_failure WITH (bucket_count = 13, bucketed_by = ARRAY['key1']) AS\n" + "SELECT key1, value1, key2, value2, key3, value3\n" + - "FROM test_table1\n" + - "JOIN test_table2\n" + + "FROM create_bucketed_table_1\n" + + "JOIN create_bucketed_table_2\n" + "ON key1 = key2\n" + - "JOIN test_table3\n" + + "JOIN create_bucketed_table_3\n" + "ON key2 = key3", 15000, ImmutableList.of( - "DROP TABLE IF EXISTS test_table1", - "DROP TABLE IF EXISTS test_table2", - "DROP TABLE IF EXISTS test_table3", - "DROP TABLE IF EXISTS test_success", - "DROP TABLE IF EXISTS test_failure")); + "DROP TABLE IF EXISTS create_bucketed_table_1", + "DROP TABLE IF EXISTS create_bucketed_table_2", + "DROP TABLE IF EXISTS create_bucketed_table_3", + "DROP TABLE IF EXISTS create_bucketed_table_success", + "DROP TABLE IF EXISTS create_bucketed_table_failure")); } - @Test(timeOut = 60_000, enabled = false) - public void testInsertBucketedTable() + @Test(timeOut = TEST_TIMEOUT, dataProvider = "writerConcurrency", invocationCount = INVOCATION_COUNT) + public void testInsertBucketedTable(int writerConcurrency) throws Exception { testRecoverableGroupedExecution( + writerConcurrency, ImmutableList.of( - "CREATE TABLE test_table1\n" + + "CREATE TABLE insert_bucketed_table_1\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key1']) AS\n" + "SELECT orderkey key1, comment value1 FROM orders", - "CREATE TABLE test_table2\n" + + "CREATE TABLE insert_bucketed_table_2\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key2']) AS\n" + "SELECT orderkey key2, comment value2 FROM orders", - "CREATE TABLE test_table3\n" + + "CREATE TABLE insert_bucketed_table_3\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key3']) AS\n" + "SELECT orderkey key3, comment value3 FROM orders", - "CREATE TABLE test_success (key BIGINT, value VARCHAR, partition_key VARCHAR)\n" + + "CREATE TABLE insert_bucketed_table_success (key BIGINT, value VARCHAR, partition_key VARCHAR)\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key'], partitioned_by = ARRAY['partition_key'])", - "CREATE TABLE test_failure (key BIGINT, value VARCHAR, partition_key VARCHAR)\n" + + "CREATE TABLE insert_bucketed_table_failure (key BIGINT, value VARCHAR, partition_key VARCHAR)\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key'], partitioned_by = ARRAY['partition_key'])"), - "INSERT INTO test_success\n" + + "INSERT INTO insert_bucketed_table_success\n" + "SELECT key1, value1, 'foo'\n" + - "FROM test_table1\n" + - "JOIN test_table2\n" + + "FROM insert_bucketed_table_1\n" + + "JOIN insert_bucketed_table_2\n" + "ON key1 = key2\n" + - "JOIN test_table3\n" + + "JOIN insert_bucketed_table_3\n" + "ON key2 = key3", - "INSERT INTO test_failure\n" + + "INSERT INTO insert_bucketed_table_failure\n" + "SELECT key1, value1, 'foo'\n" + - "FROM test_table1\n" + - "JOIN test_table2\n" + + "FROM insert_bucketed_table_1\n" + + "JOIN insert_bucketed_table_2\n" + "ON key1 = key2\n" + - "JOIN test_table3\n" + + "JOIN insert_bucketed_table_3\n" + "ON key2 = key3", 15000, ImmutableList.of( - "DROP TABLE IF EXISTS test_table1", - "DROP TABLE IF EXISTS test_table2", - "DROP TABLE IF EXISTS test_table3", - "DROP TABLE IF EXISTS test_success", - "DROP TABLE IF EXISTS test_failure")); + "DROP TABLE IF EXISTS insert_bucketed_table_1", + "DROP TABLE IF EXISTS insert_bucketed_table_2", + "DROP TABLE IF EXISTS insert_bucketed_table_3", + "DROP TABLE IF EXISTS insert_bucketed_table_success", + "DROP TABLE IF EXISTS insert_bucketed_table_failure")); } - @Test(timeOut = 60_000, enabled = false) - public void testCreateUnbucketedTableWithGroupedExecution() + @Test(timeOut = TEST_TIMEOUT, dataProvider = "writerConcurrency", invocationCount = INVOCATION_COUNT) + public void testCreateUnbucketedTableWithGroupedExecution(int writerConcurrency) throws Exception { testRecoverableGroupedExecution( + writerConcurrency, ImmutableList.of( - "CREATE TABLE test_table1\n" + + "CREATE TABLE create_unbucketed_table_with_grouped_execution_1\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key1']) AS\n" + "SELECT orderkey key1, comment value1 FROM orders", - "CREATE TABLE test_table2\n" + + "CREATE TABLE create_unbucketed_table_with_grouped_execution_2\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key2']) AS\n" + "SELECT orderkey key2, comment value2 FROM orders", - "CREATE TABLE test_table3\n" + + "CREATE TABLE create_unbucketed_table_with_grouped_execution_3\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key3']) AS\n" + "SELECT orderkey key3, comment value3 FROM orders"), - "CREATE TABLE test_success AS\n" + + "CREATE TABLE create_unbucketed_table_with_grouped_execution_success AS\n" + "SELECT key1, value1, key2, value2, key3, value3\n" + - "FROM test_table1\n" + - "JOIN test_table2\n" + + "FROM create_unbucketed_table_with_grouped_execution_1\n" + + "JOIN create_unbucketed_table_with_grouped_execution_2\n" + "ON key1 = key2\n" + - "JOIN test_table3\n" + + "JOIN create_unbucketed_table_with_grouped_execution_3\n" + "ON key2 = key3", - "CREATE TABLE test_failure AS\n" + + "CREATE TABLE create_unbucketed_table_with_grouped_execution_failure AS\n" + "SELECT key1, value1, key2, value2, key3, value3\n" + - "FROM test_table1\n" + - "JOIN test_table2\n" + + "FROM create_unbucketed_table_with_grouped_execution_1\n" + + "JOIN create_unbucketed_table_with_grouped_execution_2\n" + "ON key1 = key2\n" + - "JOIN test_table3\n" + + "JOIN create_unbucketed_table_with_grouped_execution_3\n" + "ON key2 = key3", 15000, ImmutableList.of( - "DROP TABLE IF EXISTS test_table1", - "DROP TABLE IF EXISTS test_table2", - "DROP TABLE IF EXISTS test_table3", - "DROP TABLE IF EXISTS test_success", - "DROP TABLE IF EXISTS test_failure")); + "DROP TABLE IF EXISTS create_unbucketed_table_with_grouped_execution_1", + "DROP TABLE IF EXISTS create_unbucketed_table_with_grouped_execution_2", + "DROP TABLE IF EXISTS create_unbucketed_table_with_grouped_execution_3", + "DROP TABLE IF EXISTS create_unbucketed_table_with_grouped_execution_success", + "DROP TABLE IF EXISTS create_unbucketed_table_with_grouped_execution_failure")); } - @Test(timeOut = 60_000, enabled = false) - public void testInsertUnbucketedTableWithGroupedExecution() + @Test(timeOut = TEST_TIMEOUT, dataProvider = "writerConcurrency", invocationCount = INVOCATION_COUNT) + public void testInsertUnbucketedTableWithGroupedExecution(int writerConcurrency) throws Exception { testRecoverableGroupedExecution( + writerConcurrency, ImmutableList.of( - "CREATE TABLE test_table1\n" + + "CREATE TABLE insert_unbucketed_table_with_grouped_execution_1\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key1']) AS\n" + "SELECT orderkey key1, comment value1 FROM orders", - "CREATE TABLE test_table2\n" + + "CREATE TABLE insert_unbucketed_table_with_grouped_execution_2\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key2']) AS\n" + "SELECT orderkey key2, comment value2 FROM orders", - "CREATE TABLE test_table3\n" + + "CREATE TABLE insert_unbucketed_table_with_grouped_execution_3\n" + "WITH (bucket_count = 13, bucketed_by = ARRAY['key3']) AS\n" + "SELECT orderkey key3, comment value3 FROM orders", - "CREATE TABLE test_success (key BIGINT, value VARCHAR, partition_key VARCHAR)\n" + + "CREATE TABLE insert_unbucketed_table_with_grouped_execution_success (key BIGINT, value VARCHAR, partition_key VARCHAR)\n" + "WITH (partitioned_by = ARRAY['partition_key'])", - "CREATE TABLE test_failure (key BIGINT, value VARCHAR, partition_key VARCHAR)\n" + + "CREATE TABLE insert_unbucketed_table_with_grouped_execution_failure (key BIGINT, value VARCHAR, partition_key VARCHAR)\n" + "WITH (partitioned_by = ARRAY['partition_key'])"), - "INSERT INTO test_success\n" + + "INSERT INTO insert_unbucketed_table_with_grouped_execution_success\n" + "SELECT key1, value1, 'foo'\n" + - "FROM test_table1\n" + - "JOIN test_table2\n" + + "FROM insert_unbucketed_table_with_grouped_execution_1\n" + + "JOIN insert_unbucketed_table_with_grouped_execution_2\n" + "ON key1 = key2\n" + - "JOIN test_table3\n" + + "JOIN insert_unbucketed_table_with_grouped_execution_3\n" + "ON key2 = key3", - "INSERT INTO test_failure\n" + + "INSERT INTO insert_unbucketed_table_with_grouped_execution_failure\n" + "SELECT key1, value1, 'foo'\n" + - "FROM test_table1\n" + - "JOIN test_table2\n" + + "FROM insert_unbucketed_table_with_grouped_execution_1\n" + + "JOIN insert_unbucketed_table_with_grouped_execution_2\n" + "ON key1 = key2\n" + - "JOIN test_table3\n" + + "JOIN insert_unbucketed_table_with_grouped_execution_3\n" + "ON key2 = key3", 15000, ImmutableList.of( - "DROP TABLE IF EXISTS test_table1", - "DROP TABLE IF EXISTS test_table2", - "DROP TABLE IF EXISTS test_table3", - "DROP TABLE IF EXISTS test_success", - "DROP TABLE IF EXISTS test_failure")); + "DROP TABLE IF EXISTS insert_unbucketed_table_with_grouped_execution_1", + "DROP TABLE IF EXISTS insert_unbucketed_table_with_grouped_execution_2", + "DROP TABLE IF EXISTS insert_unbucketed_table_with_grouped_execution_3", + "DROP TABLE IF EXISTS insert_unbucketed_table_with_grouped_execution_success", + "DROP TABLE IF EXISTS insert_unbucketed_table_with_grouped_execution_failure")); } - @Test(timeOut = 60_000, enabled = false) - public void testScanFilterProjectionOnlyQueryOnUnbucketedTable() + @Test(timeOut = TEST_TIMEOUT, dataProvider = "writerConcurrency", invocationCount = INVOCATION_COUNT) + public void testScanFilterProjectionOnlyQueryOnUnbucketedTable(int writerConcurrency) throws Exception { testRecoverableGroupedExecution( + writerConcurrency, ImmutableList.of( - "CREATE TABLE test_table AS\n" + + "CREATE TABLE scan_filter_projection_only_query_on_unbucketed_table AS\n" + "SELECT t.comment\n" + "FROM orders\n" + "CROSS JOIN UNNEST(REPEAT(comment, 10)) AS t (comment)"), - "CREATE TABLE test_success AS\n" + - "SELECT comment value1 FROM test_table", - "CREATE TABLE test_failure AS\n" + - "SELECT comment value1 FROM test_table", + "CREATE TABLE scan_filter_projection_only_query_on_unbucketed_table_success AS\n" + + "SELECT comment value1 FROM scan_filter_projection_only_query_on_unbucketed_table", + "CREATE TABLE scan_filter_projection_only_query_on_unbucketed_table_failure AS\n" + + "SELECT comment value1 FROM scan_filter_projection_only_query_on_unbucketed_table", 15000 * 10, ImmutableList.of( - "DROP TABLE IF EXISTS test_table", - "DROP TABLE IF EXISTS test_success", - "DROP TABLE IF EXISTS test_failure")); + "DROP TABLE IF EXISTS scan_filter_projection_only_query_on_unbucketed_table", + "DROP TABLE IF EXISTS scan_filter_projection_only_query_on_unbucketed_table_success", + "DROP TABLE IF EXISTS scan_filter_projection_only_query_on_unbucketed_table_failure")); + } + + @Test(timeOut = TEST_TIMEOUT, dataProvider = "writerConcurrency", invocationCount = INVOCATION_COUNT) + public void testUnionAll(int writerConcurrency) + throws Exception + { + testRecoverableGroupedExecution( + writerConcurrency, + ImmutableList.of( + "CREATE TABLE test_union_all AS\n" + + "SELECT t.comment\n" + + "FROM orders\n" + + "CROSS JOIN UNNEST(REPEAT(comment, 10)) AS t (comment)"), + "CREATE TABLE test_union_all_success AS\n" + + "SELECT comment value1 FROM test_union_all " + + "UNION ALL " + + "SELECT comment value1 FROM test_union_all", + "CREATE TABLE test_union_all_failure AS\n" + + "SELECT comment value1 FROM test_union_all " + + "UNION ALL " + + "SELECT comment value1 FROM test_union_all", + 30000 * 10, + ImmutableList.of( + "DROP TABLE IF EXISTS test_union_all", + "DROP TABLE IF EXISTS test_union_all_success", + "DROP TABLE IF EXISTS test_union_all_failure")); } private void testRecoverableGroupedExecution( + int writerConcurrency, List preQueries, @Language("SQL") String queryWithoutFailure, @Language("SQL") String queryWithFailure, @@ -290,71 +341,101 @@ private void testRecoverableGroupedExecution( List postQueries) throws Exception { - try (DistributedQueryRunner queryRunner = distributedQueryRunnerSupplier.get()) { - try { - for (@Language("SQL") String preQuery : preQueries) { - queryRunner.execute(recoverableSession, preQuery); - } + Session recoverableSession = createRecoverableSession(writerConcurrency); + try { + for (@Language("SQL") String preQuery : preQueries) { + queryRunner.execute(recoverableSession, preQuery); + } - // test no failure case - assertEquals(queryRunner.execute(recoverableSession, queryWithoutFailure).getUpdateCount(), OptionalLong.of(expectedUpdateCount)); + // test no failure case + Stopwatch noRecoveryStopwatch = Stopwatch.createStarted(); + assertEquals(queryRunner.execute(recoverableSession, queryWithoutFailure).getUpdateCount(), OptionalLong.of(expectedUpdateCount)); + log.info("Query with no recovery took %sms", noRecoveryStopwatch.elapsed(MILLISECONDS)); - // test failure case - ListenableFuture result = executor.submit(() -> queryRunner.execute(recoverableSession, queryWithFailure)); + // cancel all queries and tasks to make sure we are dealing only with a single running query + cancelAllQueries(queryRunner); + cancelAllTasks(queryRunner); - // wait for split scheduling starts - while (!isSplitSchedulingStarted(queryRunner)) { - sleep(10); - } + // test failure case + Stopwatch recoveryStopwatch = Stopwatch.createStarted(); + ListenableFuture result = executor.submit(() -> queryRunner.execute(recoverableSession, queryWithFailure)); - // wait for additional 300ms before close a worker - sleep(300); + List workers = queryRunner.getServers().stream() + .filter(server -> !server.isCoordinator()) + .collect(toList()); + shuffle(workers); - assertTrue(getRunningQueryId(queryRunner.getQueries()).isPresent()); - TestingPrestoServer worker = queryRunner.getServers().stream() - .filter(server -> !server.isCoordinator()) - .findFirst() - .get(); - worker.close(); + TestingPrestoServer worker1 = workers.get(0); + // kill worker1 right away, to make sure recoverable execution works in cases when the task hasn't been yet submitted + worker1.stopResponding(); - assertEquals(result.get(30, SECONDS).getUpdateCount(), OptionalLong.of(expectedUpdateCount)); - } - finally { - for (@Language("SQL") String postQuery : postQueries) { - queryRunner.execute(recoverableSession, postQuery); + // kill worker2 only after the task has been scheduled + TestingPrestoServer worker2 = workers.get(1); + Stopwatch stopwatch = Stopwatch.createStarted(); + while (true) { + // wait for a while + sleep(500); + + // if the task is already running - move ahead + if (hasTaskRunning(worker2)) { + break; + } + + // don't fail the test if task execution already finished + if (stopwatch.elapsed(SECONDS) > 5) { + break; } } + worker2.stopResponding(); + + assertEquals(result.get(60, SECONDS).getUpdateCount(), OptionalLong.of(expectedUpdateCount)); + log.info("Query with recovery took %sms", recoveryStopwatch.elapsed(MILLISECONDS)); + } + finally { + queryRunner.getServers().forEach(TestingPrestoServer::startResponding); + cancelAllQueries(queryRunner); + cancelAllTasks(queryRunner); + for (@Language("SQL") String postQuery : postQueries) { + queryRunner.execute(recoverableSession, postQuery); + } } } - private boolean isSplitSchedulingStarted(DistributedQueryRunner queryRunner) + private static void cancelAllQueries(DistributedQueryRunner queryRunner) { - Optional runningQueryId = getRunningQueryId(queryRunner.getQueries()); - if (!runningQueryId.isPresent()) { - return false; - } + queryRunner.getQueries().forEach(query -> queryRunner.getCoordinator().getQueryManager().cancelQuery(query.getQueryId())); + queryRunner.getQueries().forEach(query -> assertTrue(query.getState().isDone())); + } - return queryRunner.getQueryInfo(runningQueryId.get()).getOutputStage().get().getSubStages().stream() - .map(StageInfo::getState) - .allMatch(SPLIT_SCHEDULING_STARTED_STATES::contains); + private static void cancelAllTasks(DistributedQueryRunner queryRunner) + { + queryRunner.getServers().forEach(TestHiveRecoverableGroupedExecution::cancelAllTasks); } - private static Optional getRunningQueryId(List basicQueryInfos) + private static void cancelAllTasks(TestingPrestoServer server) { - return basicQueryInfos.stream() - .filter(basicQueryInfo -> basicQueryInfo.getState() == QueryState.RUNNING) - .map(BasicQueryInfo::getQueryId) - .collect(toOptional()); + server.getTaskManager().getAllTaskInfo().forEach(task -> server.getTaskManager().cancelTask(task.getTaskStatus().getTaskId())); + server.getTaskManager().getAllTaskInfo().forEach(task -> assertTrue(task.getTaskStatus().getState().isDone())); } - private static Session createRecoverableSession(Optional role) + private static boolean hasTaskRunning(TestingPrestoServer server) { + return server.getTaskManager().getAllTaskInfo().stream() + .anyMatch(taskInfo -> taskInfo.getTaskStatus().getState() == RUNNING); + } + + private static Session createRecoverableSession(int writerConcurrency) + { + Identity identity = new Identity( + "hive", + Optional.empty(), + Optional.of(new SelectedRole(ROLE, Optional.of("admin"))) + .map(selectedRole -> ImmutableMap.of("hive", selectedRole)) + .orElse(ImmutableMap.of()), + ImmutableMap.of()); + return testSessionBuilder() - .setIdentity(new Identity( - "hive", - Optional.empty(), - role.map(selectedRole -> ImmutableMap.of("hive", selectedRole)) - .orElse(ImmutableMap.of()))) + .setIdentity(identity) .setSystemProperty(COLOCATED_JOIN, "true") .setSystemProperty(GROUPED_EXECUTION_FOR_AGGREGATION, "true") .setSystemProperty(DYNAMIC_SCHEDULE_FOR_GROUPED_EXECUTION, "true") @@ -363,17 +444,11 @@ private static Session createRecoverableSession(Optional role) .setSystemProperty(SCALE_WRITERS, "false") .setSystemProperty(REDISTRIBUTE_WRITES, "false") .setSystemProperty(GROUPED_EXECUTION_FOR_ELIGIBLE_TABLE_SCANS, "true") - .setSystemProperty(TASK_WRITER_COUNT, "1") + .setSystemProperty(TASK_WRITER_COUNT, Integer.toString(writerConcurrency)) + .setSystemProperty(TASK_PARTITIONED_WRITER_COUNT, Integer.toString(writerConcurrency)) .setCatalogSessionProperty(HIVE_CATALOG, VIRTUAL_BUCKET_COUNT, "16") .setCatalog(HIVE_CATALOG) .setSchema(TPCH_BUCKETED_SCHEMA) .build(); } - - @FunctionalInterface - public interface DistributedQueryRunnerSupplier - { - DistributedQueryRunner get() - throws Exception; - } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveRoles.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveRoles.java index 5c8650e9b0695..1dbb65cc95eb9 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveRoles.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveRoles.java @@ -376,22 +376,22 @@ public void testSetRole() .setIdentity(new Identity("set_user_1", Optional.empty())) .build(); Session setRoleAll = Session.builder(getQueryRunner().getDefaultSession()) - .setIdentity(new Identity("set_user_1", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ALL, Optional.empty())))) + .setIdentity(new Identity("set_user_1", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ALL, Optional.empty())), ImmutableMap.of())) .build(); Session setRoleNone = Session.builder(getQueryRunner().getDefaultSession()) - .setIdentity(new Identity("set_user_1", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.NONE, Optional.empty())))) + .setIdentity(new Identity("set_user_1", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.NONE, Optional.empty())), ImmutableMap.of())) .build(); Session setRole1 = Session.builder(getQueryRunner().getDefaultSession()) - .setIdentity(new Identity("set_user_1", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ROLE, Optional.of("set_role_1"))))) + .setIdentity(new Identity("set_user_1", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ROLE, Optional.of("set_role_1"))), ImmutableMap.of())) .build(); Session setRole2 = Session.builder(getQueryRunner().getDefaultSession()) - .setIdentity(new Identity("set_user_1", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ROLE, Optional.of("set_role_2"))))) + .setIdentity(new Identity("set_user_1", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ROLE, Optional.of("set_role_2"))), ImmutableMap.of())) .build(); Session setRole3 = Session.builder(getQueryRunner().getDefaultSession()) - .setIdentity(new Identity("set_user_1", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ROLE, Optional.of("set_role_3"))))) + .setIdentity(new Identity("set_user_1", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ROLE, Optional.of("set_role_3"))), ImmutableMap.of())) .build(); Session setRole4 = Session.builder(getQueryRunner().getDefaultSession()) - .setIdentity(new Identity("set_user_1", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ROLE, Optional.of("set_role_4"))))) + .setIdentity(new Identity("set_user_1", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ROLE, Optional.of("set_role_4"))), ImmutableMap.of())) .build(); MaterializedResult actual = getQueryRunner().execute(unsetRole, "SELECT * FROM hive.information_schema.applicable_roles"); @@ -509,7 +509,7 @@ private MaterializedResult executeFromUser(String user, String sql) private Session createAdminSession() { return Session.builder(getQueryRunner().getDefaultSession()) - .setIdentity(new Identity("admin", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ROLE, Optional.of("admin"))))) + .setIdentity(new Identity("admin", Optional.empty(), ImmutableMap.of("hive", new SelectedRole(SelectedRole.Type.ROLE, Optional.of("admin"))), ImmutableMap.of())) .build(); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplit.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplit.java index 7ad2c2061fc69..f5ecaf5941a5b 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplit.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplit.java @@ -13,16 +13,21 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.block.BlockJsonSerde; import com.facebook.presto.hive.HiveColumnHandle.ColumnType; +import com.facebook.presto.hive.metastore.Column; +import com.facebook.presto.hive.metastore.Storage; +import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.metadata.HandleJsonModule; import com.facebook.presto.metadata.HandleResolver; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockEncoding; import com.facebook.presto.spi.block.BlockEncodingSerde; -import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; @@ -34,23 +39,18 @@ import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.Scopes; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonModule; import org.testng.annotations.Test; import java.util.Optional; import java.util.OptionalInt; -import java.util.Properties; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.presto.hive.HiveType.HIVE_LONG; import static com.facebook.presto.hive.HiveType.HIVE_STRING; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.google.inject.multibindings.Multibinder.newSetBinder; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static org.testng.Assert.assertEquals; public class TestHiveSplit @@ -59,10 +59,6 @@ public class TestHiveSplit public void testJsonRoundTrip() throws Exception { - Properties schema = new Properties(); - schema.setProperty("foo", "bar"); - schema.setProperty("bar", "baz"); - ImmutableList partitionKeys = ImmutableList.of(new HivePartitionKey("a", "apple"), new HivePartitionKey("b", "42")); ImmutableList addresses = ImmutableList.of(HostAddress.fromParts("127.0.0.1", 44), HostAddress.fromParts("127.0.0.1", 45)); HiveSplit expected = new HiveSplit( @@ -73,21 +69,25 @@ public void testJsonRoundTrip() 42, 87, 88, - schema, + new Storage( + StorageFormat.create("serde", "input", "output"), + "location", + Optional.empty(), + false, + ImmutableMap.of()), partitionKeys, addresses, OptionalInt.empty(), OptionalInt.empty(), true, - TupleDomain.all(), - TRUE_CONSTANT, - ImmutableMap.of(), - ImmutableMap.of(1, HIVE_STRING), + 10, + ImmutableMap.of(1, new Column("name", HIVE_STRING, Optional.empty())), Optional.of(new HiveSplit.BucketConversion( 32, 16, ImmutableList.of(new HiveColumnHandle("col", HIVE_LONG, BIGINT.getTypeSignature(), 5, ColumnType.REGULAR, Optional.of("comment"))))), - false); + false, + Optional.empty()); JsonCodec codec = getJsonCodec(); String json = codec.toJson(expected); @@ -100,10 +100,11 @@ public void testJsonRoundTrip() assertEquals(actual.getStart(), expected.getStart()); assertEquals(actual.getLength(), expected.getLength()); assertEquals(actual.getFileSize(), expected.getFileSize()); - assertEquals(actual.getSchema(), expected.getSchema()); + assertEquals(actual.getStorage(), expected.getStorage()); assertEquals(actual.getPartitionKeys(), expected.getPartitionKeys()); assertEquals(actual.getAddresses(), expected.getAddresses()); - assertEquals(actual.getColumnCoercions(), expected.getColumnCoercions()); + assertEquals(actual.getPartitionDataColumnCount(), expected.getPartitionDataColumnCount()); + assertEquals(actual.getPartitionSchemaDifference(), expected.getPartitionSchemaDifference()); assertEquals(actual.getBucketConversion(), expected.getBucketConversion()); assertEquals(actual.isForceLocalScheduling(), expected.isForceLocalScheduling()); assertEquals(actual.isS3SelectPushdownEnabled(), expected.isS3SelectPushdownEnabled()); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitSource.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitSource.java index 6c502f17f6026..c3e8bdea63818 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitSource.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplitSource.java @@ -13,14 +13,15 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.presto.hive.metastore.Storage; +import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.ConnectorSplitSource; import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.predicate.TupleDomain; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.SettableFuture; -import io.airlift.stats.CounterStat; import io.airlift.units.DataSize; import org.apache.hadoop.fs.Path; import org.testng.annotations.Test; @@ -28,16 +29,14 @@ import java.util.List; import java.util.Optional; import java.util.OptionalInt; -import java.util.Properties; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.Executors; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.testing.Assertions.assertContains; import static com.facebook.presto.hive.HiveTestUtils.SESSION; import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.testing.Assertions.assertContains; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.lang.Math.toIntExact; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -58,9 +57,6 @@ public void testOutstandingSplitCount() SESSION, "database", "table", - TupleDomain.all(), - TRUE_CONSTANT, - ImmutableMap.of(), 10, 10, new DataSize(1, MEGABYTE), @@ -94,9 +90,6 @@ public void testFail() SESSION, "database", "table", - TupleDomain.all(), - TRUE_CONSTANT, - ImmutableMap.of(), 10, 10, new DataSize(1, MEGABYTE), @@ -154,9 +147,6 @@ public void testReaderWaitsForSplits() SESSION, "database", "table", - TupleDomain.all(), - TRUE_CONSTANT, - ImmutableMap.of(), 10, 10, new DataSize(1, MEGABYTE), @@ -194,7 +184,7 @@ public void testReaderWaitsForSplits() // wait for thread to get the split ConnectorSplit split = splits.get(10, SECONDS); - assertEquals(((HiveSplit) split).getSchema().getProperty("id"), "33"); + assertEquals(((HiveSplit) split).getPartitionDataColumnCount(), 33); } finally { // make sure the thread exits @@ -210,9 +200,6 @@ public void testOutstandingSplitSize() SESSION, "database", "table", - TupleDomain.all(), - TRUE_CONSTANT, - ImmutableMap.of(), 10, 10000, maxOutstandingSplitsSize, @@ -251,9 +238,6 @@ public void testEmptyBucket() SESSION, "database", "table", - TupleDomain.all(), - TRUE_CONSTANT, - ImmutableMap.of(), 10, 10, new DataSize(1, MEGABYTE), @@ -276,9 +260,6 @@ public void testPreloadSplitsForRewindableSplitSource() SESSION, "database", "table", - TupleDomain.all(), - TRUE_CONSTANT, - ImmutableMap.of(), 10, new DataSize(1, MEGABYTE), new TestingHiveSplitLoader(), @@ -321,7 +302,7 @@ public void testPreloadSplitsForRewindableSplitSource() connectorSplits = getSplits(hiveSplitSource, OptionalInt.of(0), 10); for (int i = 0; i < 10; i++) { - assertEquals(((HiveSplit) connectorSplits.get(i)).getSchema().getProperty("id"), Integer.toString(i)); + assertEquals(((HiveSplit) connectorSplits.get(i)).getPartitionDataColumnCount(), i); } assertTrue(hiveSplitSource.isFinished()); } @@ -337,9 +318,6 @@ public void testRewindOneBucket() SESSION, "database", "table", - TupleDomain.all(), - TRUE_CONSTANT, - ImmutableMap.of(), 10, new DataSize(1, MEGABYTE), new TestingHiveSplitLoader(), @@ -375,9 +353,6 @@ public void testRewindMultipleBuckets() SESSION, "database", "table", - TupleDomain.all(), - TRUE_CONSTANT, - ImmutableMap.of(), 10, new DataSize(1, MEGABYTE), new TestingHiveSplitLoader(), @@ -453,14 +428,20 @@ private TestSplit(int id, OptionalInt bucketNumber) true, false, false, - new HiveSplitPartitionInfo(properties("id", String.valueOf(id)), new Path("path").toUri(), ImmutableList.of(), "partition-name", ImmutableMap.of(), Optional.empty())); - } - - private static Properties properties(String key, String value) - { - Properties properties = new Properties(); - properties.put(key, value); - return properties; + new HiveSplitPartitionInfo( + new Storage( + StorageFormat.create("serde", "input", "output"), + "location", + Optional.empty(), + false, + ImmutableMap.of()), + new Path("path").toUri(), + ImmutableList.of(), + "partition-name", + id, + ImmutableMap.of(), + Optional.empty()), + Optional.empty()); } } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveTableHandle.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveTableHandle.java index d0eb22960e716..88f8ab23c2456 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveTableHandle.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveTableHandle.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveTypeTranslator.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveTypeTranslator.java index 392e21aff1d70..3e6c1c2173e8c 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveTypeTranslator.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveTypeTranslator.java @@ -22,10 +22,10 @@ import java.util.HashMap; import java.util.Map; +import static com.facebook.airlift.testing.Assertions.assertContains; import static com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; -import static io.airlift.testing.Assertions.assertContains; import static java.util.Objects.requireNonNull; import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveUtil.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveUtil.java index 0747ce32edd2d..5da9559774f63 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveUtil.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveUtil.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.hive; +import com.facebook.presto.hive.authentication.NoHdfsAuthentication; +import com.facebook.presto.hive.metastore.file.FileHiveMetastore; +import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -24,15 +27,16 @@ import org.joda.time.format.DateTimeFormat; import org.testng.annotations.Test; +import java.io.File; import java.util.AbstractList; import java.util.ArrayList; import java.util.List; import java.util.Properties; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.hive.HiveUtil.getDeserializer; import static com.facebook.presto.hive.HiveUtil.parseHiveTimestamp; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; -import static io.airlift.testing.Assertions.assertInstanceOf; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_CLASS; import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT; import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB; @@ -40,6 +44,15 @@ public class TestHiveUtil { + public static FileHiveMetastore createTestingFileHiveMetastore(File catalogDirectory) + { + HiveClientConfig hiveClientConfig = new HiveClientConfig(); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig, metastoreClientConfig), ImmutableSet.of()); + HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); + return new FileHiveMetastore(hdfsEnvironment, catalogDirectory.toURI().toString(), "test"); + } + @Test public void testParseHiveTimestamp() { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveWriteUtils.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveWriteUtils.java index 4ea7d1ea0e72a..13ef2a7f1cc78 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveWriteUtils.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveWriteUtils.java @@ -31,7 +31,7 @@ public class TestHiveWriteUtils @Test public void testIsS3FileSystem() { - HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(new HiveClientConfig()); + HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(new HiveClientConfig(), new MetastoreClientConfig()); assertTrue(isS3FileSystem(CONTEXT, hdfsEnvironment, new Path("s3://test-bucket/test-folder"))); assertFalse(isS3FileSystem(CONTEXT, hdfsEnvironment, new Path("/test-dir/test-folder"))); } @@ -39,7 +39,7 @@ public void testIsS3FileSystem() @Test public void testIsViewFileSystem() { - HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(new HiveClientConfig()); + HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(new HiveClientConfig(), new MetastoreClientConfig()); Path viewfsPath = new Path("viewfs://ns-default/test-folder"); Path nonViewfsPath = new Path("hdfs://localhost/test-dir/test-folder"); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestJsonHiveHandles.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestJsonHiveHandles.java index 4ae6b82ed5d3e..7dd3b05ea49b1 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestJsonHiveHandles.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestJsonHiveHandles.java @@ -13,22 +13,22 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.type.StandardTypes; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.ObjectMapperProvider; import org.testng.annotations.Test; import java.util.Map; import java.util.Optional; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcBatchPageSourceMemoryTracking.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcBatchPageSourceMemoryTracking.java index f6eb6c6106cd1..9c89833ceda4a 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcBatchPageSourceMemoryTracking.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcBatchPageSourceMemoryTracking.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.stats.Distribution; +import com.facebook.presto.hive.metastore.Storage; +import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.hive.orc.OrcBatchPageSourceFactory; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.metadata.Split; @@ -23,13 +26,19 @@ import com.facebook.presto.operator.TableScanOperator.TableScanOperatorFactory; import com.facebook.presto.operator.project.CursorProcessor; import com.facebook.presto.operator.project.PageProcessor; +import com.facebook.presto.orc.StorageStripeMetadataSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorTableHandle; import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.RowExpression; @@ -44,7 +53,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import io.airlift.slice.Slice; -import io.airlift.stats.Distribution; import io.airlift.units.DataSize; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -88,11 +96,13 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.function.Supplier; -import java.util.stream.Collectors; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertBetweenInclusive; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.HiveTestUtils.HDFS_ENVIRONMENT; +import static com.facebook.presto.hive.HiveTestUtils.ROW_EXPRESSION_SERVICE; import static com.facebook.presto.hive.HiveTestUtils.SESSION; import static com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; @@ -104,16 +114,12 @@ import static com.google.common.base.Predicates.not; import static com.google.common.collect.Iterables.filter; import static com.google.common.collect.Iterables.transform; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertBetweenInclusive; import static io.airlift.units.DataSize.Unit.BYTE; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static java.util.stream.Collectors.toList; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT; import static org.apache.hadoop.hive.ql.io.orc.CompressionKind.ZLIB; -import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB; import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector; import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector; import static org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.COMPRESS_CODEC; @@ -400,7 +406,8 @@ public void testScanFilterAndProjectOperator() private class TestPreparer { private final FileSplit fileSplit; - private final Properties schema; + private final Storage storage; + private final TableHandle table; private final List columns; private final List types; private final List partitionKeys; @@ -417,23 +424,24 @@ public TestPreparer(String tempFilePath, List testColumns, int numRo throws Exception { OrcSerde serde = new OrcSerde(); - schema = new Properties(); - schema.setProperty("columns", - testColumns.stream() - .map(TestColumn::getName) - .collect(Collectors.joining(","))); - schema.setProperty("columns.types", - testColumns.stream() - .map(TestColumn::getType) - .collect(Collectors.joining(","))); - schema.setProperty(FILE_INPUT_FORMAT, OrcInputFormat.class.getName()); - schema.setProperty(SERIALIZATION_LIB, serde.getClass().getName()); + storage = new Storage( + StorageFormat.create(serde.getClass().getName(), OrcInputFormat.class.getName(), OrcOutputFormat.class.getName()), + "location", + Optional.empty(), + false, + ImmutableMap.of()); partitionKeys = testColumns.stream() .filter(TestColumn::isPartitionKey) .map(input -> new HivePartitionKey(input.getName(), (String) input.getWriteValue())) .collect(toList()); + table = new TableHandle( + new ConnectorId("test"), + new ConnectorTableHandle() {}, + new ConnectorTransactionHandle() {}, + Optional.empty()); + ImmutableList.Builder columnsBuilder = ImmutableList.builder(); ImmutableList.Builder typesBuilder = ImmutableList.builder(); int nextHiveColumnIndex = 0; @@ -466,7 +474,15 @@ public ConnectorPageSource newPageSource(FileFormatDataSourceStats stats) public ConnectorPageSource newPageSource(FileFormatDataSourceStats stats, ConnectorSession session) { - OrcBatchPageSourceFactory orcPageSourceFactory = new OrcBatchPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT, stats, 100); + OrcBatchPageSourceFactory orcPageSourceFactory = new OrcBatchPageSourceFactory( + TYPE_MANAGER, + false, + HDFS_ENVIRONMENT, + stats, + 100, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + new HadoopFileOpener()); return HivePageSourceProvider.createHivePageSource( ImmutableSet.of(), ImmutableSet.of(orcPageSourceFactory), @@ -477,15 +493,25 @@ public ConnectorPageSource newPageSource(FileFormatDataSourceStats stats, Connec fileSplit.getStart(), fileSplit.getLength(), fileSplit.getLength(), - schema, + storage, TupleDomain.all(), columns, + ImmutableMap.of(), partitionKeys, DateTimeZone.UTC, TYPE_MANAGER, + new SchemaTableName("schema", "table"), + ImmutableList.of(), + ImmutableList.of(), ImmutableMap.of(), + 0, + ImmutableMap.of(), + Optional.empty(), + false, Optional.empty(), - false) + null, + false, + ROW_EXPRESSION_SERVICE) .get(); } @@ -495,7 +521,8 @@ public SourceOperator newTableScanOperator(DriverContext driverContext) SourceOperatorFactory sourceOperatorFactory = new TableScanOperatorFactory( 0, new PlanNodeId("0"), - (session, split, columnHandles) -> pageSource, + (session, split, table, columnHandles) -> pageSource, + table, columns.stream().map(columnHandle -> (ColumnHandle) columnHandle).collect(toList())); SourceOperator operator = sourceOperatorFactory.createOperator(driverContext); operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit())); @@ -509,15 +536,16 @@ public SourceOperator newScanFilterAndProjectOperator(DriverContext driverContex for (int i = 0; i < types.size(); i++) { projectionsBuilder.add(field(i, types.get(i))); } - Supplier cursorProcessor = EXPRESSION_COMPILER.compileCursorProcessor(Optional.empty(), projectionsBuilder.build(), "key"); - Supplier pageProcessor = EXPRESSION_COMPILER.compilePageProcessor(Optional.empty(), projectionsBuilder.build()); + Supplier cursorProcessor = EXPRESSION_COMPILER.compileCursorProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projectionsBuilder.build(), "key"); + Supplier pageProcessor = EXPRESSION_COMPILER.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projectionsBuilder.build()); SourceOperatorFactory sourceOperatorFactory = new ScanFilterAndProjectOperatorFactory( 0, new PlanNodeId("test"), new PlanNodeId("0"), - (session, split, columnHandles) -> pageSource, + (session, split, table, columnHandles) -> pageSource, cursorProcessor, pageProcessor, + table, columns.stream().map(columnHandle -> (ColumnHandle) columnHandle).collect(toList()), types, new DataSize(0, BYTE), diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcFileWriterConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcFileWriterConfig.java index 99704be0feb24..59c06864b2c5e 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcFileWriterConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestOrcFileWriterConfig.java @@ -19,9 +19,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.KILOBYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestParquetDistributedQueries.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestParquetDistributedQueries.java new file mode 100644 index 0000000000000..62edf2610e404 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestParquetDistributedQueries.java @@ -0,0 +1,97 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestDistributedQueries; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.sql.tree.ExplainType.Type.LOGICAL; +import static com.google.common.collect.Iterables.getOnlyElement; +import static io.airlift.tpch.TpchTable.getTables; +import static org.testng.Assert.assertEquals; + +public class TestParquetDistributedQueries + extends AbstractTestDistributedQueries +{ + protected TestParquetDistributedQueries() + { + super(TestParquetDistributedQueries::createQueryRunner); + } + + private static QueryRunner createQueryRunner() + throws Exception + { + Map parquetProperties = ImmutableMap.builder() + .put("hive.storage-format", "PARQUET") + .put("hive.parquet.use-column-names", "true") + .put("hive.compression-codec", "GZIP") + .build(); + return HiveQueryRunner.createQueryRunner(getTables(), + ImmutableMap.of("experimental.pushdown-subfields-enabled", "true"), + "sql-standard", + parquetProperties, + Optional.empty()); + } + + @Test + public void testSubfieldPruning() + { + getQueryRunner().execute("CREATE TABLE test_subfield_pruning AS " + + "SELECT orderkey, linenumber, shipdate, " + + " CAST(ROW(orderkey, linenumber, ROW(day(shipdate), month(shipdate), year(shipdate))) " + + " AS ROW(orderkey BIGINT, linenumber INTEGER, shipdate ROW(ship_day TINYINT, ship_month TINYINT, ship_year INTEGER))) AS info " + + "FROM lineitem"); + + try { + assertQuery("SELECT info.orderkey, info.shipdate.ship_month FROM test_subfield_pruning", "SELECT orderkey, month(shipdate) FROM lineitem"); + + assertQuery("SELECT orderkey FROM test_subfield_pruning WHERE info.shipdate.ship_month % 2 = 0", "SELECT orderkey FROM lineitem WHERE month(shipdate) % 2 = 0"); + } + finally { + getQueryRunner().execute("DROP TABLE test_subfield_pruning"); + } + } + + @Override + protected boolean supportsNotNullColumns() + { + return false; + } + + @Override + public void testDelete() + { + // Hive connector currently does not support row-by-row delete + } + + @Override + public void testRenameColumn() + { + // Parquet field lookup use column name does not support Rename + } + + @Test + public void testExplainOfCreateTableAs() + { + String query = "CREATE TABLE copy_orders AS SELECT * FROM orders"; + MaterializedResult result = computeActual("EXPLAIN " + query); + assertEquals(getOnlyElement(result.getOnlyColumnAsSet()), getExplainPlan(query, LOGICAL)); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestParquetFileWriterConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestParquetFileWriterConfig.java index 2653ad1b9b2c7..754fd66352156 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestParquetFileWriterConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestParquetFileWriterConfig.java @@ -20,9 +20,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestPartitionUpdate.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestPartitionUpdate.java index 70d9dc6fb9545..b4b9edd124ebb 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestPartitionUpdate.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestPartitionUpdate.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.hive; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.hive.PartitionUpdate.FileWriteInfo; import com.facebook.presto.hive.PartitionUpdate.UpdateMode; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import org.apache.hadoop.fs.Path; import org.testng.annotations.Test; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; public class TestPartitionUpdate diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestSubfieldExtractor.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestSubfieldExtractor.java index 9b22b83283edd..ba67a084ca6e6 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestSubfieldExtractor.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestSubfieldExtractor.java @@ -14,37 +14,49 @@ package com.facebook.presto.hive; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Subfield; import com.facebook.presto.spi.block.TestingSession; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.MapType; +import com.facebook.presto.spi.type.NamedTypeSignature; +import com.facebook.presto.spi.type.RowFieldName; import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.relational.FunctionResolution; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import io.airlift.slice.Slices; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.util.Map; import java.util.Optional; +import java.util.function.Function; +import static com.facebook.presto.hive.HiveTestUtils.mapType; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.spi.function.OperatorType.SUBSCRIPT; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.DEREFERENCE; import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.DateType.DATE; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RealType.REAL; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.specialForm; import static com.facebook.presto.testing.assertions.Assert.assertEquals; -import static com.facebook.presto.util.Reflection.methodHandle; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.testng.Assert.assertTrue; public class TestSubfieldExtractor { @@ -59,6 +71,21 @@ public class TestSubfieldExtractor RowType.field("d", mapType(BIGINT, BIGINT)), RowType.field("e", mapType(VARCHAR, BIGINT))))); + private static final ExpressionOptimizer TEST_EXPRESSION_OPTIMIZER = new ExpressionOptimizer() + { + @Override + public RowExpression optimize(RowExpression rowExpression, Level level, ConnectorSession session) + { + return rowExpression; + } + + @Override + public Object optimize(RowExpression expression, Level level, ConnectorSession session, Function variableResolver) + { + throw new UnsupportedOperationException(); + } + }; + private FunctionManager functionManager; private SubfieldExtractor subfieldExtractor; @@ -68,7 +95,7 @@ public void setup() functionManager = createTestMetadataManager().getFunctionManager(); subfieldExtractor = new SubfieldExtractor( new FunctionResolution(functionManager), - (rowExpression, level, session) -> rowExpression, + TEST_EXPRESSION_OPTIMIZER, TestingSession.SESSION); } @@ -77,15 +104,42 @@ public void test() { assertSubfieldExtract(C_BIGINT, "c_bigint"); assertSubfieldExtract(arraySubscript(C_BIGINT_ARRAY, 5), "c_bigint_array[5]"); - assertSubfieldExtract(mapSubscript(C_BIGINT_TO_BIGINT_MAP, constant(5, BIGINT)), "c_bigint_to_bigint_map[5]"); + assertSubfieldExtract(mapSubscript(C_BIGINT_TO_BIGINT_MAP, constant(5L, BIGINT)), "c_bigint_to_bigint_map[5]"); assertSubfieldExtract(mapSubscript(C_VARCHAR_TO_BIGINT_MAP, constant(Slices.utf8Slice("foo"), VARCHAR)), "c_varchar_to_bigint_map[\"foo\"]"); assertSubfieldExtract(dereference(C_STRUCT, 0), "c_struct.a"); assertSubfieldExtract(dereference(dereference(C_STRUCT, 1), 0), "c_struct.b.x"); assertSubfieldExtract(arraySubscript(dereference(C_STRUCT, 2), 5), "c_struct.c[5]"); - assertSubfieldExtract(mapSubscript(dereference(C_STRUCT, 3), constant(5, BIGINT)), "c_struct.d[5]"); + assertSubfieldExtract(mapSubscript(dereference(C_STRUCT, 3), constant(5L, BIGINT)), "c_struct.d[5]"); assertSubfieldExtract(mapSubscript(dereference(C_STRUCT, 4), constant(Slices.utf8Slice("foo"), VARCHAR)), "c_struct.e[\"foo\"]"); - assertEquals(subfieldExtractor.extract(constant(2, INTEGER)), Optional.empty()); + assertEquals(subfieldExtractor.extract(constant(2L, INTEGER)), Optional.empty()); + } + + @Test + public void testToRowExpression() + { + assertToRowExpression("a", DATE); + assertToRowExpression("a[1]", new ArrayType(INTEGER)); + assertToRowExpression("a.b", rowType(ImmutableMap.of("b", VARCHAR, "c", DOUBLE))); + assertToRowExpression("a[1]", mapType(BIGINT, DOUBLE)); + assertToRowExpression("a[\"hello\"]", mapType(VARCHAR, REAL)); + assertToRowExpression("a[\"hello\"].b", mapType(VARCHAR, rowType(ImmutableMap.of("b", BIGINT)))); + assertToRowExpression("a[\"hello\"].b.c", mapType(VARCHAR, rowType(ImmutableMap.of("b", rowType(ImmutableMap.of("c", BIGINT)))))); + } + + private void assertToRowExpression(String subfieldPath, Type type) + { + Subfield subfield = new Subfield(subfieldPath); + Optional recreatedSubfield = subfieldExtractor.extract(subfieldExtractor.toRowExpression(subfield, type)); + assertTrue(recreatedSubfield.isPresent()); + assertEquals(recreatedSubfield.get(), subfield); + } + + private static RowType rowType(Map fields) + { + return HiveTestUtils.rowType(fields.entrySet().stream() + .map(entry -> new NamedTypeSignature(Optional.of(new RowFieldName(entry.getKey(), false)), entry.getValue().getTypeSignature())) + .collect(toImmutableList())); } private void assertSubfieldExtract(RowExpression expression, String subfield) @@ -96,7 +150,7 @@ private void assertSubfieldExtract(RowExpression expression, String subfield) private RowExpression dereference(RowExpression base, int field) { Type fieldType = base.getType().getTypeParameters().get(field); - return specialForm(DEREFERENCE, fieldType, ImmutableList.of(base, new ConstantExpression(field, INTEGER))); + return specialForm(DEREFERENCE, fieldType, ImmutableList.of(base, new ConstantExpression((long) field, INTEGER))); } private RowExpression arraySubscript(RowExpression arrayExpression, int index) @@ -106,7 +160,7 @@ private RowExpression arraySubscript(RowExpression arrayExpression, int index) return call(SUBSCRIPT.name(), operator(SUBSCRIPT, arrayType, elementType), elementType, - ImmutableList.of(arrayExpression, constant(index, INTEGER))); + ImmutableList.of(arrayExpression, constant((long) index, INTEGER))); } private RowExpression mapSubscript(RowExpression mapExpression, RowExpression keyExpression) @@ -118,22 +172,6 @@ private RowExpression mapSubscript(RowExpression mapExpression, RowExpression ke ImmutableList.of(mapExpression, keyExpression)); } - private static MapType mapType(Type keyType, Type valueType) - { - return new MapType( - keyType, - valueType, - methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException"), - methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException"), - methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException"), - methodHandle(TestDomainTranslator.class, "throwUnsupportedOperationException")); - } - - public static void throwUnsupportedOperationException() - { - throw new UnsupportedOperationException(); - } - private FunctionHandle operator(OperatorType operatorType, Type... types) { return functionManager.resolveOperator(operatorType, fromTypes(types)); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java new file mode 100644 index 0000000000000..af37473cc83b2 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java @@ -0,0 +1,340 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.hive; + +import com.facebook.presto.hive.authentication.NoHdfsAuthentication; +import com.facebook.presto.hive.metastore.Database; +import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; +import com.facebook.presto.hive.metastore.HivePageSinkMetadata; +import com.facebook.presto.hive.metastore.HivePrivilegeInfo; +import com.facebook.presto.hive.metastore.HiveTableName; +import com.facebook.presto.hive.metastore.Partition; +import com.facebook.presto.hive.metastore.PartitionStatistics; +import com.facebook.presto.hive.metastore.PrincipalPrivileges; +import com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore; +import com.facebook.presto.hive.metastore.Table; +import com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore; +import com.facebook.presto.hive.metastore.thrift.HiveCluster; +import com.facebook.presto.hive.metastore.thrift.TestingHiveCluster; +import com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.security.PrestoPrincipal; +import com.facebook.presto.spi.security.RoleGrant; +import com.facebook.presto.spi.statistics.ColumnStatisticType; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.ListeningExecutorService; +import org.apache.hadoop.fs.Path; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; + +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; +import static java.util.concurrent.Executors.newCachedThreadPool; + +public class TestingSemiTransactionalHiveMetastore + extends SemiTransactionalHiveMetastore +{ + private static final String HOST = "dummy"; + private static final int PORT = 1111; + + private final Map tablesMap = new HashMap<>(); + private final Map> partitionsMap = new HashMap<>(); + + private TestingSemiTransactionalHiveMetastore(HdfsEnvironment hdfsEnvironment, ExtendedHiveMetastore delegate, ListeningExecutorService renameExecutor, boolean skipDeletionForAlter, boolean skipTargetCleanupOnRollback) + { + super(hdfsEnvironment, delegate, renameExecutor, skipDeletionForAlter, skipTargetCleanupOnRollback); + } + + public static TestingSemiTransactionalHiveMetastore create() + { + // none of these values matter, as we never use them + HiveClientConfig config = new HiveClientConfig(); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config, metastoreClientConfig), ImmutableSet.of()); + HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); + HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, HOST, PORT); + ExtendedHiveMetastore delegate = new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster)); + ExecutorService executor = newCachedThreadPool(daemonThreadsNamed("hive-%s")); + ListeningExecutorService renameExecutor = listeningDecorator(executor); + + return new TestingSemiTransactionalHiveMetastore(hdfsEnvironment, delegate, renameExecutor, false, false); + } + + public void addTable(String database, String tableName, Table table, List partitions) + { + HiveTableName hiveTableName = new HiveTableName(database, tableName); + tablesMap.put(hiveTableName, table); + partitionsMap.put(hiveTableName, partitions); + } + + @Override + public synchronized List getAllDatabases() + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized Optional getDatabase(String databaseName) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized Optional> getAllTables(String databaseName) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized Optional
getTable(String databaseName, String tableName) + { + return Optional.ofNullable(tablesMap.get(new HiveTableName(databaseName, tableName))); + } + + @Override + public synchronized Set getSupportedColumnStatistics(Type type) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized PartitionStatistics getTableStatistics(String databaseName, String tableName) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized Map getPartitionStatistics(String databaseName, String tableName, Set partitionNames) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized HivePageSinkMetadata generatePageSinkMetadata(SchemaTableName schemaTableName) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized Optional> getAllViews(String databaseName) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void createDatabase(Database database) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void dropDatabase(String schemaName) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void renameDatabase(String source, String target) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void setTableStatistics(Table table, PartitionStatistics tableStatistics) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void setPartitionStatistics(Table table, Map, PartitionStatistics> partitionStatisticsMap) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void createTable(ConnectorSession session, Table table, PrincipalPrivileges principalPrivileges, Optional currentPath, boolean ignoreExisting, PartitionStatistics statistics) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void dropTable(ConnectorSession session, String databaseName, String tableName) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void replaceView(String databaseName, String tableName, Table table, PrincipalPrivileges principalPrivileges) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void renameTable(String databaseName, String tableName, String newDatabaseName, String newTableName) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void addColumn(String databaseName, String tableName, String columnName, HiveType columnType, String columnComment) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void renameColumn(String databaseName, String tableName, String oldColumnName, String newColumnName) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void dropColumn(String databaseName, String tableName, String columnName) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void finishInsertIntoExistingTable(ConnectorSession session, String databaseName, String tableName, Path currentLocation, List fileNames, PartitionStatistics statisticsUpdate) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void truncateUnpartitionedTable(ConnectorSession session, String databaseName, String tableName) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized Optional> getPartitionNames(String databaseName, String tableName) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized Optional> getPartitionNamesByParts(String databaseName, String tableName, List parts) + { + return Optional.ofNullable(partitionsMap.get(new HiveTableName(databaseName, tableName))); + } + + @Override + public synchronized Optional getPartition(String databaseName, String tableName, List partitionValues) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized Map> getPartitionsByNames(String databaseName, String tableName, List partitionNames) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void addPartition(ConnectorSession session, String databaseName, String tableName, Partition partition, Path currentLocation, PartitionStatistics statistics) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void dropPartition(ConnectorSession session, String databaseName, String tableName, List partitionValues) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void finishInsertIntoExistingPartition(ConnectorSession session, String databaseName, String tableName, List partitionValues, Path currentLocation, List fileNames, PartitionStatistics statisticsUpdate) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void createRole(String role, String grantor) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void dropRole(String role) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized Set listRoles() + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void grantRoles(Set roles, Set grantees, boolean withAdminOption, PrestoPrincipal grantor) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void revokeRoles(Set roles, Set grantees, boolean adminOptionFor, PrestoPrincipal grantor) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized Set listRoleGrants(PrestoPrincipal principal) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized Set listTablePrivileges(String databaseName, String tableName, PrestoPrincipal principal) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void grantTablePrivileges(String databaseName, String tableName, PrestoPrincipal grantee, Set privileges) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void revokeTablePrivileges(String databaseName, String tableName, PrestoPrincipal grantee, Set privileges) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void declareIntentionToWrite(ConnectorSession session, LocationHandle.WriteMode writeMode, Path stagingPathRoot, Optional tempPathRoot, String filePrefix, SchemaTableName schemaTableName, boolean temporaryTable) + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void commit() + { + throw new UnsupportedOperationException("method not implemented"); + } + + @Override + public synchronized void rollback() + { + throw new UnsupportedOperationException("method not implemented"); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/authentication/TestHdfsKerberosConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/authentication/TestHdfsKerberosConfig.java index 74a7e4ccb5497..1bde8be0c82be 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/authentication/TestHdfsKerberosConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/authentication/TestHdfsKerberosConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive.authentication; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import org.testng.annotations.Test; import java.util.Map; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/authentication/TestMetastoreKerberosConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/authentication/TestMetastoreKerberosConfig.java index 460fae5534f31..d2c56ef74ec84 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/authentication/TestMetastoreKerberosConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/authentication/TestMetastoreKerberosConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.hive.authentication; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import org.testng.annotations.Test; import java.util.Map; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/benchmark/FileFormat.java b/presto-hive/src/test/java/com/facebook/presto/hive/benchmark/FileFormat.java index 6091a4187f78e..c12fef7f69c6b 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/benchmark/FileFormat.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/benchmark/FileFormat.java @@ -15,6 +15,7 @@ import com.facebook.presto.hive.FileFormatDataSourceStats; import com.facebook.presto.hive.GenericHiveRecordCursorProvider; +import com.facebook.presto.hive.HadoopFileOpener; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HiveBatchPageSourceFactory; import com.facebook.presto.hive.HiveColumnHandle; @@ -27,6 +28,8 @@ import com.facebook.presto.hive.RecordFileWriter; import com.facebook.presto.hive.TypeTranslator; import com.facebook.presto.hive.benchmark.HiveFileFormatBenchmark.TestData; +import com.facebook.presto.hive.metastore.Storage; +import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.hive.orc.DwrfBatchPageSourceFactory; import com.facebook.presto.hive.orc.OrcBatchPageSourceFactory; import com.facebook.presto.hive.parquet.ParquetPageSourceFactory; @@ -35,6 +38,8 @@ import com.facebook.presto.orc.OrcWriterOptions; import com.facebook.presto.orc.OrcWriterStats; import com.facebook.presto.orc.OutputStreamOrcDataSink; +import com.facebook.presto.orc.StorageStripeMetadataSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.rcfile.AircompressorCodecFactory; import com.facebook.presto.rcfile.HadoopCodecFactory; import com.facebook.presto.rcfile.RcFileEncoding; @@ -84,7 +89,7 @@ public enum FileFormat @Override public ConnectorPageSource createFileFormatReader(ConnectorSession session, HdfsEnvironment hdfsEnvironment, File targetFile, List columnNames, List columnTypes) { - HiveBatchPageSourceFactory pageSourceFactory = new RcFilePageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats()); + HiveBatchPageSourceFactory pageSourceFactory = new RcFilePageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats(), new HadoopFileOpener()); return createPageSource(pageSourceFactory, session, targetFile, columnNames, columnTypes, HiveStorageFormat.RCBINARY); } @@ -109,7 +114,7 @@ public FormatWriter createFileFormatWriter( @Override public ConnectorPageSource createFileFormatReader(ConnectorSession session, HdfsEnvironment hdfsEnvironment, File targetFile, List columnNames, List columnTypes) { - HiveBatchPageSourceFactory pageSourceFactory = new RcFilePageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats()); + HiveBatchPageSourceFactory pageSourceFactory = new RcFilePageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats(), new HadoopFileOpener()); return createPageSource(pageSourceFactory, session, targetFile, columnNames, columnTypes, HiveStorageFormat.RCTEXT); } @@ -134,7 +139,15 @@ public FormatWriter createFileFormatWriter( @Override public ConnectorPageSource createFileFormatReader(ConnectorSession session, HdfsEnvironment hdfsEnvironment, File targetFile, List columnNames, List columnTypes) { - HiveBatchPageSourceFactory pageSourceFactory = new OrcBatchPageSourceFactory(TYPE_MANAGER, false, hdfsEnvironment, new FileFormatDataSourceStats(), 100); + HiveBatchPageSourceFactory pageSourceFactory = new OrcBatchPageSourceFactory( + TYPE_MANAGER, + false, + hdfsEnvironment, + new FileFormatDataSourceStats(), + 100, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + new HadoopFileOpener()); return createPageSource(pageSourceFactory, session, targetFile, columnNames, columnTypes, HiveStorageFormat.ORC); } @@ -160,7 +173,7 @@ public FormatWriter createFileFormatWriter( @Override public ConnectorPageSource createFileFormatReader(ConnectorSession session, HdfsEnvironment hdfsEnvironment, File targetFile, List columnNames, List columnTypes) { - HiveBatchPageSourceFactory pageSourceFactory = new DwrfBatchPageSourceFactory(TYPE_MANAGER, HIVE_CLIENT_CONFIG, hdfsEnvironment, new FileFormatDataSourceStats()); + HiveBatchPageSourceFactory pageSourceFactory = new DwrfBatchPageSourceFactory(TYPE_MANAGER, HIVE_CLIENT_CONFIG, hdfsEnvironment, new FileFormatDataSourceStats(), new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener()); return createPageSource(pageSourceFactory, session, targetFile, columnNames, columnTypes, HiveStorageFormat.DWRF); } @@ -192,7 +205,7 @@ public boolean supportsDate() @Override public ConnectorPageSource createFileFormatReader(ConnectorSession session, HdfsEnvironment hdfsEnvironment, File targetFile, List columnNames, List columnTypes) { - HiveBatchPageSourceFactory pageSourceFactory = new ParquetPageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats()); + HiveBatchPageSourceFactory pageSourceFactory = new ParquetPageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats(), new HadoopFileOpener()); return createPageSource(pageSourceFactory, session, targetFile, columnNames, columnTypes, HiveStorageFormat.PARQUET); } @@ -298,7 +311,7 @@ public boolean supportsDate() @Override public ConnectorPageSource createFileFormatReader(ConnectorSession session, HdfsEnvironment hdfsEnvironment, File targetFile, List columnNames, List columnTypes) { - HiveBatchPageSourceFactory pageSourceFactory = new ParquetPageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats()); + HiveBatchPageSourceFactory pageSourceFactory = new ParquetPageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats(), new HadoopFileOpener()); return createPageSource(pageSourceFactory, session, targetFile, columnNames, columnTypes, HiveStorageFormat.PARQUET); } @@ -400,10 +413,17 @@ private static ConnectorPageSource createPageSource( 0, targetFile.length(), targetFile.length(), - createSchema(format, columnNames, columnTypes), + new Storage( + StorageFormat.create(format.getSerDe(), format.getInputFormat(), format.getOutputFormat()), + "location", + Optional.empty(), + false, + ImmutableMap.of()), + ImmutableMap.of(), columnHandles, TupleDomain.all(), - DateTimeZone.forID(session.getTimeZoneKey().getId())) + DateTimeZone.forID(session.getTimeZoneKey().getId()), + Optional.empty()) .get(); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/benchmark/HiveFileFormatBenchmark.java b/presto-hive/src/test/java/com/facebook/presto/hive/benchmark/HiveFileFormatBenchmark.java index 50a7cc516b114..6bf7d2b380b5c 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/benchmark/HiveFileFormatBenchmark.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/benchmark/HiveFileFormatBenchmark.java @@ -63,6 +63,7 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; +import static com.facebook.presto.hive.HiveTestUtils.METASTORE_CLIENT_CONFIG; import static com.facebook.presto.hive.HiveTestUtils.createTestHdfsEnvironment; import static com.facebook.presto.hive.HiveTestUtils.mapType; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -99,7 +100,7 @@ public class HiveFileFormatBenchmark private static final ConnectorSession SESSION = new TestingConnectorSession(new HiveSessionProperties(CONFIG, new OrcFileWriterConfig(), new ParquetFileWriterConfig()) .getSessionProperties()); - private static final HdfsEnvironment HDFS_ENVIRONMENT = createTestHdfsEnvironment(CONFIG); + private static final HdfsEnvironment HDFS_ENVIRONMENT = createTestHdfsEnvironment(CONFIG, METASTORE_CLIENT_CONFIG); @Param({ "LINEITEM", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/gcs/TestHiveGcsConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/gcs/TestHiveGcsConfig.java new file mode 100644 index 0000000000000..e52f2a7801a5e --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/hive/gcs/TestHiveGcsConfig.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.gcs; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestHiveGcsConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(HiveGcsConfig.class) + .setJsonKeyFilePath(null) + .setUseGcsAccessToken(false)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("hive.gcs.json-key-file-path", "/tmp/key.json") + .put("hive.gcs.use-access-token", "true") + .build(); + + HiveGcsConfig expected = new HiveGcsConfig() + .setJsonKeyFilePath("/tmp/key.json") + .setUseGcsAccessToken(true); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestHiveClientGlueMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestHiveClientGlueMetastore.java index 701260f8985ff..4bf99a018f1a9 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestHiveClientGlueMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestHiveClientGlueMetastore.java @@ -15,26 +15,53 @@ import com.facebook.presto.hive.AbstractTestHiveClientLocal; import com.facebook.presto.hive.HdfsConfiguration; -import com.facebook.presto.hive.HdfsConfigurationUpdater; +import com.facebook.presto.hive.HdfsConfigurationInitializer; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.HiveHdfsConfiguration; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; import java.io.File; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Locale.ENGLISH; import static java.util.UUID.randomUUID; +import static java.util.concurrent.Executors.newCachedThreadPool; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; public class TestHiveClientGlueMetastore extends AbstractTestHiveClientLocal { + private ExecutorService executorService; + public TestHiveClientGlueMetastore() { super("test_glue" + randomUUID().toString().toLowerCase(ENGLISH).replace("-", "")); } + @BeforeClass + public void setUp() + { + executorService = newCachedThreadPool(daemonThreadsNamed("hive-glue-%s")); + } + + @AfterClass(alwaysRun = true) + public void tearDown() + { + executorService.shutdownNow(); + } + /** * GlueHiveMetastore currently uses AWS Default Credential Provider Chain, * See https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default @@ -44,12 +71,13 @@ public TestHiveClientGlueMetastore() protected ExtendedHiveMetastore createMetastore(File tempDir) { HiveClientConfig hiveClientConfig = new HiveClientConfig(); - HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationUpdater(hiveClientConfig)); - HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication()); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig, metastoreClientConfig), ImmutableSet.of()); + HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); GlueHiveMetastoreConfig glueConfig = new GlueHiveMetastoreConfig(); glueConfig.setDefaultWarehouseDir(tempDir.toURI().toString()); - return new GlueHiveMetastore(hdfsEnvironment, glueConfig); + return new GlueHiveMetastore(hdfsEnvironment, glueConfig, executor); } @Override @@ -95,4 +123,18 @@ public void testStorePartitionWithStatistics() { testStorePartitionWithStatistics(STATISTICS_PARTITIONED_TABLE_COLUMNS, BASIC_STATISTICS_1, BASIC_STATISTICS_2, BASIC_STATISTICS_1, EMPTY_TABLE_STATISTICS); } + + @Test + public void testGetPartitions() throws Exception + { + try { + createDummyPartitionedTable(tablePartitionFormat, CREATE_TABLE_COLUMNS_PARTITIONED); + Optional> partitionNames = getMetastoreClient().getPartitionNames(tablePartitionFormat.getSchemaName(), tablePartitionFormat.getTableName()); + assertTrue(partitionNames.isPresent()); + assertEquals(partitionNames.get(), ImmutableList.of("ds=2016-01-01", "ds=2016-01-02")); + } + finally { + dropTable(tablePartitionFormat); + } + } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/parquet/AbstractTestParquetReader.java b/presto-hive/src/test/java/com/facebook/presto/hive/parquet/AbstractTestParquetReader.java index 0a24e81918550..41611a3208998 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/parquet/AbstractTestParquetReader.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/parquet/AbstractTestParquetReader.java @@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Range; import com.google.common.primitives.Shorts; +import io.airlift.units.DataSize; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaHiveDecimalObjectInspector; @@ -1492,6 +1493,43 @@ private void setParquetLogging() Logger.getLogger("parquet.hadoop.ColumnChunkPageWriteStore").setLevel(Level.WARNING); } + @Test + public void testStructMaxReadBytes() + throws Exception + { + DataSize maxReadBlockSize = new DataSize(1_000, DataSize.Unit.BYTE); + List structValues = createTestStructs( + Collections.nCopies(500, String.join("", Collections.nCopies(33, "test"))), + Collections.nCopies(500, String.join("", Collections.nCopies(1, "test")))); + List structFieldNames = asList("a", "b"); + Type structType = RowType.from(asList(field("a", VARCHAR), field("b", VARCHAR))); + + tester.testMaxReadBytes( + getStandardStructObjectInspector(structFieldNames, asList(javaStringObjectInspector, javaStringObjectInspector)), + structValues, + structValues, + structType, + maxReadBlockSize); + } + + @Test + public void testArrayMaxReadBytes() + throws Exception + { + DataSize maxReadBlockSize = new DataSize(1_000, DataSize.Unit.BYTE); + Iterable> values = createFixedTestArrays(limit(cycle(asList(1, null, 3, 5, null, null, null, 7, 11, null, 13, 17)), 30_000)); + tester.testMaxReadBytes(getStandardListObjectInspector(javaIntObjectInspector), values, values, new ArrayType(INTEGER), maxReadBlockSize); + } + + @Test + public void testMapMaxReadBytes() + throws Exception + { + DataSize maxReadBlockSize = new DataSize(1_000, DataSize.Unit.BYTE); + Iterable> values = createFixedTestMaps(Collections.nCopies(5_000, String.join("", Collections.nCopies(33, "test"))), longsBetween(0, 5_000)); + tester.testMaxReadBytes(getStandardMapObjectInspector(javaStringObjectInspector, javaLongObjectInspector), values, values, mapType(VARCHAR, BIGINT), maxReadBlockSize); + } + private static Iterable repeatEach(int n, Iterable iterable) { return () -> new AbstractIterator() @@ -1611,6 +1649,47 @@ private Iterable> createNullableTestArrays(Iterable values) return insertNullEvery(ThreadLocalRandom.current().nextInt(2, 5), createTestArrays(values)); } + private List> createFixedTestArrays(Iterable values) + { + List> arrays = new ArrayList<>(); + Iterator valuesIter = values.iterator(); + List array = new ArrayList<>(); + int count = 1; + while (valuesIter.hasNext()) { + if (count % 10 == 0) { + arrays.add(array); + array = new ArrayList<>(); + } + if (count % 20 == 0) { + arrays.add(Collections.emptyList()); + } + array.add(valuesIter.next()); + ++count; + } + return arrays; + } + + private Iterable> createFixedTestMaps(Iterable keys, Iterable values) + { + List> maps = new ArrayList<>(); + Iterator keysIterator = keys.iterator(); + Iterator valuesIterator = values.iterator(); + Map map = new HashMap<>(); + int count = 1; + while (keysIterator.hasNext() && valuesIterator.hasNext()) { + if (count % 5 == 0) { + maps.add(map); + map = new HashMap<>(); + } + if (count % 10 == 0) { + maps.add(Collections.emptyMap()); + } + map.put(keysIterator.next(), valuesIterator.next()); + ++count; + } + return maps; + } + private Iterable> createTestMaps(Iterable keys, Iterable values) { List> maps = new ArrayList<>(); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/parquet/ParquetTester.java b/presto-hive/src/test/java/com/facebook/presto/hive/parquet/ParquetTester.java index 92011928657e2..0545734891644 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/parquet/ParquetTester.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/parquet/ParquetTester.java @@ -82,11 +82,13 @@ import java.util.Set; import static com.facebook.presto.hive.AbstractTestHiveFileFormats.getFieldFromCursor; +import static com.facebook.presto.hive.HiveSessionProperties.getParquetMaxReadBlockSize; +import static com.facebook.presto.hive.HiveTestUtils.METASTORE_CLIENT_CONFIG; import static com.facebook.presto.hive.HiveTestUtils.createTestHdfsEnvironment; -import static com.facebook.presto.hive.HiveUtil.isArrayType; -import static com.facebook.presto.hive.HiveUtil.isMapType; -import static com.facebook.presto.hive.HiveUtil.isRowType; import static com.facebook.presto.hive.HiveUtil.isStructuralType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isArrayType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isMapType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isRowType; import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.spi.type.Varchars.isVarcharType; @@ -113,7 +115,7 @@ public class ParquetTester public static final DateTimeZone HIVE_STORAGE_TIME_ZONE = DateTimeZone.forID("America/Bahia_Banderas"); private static final boolean OPTIMIZED = true; private static final HiveClientConfig HIVE_CLIENT_CONFIG = createHiveClientConfig(false); - private static final HdfsEnvironment HDFS_ENVIRONMENT = createTestHdfsEnvironment(HIVE_CLIENT_CONFIG); + private static final HdfsEnvironment HDFS_ENVIRONMENT = createTestHdfsEnvironment(HIVE_CLIENT_CONFIG, METASTORE_CLIENT_CONFIG); private static final TestingConnectorSession SESSION = new TestingConnectorSession(new HiveSessionProperties(HIVE_CLIENT_CONFIG, new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties()); private static final TestingConnectorSession SESSION_USE_NAME = new TestingConnectorSession(new HiveSessionProperties(createHiveClientConfig(true), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties()); private static final List TEST_COLUMN = singletonList("test"); @@ -204,13 +206,27 @@ public void testSingleLevelArrayRoundTrip(ObjectInspector objectInspector, Itera public void testRoundTrip(ObjectInspector objectInspector, Iterable writeValues, Iterable readValues, String columnName, Type type, Optional parquetSchema) throws Exception { - testRoundTrip(singletonList(objectInspector), new Iterable[] {writeValues}, new Iterable[] {readValues}, singletonList(columnName), singletonList(type), parquetSchema, false); + testRoundTrip( + singletonList(objectInspector), + new Iterable[] {writeValues}, + new Iterable[] {readValues}, + singletonList(columnName), + singletonList(type), + parquetSchema, + false); } public void testSingleLevelArrayRoundTrip(ObjectInspector objectInspector, Iterable writeValues, Iterable readValues, String columnName, Type type, Optional parquetSchema) throws Exception { - testRoundTrip(singletonList(objectInspector), new Iterable[] {writeValues}, new Iterable[] {readValues}, singletonList(columnName), singletonList(type), parquetSchema, true); + testRoundTrip( + singletonList(objectInspector), + new Iterable[] {writeValues}, + new Iterable[] {readValues}, + singletonList(columnName), + singletonList(type), + parquetSchema, + true); } public void testRoundTrip(List objectInspectors, Iterable[] writeValues, Iterable[] readValues, List columnNames, List columnTypes, Optional parquetSchema, boolean singleLevelArray) @@ -223,8 +239,14 @@ public void testRoundTrip(List objectInspectors, Iterable[] assertRoundTrip(objectInspectors, transformToNulls(writeValues), transformToNulls(readValues), columnNames, columnTypes, parquetSchema, singleLevelArray); } - private void testRoundTripType(List objectInspectors, Iterable[] writeValues, Iterable[] readValues, - List columnNames, List columnTypes, Optional parquetSchema, boolean singleLevelArray) + private void testRoundTripType( + List objectInspectors, + Iterable[] writeValues, + Iterable[] readValues, + List columnNames, + List columnTypes, + Optional parquetSchema, + boolean singleLevelArray) throws Exception { // forward order @@ -240,7 +262,8 @@ private void testRoundTripType(List objectInspectors, Iterable< assertRoundTrip(objectInspectors, insertNullEvery(5, reverse(writeValues)), insertNullEvery(5, reverse(readValues)), columnNames, columnTypes, parquetSchema, singleLevelArray); } - void assertRoundTrip(List objectInspectors, + void assertRoundTrip( + List objectInspectors, Iterable[] writeValues, Iterable[] readValues, List columnNames, @@ -251,7 +274,8 @@ void assertRoundTrip(List objectInspectors, assertRoundTrip(objectInspectors, writeValues, readValues, columnNames, columnTypes, parquetSchema, false); } - void assertRoundTrip(List objectInspectors, + void assertRoundTrip( + List objectInspectors, Iterable[] writeValues, Iterable[] readValues, List columnNames, @@ -289,6 +313,69 @@ void assertRoundTrip(List objectInspectors, } } + void testMaxReadBytes(ObjectInspector objectInspector, Iterable writeValues, Iterable readValues, Type type, DataSize maxReadBlockSize) + throws Exception + { + assertMaxReadBytes( + singletonList(objectInspector), + new Iterable[] {writeValues}, + new Iterable[] {readValues}, + TEST_COLUMN, + singletonList(type), + Optional.empty(), + maxReadBlockSize); + } + + void assertMaxReadBytes( + List objectInspectors, + Iterable[] writeValues, + Iterable[] readValues, + List columnNames, + List columnTypes, + Optional parquetSchema, + DataSize maxReadBlockSize) + throws Exception + { + WriterVersion version = PARQUET_1_0; + CompressionCodecName compressionCodecName = UNCOMPRESSED; + HiveClientConfig config = new HiveClientConfig() + .setHiveStorageFormat(HiveStorageFormat.PARQUET) + .setUseParquetColumnNames(false) + .setParquetMaxReadBlockSize(maxReadBlockSize); + ConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(config, new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties()); + + try (TempFile tempFile = new TempFile("test", "parquet")) { + JobConf jobConf = new JobConf(); + jobConf.setEnum(COMPRESSION, compressionCodecName); + jobConf.setBoolean(ENABLE_DICTIONARY, true); + jobConf.setEnum(WRITER_VERSION, version); + writeParquetColumn( + jobConf, + tempFile.getFile(), + compressionCodecName, + createTableProperties(columnNames, objectInspectors), + getStandardStructObjectInspector(columnNames, objectInspectors), + getIterators(writeValues), + parquetSchema, + false); + + Iterator[] expectedValues = getIterators(readValues); + try (ConnectorPageSource pageSource = getFileFormat().createFileFormatReader( + session, + HDFS_ENVIRONMENT, + tempFile.getFile(), + columnNames, + columnTypes)) { + assertPageSource( + columnTypes, + expectedValues, + pageSource, + Optional.of(getParquetMaxReadBlockSize(session).toBytes())); + assertFalse(stream(expectedValues).allMatch(Iterator::hasNext)); + } + } + } + private static void assertFileContents( ConnectorSession session, File dataFile, @@ -314,9 +401,18 @@ private static void assertFileContents( } private static void assertPageSource(List types, Iterator[] valuesByField, ConnectorPageSource pageSource) + { + assertPageSource(types, valuesByField, pageSource, Optional.empty()); + } + + private static void assertPageSource(List types, Iterator[] valuesByField, ConnectorPageSource pageSource, Optional maxReadBlockSize) { Page page; while ((page = pageSource.getNextPage()) != null) { + if (maxReadBlockSize.isPresent()) { + assertTrue(page.getPositionCount() == 1 || page.getSizeInBytes() <= maxReadBlockSize.get()); + } + for (int field = 0; field < page.getChannelCount(); field++) { Block block = page.getBlock(field); for (int i = 0; i < block.getPositionCount(); i++) { @@ -420,7 +516,8 @@ private static FileFormat getFileFormat() return OPTIMIZED ? FileFormat.PRESTO_PARQUET : FileFormat.HIVE_PARQUET; } - private static DataSize writeParquetColumn(JobConf jobConf, + private static DataSize writeParquetColumn( + JobConf jobConf, File outputFile, CompressionCodecName compressionCodecName, Properties tableProperties, diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/parquet/write/TestDataWritableWriter.java b/presto-hive/src/test/java/com/facebook/presto/hive/parquet/write/TestDataWritableWriter.java index c9ffd88f26de6..639df16a07c03 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/parquet/write/TestDataWritableWriter.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/parquet/write/TestDataWritableWriter.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive.parquet.write; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/s3/MockAmazonS3.java b/presto-hive/src/test/java/com/facebook/presto/hive/s3/MockAmazonS3.java index e7c248dbbc6c1..876758701fe14 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/s3/MockAmazonS3.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/s3/MockAmazonS3.java @@ -29,7 +29,7 @@ import java.util.Date; -import static org.apache.http.HttpStatus.SC_OK; +import static java.net.HttpURLConnection.HTTP_OK; public class MockAmazonS3 extends AbstractAmazonS3 @@ -37,8 +37,8 @@ public class MockAmazonS3 private static final String STANDARD_OBJECT_KEY = "test/standard"; private static final String GLACIER_OBJECT_KEY = "test/glacier"; - private int getObjectHttpCode = SC_OK; - private int getObjectMetadataHttpCode = SC_OK; + private int getObjectHttpCode = HTTP_OK; + private int getObjectMetadataHttpCode = HTTP_OK; private GetObjectMetadataRequest getObjectMetadataRequest; private CannedAccessControlList acl; private boolean hasGlacierObjects; @@ -72,7 +72,7 @@ public GetObjectMetadataRequest getGetObjectMetadataRequest() public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetadataRequest) { this.getObjectMetadataRequest = getObjectMetadataRequest; - if (getObjectMetadataHttpCode != SC_OK) { + if (getObjectMetadataHttpCode != HTTP_OK) { AmazonS3Exception exception = new AmazonS3Exception("Failing getObjectMetadata call with " + getObjectMetadataHttpCode); exception.setStatusCode(getObjectMetadataHttpCode); throw exception; @@ -83,7 +83,7 @@ public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetada @Override public S3Object getObject(GetObjectRequest getObjectRequest) { - if (getObjectHttpCode != SC_OK) { + if (getObjectHttpCode != HTTP_OK) { AmazonS3Exception exception = new AmazonS3Exception("Failing getObject call with " + getObjectHttpCode); exception.setStatusCode(getObjectHttpCode); throw exception; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/s3/TestHiveS3Config.java b/presto-hive/src/test/java/com/facebook/presto/hive/s3/TestHiveS3Config.java index 6a8a1c1b56173..c4e43f804f69e 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/s3/TestHiveS3Config.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/s3/TestHiveS3Config.java @@ -24,9 +24,9 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestHiveS3Config { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/s3/TestPrestoS3FileSystem.java b/presto-hive/src/test/java/com/facebook/presto/hive/s3/TestPrestoS3FileSystem.java index 76f21fabe3d14..769f25f919857 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/s3/TestPrestoS3FileSystem.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/s3/TestPrestoS3FileSystem.java @@ -27,7 +27,10 @@ import com.amazonaws.services.s3.model.EncryptionMaterials; import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; import com.amazonaws.services.s3.model.GetObjectMetadataRequest; +import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.facebook.presto.hive.s3.PrestoS3FileSystem.UnrecoverableS3OperationException; import com.google.common.base.VerifyException; import org.apache.hadoop.conf.Configuration; @@ -41,13 +44,16 @@ import javax.crypto.spec.SecretKeySpec; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.lang.reflect.Field; import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.hive.s3.PrestoS3FileSystem.S3_DIRECTORY_OBJECT_CONTENT_TYPE; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_ACCESS_KEY; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_ACL_TYPE; @@ -70,18 +76,18 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.testing.Assertions.assertInstanceOf; +import static java.net.HttpURLConnection.HTTP_FORBIDDEN; +import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.nio.file.Files.createTempDirectory; import static java.nio.file.Files.createTempFile; -import static org.apache.http.HttpStatus.SC_FORBIDDEN; -import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR; -import static org.apache.http.HttpStatus.SC_NOT_FOUND; -import static org.apache.http.HttpStatus.SC_REQUESTED_RANGE_NOT_SATISFIABLE; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; public class TestPrestoS3FileSystem { + private static final int HTTP_RANGE_NOT_SATISFIABLE = 416; + @Test public void testStaticCredentials() throws Exception @@ -190,7 +196,7 @@ public void testReadRetryCounters() try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { int maxRetries = 2; MockAmazonS3 s3 = new MockAmazonS3(); - s3.setGetObjectHttpErrorCode(SC_INTERNAL_SERVER_ERROR); + s3.setGetObjectHttpErrorCode(HTTP_INTERNAL_ERROR); Configuration configuration = new Configuration(); configuration.set(S3_MAX_BACKOFF_TIME, "1ms"); configuration.set(S3_MAX_RETRY_TIME, "5s"); @@ -202,7 +208,7 @@ public void testReadRetryCounters() } catch (Throwable expected) { assertInstanceOf(expected, AmazonS3Exception.class); - assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR); + assertEquals(((AmazonS3Exception) expected).getStatusCode(), HTTP_INTERNAL_ERROR); assertEquals(PrestoS3FileSystem.getFileSystemStats().getReadRetries().getTotalCount(), maxRetries); assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetObjectRetries().getTotalCount(), (maxRetries + 1L) * maxRetries); } @@ -216,7 +222,7 @@ public void testGetMetadataRetryCounter() int maxRetries = 2; try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); - s3.setGetObjectMetadataHttpCode(SC_INTERNAL_SERVER_ERROR); + s3.setGetObjectMetadataHttpCode(HTTP_INTERNAL_ERROR); Configuration configuration = new Configuration(); configuration.set(S3_MAX_BACKOFF_TIME, "1ms"); configuration.set(S3_MAX_RETRY_TIME, "5s"); @@ -227,19 +233,19 @@ public void testGetMetadataRetryCounter() } catch (Throwable expected) { assertInstanceOf(expected, AmazonS3Exception.class); - assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR); + assertEquals(((AmazonS3Exception) expected).getStatusCode(), HTTP_INTERNAL_ERROR); assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetMetadataRetries().getTotalCount(), maxRetries); } } @SuppressWarnings("ResultOfMethodCallIgnored") - @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + SC_NOT_FOUND + ".*") + @Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + HTTP_NOT_FOUND + ".*") public void testReadNotFound() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); - s3.setGetObjectHttpErrorCode(SC_NOT_FOUND); + s3.setGetObjectHttpErrorCode(HTTP_NOT_FOUND); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { @@ -249,13 +255,13 @@ public void testReadNotFound() } @SuppressWarnings("ResultOfMethodCallIgnored") - @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + SC_FORBIDDEN + ".*") + @Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + HTTP_FORBIDDEN + ".*") public void testReadForbidden() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); - s3.setGetObjectHttpErrorCode(SC_FORBIDDEN); + s3.setGetObjectHttpErrorCode(HTTP_FORBIDDEN); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { @@ -348,7 +354,7 @@ public void testReadRequestRangeNotSatisfiable() { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); - s3.setGetObjectHttpErrorCode(SC_REQUESTED_RANGE_NOT_SATISFIABLE); + s3.setGetObjectHttpErrorCode(HTTP_RANGE_NOT_SATISFIABLE); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { @@ -357,13 +363,13 @@ public void testReadRequestRangeNotSatisfiable() } } - @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Failing getObjectMetadata call with " + SC_FORBIDDEN + ".*") + @Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = ".*Failing getObjectMetadata call with " + HTTP_FORBIDDEN + ".*") public void testGetMetadataForbidden() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); - s3.setGetObjectMetadataHttpCode(SC_FORBIDDEN); + s3.setGetObjectMetadataHttpCode(HTTP_FORBIDDEN); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); fs.getS3ObjectMetadata(new Path("s3n://test-bucket/test")); @@ -376,7 +382,7 @@ public void testGetMetadataNotFound() { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); - s3.setGetObjectMetadataHttpCode(SC_NOT_FOUND); + s3.setGetObjectMetadataHttpCode(HTTP_NOT_FOUND); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); assertEquals(fs.getS3ObjectMetadata(new Path("s3n://test-bucket/test")), null); @@ -410,7 +416,7 @@ public void testKMSEncryptionMaterialsProvider() } @Test(expectedExceptions = UnrecoverableS3OperationException.class, expectedExceptionsMessageRegExp = ".*\\Q (Path: /tmp/test/path)\\E") - public void testUnrecoverableS3ExceptionMessage() + public void testUnrecoverableS3ExceptionMessage() throws Exception { throw new UnrecoverableS3OperationException(new Path("/tmp/test/path"), new IOException("test io exception")); } @@ -625,4 +631,35 @@ public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetada assertTrue(fileStatus.isDirectory()); } } + + @Test + public void testPrestoS3InputStreamEOS() throws Exception + { + try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { + AtomicInteger readableBytes = new AtomicInteger(1); + MockAmazonS3 s3 = new MockAmazonS3() + { + @Override + public S3Object getObject(GetObjectRequest req) + { + return new S3Object() + { + @Override + public S3ObjectInputStream getObjectContent() + { + return new S3ObjectInputStream(new ByteArrayInputStream(new byte[readableBytes.get()]), null); + } + }; + } + }; + fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); + fs.setS3Client(s3); + + try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { + assertEquals(inputStream.read(0, new byte[2], 0, 2), 1); + readableBytes.set(0); + assertEquals(inputStream.read(0, new byte[1], 0, 1), -1); + } + } + } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/security/TestLegacySecurityConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/security/TestLegacySecurityConfig.java index 0ed1a6896f900..0de7094b463e7 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/security/TestLegacySecurityConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/security/TestLegacySecurityConfig.java @@ -18,9 +18,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestLegacySecurityConfig { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestMetastoreHiveStatisticsProvider.java b/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestMetastoreHiveStatisticsProvider.java index 219af9e77ce1b..3dc9f85293647 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestMetastoreHiveStatisticsProvider.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestMetastoreHiveStatisticsProvider.java @@ -20,12 +20,12 @@ import com.facebook.presto.hive.HiveSessionProperties; import com.facebook.presto.hive.OrcFileWriterConfig; import com.facebook.presto.hive.ParquetFileWriterConfig; -import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.metastore.DateStatistics; import com.facebook.presto.hive.metastore.DecimalStatistics; import com.facebook.presto.hive.metastore.DoubleStatistics; import com.facebook.presto.hive.metastore.HiveColumnStatistics; import com.facebook.presto.hive.metastore.IntegerStatistics; +import com.facebook.presto.hive.metastore.PartitionStatistics; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.statistics.ColumnStatistics; @@ -48,12 +48,12 @@ import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_CORRUPTED_COLUMN_STATISTICS; import static com.facebook.presto.hive.HivePartition.UNPARTITIONED_ID; import static com.facebook.presto.hive.HivePartitionManager.parsePartition; import static com.facebook.presto.hive.HiveType.HIVE_LONG; import static com.facebook.presto.hive.HiveType.HIVE_STRING; import static com.facebook.presto.hive.HiveUtil.parsePartitionValue; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_CORRUPTED_COLUMN_STATISTICS; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createDateColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createDecimalColumnStatistics; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/util/TestAsyncQueue.java b/presto-hive/src/test/java/com/facebook/presto/hive/util/TestAsyncQueue.java index 998f7cb6d1285..762d4477267ab 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/util/TestAsyncQueue.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/util/TestAsyncQueue.java @@ -14,10 +14,10 @@ package com.facebook.presto.hive.util; +import com.facebook.airlift.concurrent.Threads; import com.facebook.presto.hive.util.AsyncQueue.BorrowResult; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.concurrent.Threads; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -29,8 +29,8 @@ import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicBoolean; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.testing.Assertions.assertContains; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.testing.Assertions.assertContains; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/util/TestStatistics.java b/presto-hive/src/test/java/com/facebook/presto/hive/util/TestStatistics.java index 517ee3a26ce84..0db19d3ad415d 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/util/TestStatistics.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/util/TestStatistics.java @@ -36,10 +36,10 @@ import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createDoubleColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics; -import static com.facebook.presto.hive.util.Statistics.ReduceOperator.ADD; -import static com.facebook.presto.hive.util.Statistics.ReduceOperator.SUBTRACT; -import static com.facebook.presto.hive.util.Statistics.merge; -import static com.facebook.presto.hive.util.Statistics.reduce; +import static com.facebook.presto.hive.metastore.Statistics.ReduceOperator.ADD; +import static com.facebook.presto.hive.metastore.Statistics.ReduceOperator.SUBTRACT; +import static com.facebook.presto.hive.metastore.Statistics.merge; +import static com.facebook.presto.hive.metastore.Statistics.reduce; import static org.assertj.core.api.Assertions.assertThat; public class TestStatistics diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 6828447a9e4b5..0f33a6c9061bc 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-jdbc @@ -33,7 +33,7 @@ - io.airlift + com.facebook.airlift security @@ -43,7 +43,7 @@ - io.airlift + com.facebook.airlift json @@ -119,23 +119,29 @@ - io.airlift + com.facebook.airlift testing test - io.airlift + com.facebook.airlift log-manager test - io.airlift + com.facebook.airlift concurrent test + + com.google.inject + guice + test + + com.squareup.okhttp3 mockwebserver @@ -218,9 +224,14 @@ com.google.thirdparty ${shadeBase}.guava + io.airlift - ${shadeBase}.airlift + ${shadeBase}.io.airlift + + + com.facebook.airlift + ${shadeBase}.com.facebook.airlift javax.annotation diff --git a/presto-jdbc/src/main/java/com/facebook/presto/jdbc/PrestoConnection.java b/presto-jdbc/src/main/java/com/facebook/presto/jdbc/PrestoConnection.java index 2ca183dbfba03..52d417068d4ff 100644 --- a/presto-jdbc/src/main/java/com/facebook/presto/jdbc/PrestoConnection.java +++ b/presto-jdbc/src/main/java/com/facebook/presto/jdbc/PrestoConnection.java @@ -75,7 +75,6 @@ public class PrestoConnection private final AtomicBoolean readOnly = new AtomicBoolean(); private final AtomicReference catalog = new AtomicReference<>(); private final AtomicReference schema = new AtomicReference<>(); - private final AtomicReference path = new AtomicReference<>(); private final AtomicReference timeZoneId = new AtomicReference<>(); private final AtomicReference locale = new AtomicReference<>(); private final AtomicReference networkTimeoutMillis = new AtomicReference<>(Ints.saturatedCast(MINUTES.toMillis(2))); @@ -696,7 +695,6 @@ else if (applicationName != null) { clientInfo.get("ClientInfo"), catalog.get(), schema.get(), - path.get(), timeZoneId.get(), locale.get(), ImmutableMap.of(), @@ -720,7 +718,6 @@ void updateSession(StatementClient client) client.getSetCatalog().ifPresent(catalog::set); client.getSetSchema().ifPresent(schema::set); - client.getSetPath().ifPresent(path::set); if (client.getStartedTransactionId() != null) { transactionId.set(client.getStartedTransactionId()); diff --git a/presto-jdbc/src/main/java/com/facebook/presto/jdbc/QueryExecutor.java b/presto-jdbc/src/main/java/com/facebook/presto/jdbc/QueryExecutor.java index fa5a0a1815f2c..e7efae61e0787 100644 --- a/presto-jdbc/src/main/java/com/facebook/presto/jdbc/QueryExecutor.java +++ b/presto-jdbc/src/main/java/com/facebook/presto/jdbc/QueryExecutor.java @@ -13,20 +13,20 @@ */ package com.facebook.presto.jdbc; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.client.ClientException; import com.facebook.presto.client.ClientSession; import com.facebook.presto.client.JsonResponse; import com.facebook.presto.client.ServerInfo; import com.facebook.presto.client.StatementClient; -import io.airlift.json.JsonCodec; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import okhttp3.Request; import java.net.URI; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.client.StatementClientFactory.newStatementClient; -import static io.airlift.json.JsonCodec.jsonCodec; import static java.lang.String.format; import static java.util.Objects.requireNonNull; diff --git a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcConnection.java b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcConnection.java index b4421501d2079..146c2336b85cd 100644 --- a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcConnection.java +++ b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcConnection.java @@ -13,12 +13,23 @@ */ package com.facebook.presto.jdbc; +import com.facebook.airlift.log.Logging; import com.facebook.presto.hive.HiveHadoop2Plugin; import com.facebook.presto.server.testing.TestingPrestoServer; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.InMemoryRecordSet; +import com.facebook.presto.spi.RecordCursor; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.SystemTable; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.predicate.TupleDomain; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logging; +import com.google.inject.Module; +import com.google.inject.Scopes; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -33,6 +44,10 @@ import java.util.Set; import static com.facebook.presto.jdbc.TestPrestoDriver.closeQuietly; +import static com.facebook.presto.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder; +import static com.facebook.presto.spi.SystemTable.Distribution.ALL_NODES; +import static com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType; +import static com.google.inject.multibindings.Multibinder.newSetBinder; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; @@ -48,7 +63,9 @@ public void setupServer() throws Exception { Logging.initialize(); - server = new TestingPrestoServer(); + Module systemTables = binder -> newSetBinder(binder, SystemTable.class) + .addBinding().to(ExtraCredentialsSystemTable.class).in(Scopes.SINGLETON); + server = new TestingPrestoServer(ImmutableList.of(systemTables)); server.installPlugin(new HiveHadoop2Plugin()); server.createCatalog("hive", "hive-hadoop2", ImmutableMap.builder() @@ -201,11 +218,12 @@ public void testExtraCredentials() throws SQLException { Map credentials = ImmutableMap.of("test.token.foo", "bar", "test.token.abc", "xyz"); - Connection connection = DriverManager.getConnection("jdbc:presto://localhost:8080?extraCredentials=test.token.foo:bar;test.token.abc:xyz", "admin", null); + Connection connection = createConnection("extraCredentials=test.token.foo:bar;test.token.abc:xyz"); assertTrue(connection instanceof PrestoConnection); - PrestoConnection prestoConnection = (PrestoConnection) connection; + PrestoConnection prestoConnection = connection.unwrap(PrestoConnection.class); assertEquals(prestoConnection.getExtraCredentials(), credentials); + assertEquals(listExtraCredentials(connection), credentials); } private Connection createConnection() @@ -250,6 +268,17 @@ private static Set listSession(Connection connection) return set.build(); } + private static Map listExtraCredentials(Connection connection) + throws SQLException + { + ResultSet rs = connection.createStatement().executeQuery("SELECT * FROM system.test.extra_credentials"); + ImmutableMap.Builder builder = ImmutableMap.builder(); + while (rs.next()) { + builder.put(rs.getString("name"), rs.getString("value")); + } + return builder.build(); + } + private static void assertConnectionSource(Connection connection, String expectedSource) throws SQLException { @@ -269,4 +298,35 @@ private static void assertConnectionSource(Connection connection, String expecte } } } + + private static class ExtraCredentialsSystemTable + implements SystemTable + { + private static final SchemaTableName NAME = new SchemaTableName("test", "extra_credentials"); + + public static final ConnectorTableMetadata METADATA = tableMetadataBuilder(NAME) + .column("name", createUnboundedVarcharType()) + .column("value", createUnboundedVarcharType()) + .build(); + + @Override + public Distribution getDistribution() + { + return ALL_NODES; + } + + @Override + public ConnectorTableMetadata getTableMetadata() + { + return METADATA; + } + + @Override + public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain constraint) + { + InMemoryRecordSet.Builder table = InMemoryRecordSet.builder(METADATA); + session.getIdentity().getExtraCredentials().forEach(table::addRow); + return table.build().cursor(); + } + } } diff --git a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcPreparedStatement.java b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcPreparedStatement.java index ab9f694764ed5..a894d9fbcabb5 100644 --- a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcPreparedStatement.java +++ b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcPreparedStatement.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.jdbc; +import com.facebook.airlift.log.Logging; import com.facebook.presto.plugin.blackhole.BlackHolePlugin; import com.facebook.presto.server.testing.TestingPrestoServer; -import io.airlift.log.Logging; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; diff --git a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcResultSet.java b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcResultSet.java index e00d46eabb889..2bb6470b0cba8 100644 --- a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcResultSet.java +++ b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcResultSet.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.jdbc; +import com.facebook.airlift.log.Logging; import com.facebook.presto.server.testing.TestingPrestoServer; -import io.airlift.log.Logging; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; diff --git a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcWarnings.java b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcWarnings.java index 2670c872f940b..ec75443533f4c 100644 --- a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcWarnings.java +++ b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestJdbcWarnings.java @@ -46,9 +46,9 @@ import java.util.Set; import java.util.concurrent.ExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.jdbc.TestPrestoDriver.closeQuietly; import static com.facebook.presto.jdbc.TestPrestoDriver.waitForNodeRefresh; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newSingleThreadExecutor; diff --git a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDatabaseMetaData.java b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDatabaseMetaData.java index d3459db7f4933..2a33623324efa 100644 --- a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDatabaseMetaData.java +++ b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDatabaseMetaData.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.jdbc; +import com.facebook.airlift.log.Logging; import com.facebook.presto.server.BasicQueryInfo; import com.facebook.presto.server.testing.TestingPrestoServer; import com.facebook.presto.spi.QueryId; -import io.airlift.log.Logging; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; @@ -32,10 +32,10 @@ import java.util.Set; import java.util.concurrent.Callable; +import static com.facebook.airlift.testing.Assertions.assertContains; import static com.facebook.presto.jdbc.TestPrestoDriver.closeQuietly; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.testing.Assertions.assertContains; import static java.lang.String.format; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; diff --git a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDriver.java b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDriver.java index fdcb6948538e6..39605d81b466a 100644 --- a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDriver.java +++ b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDriver.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.jdbc; +import com.facebook.airlift.log.Logging; import com.facebook.presto.execution.QueryState; import com.facebook.presto.plugin.blackhole.BlackHolePlugin; import com.facebook.presto.server.testing.TestingPrestoServer; @@ -39,7 +40,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logging; import io.airlift.units.Duration; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -74,16 +74,16 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertContains; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static com.facebook.presto.execution.QueryState.FAILED; import static com.facebook.presto.execution.QueryState.RUNNING; import static com.facebook.presto.spi.type.CharType.createCharType; import static com.facebook.presto.spi.type.DecimalType.createDecimalType; import static com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType; import static com.facebook.presto.spi.type.VarcharType.createVarcharType; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertContains; -import static io.airlift.testing.Assertions.assertInstanceOf; -import static io.airlift.testing.Assertions.assertLessThan; import static io.airlift.units.Duration.nanosSince; import static java.lang.Float.POSITIVE_INFINITY; import static java.lang.String.format; diff --git a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDriverAuth.java b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDriverAuth.java index 399e6ad85019e..3a6314c8f793c 100644 --- a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDriverAuth.java +++ b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDriverAuth.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.jdbc; +import com.facebook.airlift.log.Logging; +import com.facebook.airlift.security.pem.PemReader; import com.facebook.presto.server.testing.TestingPrestoServer; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logging; -import io.airlift.security.pem.PemReader; import io.jsonwebtoken.Jwts; import io.jsonwebtoken.SignatureAlgorithm; import org.testng.annotations.AfterClass; diff --git a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestProgressMonitor.java b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestProgressMonitor.java index 6730683cdd148..94c88574f0071 100644 --- a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestProgressMonitor.java +++ b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestProgressMonitor.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.jdbc; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.client.Column; import com.facebook.presto.client.QueryResults; import com.facebook.presto.client.StatementStats; import com.facebook.presto.spi.type.BigintType; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import org.testng.annotations.AfterMethod; @@ -35,10 +35,10 @@ import java.util.List; import java.util.function.Consumer; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual; import static com.google.common.base.Preconditions.checkState; import static com.google.common.net.HttpHeaders.CONTENT_TYPE; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.testing.Assertions.assertGreaterThanOrEqual; import static java.lang.String.format; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; diff --git a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestQueryExecutor.java b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestQueryExecutor.java index 38a8432609cc9..71e6c5d042941 100644 --- a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestQueryExecutor.java +++ b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestQueryExecutor.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.jdbc; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.client.ServerInfo; -import io.airlift.json.JsonCodec; import io.airlift.units.Duration; import okhttp3.OkHttpClient; import okhttp3.mockwebserver.MockResponse; @@ -26,9 +26,9 @@ import java.io.IOException; import java.util.Optional; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.client.NodeVersion.UNKNOWN; import static com.google.common.net.HttpHeaders.CONTENT_TYPE; -import static io.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; @Test(singleThreaded = true) diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index efc3b5af20edb..53e9d5487eedd 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-jmx @@ -22,22 +22,22 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift configuration - io.airlift + com.facebook.airlift concurrent @@ -105,13 +105,13 @@ - io.airlift + com.facebook.airlift testing test - io.airlift + com.facebook.airlift json test diff --git a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnector.java b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnector.java index 5c9dc275920a4..23b2594970ba3 100644 --- a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnector.java +++ b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnector.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.connector.jmx; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.transaction.IsolationLevel; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnectorConfig.java b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnectorConfig.java index 5fe078afeff68..d220c67c71484 100644 --- a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnectorConfig.java +++ b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnectorConfig.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.connector.jmx; +import com.facebook.airlift.configuration.Config; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableSet; -import io.airlift.configuration.Config; import io.airlift.units.Duration; import io.airlift.units.MinDuration; diff --git a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnectorFactory.java b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnectorFactory.java index 9e29a3c041a70..cb4db73bad28f 100644 --- a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnectorFactory.java +++ b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnectorFactory.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.jmx; +import com.facebook.airlift.bootstrap.Bootstrap; import com.facebook.presto.connector.jmx.util.RebindSafeMBeanServer; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.NodeManager; @@ -21,14 +22,13 @@ import com.facebook.presto.spi.connector.ConnectorFactory; import com.google.inject.Injector; import com.google.inject.Scopes; -import io.airlift.bootstrap.Bootstrap; import javax.management.MBeanServer; import java.util.Map; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static com.google.common.base.Throwables.throwIfUnchecked; -import static io.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; public class JmxConnectorFactory diff --git a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxPeriodicSampler.java b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxPeriodicSampler.java index 0a1e2466d8086..a92b6e2604da6 100644 --- a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxPeriodicSampler.java +++ b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxPeriodicSampler.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.connector.jmx; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.SchemaTableName; import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; import javax.annotation.PostConstruct; import javax.inject.Inject; @@ -23,7 +23,7 @@ import java.util.List; import java.util.concurrent.ScheduledExecutorService; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.format; import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; diff --git a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/util/RebindSafeMBeanServer.java b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/util/RebindSafeMBeanServer.java index 672b407e86c79..7579c1f0c9dbe 100644 --- a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/util/RebindSafeMBeanServer.java +++ b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/util/RebindSafeMBeanServer.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.connector.jmx.util; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import javax.annotation.concurrent.ThreadSafe; import javax.management.Attribute; diff --git a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/JmxQueryRunner.java b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/JmxQueryRunner.java index cf6c34e96af82..7c5d9f6009fed 100644 --- a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/JmxQueryRunner.java +++ b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/JmxQueryRunner.java @@ -16,9 +16,9 @@ import com.facebook.presto.Session; import com.facebook.presto.tests.DistributedQueryRunner; +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; import static com.facebook.presto.connector.jmx.JmxMetadata.JMX_SCHEMA_NAME; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.testing.Closeables.closeAllSuppress; public final class JmxQueryRunner { diff --git a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/MetadataUtil.java b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/MetadataUtil.java index d80f86afeb384..dc006e60327ae 100644 --- a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/MetadataUtil.java +++ b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/MetadataUtil.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.connector.jmx; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; import java.util.Map; diff --git a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxColumnHandle.java b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxColumnHandle.java index e540d8b844eab..fa6e4ea6d9109 100644 --- a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxColumnHandle.java +++ b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxColumnHandle.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.connector.jmx; -import io.airlift.testing.EquivalenceTester; +import com.facebook.airlift.testing.EquivalenceTester; import org.testng.annotations.Test; import static com.facebook.presto.connector.jmx.MetadataUtil.COLUMN_CODEC; diff --git a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxConnectorConfig.java b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxConnectorConfig.java index d0d3b69f41805..950c39de138af 100644 --- a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxConnectorConfig.java +++ b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxConnectorConfig.java @@ -13,18 +13,18 @@ */ package com.facebook.presto.connector.jmx; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.Duration; import org.testng.annotations.Test; import java.util.Map; import java.util.concurrent.TimeUnit; -import static io.airlift.configuration.testing.ConfigAssertions.assertDeprecatedEquivalence; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertDeprecatedEquivalence; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; public class TestJmxConnectorConfig { diff --git a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxTableHandle.java b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxTableHandle.java index 130c23754cb34..8024fbeb21645 100644 --- a/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxTableHandle.java +++ b/presto-jmx/src/test/java/com/facebook/presto/connector/jmx/TestJmxTableHandle.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.connector.jmx; +import com.facebook.airlift.testing.EquivalenceTester; import com.facebook.presto.spi.SchemaTableName; import com.google.common.collect.ImmutableList; -import io.airlift.testing.EquivalenceTester; import org.testng.annotations.Test; import java.util.List; diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index fe2875b738fba..fdd1932130bec 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-kafka @@ -18,22 +18,22 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift configuration @@ -120,7 +120,7 @@ - io.airlift + com.facebook.airlift log-manager runtime @@ -156,7 +156,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnector.java b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnector.java index 69cfe62e87379..fbe644e09053d 100644 --- a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnector.java +++ b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnector.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.kafka; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.transaction.IsolationLevel; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnectorConfig.java b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnectorConfig.java index ae5671f6512ea..1333e708c4d17 100644 --- a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnectorConfig.java +++ b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnectorConfig.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.kafka; +import com.facebook.airlift.configuration.Config; import com.facebook.presto.spi.HostAddress; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableSet; -import io.airlift.configuration.Config; import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import io.airlift.units.Duration; diff --git a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnectorFactory.java b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnectorFactory.java index 03e8d396cc151..fdaf9d1b30090 100644 --- a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnectorFactory.java +++ b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnectorFactory.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.kafka; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.SchemaTableName; @@ -23,8 +25,6 @@ import com.google.inject.Injector; import com.google.inject.Scopes; import com.google.inject.TypeLiteral; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; import java.util.Optional; diff --git a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnectorModule.java b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnectorModule.java index 8afa5068eb82e..b49eb74c963e6 100644 --- a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnectorModule.java +++ b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaConnectorModule.java @@ -24,10 +24,10 @@ import javax.inject.Inject; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.util.Objects.requireNonNull; /** diff --git a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaRecordSet.java b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaRecordSet.java index fa4cccbd2e16c..b4ba6ac3ac639 100644 --- a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaRecordSet.java +++ b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaRecordSet.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.kafka; +import com.facebook.airlift.log.Logger; import com.facebook.presto.decoder.DecoderColumnHandle; import com.facebook.presto.decoder.FieldValueProvider; import com.facebook.presto.decoder.RowDecoder; @@ -23,7 +24,6 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import kafka.api.FetchRequest; import kafka.api.FetchRequestBuilder; diff --git a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaSimpleConsumerManager.java b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaSimpleConsumerManager.java index 2d171ff1dabdb..273e323434d01 100644 --- a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaSimpleConsumerManager.java +++ b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaSimpleConsumerManager.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.kafka; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.NodeManager; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import io.airlift.log.Logger; import kafka.javaapi.consumer.SimpleConsumer; import javax.annotation.PreDestroy; diff --git a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaSplitManager.java b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaSplitManager.java index 8061b32724d56..d599ea9c7462c 100644 --- a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaSplitManager.java +++ b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaSplitManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.kafka; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.ConnectorSplitSource; @@ -26,7 +27,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.io.CharStreams; -import io.airlift.log.Logger; import kafka.api.PartitionOffsetRequestInfo; import kafka.cluster.BrokerEndPoint; import kafka.common.TopicAndPartition; diff --git a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaTableDescriptionSupplier.java b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaTableDescriptionSupplier.java index 682a32b471c35..3b813e8a328e2 100644 --- a/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaTableDescriptionSupplier.java +++ b/presto-kafka/src/main/java/com/facebook/presto/kafka/KafkaTableDescriptionSupplier.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.kafka; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.facebook.presto.decoder.dummy.DummyRowDecoder; import com.facebook.presto.spi.SchemaTableName; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-kafka/src/test/java/com/facebook/presto/kafka/KafkaQueryRunner.java b/presto-kafka/src/test/java/com/facebook/presto/kafka/KafkaQueryRunner.java index 1f760f8a4c46d..0254a78c03be8 100644 --- a/presto-kafka/src/test/java/com/facebook/presto/kafka/KafkaQueryRunner.java +++ b/presto-kafka/src/test/java/com/facebook/presto/kafka/KafkaQueryRunner.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.kafka; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.log.Logging; import com.facebook.presto.Session; import com.facebook.presto.kafka.util.CodecSupplier; import com.facebook.presto.kafka.util.EmbeddedKafka; @@ -25,18 +28,15 @@ import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; -import io.airlift.log.Logging; import io.airlift.tpch.TpchTable; import java.util.Map; +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; import static com.facebook.presto.kafka.util.TestUtils.installKafkaPlugin; import static com.facebook.presto.kafka.util.TestUtils.loadTpchTopicDescription; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static io.airlift.testing.Closeables.closeAllSuppress; import static io.airlift.units.Duration.nanosSince; import static java.util.Locale.ENGLISH; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-kafka/src/test/java/com/facebook/presto/kafka/TestKafkaConnectorConfig.java b/presto-kafka/src/test/java/com/facebook/presto/kafka/TestKafkaConnectorConfig.java index 92bc30a9f160c..e161e8fb52552 100644 --- a/presto-kafka/src/test/java/com/facebook/presto/kafka/TestKafkaConnectorConfig.java +++ b/presto-kafka/src/test/java/com/facebook/presto/kafka/TestKafkaConnectorConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.kafka; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import org.testng.annotations.Test; import java.io.File; diff --git a/presto-kafka/src/test/java/com/facebook/presto/kafka/TestKafkaPlugin.java b/presto-kafka/src/test/java/com/facebook/presto/kafka/TestKafkaPlugin.java index ad5e19b0eef4f..69cfdff8f2657 100644 --- a/presto-kafka/src/test/java/com/facebook/presto/kafka/TestKafkaPlugin.java +++ b/presto-kafka/src/test/java/com/facebook/presto/kafka/TestKafkaPlugin.java @@ -19,8 +19,8 @@ import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.testing.Assertions.assertInstanceOf; import static org.testng.Assert.assertNotNull; @Test diff --git a/presto-kafka/src/test/java/com/facebook/presto/kafka/util/CodecSupplier.java b/presto-kafka/src/test/java/com/facebook/presto/kafka/util/CodecSupplier.java index 05f7a035869f3..568e38d5a2b15 100644 --- a/presto-kafka/src/test/java/com/facebook/presto/kafka/util/CodecSupplier.java +++ b/presto-kafka/src/test/java/com/facebook/presto/kafka/util/CodecSupplier.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.kafka.util; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; import java.util.function.Supplier; diff --git a/presto-kafka/src/test/java/com/facebook/presto/kafka/util/TestUtils.java b/presto-kafka/src/test/java/com/facebook/presto/kafka/util/TestUtils.java index 98ed0bb58f788..755a54f35bab4 100644 --- a/presto-kafka/src/test/java/com/facebook/presto/kafka/util/TestUtils.java +++ b/presto-kafka/src/test/java/com/facebook/presto/kafka/util/TestUtils.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.kafka.util; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.kafka.KafkaPlugin; import com.facebook.presto.kafka.KafkaTopicDescription; import com.facebook.presto.metadata.QualifiedObjectName; @@ -22,7 +23,6 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; -import io.airlift.json.JsonCodec; import java.io.IOException; import java.net.ServerSocket; diff --git a/presto-kudu/pom.xml b/presto-kudu/pom.xml index fe06ce98b5fb3..21ebbaf825c18 100644 --- a/presto-kudu/pom.xml +++ b/presto-kudu/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-kudu @@ -24,17 +24,17 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift log @@ -60,7 +60,7 @@ - io.airlift + com.facebook.airlift configuration @@ -141,7 +141,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduClientConfig.java b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduClientConfig.java index 684bc80182243..c451eefb47051 100755 --- a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduClientConfig.java +++ b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduClientConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.kudu; +import com.facebook.airlift.configuration.Config; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import io.airlift.configuration.Config; import io.airlift.units.Duration; import io.airlift.units.MaxDuration; import io.airlift.units.MinDuration; diff --git a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduClientSession.java b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduClientSession.java index 61e13f2ff4e12..ff841c9d2e876 100644 --- a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduClientSession.java +++ b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduClientSession.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.kudu; +import com.facebook.airlift.log.Logger; import com.facebook.presto.kudu.properties.ColumnDesign; import com.facebook.presto.kudu.properties.HashPartitionDefinition; import com.facebook.presto.kudu.properties.KuduTableProperties; @@ -38,7 +39,6 @@ import com.facebook.presto.spi.predicate.ValueSet; import com.facebook.presto.spi.type.DecimalType; import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; import org.apache.kudu.ColumnSchema; import org.apache.kudu.ColumnTypeAttributes; import org.apache.kudu.Schema; diff --git a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduConnector.java b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduConnector.java index 7c8ade0ecbc3a..7cdb99f0932c2 100755 --- a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduConnector.java +++ b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduConnector.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.kudu; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.kudu.properties.KuduTableProperties; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorMetadata; @@ -25,8 +27,6 @@ import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spi.transaction.IsolationLevel; import com.google.common.collect.ImmutableSet; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduConnectorFactory.java b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduConnectorFactory.java index bb8df46a2b4a0..73d6221a448a8 100755 --- a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduConnectorFactory.java +++ b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduConnectorFactory.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.kudu; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.connector.ConnectorFactory; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; diff --git a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduMetadata.java b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduMetadata.java index e00155c186889..5aa63dc0b5062 100755 --- a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduMetadata.java +++ b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduMetadata.java @@ -368,7 +368,7 @@ public void finishDelete(ConnectorSession session, ConnectorTableHandle tableHan } @Override - public boolean supportsMetadataDelete(ConnectorSession session, ConnectorTableHandle tableHandle) + public boolean supportsMetadataDelete(ConnectorSession session, ConnectorTableHandle tableHandle, Optional tableLayoutHandle) { return false; } diff --git a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduModule.java b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduModule.java index 22ba2683cd5c1..4cd0530bf4f63 100755 --- a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduModule.java +++ b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduModule.java @@ -33,7 +33,7 @@ import javax.inject.Singleton; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; public class KuduModule diff --git a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduRecordCursor.java b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduRecordCursor.java index df29f070f06ff..8f73510fb95c8 100755 --- a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduRecordCursor.java +++ b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduRecordCursor.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.kudu; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.RecordCursor; import com.facebook.presto.spi.type.Type; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import org.apache.kudu.client.KuduException; import org.apache.kudu.client.KuduScanner; diff --git a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduUpdatablePageSource.java b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduUpdatablePageSource.java index 374c199b70df9..798a1762810b6 100644 --- a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduUpdatablePageSource.java +++ b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduUpdatablePageSource.java @@ -85,6 +85,12 @@ public long getCompletedBytes() return inner.getCompletedBytes(); } + @Override + public long getCompletedPositions() + { + return inner.getCompletedPositions(); + } + @Override public long getReadTimeNanos() { diff --git a/presto-kudu/src/test/java/com/facebook/presto/kudu/KuduQueryRunnerFactory.java b/presto-kudu/src/test/java/com/facebook/presto/kudu/KuduQueryRunnerFactory.java index f53555db2892a..08672f5d076ea 100644 --- a/presto-kudu/src/test/java/com/facebook/presto/kudu/KuduQueryRunnerFactory.java +++ b/presto-kudu/src/test/java/com/facebook/presto/kudu/KuduQueryRunnerFactory.java @@ -23,11 +23,11 @@ import java.util.Map; +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tests.QueryAssertions.copyTpchTables; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static io.airlift.testing.Closeables.closeAllSuppress; import static java.util.Locale.ENGLISH; public class KuduQueryRunnerFactory diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index bfb41ae31a1a0..ccaefcbb32948 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-local-file @@ -18,22 +18,22 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift configuration - io.airlift + com.facebook.airlift json @@ -96,7 +96,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileConfig.java b/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileConfig.java index 50058b4be86b0..6630a5e3f8bc3 100644 --- a/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileConfig.java +++ b/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.localfile; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; public class LocalFileConfig { diff --git a/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileConnector.java b/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileConnector.java index 500ddf6266ae2..cf81f1549f2b3 100644 --- a/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileConnector.java +++ b/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileConnector.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.localfile; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.transaction.IsolationLevel; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileConnectorFactory.java b/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileConnectorFactory.java index b1ffe7041d9e8..ffebb847e969f 100644 --- a/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileConnectorFactory.java +++ b/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileConnectorFactory.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.localfile; +import com.facebook.airlift.bootstrap.Bootstrap; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.connector.ConnectorFactory; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; import java.util.Map; diff --git a/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileModule.java b/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileModule.java index 06daaa70cbdd7..c69080fa61ae9 100644 --- a/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileModule.java +++ b/presto-local-file/src/main/java/com/facebook/presto/localfile/LocalFileModule.java @@ -17,7 +17,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; public class LocalFileModule diff --git a/presto-local-file/src/test/java/com/facebook/presto/localfile/MetadataUtil.java b/presto-local-file/src/test/java/com/facebook/presto/localfile/MetadataUtil.java index 6d13e3e725621..fe8397b6c3733 100644 --- a/presto-local-file/src/test/java/com/facebook/presto/localfile/MetadataUtil.java +++ b/presto-local-file/src/test/java/com/facebook/presto/localfile/MetadataUtil.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.localfile; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; import java.util.Map; diff --git a/presto-local-file/src/test/java/com/facebook/presto/localfile/TestLocalFileConfig.java b/presto-local-file/src/test/java/com/facebook/presto/localfile/TestLocalFileConfig.java index 02ed863ffdee3..472da074a83a7 100644 --- a/presto-local-file/src/test/java/com/facebook/presto/localfile/TestLocalFileConfig.java +++ b/presto-local-file/src/test/java/com/facebook/presto/localfile/TestLocalFileConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.localfile; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import org.testng.annotations.Test; import java.util.Map; diff --git a/presto-local-file/src/test/java/com/facebook/presto/localfile/TestLocalFileSplit.java b/presto-local-file/src/test/java/com/facebook/presto/localfile/TestLocalFileSplit.java index 818a44dbe4ddb..73a4ced16817b 100644 --- a/presto-local-file/src/test/java/com/facebook/presto/localfile/TestLocalFileSplit.java +++ b/presto-local-file/src/test/java/com/facebook/presto/localfile/TestLocalFileSplit.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.localfile; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.predicate.TupleDomain; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; public class TestLocalFileSplit diff --git a/presto-main/etc/catalog/raptor.properties b/presto-main/etc/catalog/raptor.properties index 59c12f5e1145b..daf7cce1b44b2 100644 --- a/presto-main/etc/catalog/raptor.properties +++ b/presto-main/etc/catalog/raptor.properties @@ -10,4 +10,4 @@ metadata.db.type=h2 metadata.db.filename=mem:raptor;DB_CLOSE_DELAY=-1 #metadata.db.type=mysql #metadata.db.connections.max=500 -storage.data-directory=var/data +storage.data-directory=file:///var/data diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 10a849be081d9..f98272332b82f 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-main @@ -16,11 +16,6 @@ - - org.locationtech.jts - jts-core - - com.esri.geometry esri-geometry-api @@ -76,13 +71,23 @@ presto-memory-context + + com.facebook.presto + presto-sql-function + + + + com.facebook.presto + presto-expressions + + com.google.code.findbugs jsr305 - io.airlift + com.facebook.airlift bootstrap @@ -97,62 +102,62 @@ - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift node - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift configuration - io.airlift + com.facebook.airlift discovery - io.airlift + com.facebook.airlift event - io.airlift + com.facebook.airlift http-server - io.airlift + com.facebook.airlift jaxrs - io.airlift + com.facebook.airlift jmx - io.airlift + com.facebook.airlift jmx-http - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift log-manager @@ -162,17 +167,17 @@ - io.airlift + com.facebook.airlift security - io.airlift + com.facebook.airlift stats - io.airlift + com.facebook.airlift trace-token @@ -197,7 +202,7 @@ - io.airlift.discovery + com.facebook.airlift.discovery discovery-server @@ -277,7 +282,7 @@ - io.airlift + com.facebook.airlift http-client @@ -317,12 +322,6 @@ jjwt - - - org.apache.bval - bval-jsr - - org.testng @@ -331,7 +330,7 @@ - io.airlift + com.facebook.airlift testing test @@ -387,7 +386,7 @@ - io.airlift + com.facebook.airlift jaxrs-testing test diff --git a/presto-main/src/main/java/com/facebook/presto/FullConnectorSession.java b/presto-main/src/main/java/com/facebook/presto/FullConnectorSession.java index 8b66cf84ada2e..58a359f73a2ba 100644 --- a/presto-main/src/main/java/com/facebook/presto/FullConnectorSession.java +++ b/presto-main/src/main/java/com/facebook/presto/FullConnectorSession.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.security.ConnectorIdentity; import com.facebook.presto.spi.type.TimeZoneKey; import com.google.common.collect.ImmutableMap; @@ -39,7 +40,7 @@ public class FullConnectorSession private final ConnectorId connectorId; private final String catalog; private final SessionPropertyManager sessionPropertyManager; - private final boolean isLegacyTimestamp; + private final SqlFunctionProperties sqlFunctionProperties; public FullConnectorSession(Session session, ConnectorIdentity identity) { @@ -49,7 +50,7 @@ public FullConnectorSession(Session session, ConnectorIdentity identity) this.connectorId = null; this.catalog = null; this.sessionPropertyManager = null; - this.isLegacyTimestamp = SystemSessionProperties.isLegacyTimestamp(session); + this.sqlFunctionProperties = session.getSqlFunctionProperties(); } public FullConnectorSession( @@ -66,7 +67,7 @@ public FullConnectorSession( this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.catalog = requireNonNull(catalog, "catalog is null"); this.sessionPropertyManager = requireNonNull(sessionPropertyManager, "sessionPropertyManager is null"); - this.isLegacyTimestamp = SystemSessionProperties.isLegacyTimestamp(session); + this.sqlFunctionProperties = session.getSqlFunctionProperties(); } public Session getSession() @@ -116,10 +117,22 @@ public Optional getTraceToken() return session.getTraceToken(); } + @Override + public Optional getClientInfo() + { + return session.getClientInfo(); + } + @Override public boolean isLegacyTimestamp() { - return isLegacyTimestamp; + return sqlFunctionProperties.isLegacyTimestamp(); + } + + @Override + public SqlFunctionProperties getSqlFunctionProperties() + { + return sqlFunctionProperties; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/Session.java b/presto-main/src/main/java/com/facebook/presto/Session.java index dfe7d4887dc6b..fe4f05ea2e3da 100644 --- a/presto-main/src/main/java/com/facebook/presto/Session.java +++ b/presto-main/src/main/java/com/facebook/presto/Session.java @@ -19,11 +19,11 @@ import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.security.SelectedRole; import com.facebook.presto.spi.session.ResourceEstimates; import com.facebook.presto.spi.type.TimeZoneKey; -import com.facebook.presto.sql.SqlPath; import com.facebook.presto.sql.tree.Execute; import com.facebook.presto.transaction.TransactionId; import com.facebook.presto.transaction.TransactionManager; @@ -44,6 +44,9 @@ import java.util.TimeZone; import java.util.stream.Collectors; +import static com.facebook.presto.SystemSessionProperties.isLegacyRowFieldOrdinalAccessEnabled; +import static com.facebook.presto.SystemSessionProperties.isLegacyTimestamp; +import static com.facebook.presto.SystemSessionProperties.isParseDecimalLiteralsAsDouble; import static com.facebook.presto.spi.ConnectorId.createInformationSchemaConnectorId; import static com.facebook.presto.spi.ConnectorId.createSystemTablesConnectorId; import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; @@ -62,7 +65,6 @@ public final class Session private final Optional source; private final Optional catalog; private final Optional schema; - private final SqlPath path; private final TimeZoneKey timeZoneKey; private final Locale locale; private final Optional remoteUserAddress; @@ -70,7 +72,6 @@ public final class Session private final Optional clientInfo; private final Optional traceToken; private final Set clientTags; - private final Set clientCapabilities; private final ResourceEstimates resourceEstimates; private final long startTime; private final Map systemProperties; @@ -87,7 +88,6 @@ public Session( Optional source, Optional catalog, Optional schema, - SqlPath path, Optional traceToken, TimeZoneKey timeZoneKey, Locale locale, @@ -95,7 +95,6 @@ public Session( Optional userAgent, Optional clientInfo, Set clientTags, - Set clientCapabilities, ResourceEstimates resourceEstimates, long startTime, Map systemProperties, @@ -111,7 +110,6 @@ public Session( this.source = requireNonNull(source, "source is null"); this.catalog = requireNonNull(catalog, "catalog is null"); this.schema = requireNonNull(schema, "schema is null"); - this.path = requireNonNull(path, "path is null"); this.traceToken = requireNonNull(traceToken, "traceToken is null"); this.timeZoneKey = requireNonNull(timeZoneKey, "timeZoneKey is null"); this.locale = requireNonNull(locale, "locale is null"); @@ -119,7 +117,6 @@ public Session( this.userAgent = requireNonNull(userAgent, "userAgent is null"); this.clientInfo = requireNonNull(clientInfo, "clientInfo is null"); this.clientTags = ImmutableSet.copyOf(requireNonNull(clientTags, "clientTags is null")); - this.clientCapabilities = ImmutableSet.copyOf(requireNonNull(clientCapabilities, "clientCapabilities is null")); this.resourceEstimates = requireNonNull(resourceEstimates, "resourceEstimates is null"); this.startTime = startTime; this.systemProperties = ImmutableMap.copyOf(requireNonNull(systemProperties, "systemProperties is null")); @@ -173,11 +170,6 @@ public Optional getSchema() return schema; } - public SqlPath getPath() - { - return path; - } - public TimeZoneKey getTimeZoneKey() { return timeZoneKey; @@ -208,11 +200,6 @@ public Set getClientTags() return clientTags; } - public Set getClientCapabilities() - { - return clientCapabilities; - } - public Optional getTraceToken() { return traceToken; @@ -352,11 +339,10 @@ public Session beginTransactionId(TransactionId transactionId, TransactionManage queryId, Optional.of(transactionId), clientTransactionSupport, - new Identity(identity.getUser(), identity.getPrincipal(), roles.build()), + new Identity(identity.getUser(), identity.getPrincipal(), roles.build(), identity.getExtraCredentials()), source, catalog, schema, - path, traceToken, timeZoneKey, locale, @@ -364,7 +350,6 @@ public Session beginTransactionId(TransactionId transactionId, TransactionManage userAgent, clientInfo, clientTags, - clientCapabilities, resourceEstimates, startTime, systemProperties, @@ -407,7 +392,6 @@ public Session withDefaultProperties(Map systemPropertyDefaults, source, catalog, schema, - path, traceToken, timeZoneKey, locale, @@ -415,7 +399,6 @@ public Session withDefaultProperties(Map systemPropertyDefaults, userAgent, clientInfo, clientTags, - clientCapabilities, resourceEstimates, startTime, systemProperties, @@ -430,6 +413,16 @@ public ConnectorSession toConnectorSession() return new FullConnectorSession(this, identity.toConnectorIdentity()); } + public SqlFunctionProperties getSqlFunctionProperties() + { + return SqlFunctionProperties.builder() + .setTimeZoneKey(timeZoneKey) + .setLegacyRowFieldOrdinalAccessEnabled(isLegacyRowFieldOrdinalAccessEnabled(this)) + .setLegacyTimestamp(isLegacyTimestamp(this)) + .setParseDecimalLiteralAsDouble(isParseDecimalLiteralsAsDouble(this)) + .build(); + } + public ConnectorSession toConnectorSession(ConnectorId connectorId) { requireNonNull(connectorId, "connectorId is null"); @@ -454,7 +447,6 @@ public SessionRepresentation toSessionRepresentation() source, catalog, schema, - path, traceToken, timeZoneKey, locale, @@ -462,7 +454,6 @@ public SessionRepresentation toSessionRepresentation() userAgent, clientInfo, clientTags, - clientCapabilities, resourceEstimates, startTime, systemProperties, @@ -483,7 +474,6 @@ public String toString() .add("source", source.orElse(null)) .add("catalog", catalog.orElse(null)) .add("schema", schema.orElse(null)) - .add("path", path) .add("traceToken", traceToken.orElse(null)) .add("timeZoneKey", timeZoneKey) .add("locale", locale) @@ -491,7 +481,6 @@ public String toString() .add("userAgent", userAgent.orElse(null)) .add("clientInfo", clientInfo.orElse(null)) .add("clientTags", clientTags) - .add("clientCapabilities", clientCapabilities) .add("resourceEstimates", resourceEstimates) .add("startTime", startTime) .omitNullValues() @@ -518,7 +507,6 @@ public static class SessionBuilder private String source; private String catalog; private String schema; - private SqlPath path = new SqlPath(Optional.empty()); private Optional traceToken = Optional.empty(); private TimeZoneKey timeZoneKey = TimeZoneKey.getTimeZoneKey(TimeZone.getDefault().getID()); private Locale locale = Locale.getDefault(); @@ -526,7 +514,6 @@ public static class SessionBuilder private String userAgent; private String clientInfo; private Set clientTags = ImmutableSet.of(); - private Set clientCapabilities = ImmutableSet.of(); private ResourceEstimates resourceEstimates; private long startTime = System.currentTimeMillis(); private final Map systemProperties = new HashMap<>(); @@ -550,7 +537,6 @@ private SessionBuilder(Session session) this.identity = session.identity; this.source = session.source.orElse(null); this.catalog = session.catalog.orElse(null); - this.path = session.path; this.schema = session.schema.orElse(null); this.traceToken = requireNonNull(session.traceToken, "traceToken is null"); this.timeZoneKey = session.timeZoneKey; @@ -608,12 +594,6 @@ public SessionBuilder setSchema(String schema) return this; } - public SessionBuilder setPath(SqlPath path) - { - this.path = path; - return this; - } - public SessionBuilder setSource(String source) { this.source = source; @@ -662,12 +642,6 @@ public SessionBuilder setClientTags(Set clientTags) return this; } - public SessionBuilder setClientCapabilities(Set clientCapabilities) - { - this.clientCapabilities = ImmutableSet.copyOf(clientCapabilities); - return this; - } - public SessionBuilder setResourceEstimates(ResourceEstimates resourceEstimates) { this.resourceEstimates = resourceEstimates; @@ -711,7 +685,6 @@ public Session build() Optional.ofNullable(source), Optional.ofNullable(catalog), Optional.ofNullable(schema), - path, traceToken, timeZoneKey, locale, @@ -719,7 +692,6 @@ public Session build() Optional.ofNullable(userAgent), Optional.ofNullable(clientInfo), clientTags, - clientCapabilities, Optional.ofNullable(resourceEstimates).orElse(new ResourceEstimateBuilder().build()), startTime, systemProperties, diff --git a/presto-main/src/main/java/com/facebook/presto/SessionRepresentation.java b/presto-main/src/main/java/com/facebook/presto/SessionRepresentation.java index 1d2af44e948af..d481ab0f6b103 100644 --- a/presto-main/src/main/java/com/facebook/presto/SessionRepresentation.java +++ b/presto-main/src/main/java/com/facebook/presto/SessionRepresentation.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.security.SelectedRole; import com.facebook.presto.spi.session.ResourceEstimates; import com.facebook.presto.spi.type.TimeZoneKey; -import com.facebook.presto.sql.SqlPath; import com.facebook.presto.transaction.TransactionId; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -46,7 +45,6 @@ public final class SessionRepresentation private final Optional source; private final Optional catalog; private final Optional schema; - private final SqlPath path; private final Optional traceToken; private final TimeZoneKey timeZoneKey; private final Locale locale; @@ -54,7 +52,6 @@ public final class SessionRepresentation private final Optional userAgent; private final Optional clientInfo; private final Set clientTags; - private final Set clientCapabilities; private final long startTime; private final ResourceEstimates resourceEstimates; private final Map systemProperties; @@ -73,7 +70,6 @@ public SessionRepresentation( @JsonProperty("source") Optional source, @JsonProperty("catalog") Optional catalog, @JsonProperty("schema") Optional schema, - @JsonProperty("path") SqlPath path, @JsonProperty("traceToken") Optional traceToken, @JsonProperty("timeZoneKey") TimeZoneKey timeZoneKey, @JsonProperty("locale") Locale locale, @@ -81,7 +77,6 @@ public SessionRepresentation( @JsonProperty("userAgent") Optional userAgent, @JsonProperty("clientInfo") Optional clientInfo, @JsonProperty("clientTags") Set clientTags, - @JsonProperty("clientCapabilities") Set clientCapabilities, @JsonProperty("resourceEstimates") ResourceEstimates resourceEstimates, @JsonProperty("startTime") long startTime, @JsonProperty("systemProperties") Map systemProperties, @@ -98,7 +93,6 @@ public SessionRepresentation( this.source = requireNonNull(source, "source is null"); this.catalog = requireNonNull(catalog, "catalog is null"); this.schema = requireNonNull(schema, "schema is null"); - this.path = requireNonNull(path, "path is null"); this.traceToken = requireNonNull(traceToken, "traceToken is null"); this.timeZoneKey = requireNonNull(timeZoneKey, "timeZoneKey is null"); this.locale = requireNonNull(locale, "locale is null"); @@ -106,7 +100,6 @@ public SessionRepresentation( this.userAgent = requireNonNull(userAgent, "userAgent is null"); this.clientInfo = requireNonNull(clientInfo, "clientInfo is null"); this.clientTags = requireNonNull(clientTags, "clientTags is null"); - this.clientCapabilities = requireNonNull(clientCapabilities, "clientCapabilities is null"); this.resourceEstimates = requireNonNull(resourceEstimates, "resourceEstimates is null"); this.startTime = startTime; this.systemProperties = ImmutableMap.copyOf(systemProperties); @@ -180,12 +173,6 @@ public Optional getSchema() return schema; } - @JsonProperty - public SqlPath getPath() - { - return path; - } - @JsonProperty public TimeZoneKey getTimeZoneKey() { @@ -222,12 +209,6 @@ public Set getClientTags() return clientTags; } - @JsonProperty - public Set getClientCapabilities() - { - return clientCapabilities; - } - @JsonProperty public long getStartTime() { @@ -285,7 +266,6 @@ public Session toSession(SessionPropertyManager sessionPropertyManager, Map> sessionProperties; @@ -246,7 +252,16 @@ public SystemSessionProperties( Integer.class, taskManagerConfig.getWriterCount(), false, - value -> validateValueIsPowerOfTwo(value, TASK_WRITER_COUNT), + value -> validateValueIsPowerOfTwo(requireNonNull(value, "value is null"), TASK_WRITER_COUNT), + value -> value), + new PropertyMetadata<>( + TASK_PARTITIONED_WRITER_COUNT, + "Number of writers per task for partitioned writes. If not set, the number set by task.writer-count will be used", + BIGINT, + Integer.class, + taskManagerConfig.getPartitionedWriterCount(), + false, + value -> validateValueIsPowerOfTwo(value, TASK_PARTITIONED_WRITER_COUNT), value -> value), booleanProperty( REDISTRIBUTE_WRITES, @@ -279,7 +294,7 @@ public SystemSessionProperties( Integer.class, taskManagerConfig.getTaskConcurrency(), false, - value -> validateValueIsPowerOfTwo(value, TASK_CONCURRENCY), + value -> validateValueIsPowerOfTwo(requireNonNull(value, "value is null"), TASK_CONCURRENCY), value -> value), booleanProperty( TASK_SHARE_INDEX_LOADING, @@ -626,6 +641,35 @@ public SystemSessionProperties( PUSHDOWN_SUBFIELDS_ENABLED, "Experimental: enable subfield pruning", featuresConfig.isPushdownSubfieldsEnabled(), + false), + booleanProperty( + TABLE_WRITER_MERGE_OPERATOR_ENABLED, + "Experimental: enable table writer merge operator", + featuresConfig.isTableWriterMergeOperatorEnabled(), + false), + booleanProperty( + OPTIMIZE_FULL_OUTER_JOIN_WITH_COALESCE, + "optimize partition properties for queries using COALESCE + FULL OUTER JOIN", + featuresConfig.isOptimizeFullOuterJoinWithCoalesce(), + false), + new PropertyMetadata<>( + INDEX_LOADER_TIMEOUT, + "Timeout for loading indexes for index joins", + VARCHAR, + Duration.class, + featuresConfig.getIndexLoaderTimeout(), + false, + value -> Duration.valueOf((String) value), + Duration::toString), + booleanProperty( + OPTIMIZED_REPARTITIONING_ENABLED, + "Experimental: Use optimized repartitioning", + featuresConfig.isOptimizedRepartitioningEnabled(), + false), + booleanProperty( + SIMPLIFY_ARRAY_OPERATIONS, + "Simplify and optimize array operations.", + true, false)); } @@ -718,6 +762,15 @@ public static int getTaskWriterCount(Session session) return session.getSystemProperty(TASK_WRITER_COUNT, Integer.class); } + public static int getTaskPartitionedWriterCount(Session session) + { + Integer partitionedWriterCount = session.getSystemProperty(TASK_PARTITIONED_WRITER_COUNT, Integer.class); + if (partitionedWriterCount != null) { + return partitionedWriterCount; + } + return getTaskWriterCount(session); + } + public static boolean isRedistributeWrites(Session session) { return session.getSystemProperty(REDISTRIBUTE_WRITES, Boolean.class); @@ -991,9 +1044,13 @@ public static int getMaxTasksPerStage(Session session) return session.getSystemProperty(MAX_TASKS_PER_STAGE, Integer.class); } - private static int validateValueIsPowerOfTwo(Object value, String property) + private static Integer validateValueIsPowerOfTwo(Object value, String property) { - int intValue = ((Number) requireNonNull(value, "value is null")).intValue(); + Number number = (Number) value; + if (number == null) { + return null; + } + int intValue = number.intValue(); if (Integer.bitCount(intValue) != 1) { throw new PrestoException( INVALID_SESSION_PROPERTY, @@ -1063,4 +1120,28 @@ public static boolean isPushdownSubfieldsEnabled(Session session) { return session.getSystemProperty(PUSHDOWN_SUBFIELDS_ENABLED, Boolean.class); } + + public static boolean isTableWriterMergeOperatorEnabled(Session session) + { + return session.getSystemProperty(TABLE_WRITER_MERGE_OPERATOR_ENABLED, Boolean.class); + } + + public static boolean isOptimizeFullOuterJoinWithCoalesce(Session session) + { + return session.getSystemProperty(OPTIMIZE_FULL_OUTER_JOIN_WITH_COALESCE, Boolean.class); + } + + public static Duration getIndexLoaderTimeout(Session session) + { + return session.getSystemProperty(INDEX_LOADER_TIMEOUT, Duration.class); + } + + public static boolean isOptimizedRepartitioningEnabled(Session session) + { + return session.getSystemProperty(OPTIMIZED_REPARTITIONING_ENABLED, Boolean.class); + } + public static boolean isSimplifyArrayOperations(Session session) + { + return session.getSystemProperty(SIMPLIFY_ARRAY_OPERATIONS, Boolean.class); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java index 3d13e18766c39..69cad15ce3c44 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorContextInstance.java @@ -19,6 +19,7 @@ import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.FilterStatsCalculatorService; import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; @@ -34,6 +35,7 @@ public class ConnectorContextInstance private final PageSorter pageSorter; private final PageIndexerFactory pageIndexerFactory; private final RowExpressionService rowExpressionService; + private final FilterStatsCalculatorService filterStatsCalculatorService; public ConnectorContextInstance( NodeManager nodeManager, @@ -42,7 +44,8 @@ public ConnectorContextInstance( StandardFunctionResolution functionResolution, PageSorter pageSorter, PageIndexerFactory pageIndexerFactory, - RowExpressionService rowExpressionService) + RowExpressionService rowExpressionService, + FilterStatsCalculatorService filterStatsCalculatorService) { this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); @@ -51,6 +54,7 @@ public ConnectorContextInstance( this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null"); + this.filterStatsCalculatorService = requireNonNull(filterStatsCalculatorService, "filterStatsCalculatorService is null"); } @Override @@ -94,4 +98,10 @@ public RowExpressionService getRowExpressionService() { return rowExpressionService; } + + @Override + public FilterStatsCalculatorService getFilterStatsCalculatorService() + { + return filterStatsCalculatorService; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java index 9b9db7152ca14..0fccacfcff7ee 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java @@ -13,12 +13,16 @@ */ package com.facebook.presto.connector; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.connector.informationSchema.InformationSchemaConnector; import com.facebook.presto.connector.system.DelegatingSystemTablesProvider; import com.facebook.presto.connector.system.MetadataBasedSystemTablesProvider; import com.facebook.presto.connector.system.StaticSystemTablesProvider; import com.facebook.presto.connector.system.SystemConnector; import com.facebook.presto.connector.system.SystemTablesProvider; +import com.facebook.presto.cost.ConnectorFilterStatsCalculatorService; +import com.facebook.presto.cost.FilterStatsCalculator; import com.facebook.presto.index.IndexManager; import com.facebook.presto.metadata.Catalog; import com.facebook.presto.metadata.CatalogManager; @@ -53,15 +57,14 @@ import com.facebook.presto.split.RecordPageSourceProvider; import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; -import com.facebook.presto.sql.planner.NodePartitioningManager; +import com.facebook.presto.sql.planner.PartitioningProviderManager; +import com.facebook.presto.sql.planner.planPrinter.RowExpressionFormatter; import com.facebook.presto.sql.relational.ConnectorRowExpressionService; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; -import io.airlift.node.NodeInfo; import javax.annotation.PreDestroy; import javax.annotation.concurrent.GuardedBy; @@ -94,7 +97,7 @@ public class ConnectorManager private final SplitManager splitManager; private final PageSourceManager pageSourceManager; private final IndexManager indexManager; - private final NodePartitioningManager nodePartitioningManager; + private final PartitioningProviderManager partitioningProviderManager; private final ConnectorPlanOptimizerManager connectorPlanOptimizerManager; private final PageSinkManager pageSinkManager; @@ -108,6 +111,7 @@ public class ConnectorManager private final DomainTranslator domainTranslator; private final PredicateCompiler predicateCompiler; private final DeterminismEvaluator determinismEvaluator; + private final FilterStatsCalculator filterStatsCalculator; @GuardedBy("this") private final ConcurrentMap connectorFactories = new ConcurrentHashMap<>(); @@ -125,7 +129,7 @@ public ConnectorManager( SplitManager splitManager, PageSourceManager pageSourceManager, IndexManager indexManager, - NodePartitioningManager nodePartitioningManager, + PartitioningProviderManager partitioningProviderManager, ConnectorPlanOptimizerManager connectorPlanOptimizerManager, PageSinkManager pageSinkManager, HandleResolver handleResolver, @@ -137,27 +141,29 @@ public ConnectorManager( TransactionManager transactionManager, DomainTranslator domainTranslator, PredicateCompiler predicateCompiler, - DeterminismEvaluator determinismEvaluator) + DeterminismEvaluator determinismEvaluator, + FilterStatsCalculator filterStatsCalculator) { - this.metadataManager = metadataManager; - this.catalogManager = catalogManager; - this.accessControlManager = accessControlManager; - this.splitManager = splitManager; - this.pageSourceManager = pageSourceManager; - this.indexManager = indexManager; - this.nodePartitioningManager = nodePartitioningManager; - this.connectorPlanOptimizerManager = connectorPlanOptimizerManager; - this.pageSinkManager = pageSinkManager; - this.handleResolver = handleResolver; - this.nodeManager = nodeManager; - this.typeManager = typeManager; - this.pageSorter = pageSorter; - this.pageIndexerFactory = pageIndexerFactory; - this.nodeInfo = nodeInfo; - this.transactionManager = transactionManager; - this.domainTranslator = domainTranslator; - this.predicateCompiler = predicateCompiler; - this.determinismEvaluator = determinismEvaluator; + this.metadataManager = requireNonNull(metadataManager, "metadataManager is null"); + this.catalogManager = requireNonNull(catalogManager, "catalogManager is null"); + this.accessControlManager = requireNonNull(accessControlManager, "accessControlManager is null"); + this.splitManager = requireNonNull(splitManager, "splitManager is null"); + this.pageSourceManager = requireNonNull(pageSourceManager, "pageSourceManager is null"); + this.indexManager = requireNonNull(indexManager, "indexManager is null"); + this.partitioningProviderManager = requireNonNull(partitioningProviderManager, "partitioningProviderManager is null"); + this.connectorPlanOptimizerManager = requireNonNull(connectorPlanOptimizerManager, "connectorPlanOptimizerManager is null"); + this.pageSinkManager = requireNonNull(pageSinkManager, "pageSinkManager is null"); + this.handleResolver = requireNonNull(handleResolver, "handleResolver is null"); + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); + this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); + this.nodeInfo = requireNonNull(nodeInfo, "nodeInfo is null"); + this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); + this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); + this.predicateCompiler = requireNonNull(predicateCompiler, "predicateCompiler is null"); + this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); + this.filterStatsCalculator = requireNonNull(filterStatsCalculator, "filterStatsCalculator is null"); } @PreDestroy @@ -279,7 +285,7 @@ private synchronized void addConnectorInternal(MaterializedConnector connector) .ifPresent(indexProvider -> indexManager.addIndexProvider(connectorId, indexProvider)); connector.getPartitioningProvider() - .ifPresent(partitioningProvider -> nodePartitioningManager.addPartitioningProvider(connectorId, partitioningProvider)); + .ifPresent(partitioningProvider -> partitioningProviderManager.addPartitioningProvider(connectorId, partitioningProvider)); if (nodeManager.getCurrentNode().isCoordinator()) { connector.getPlanOptimizerProvider() @@ -316,7 +322,7 @@ private synchronized void removeConnectorInternal(ConnectorId connectorId) pageSourceManager.removeConnectorPageSourceProvider(connectorId); pageSinkManager.removeConnectorPageSinkProvider(connectorId); indexManager.removeIndexProvider(connectorId); - nodePartitioningManager.removePartitioningProvider(connectorId); + partitioningProviderManager.removePartitioningProvider(connectorId); metadataManager.getProcedureRegistry().removeProcedures(connectorId); accessControlManager.removeCatalogAccessControl(connectorId); metadataManager.getTablePropertyManager().removeProperties(connectorId); @@ -346,7 +352,13 @@ private Connector createConnector(ConnectorId connectorId, ConnectorFactory fact new FunctionResolution(metadataManager.getFunctionManager()), pageSorter, pageIndexerFactory, - new ConnectorRowExpressionService(domainTranslator, new RowExpressionOptimizer(metadataManager), predicateCompiler, determinismEvaluator)); + new ConnectorRowExpressionService( + domainTranslator, + new RowExpressionOptimizer(metadataManager), + predicateCompiler, + determinismEvaluator, + new RowExpressionFormatter(metadataManager.getFunctionManager())), + new ConnectorFilterStatsCalculatorService(filterStatsCalculator)); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(factory.getClass().getClassLoader())) { return factory.create(connectorId.getCatalogName(), properties, context); diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/GlobalSystemConnector.java b/presto-main/src/main/java/com/facebook/presto/connector/system/GlobalSystemConnector.java index daaea49eadd28..7b47ae6394972 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/GlobalSystemConnector.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/GlobalSystemConnector.java @@ -15,7 +15,9 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.ConnectorTableHandle; import com.facebook.presto.spi.ConnectorTableLayout; import com.facebook.presto.spi.ConnectorTableLayoutHandle; @@ -138,8 +140,12 @@ public ConnectorSplitManager getSplitManager() @Override public ConnectorPageSourceProvider getPageSourceProvider() { - return (transactionHandle, session, split, columns) -> { - throw new UnsupportedOperationException(); + return new ConnectorPageSourceProvider() { + @Override + public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, ConnectorTableLayoutHandle layout, List columns) + { + throw new UnsupportedOperationException(); + } }; } diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemConnectorSessionUtil.java b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemConnectorSessionUtil.java index cc3607d3d244a..3cf5998a653f8 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/SystemConnectorSessionUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/SystemConnectorSessionUtil.java @@ -21,11 +21,8 @@ import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.security.ConnectorIdentity; import com.facebook.presto.spi.security.Identity; -import com.facebook.presto.sql.SqlPath; import com.facebook.presto.transaction.TransactionId; -import java.util.Optional; - public final class SystemConnectorSessionUtil { private static final SystemSessionProperties SYSTEM_SESSION_PROPERTIES = new SystemSessionProperties(); @@ -43,7 +40,6 @@ public static Session toSession(ConnectorTransactionHandle transactionHandle, Co .setTransactionId(transactionId) .setCatalog("catalog") .setSchema("schema") - .setPath(new SqlPath(Optional.of("path"))) .setIdentity(identity) .setTimeZoneKey(session.getTimeZoneKey()) .setLocale(session.getLocale()) diff --git a/presto-main/src/main/java/com/facebook/presto/connector/system/TaskSystemTable.java b/presto-main/src/main/java/com/facebook/presto/connector/system/TaskSystemTable.java index fa08ad5864011..ff8cf89392c0c 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/system/TaskSystemTable.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/system/TaskSystemTable.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.system; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.execution.TaskInfo; import com.facebook.presto.execution.TaskManager; import com.facebook.presto.execution.TaskStatus; @@ -26,7 +27,6 @@ import com.facebook.presto.spi.SystemTable; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.predicate.TupleDomain; -import io.airlift.node.NodeInfo; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -48,6 +48,7 @@ public class TaskSystemTable .column("node_id", createUnboundedVarcharType()) .column("task_id", createUnboundedVarcharType()) + .column("stage_execution_id", createUnboundedVarcharType()) .column("stage_id", createUnboundedVarcharType()) .column("query_id", createUnboundedVarcharType()) .column("state", createUnboundedVarcharType()) @@ -111,7 +112,8 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect nodeId, taskStatus.getTaskId().toString(), - taskStatus.getTaskId().getStageId().toString(), + taskStatus.getTaskId().getStageExecutionId().toString(), + taskStatus.getTaskId().getStageExecutionId().getStageId().toString(), taskStatus.getTaskId().getQueryId().toString(), taskStatus.getState().toString(), diff --git a/presto-main/src/main/java/com/facebook/presto/cost/AggregationStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/AggregationStatsRule.java index dbd7016c64c8c..866eb21b09cac 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/AggregationStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/AggregationStatsRule.java @@ -15,17 +15,17 @@ import com.facebook.presto.Session; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Lookup; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import java.util.Collection; import java.util.Map; import java.util.Optional; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; import static java.lang.Math.min; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/cost/CachingCostProvider.java b/presto-main/src/main/java/com/facebook/presto/cost/CachingCostProvider.java index caf980a922582..400ca281a3bcc 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/CachingCostProvider.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/CachingCostProvider.java @@ -13,12 +13,11 @@ */ package com.facebook.presto.cost; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.GroupReference; import com.facebook.presto.sql.planner.iterative.Memo; -import io.airlift.log.Logger; import java.util.IdentityHashMap; import java.util.Map; @@ -38,22 +37,20 @@ public class CachingCostProvider private final StatsProvider statsProvider; private final Optional memo; private final Session session; - private final TypeProvider types; private final Map cache = new IdentityHashMap<>(); - public CachingCostProvider(CostCalculator costCalculator, StatsProvider statsProvider, Session session, TypeProvider types) + public CachingCostProvider(CostCalculator costCalculator, StatsProvider statsProvider, Session session) { - this(costCalculator, statsProvider, Optional.empty(), session, types); + this(costCalculator, statsProvider, Optional.empty(), session); } - public CachingCostProvider(CostCalculator costCalculator, StatsProvider statsProvider, Optional memo, Session session, TypeProvider types) + public CachingCostProvider(CostCalculator costCalculator, StatsProvider statsProvider, Optional memo, Session session) { this.costCalculator = requireNonNull(costCalculator, "costCalculator is null"); this.statsProvider = requireNonNull(statsProvider, "statsProvider is null"); this.memo = requireNonNull(memo, "memo is null"); this.session = requireNonNull(session, "session is null"); - this.types = requireNonNull(types, "types is null"); } @Override @@ -106,6 +103,6 @@ private PlanCostEstimate getGroupCost(GroupReference groupReference) private PlanCostEstimate calculateCost(PlanNode node) { - return costCalculator.calculateCost(node, statsProvider, this, session, types); + return costCalculator.calculateCost(node, statsProvider, this, session); } } diff --git a/presto-main/src/main/java/com/facebook/presto/cost/CachingStatsProvider.java b/presto-main/src/main/java/com/facebook/presto/cost/CachingStatsProvider.java index d2c6e68c743a7..366de73026f29 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/CachingStatsProvider.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/CachingStatsProvider.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.cost; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.GroupReference; import com.facebook.presto.sql.planner.iterative.Lookup; import com.facebook.presto.sql.planner.iterative.Memo; -import io.airlift.log.Logger; import java.util.IdentityHashMap; import java.util.Map; diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ConnectorFilterStatsCalculatorService.java b/presto-main/src/main/java/com/facebook/presto/cost/ConnectorFilterStatsCalculatorService.java new file mode 100644 index 0000000000000..c801abbb4eb4a --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/cost/ConnectorFilterStatsCalculatorService.java @@ -0,0 +1,123 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.cost; + +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.plan.FilterStatsCalculatorService; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.statistics.ColumnStatistics; +import com.facebook.presto.spi.statistics.ColumnStatistics.Builder; +import com.facebook.presto.spi.statistics.DoubleRange; +import com.facebook.presto.spi.statistics.Estimate; +import com.facebook.presto.spi.statistics.TableStatistics; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableBiMap; + +import java.util.Map; + +import static com.facebook.presto.cost.FilterStatsCalculator.UNKNOWN_FILTER_COEFFICIENT; +import static com.facebook.presto.cost.StatsUtil.toVariableStatsEstimate; +import static java.util.Objects.requireNonNull; + +public class ConnectorFilterStatsCalculatorService + implements FilterStatsCalculatorService +{ + private final FilterStatsCalculator filterStatsCalculator; + + public ConnectorFilterStatsCalculatorService(FilterStatsCalculator filterStatsCalculator) + { + this.filterStatsCalculator = requireNonNull(filterStatsCalculator, "filterStatsCalculator is null"); + } + + @Override + public TableStatistics filterStats( + TableStatistics tableStatistics, + RowExpression predicate, + ConnectorSession session, + Map columnNames, + Map columnTypes) + { + PlanNodeStatsEstimate tableStats = toPlanNodeStats(tableStatistics, columnNames, columnTypes); + PlanNodeStatsEstimate filteredStats = filterStatsCalculator.filterStats(tableStats, predicate, session); + + if (filteredStats.isOutputRowCountUnknown()) { + filteredStats = tableStats.mapOutputRowCount(sourceRowCount -> tableStats.getOutputRowCount() * UNKNOWN_FILTER_COEFFICIENT); + } + + return toTableStatistics(filteredStats, ImmutableBiMap.copyOf(columnNames).inverse()); + } + + private static PlanNodeStatsEstimate toPlanNodeStats( + TableStatistics tableStatistics, + Map columnNames, + Map columnTypes) + { + PlanNodeStatsEstimate.Builder builder = PlanNodeStatsEstimate.builder() + .setOutputRowCount(tableStatistics.getRowCount().getValue()); + + for (Map.Entry entry : tableStatistics.getColumnStatistics().entrySet()) { + String columnName = columnNames.get(entry.getKey()); + VariableReferenceExpression variable = new VariableReferenceExpression(columnName, columnTypes.get(columnName)); + builder.addVariableStatistics(variable, toVariableStatsEstimate(tableStatistics, entry.getValue())); + } + return builder.build(); + } + + private static TableStatistics toTableStatistics(PlanNodeStatsEstimate planNodeStats, Map columnByName) + { + TableStatistics.Builder builder = TableStatistics.builder(); + if (planNodeStats.isOutputRowCountUnknown()) { + builder.setRowCount(Estimate.unknown()); + return builder.build(); + } + + double rowCount = planNodeStats.getOutputRowCount(); + builder.setRowCount(Estimate.of(rowCount)); + for (Map.Entry entry : planNodeStats.getVariableStatistics().entrySet()) { + builder.setColumnStatistics(columnByName.get(entry.getKey().getName()), toColumnStatistics(entry.getValue(), rowCount)); + } + return builder.build(); + } + + private static ColumnStatistics toColumnStatistics(VariableStatsEstimate variableStatsEstimate, double rowCount) + { + if (variableStatsEstimate.isUnknown()) { + return ColumnStatistics.empty(); + } + + double nullsFractionDouble = variableStatsEstimate.getNullsFraction(); + double nonNullRowsCount = rowCount * (1.0 - nullsFractionDouble); + + Builder builder = ColumnStatistics.builder(); + if (!Double.isNaN(nullsFractionDouble)) { + builder.setNullsFraction(Estimate.of(nullsFractionDouble)); + } + + if (!Double.isNaN(variableStatsEstimate.getDistinctValuesCount())) { + builder.setDistinctValuesCount(Estimate.of(variableStatsEstimate.getDistinctValuesCount())); + } + + if (!Double.isNaN(variableStatsEstimate.getAverageRowSize())) { + builder.setDataSize(Estimate.of(variableStatsEstimate.getAverageRowSize() * nonNullRowsCount)); + } + + if (!Double.isNaN(variableStatsEstimate.getLowValue()) && !Double.isNaN(variableStatsEstimate.getHighValue())) { + builder.setRange(new DoubleRange(variableStatsEstimate.getLowValue(), variableStatsEstimate.getHighValue())); + } + return builder.build(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/cost/CostCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/CostCalculator.java index 9aa7d152af0da..e681c26ce7f52 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/CostCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/CostCalculator.java @@ -16,7 +16,6 @@ import com.facebook.presto.Session; import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.sql.planner.TypeProvider; import com.google.inject.BindingAnnotation; import javax.annotation.concurrent.ThreadSafe; @@ -32,7 +31,6 @@ public interface CostCalculator { /** * Calculates cumulative cost of a node. - * * @param node The node to compute cost for. * @param stats The stats provider for node's stats and child nodes' stats, to be used if stats are needed to compute cost for the {@code node} */ @@ -40,8 +38,7 @@ PlanCostEstimate calculateCost( PlanNode node, StatsProvider stats, CostProvider sourcesCosts, - Session session, - TypeProvider types); + Session session); @BindingAnnotation @Target(PARAMETER) diff --git a/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorUsingExchanges.java b/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorUsingExchanges.java index 8d051546336da..c693887e85f93 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorUsingExchanges.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorUsingExchanges.java @@ -15,27 +15,26 @@ package com.facebook.presto.cost; import com.facebook.presto.Session; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.GroupReference; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.google.common.collect.ImmutableList; import javax.annotation.concurrent.ThreadSafe; @@ -52,8 +51,8 @@ import static com.facebook.presto.cost.CostCalculatorWithEstimatedExchanges.calculateRemoteRepartitionCost; import static com.facebook.presto.cost.CostCalculatorWithEstimatedExchanges.calculateRemoteReplicateCost; import static com.facebook.presto.cost.LocalCostEstimate.addPartialComponents; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.FINAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.Math.max; @@ -75,9 +74,9 @@ public CostCalculatorUsingExchanges(TaskCountEstimator taskCountEstimator) } @Override - public PlanCostEstimate calculateCost(PlanNode node, StatsProvider stats, CostProvider sourcesCosts, Session session, TypeProvider types) + public PlanCostEstimate calculateCost(PlanNode node, StatsProvider stats, CostProvider sourcesCosts, Session session) { - CostEstimator costEstimator = new CostEstimator(stats, sourcesCosts, types, taskCountEstimator); + CostEstimator costEstimator = new CostEstimator(stats, sourcesCosts, taskCountEstimator); return node.accept(costEstimator, null); } @@ -86,14 +85,12 @@ private static class CostEstimator { private final StatsProvider stats; private final CostProvider sourcesCosts; - private final TypeProvider types; private final TaskCountEstimator taskCountEstimator; - CostEstimator(StatsProvider stats, CostProvider sourcesCosts, TypeProvider types, TaskCountEstimator taskCountEstimator) + CostEstimator(StatsProvider stats, CostProvider sourcesCosts, TaskCountEstimator taskCountEstimator) { this.stats = requireNonNull(stats, "stats is null"); this.sourcesCosts = requireNonNull(sourcesCosts, "sourcesCosts is null"); - this.types = requireNonNull(types, "types is null"); this.taskCountEstimator = requireNonNull(taskCountEstimator, "taskCountEstimator is null"); } @@ -195,7 +192,6 @@ private LocalCostEstimate calculateJoinCost(PlanNode join, PlanNode probe, PlanN probe, build, stats, - types, replicated, taskCountEstimator.estimateSourceDistributedTaskCount()); LocalCostEstimate joinOutputCost = calculateJoinOutputCost(join); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorWithEstimatedExchanges.java b/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorWithEstimatedExchanges.java index f2f1b0ae40803..0cede994d16ae 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorWithEstimatedExchanges.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/CostCalculatorWithEstimatedExchanges.java @@ -15,17 +15,16 @@ package com.facebook.presto.cost; import com.facebook.presto.Session; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.sql.planner.iterative.GroupReference; import com.facebook.presto.sql.planner.iterative.rule.DetermineJoinDistributionType; import com.facebook.presto.sql.planner.iterative.rule.ReorderJoins; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import javax.annotation.concurrent.ThreadSafe; import javax.inject.Inject; @@ -58,10 +57,10 @@ public CostCalculatorWithEstimatedExchanges(CostCalculator costCalculator, TaskC } @Override - public PlanCostEstimate calculateCost(PlanNode node, StatsProvider stats, CostProvider sourcesCosts, Session session, TypeProvider types) + public PlanCostEstimate calculateCost(PlanNode node, StatsProvider stats, CostProvider sourcesCosts, Session session) { - ExchangeCostEstimator exchangeCostEstimator = new ExchangeCostEstimator(stats, types, taskCountEstimator); - PlanCostEstimate costEstimate = costCalculator.calculateCost(node, stats, sourcesCosts, session, types); + ExchangeCostEstimator exchangeCostEstimator = new ExchangeCostEstimator(stats, taskCountEstimator); + PlanCostEstimate costEstimate = costCalculator.calculateCost(node, stats, sourcesCosts, session); LocalCostEstimate estimatedExchangeCost = node.accept(exchangeCostEstimator, null); return addExchangeCost(costEstimate, estimatedExchangeCost); } @@ -87,13 +86,11 @@ private static class ExchangeCostEstimator extends InternalPlanVisitor { private final StatsProvider stats; - private final TypeProvider types; private final TaskCountEstimator taskCountEstimator; - ExchangeCostEstimator(StatsProvider stats, TypeProvider types, TaskCountEstimator taskCountEstimator) + ExchangeCostEstimator(StatsProvider stats, TaskCountEstimator taskCountEstimator) { this.stats = requireNonNull(stats, "stats is null"); - this.types = requireNonNull(types, "types is null"); this.taskCountEstimator = requireNonNull(taskCountEstimator, "taskCountEstimator is null"); } @@ -130,7 +127,6 @@ public LocalCostEstimate visitJoin(JoinNode node, Void context) node.getLeft(), node.getRight(), stats, - types, Objects.equals(node.getDistributionType(), Optional.of(JoinNode.DistributionType.REPLICATED)), taskCountEstimator.estimateSourceDistributedTaskCount()); } @@ -142,7 +138,6 @@ public LocalCostEstimate visitSemiJoin(SemiJoinNode node, Void context) node.getSource(), node.getFilteringSource(), stats, - types, Objects.equals(node.getDistributionType(), Optional.of(SemiJoinNode.DistributionType.REPLICATED)), taskCountEstimator.estimateSourceDistributedTaskCount()); } @@ -154,7 +149,6 @@ public LocalCostEstimate visitSpatialJoin(SpatialJoinNode node, Void context) node.getLeft(), node.getRight(), stats, - types, node.getDistributionType() == SpatialJoinNode.DistributionType.REPLICATED, taskCountEstimator.estimateSourceDistributedTaskCount()); } @@ -200,7 +194,6 @@ public static LocalCostEstimate calculateJoinCostWithoutOutput( PlanNode probe, PlanNode build, StatsProvider stats, - TypeProvider types, boolean replicated, int estimatedSourceDistributedTaskCount) { @@ -208,14 +201,12 @@ public static LocalCostEstimate calculateJoinCostWithoutOutput( probe, build, stats, - types, replicated, estimatedSourceDistributedTaskCount); LocalCostEstimate inputCost = calculateJoinInputCost( probe, build, stats, - types, replicated, estimatedSourceDistributedTaskCount); return addPartialComponents(exchangesCost, inputCost); @@ -225,7 +216,6 @@ private static LocalCostEstimate calculateJoinExchangeCost( PlanNode probe, PlanNode build, StatsProvider stats, - TypeProvider types, boolean replicated, int estimatedSourceDistributedTaskCount) { @@ -250,7 +240,6 @@ public static LocalCostEstimate calculateJoinInputCost( PlanNode probe, PlanNode build, StatsProvider stats, - TypeProvider types, boolean replicated, int estimatedSourceDistributedTaskCount) { diff --git a/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java index 447870b6cf07e..2cc7c2436c2e8 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java @@ -17,6 +17,7 @@ import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.relation.CallExpression; @@ -33,7 +34,7 @@ import com.facebook.presto.sql.planner.ExpressionInterpreter; import com.facebook.presto.sql.planner.LiteralEncoder; import com.facebook.presto.sql.planner.LiteralInterpreter; -import com.facebook.presto.sql.planner.NoOpSymbolResolver; +import com.facebook.presto.sql.planner.NoOpVariableResolver; import com.facebook.presto.sql.planner.RowExpressionInterpreter; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.relational.FunctionResolution; @@ -56,6 +57,7 @@ import com.google.common.collect.ImmutableList; import javax.annotation.Nullable; +import javax.inject.Inject; import java.util.List; import java.util.Map; @@ -68,7 +70,8 @@ import static com.facebook.presto.cost.PlanNodeStatsEstimateMath.capStats; import static com.facebook.presto.cost.PlanNodeStatsEstimateMath.subtractSubsetStats; import static com.facebook.presto.cost.StatsUtil.toStatsRepresentation; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.and; +import static com.facebook.presto.expressions.LogicalRowExpressions.and; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IS_NULL; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.sql.ExpressionUtils.and; @@ -103,6 +106,7 @@ public class FilterStatsCalculator private final LiteralEncoder literalEncoder; private final FunctionResolution functionResolution; + @Inject public FilterStatsCalculator(Metadata metadata, ScalarStatsCalculator scalarStatsCalculator, StatsNormalizer normalizer) { this.metadata = requireNonNull(metadata, "metadata is null"); @@ -124,15 +128,21 @@ public PlanNodeStatsEstimate filterStats( .process(simplifiedExpression); } - // TODO: remove types once we have type info in PlanNodeStatsEstimate public PlanNodeStatsEstimate filterStats( PlanNodeStatsEstimate statsEstimate, RowExpression predicate, - Session session, - TypeProvider types) + ConnectorSession session) { RowExpression simplifiedExpression = simplifyExpression(session, predicate); - return new FilterRowExpressionStatsCalculatingVisitor(statsEstimate, session, metadata.getFunctionManager(), types).process(simplifiedExpression); + return new FilterRowExpressionStatsCalculatingVisitor(statsEstimate, session, metadata.getFunctionManager()).process(simplifiedExpression); + } + + public PlanNodeStatsEstimate filterStats( + PlanNodeStatsEstimate statsEstimate, + RowExpression predicate, + Session session) + { + return filterStats(statsEstimate, predicate, session.toConnectorSession()); } private Expression simplifyExpression(Session session, Expression predicate, TypeProvider types) @@ -141,7 +151,7 @@ private Expression simplifyExpression(Session session, Expression predicate, Typ Map, Type> expressionTypes = getExpressionTypes(session, predicate, types); ExpressionInterpreter interpreter = ExpressionInterpreter.expressionOptimizer(predicate, metadata, session, expressionTypes); - Object value = interpreter.optimize(NoOpSymbolResolver.INSTANCE); + Object value = interpreter.optimize(NoOpVariableResolver.INSTANCE); if (value == null) { // Expression evaluates to SQL null, which in Filter is equivalent to false. This assumes the expression is a top-level expression (eg. not in NOT). @@ -150,9 +160,9 @@ private Expression simplifyExpression(Session session, Expression predicate, Typ return literalEncoder.toExpression(value, BOOLEAN); } - private RowExpression simplifyExpression(Session session, RowExpression predicate) + private RowExpression simplifyExpression(ConnectorSession session, RowExpression predicate) { - RowExpressionInterpreter interpreter = new RowExpressionInterpreter(predicate, metadata, session.toConnectorSession(), true); + RowExpressionInterpreter interpreter = new RowExpressionInterpreter(predicate, metadata, session, OPTIMIZED); Object value = interpreter.optimize(); if (value == null) { @@ -480,16 +490,14 @@ private class FilterRowExpressionStatsCalculatingVisitor implements RowExpressionVisitor { private final PlanNodeStatsEstimate input; - private final Session session; + private final ConnectorSession session; private final FunctionManager functionManager; - private final TypeProvider types; - FilterRowExpressionStatsCalculatingVisitor(PlanNodeStatsEstimate input, Session session, FunctionManager functionManager, TypeProvider types) + FilterRowExpressionStatsCalculatingVisitor(PlanNodeStatsEstimate input, ConnectorSession session, FunctionManager functionManager) { this.input = requireNonNull(input, "input is null"); this.session = requireNonNull(session, "session is null"); this.functionManager = requireNonNull(functionManager, "functionManager is null"); - this.types = requireNonNull(types, "types is null"); } @Override @@ -575,7 +583,7 @@ public PlanNodeStatsEstimate visitCall(CallExpression node, Void context) if (rightValue == null) { return visitConstant(constantNull(BOOLEAN), null); } - OptionalDouble literal = toStatsRepresentation(metadata, session, right.getType(), rightValue); + OptionalDouble literal = toStatsRepresentation(metadata.getFunctionManager(), session, right.getType(), rightValue); return estimateExpressionToLiteralComparison(input, leftStats, leftVariable, literal, getComparisonOperator(operatorType)); } @@ -660,7 +668,7 @@ public PlanNodeStatsEstimate visitInputReference(InputReferenceExpression node, private FilterRowExpressionStatsCalculatingVisitor newEstimate(PlanNodeStatsEstimate input) { - return new FilterRowExpressionStatsCalculatingVisitor(input, session, functionManager, types); + return new FilterRowExpressionStatsCalculatingVisitor(input, session, functionManager); } private PlanNodeStatsEstimate process(RowExpression rowExpression) diff --git a/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsRule.java index 372763c4c69c4..a36bd8dfe62de 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/FilterStatsRule.java @@ -55,7 +55,7 @@ public Optional doCalculate(FilterNode node, StatsProvide estimate = filterStatsCalculator.filterStats(sourceStats, castToExpression(node.getPredicate()), session, types); } else { - estimate = filterStatsCalculator.filterStats(sourceStats, node.getPredicate(), session, types); + estimate = filterStatsCalculator.filterStats(sourceStats, node.getPredicate(), session); } if (isDefaultFilterFactorEnabled(session) && estimate.isOutputRowCountUnknown()) { estimate = sourceStats.mapOutputRowCount(sourceRowCount -> sourceStats.getOutputRowCount() * UNKNOWN_FILTER_COEFFICIENT); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/JoinStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/JoinStatsRule.java index aa51f7484ce1a..61d358decd828 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/JoinStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/JoinStatsRule.java @@ -14,8 +14,8 @@ package com.facebook.presto.cost; import com.facebook.presto.Session; +import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.matching.Pattern; -import com.facebook.presto.spi.relation.LogicalRowExpressions; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; @@ -160,7 +160,7 @@ private PlanNodeStatsEstimate computeInnerJoinStats(JoinNode node, PlanNodeStats return filterStatsCalculator.filterStats(crossJoinStats, castToExpression(node.getFilter().get()), session, types); } else { - return filterStatsCalculator.filterStats(crossJoinStats, node.getFilter().get(), session, types); + return filterStatsCalculator.filterStats(crossJoinStats, node.getFilter().get(), session); } } @@ -179,7 +179,7 @@ private PlanNodeStatsEstimate computeInnerJoinStats(JoinNode node, PlanNodeStats filteredEquiJoinEstimate = filterStatsCalculator.filterStats(equiJoinEstimate, castToExpression(node.getFilter().get()), session, types); } else { - filteredEquiJoinEstimate = filterStatsCalculator.filterStats(equiJoinEstimate, node.getFilter().get(), session, types); + filteredEquiJoinEstimate = filterStatsCalculator.filterStats(equiJoinEstimate, node.getFilter().get(), session); } if (filteredEquiJoinEstimate.isOutputRowCountUnknown()) { diff --git a/presto-main/src/main/java/com/facebook/presto/cost/LimitStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/LimitStatsRule.java index c8a4e6e166d9a..29479668357f0 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/LimitStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/LimitStatsRule.java @@ -15,9 +15,9 @@ import com.facebook.presto.Session; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Lookup; -import com.facebook.presto.sql.planner.plan.LimitNode; import java.util.Optional; diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ProjectStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/ProjectStatsRule.java index ea2406ef7a87f..3f9fa4665d303 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ProjectStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ProjectStatsRule.java @@ -15,11 +15,11 @@ import com.facebook.presto.Session; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Lookup; -import com.facebook.presto.sql.planner.plan.ProjectNode; import java.util.Map; import java.util.Optional; diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java index 59d21878bc578..927254e5908ec 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.relation.CallExpression; @@ -31,10 +32,10 @@ import com.facebook.presto.sql.analyzer.ExpressionAnalyzer; import com.facebook.presto.sql.analyzer.Scope; import com.facebook.presto.sql.planner.ExpressionInterpreter; -import com.facebook.presto.sql.planner.NoOpSymbolResolver; -import com.facebook.presto.sql.planner.RowExpressionInterpreter; +import com.facebook.presto.sql.planner.NoOpVariableResolver; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; import com.facebook.presto.sql.tree.ArithmeticUnaryExpression; import com.facebook.presto.sql.tree.AstVisitor; @@ -58,8 +59,10 @@ import static com.facebook.presto.cost.StatsUtil.toStatsRepresentation; import static com.facebook.presto.spi.function.OperatorType.DIVIDE; import static com.facebook.presto.spi.function.OperatorType.MODULUS; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.COALESCE; import static com.facebook.presto.sql.planner.LiteralInterpreter.evaluate; +import static com.facebook.presto.sql.relational.Expressions.isNull; import static com.facebook.presto.util.MoreMath.max; import static com.facebook.presto.util.MoreMath.min; import static com.google.common.base.Preconditions.checkState; @@ -88,6 +91,11 @@ public VariableStatsEstimate calculate(Expression scalarExpression, PlanNodeStat } public VariableStatsEstimate calculate(RowExpression scalarExpression, PlanNodeStatsEstimate inputStatistics, Session session) + { + return scalarExpression.accept(new RowExpressionStatsVisitor(inputStatistics, session.toConnectorSession()), null); + } + + public VariableStatsEstimate calculate(RowExpression scalarExpression, PlanNodeStatsEstimate inputStatistics, ConnectorSession session) { return scalarExpression.accept(new RowExpressionStatsVisitor(inputStatistics, session), null); } @@ -96,10 +104,10 @@ private class RowExpressionStatsVisitor implements RowExpressionVisitor { private final PlanNodeStatsEstimate input; - private final Session session; + private final ConnectorSession session; private final FunctionResolution resolution = new FunctionResolution(metadata.getFunctionManager()); - public RowExpressionStatsVisitor(PlanNodeStatsEstimate input, Session session) + public RowExpressionStatsVisitor(PlanNodeStatsEstimate input, ConnectorSession session) { this.input = requireNonNull(input, "input is null"); this.session = requireNonNull(session, "session is null"); @@ -108,10 +116,6 @@ public RowExpressionStatsVisitor(PlanNodeStatsEstimate input, Session session) @Override public VariableStatsEstimate visitCall(CallExpression call, Void context) { - if (resolution.isCastFunction(call.getFunctionHandle())) { - return computeCastStatistics(call, context); - } - if (resolution.isNegateFunction(call.getFunctionHandle())) { return computeNegationStatistics(call, context); } @@ -121,22 +125,21 @@ public VariableStatsEstimate visitCall(CallExpression call, Void context) return computeArithmeticBinaryStatistics(call, context); } - Object value = new RowExpressionInterpreter(call, metadata, session.toConnectorSession(), true).optimize(); + RowExpression value = new RowExpressionOptimizer(metadata).optimize(call, OPTIMIZED, session); - if (value == null) { + if (isNull(value)) { return nullStatsEstimate(); } - if (value instanceof RowExpression) { - // value is not a constant - return VariableStatsEstimate.unknown(); + if (value instanceof ConstantExpression) { + return value.accept(this, context); } - // value is a constant - return VariableStatsEstimate.builder() - .setNullsFraction(0) - .setDistinctValuesCount(1) - .build(); + // value is not a constant but we can still propagate estimation through cast + if (resolution.isCastFunction(call.getFunctionHandle())) { + return computeCastStatistics(call, context); + } + return VariableStatsEstimate.unknown(); } @Override @@ -152,7 +155,7 @@ public VariableStatsEstimate visitConstant(ConstantExpression literal, Void cont return nullStatsEstimate(); } - OptionalDouble doubleValue = toStatsRepresentation(metadata, session, literal.getType(), literal.getValue()); + OptionalDouble doubleValue = toStatsRepresentation(metadata.getFunctionManager(), session, literal.getType(), literal.getValue()); VariableStatsEstimate.Builder estimate = VariableStatsEstimate.builder() .setNullsFraction(0) .setDistinctValuesCount(1); @@ -370,7 +373,7 @@ protected VariableStatsEstimate visitFunctionCall(FunctionCall node, Void contex { Map, Type> expressionTypes = getExpressionTypes(session, node, types); ExpressionInterpreter interpreter = ExpressionInterpreter.expressionOptimizer(node, metadata, session, expressionTypes); - Object value = interpreter.optimize(NoOpSymbolResolver.INSTANCE); + Object value = interpreter.optimize(NoOpVariableResolver.INSTANCE); if (value == null || value instanceof NullLiteral) { return nullStatsEstimate(); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/SimpleFilterProjectSemiJoinStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/SimpleFilterProjectSemiJoinStatsRule.java index e7357b4ae717c..0c885f20cff3e 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/SimpleFilterProjectSemiJoinStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/SimpleFilterProjectSemiJoinStatsRule.java @@ -14,17 +14,17 @@ package com.facebook.presto.cost; import com.facebook.presto.Session; +import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; -import com.facebook.presto.spi.relation.LogicalRowExpressions; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Lookup; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; @@ -68,7 +68,7 @@ public SimpleFilterProjectSemiJoinStatsRule(StatsNormalizer normalizer, FilterSt super(normalizer); this.filterStatsCalculator = requireNonNull(filterStatsCalculator, "filterStatsCalculator can not be null"); requireNonNull(functionManager, "functionManager can not be null"); - this.logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(functionManager), new FunctionResolution(functionManager)); + this.logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(functionManager), new FunctionResolution(functionManager), functionManager); this.functionResolution = new FunctionResolution(functionManager); } @@ -142,7 +142,7 @@ private Optional calculate(FilterNode filterNode, SemiJoi filteredStats = filterStatsCalculator.filterStats(semiJoinStats, castToExpression(semiJoinOutputFilter.get().getRemainingPredicate()), session, types); } else { - filteredStats = filterStatsCalculator.filterStats(semiJoinStats, semiJoinOutputFilter.get().getRemainingPredicate(), session, types); + filteredStats = filterStatsCalculator.filterStats(semiJoinStats, semiJoinOutputFilter.get().getRemainingPredicate(), session); } if (filteredStats.isOutputRowCountUnknown()) { diff --git a/presto-main/src/main/java/com/facebook/presto/cost/SpatialJoinStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/SpatialJoinStatsRule.java index fb73ab05463ad..e5eb26931b257 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/SpatialJoinStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/SpatialJoinStatsRule.java @@ -52,7 +52,7 @@ protected Optional doCalculate(SpatialJoinNode node, Stat return Optional.of(statsCalculator.filterStats(crossJoinStats, castToExpression(node.getFilter()), session, types)); } else { - return Optional.of(statsCalculator.filterStats(crossJoinStats, node.getFilter(), session, types)); + return Optional.of(statsCalculator.filterStats(crossJoinStats, node.getFilter(), session)); } case LEFT: return Optional.of(PlanNodeStatsEstimate.unknown()); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/StatsCalculatorModule.java b/presto-main/src/main/java/com/facebook/presto/cost/StatsCalculatorModule.java index 51d32a4965368..f002089b789f8 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/StatsCalculatorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/StatsCalculatorModule.java @@ -18,6 +18,7 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; +import com.google.inject.Scopes; import javax.inject.Singleton; @@ -27,16 +28,19 @@ public class StatsCalculatorModule @Override public void configure(Binder binder) { + binder.bind(ScalarStatsCalculator.class).in(Scopes.SINGLETON); + binder.bind(StatsNormalizer.class).in(Scopes.SINGLETON); + binder.bind(FilterStatsCalculator.class).in(Scopes.SINGLETON); } @Provides @Singleton - public static StatsCalculator createNewStatsCalculator(Metadata metadata) + public static StatsCalculator createNewStatsCalculator( + Metadata metadata, + ScalarStatsCalculator scalarStatsCalculator, + StatsNormalizer normalizer, + FilterStatsCalculator filterStatsCalculator) { - StatsNormalizer normalizer = new StatsNormalizer(); - ScalarStatsCalculator scalarStatsCalculator = new ScalarStatsCalculator(metadata); - FilterStatsCalculator filterStatsCalculator = new FilterStatsCalculator(metadata, scalarStatsCalculator, normalizer); - ImmutableList.Builder> rules = ImmutableList.builder(); rules.add(new OutputStatsRule()); rules.add(new TableScanStatsRule(metadata, normalizer)); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/StatsUtil.java b/presto-main/src/main/java/com/facebook/presto/cost/StatsUtil.java index 6ca66e2600aa3..d1a36cd985f5a 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/StatsUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/StatsUtil.java @@ -18,6 +18,8 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.statistics.ColumnStatistics; +import com.facebook.presto.spi.statistics.TableStatistics; import com.facebook.presto.spi.type.BigintType; import com.facebook.presto.spi.type.BooleanType; import com.facebook.presto.spi.type.DateType; @@ -74,4 +76,20 @@ private static boolean convertibleToDoubleWithCast(Type type) || TinyintType.TINYINT.equals(type) || BooleanType.BOOLEAN.equals(type); } + + public static VariableStatsEstimate toVariableStatsEstimate(TableStatistics tableStatistics, ColumnStatistics columnStatistics) + { + double nullsFraction = columnStatistics.getNullsFraction().getValue(); + double nonNullRowsCount = tableStatistics.getRowCount().getValue() * (1.0 - nullsFraction); + double averageRowSize = nonNullRowsCount == 0 ? 0 : columnStatistics.getDataSize().getValue() / nonNullRowsCount; + VariableStatsEstimate.Builder result = VariableStatsEstimate.builder(); + result.setNullsFraction(nullsFraction); + result.setDistinctValuesCount(columnStatistics.getDistinctValuesCount().getValue()); + result.setAverageRowSize(averageRowSize); + columnStatistics.getRange().ifPresent(range -> { + result.setLowValue(range.getMin()); + result.setHighValue(range.getMax()); + }); + return result.build(); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/cost/TableScanStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/TableScanStatsRule.java index 165cd6e13c38f..468b36dcbc5dd 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/TableScanStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/TableScanStatsRule.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.statistics.TableStatistics; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Lookup; +import com.google.common.collect.ImmutableList; import java.util.HashMap; import java.util.Map; @@ -57,12 +58,12 @@ protected Optional doCalculate(TableScanNode node, StatsP // TODO Construct predicate like AddExchanges's LayoutConstraintEvaluator Constraint constraint = new Constraint<>(node.getCurrentConstraint()); - TableStatistics tableStatistics = metadata.getTableStatistics(session, node.getTable(), constraint); + TableStatistics tableStatistics = metadata.getTableStatistics(session, node.getTable(), ImmutableList.copyOf(node.getAssignments().values()), constraint); Map outputVariableStats = new HashMap<>(); for (Map.Entry entry : node.getAssignments().entrySet()) { Optional columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getValue())); - outputVariableStats.put(entry.getKey(), columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics)).orElse(VariableStatsEstimate.unknown())); + outputVariableStats.put(entry.getKey(), columnStatistics.map(statistics -> StatsUtil.toVariableStatsEstimate(tableStatistics, statistics)).orElse(VariableStatsEstimate.unknown())); } return Optional.of(PlanNodeStatsEstimate.builder() @@ -70,20 +71,4 @@ protected Optional doCalculate(TableScanNode node, StatsP .addVariableStatistics(outputVariableStats) .build()); } - - private VariableStatsEstimate toSymbolStatistics(TableStatistics tableStatistics, ColumnStatistics columnStatistics) - { - double nullsFraction = columnStatistics.getNullsFraction().getValue(); - double nonNullRowsCount = tableStatistics.getRowCount().getValue() * (1.0 - nullsFraction); - double averageRowSize = nonNullRowsCount == 0 ? 0 : columnStatistics.getDataSize().getValue() / nonNullRowsCount; - VariableStatsEstimate.Builder result = VariableStatsEstimate.builder(); - result.setNullsFraction(nullsFraction); - result.setDistinctValuesCount(columnStatistics.getDistinctValuesCount().getValue()); - result.setAverageRowSize(averageRowSize); - columnStatistics.getRange().ifPresent(range -> { - result.setLowValue(range.getMin()); - result.setHighValue(range.getMax()); - }); - return result.build(); - } } diff --git a/presto-main/src/main/java/com/facebook/presto/cost/UnionStatsRule.java b/presto-main/src/main/java/com/facebook/presto/cost/UnionStatsRule.java index 0789872a748f0..d5b9ebad66589 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/UnionStatsRule.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/UnionStatsRule.java @@ -17,11 +17,9 @@ import com.facebook.presto.Session; import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Lookup; -import com.facebook.presto.sql.planner.plan.UnionNode; -import com.google.common.collect.ListMultimap; import java.util.Optional; @@ -55,7 +53,7 @@ protected final Optional doCalculate(UnionNode node, Stat PlanNode source = node.getSources().get(i); PlanNodeStatsEstimate sourceStats = statsProvider.getStats(source); - PlanNodeStatsEstimate sourceStatsWithMappedSymbols = mapToOutputSymbols(sourceStats, node.getVariableMapping(), i); + PlanNodeStatsEstimate sourceStatsWithMappedSymbols = mapToOutputSymbols(sourceStats, node, i); if (estimate.isPresent()) { estimate = Optional.of(addStatsAndCollapseDistinctValues(estimate.get(), sourceStatsWithMappedSymbols)); @@ -68,13 +66,10 @@ protected final Optional doCalculate(UnionNode node, Stat return estimate; } - private PlanNodeStatsEstimate mapToOutputSymbols(PlanNodeStatsEstimate estimate, ListMultimap mapping, int index) + private PlanNodeStatsEstimate mapToOutputSymbols(PlanNodeStatsEstimate estimate, UnionNode node, int index) { - PlanNodeStatsEstimate.Builder mapped = PlanNodeStatsEstimate.builder() - .setOutputRowCount(estimate.getOutputRowCount()); - - mapping.keySet().stream() - .forEach(variable -> mapped.addVariableStatistics(variable, estimate.getVariableStatistics(mapping.get(variable).get(index)))); + PlanNodeStatsEstimate.Builder mapped = PlanNodeStatsEstimate.builder().setOutputRowCount(estimate.getOutputRowCount()); + node.getOutputVariables().forEach(variable -> mapped.addVariableStatistics(variable, estimate.getVariableStatistics(node.getVariableMapping().get(variable).get(index)))); return mapped.build(); } diff --git a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java index 086701f1c22ac..d38b4949fea64 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java @@ -13,6 +13,11 @@ */ package com.facebook.presto.event; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.node.NodeInfo; +import com.facebook.airlift.stats.Distribution; +import com.facebook.airlift.stats.Distribution.DistributionSnapshot; import com.facebook.presto.SessionRepresentation; import com.facebook.presto.client.NodeVersion; import com.facebook.presto.eventlistener.EventListenerManager; @@ -43,7 +48,7 @@ import com.facebook.presto.spi.eventlistener.QueryMetadata; import com.facebook.presto.spi.eventlistener.QueryOutputMetadata; import com.facebook.presto.spi.eventlistener.QueryStatistics; -import com.facebook.presto.spi.eventlistener.StageCpuDistribution; +import com.facebook.presto.spi.eventlistener.ResourceDistribution; import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import com.facebook.presto.sql.planner.plan.PlanFragmentId; import com.facebook.presto.transaction.TransactionId; @@ -53,11 +58,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedMap; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; -import io.airlift.node.NodeInfo; -import io.airlift.stats.Distribution; -import io.airlift.stats.Distribution.DistributionSnapshot; import org.joda.time.DateTime; import javax.inject.Inject; @@ -164,10 +164,12 @@ public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailur 0, 0, 0, + 0, ImmutableList.of(), 0, true, ImmutableList.of(), + ImmutableList.of(), ImmutableList.of()), createQueryContext(queryInfo.getSession(), queryInfo.getResourceGroupId()), new QueryIOMetadata(ImmutableList.of(), Optional.empty()), @@ -204,6 +206,26 @@ public void queryCompletedEvent(QueryInfo queryInfo) logQueryTimeline(queryInfo); } + public static ResourceDistribution createResourceDistribution( + int stageId, + int tasks, + DistributionSnapshot distributionSnapshot) + { + return new ResourceDistribution( + stageId, + tasks, + distributionSnapshot.getP25(), + distributionSnapshot.getP50(), + distributionSnapshot.getP75(), + distributionSnapshot.getP90(), + distributionSnapshot.getP95(), + distributionSnapshot.getP99(), + distributionSnapshot.getMin(), + distributionSnapshot.getMax(), + (long) distributionSnapshot.getTotal(), + distributionSnapshot.getTotal() / distributionSnapshot.getCount()); + } + private QueryMetadata createQueryMetadata(QueryInfo queryInfo) { return new QueryMetadata( @@ -225,11 +247,19 @@ private QueryStatistics createQueryStatistics(QueryInfo queryInfo) } QueryStats queryStats = queryInfo.getQueryStats(); + + ImmutableList.Builder cpuDistributionBuilder = ImmutableList.builder(); + ImmutableList.Builder memoryDistributionBuilder = ImmutableList.builder(); + if (queryInfo.getOutputStage().isPresent()) { + computeCpuAndMemoryDistributions(queryInfo.getOutputStage().get(), cpuDistributionBuilder, memoryDistributionBuilder); + } + return new QueryStatistics( ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getTotalScheduledTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.of(ofMillis(queryStats.getAnalysisTime().toMillis())), + queryStats.getPeakRunningTasks(), queryStats.getPeakUserMemoryReservation().toBytes(), queryStats.getPeakTotalMemoryReservation().toBytes(), queryStats.getPeakTaskUserMemory().toBytes(), @@ -245,7 +275,8 @@ private QueryStatistics createQueryStatistics(QueryInfo queryInfo) queryStats.getStageGcStatistics(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), - getCpuDistributions(queryInfo), + cpuDistributionBuilder.build(), + memoryDistributionBuilder.build(), operatorSummaries.build()); } @@ -258,7 +289,6 @@ private QueryContext createQueryContext(SessionRepresentation session, Optional< session.getUserAgent(), session.getClientInfo(), session.getClientTags(), - session.getClientCapabilities(), session.getSource(), session.getCatalog(), session.getSchema(), @@ -336,7 +366,7 @@ private static QueryIOMetadata getQueryIOMetadata(QueryInfo queryInfo) queryInfo.getOutput().get().getConnectorId().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable(), - tableFinishInfo.map(TableFinishInfo::getConnectorOutputMetadata), + tableFinishInfo.map(TableFinishInfo::getSerializedConnectorOutputMetadata), tableFinishInfo.map(TableFinishInfo::isJsonLengthLimitExceeded))); } return new QueryIOMetadata(inputs.build(), output); @@ -367,7 +397,7 @@ private static Optional findFailedTask(StageInfo stageInfo) return task; } } - return stageInfo.getTasks().stream() + return stageInfo.getLatestAttemptExecutionInfo().getTasks().stream() .filter(taskInfo -> taskInfo.getTaskStatus().getState() == TaskState.FAILED) .findFirst(); } @@ -417,7 +447,7 @@ private static void logQueryTimeline(QueryInfo queryInfo) continue; } - for (TaskInfo taskInfo : stage.getTasks()) { + for (TaskInfo taskInfo : stage.getLatestAttemptExecutionInfo().getTasks()) { TaskStats taskStats = taskInfo.getStats(); DateTime firstStartTime = taskStats.getFirstStartTime(); @@ -505,49 +535,32 @@ private static void logQueryTimeline( queryEndTime); } - private static List getCpuDistributions(QueryInfo queryInfo) - { - if (!queryInfo.getOutputStage().isPresent()) { - return ImmutableList.of(); - } - - ImmutableList.Builder builder = ImmutableList.builder(); - populateDistribution(queryInfo.getOutputStage().get(), builder); - - return builder.build(); - } - - private static void populateDistribution(StageInfo stageInfo, ImmutableList.Builder distributions) - { - distributions.add(computeCpuDistribution(stageInfo)); - for (StageInfo subStage : stageInfo.getSubStages()) { - populateDistribution(subStage, distributions); - } - } - - private static StageCpuDistribution computeCpuDistribution(StageInfo stageInfo) + private static void computeCpuAndMemoryDistributions( + StageInfo stageInfo, + ImmutableList.Builder cpuDistributionBuilder, + ImmutableList.Builder memoryDistributionBuilder) { Distribution cpuDistribution = new Distribution(); + Distribution memoryDistribution = new Distribution(); - for (TaskInfo taskInfo : stageInfo.getTasks()) { + for (TaskInfo taskInfo : stageInfo.getLatestAttemptExecutionInfo().getTasks()) { cpuDistribution.add(taskInfo.getStats().getTotalCpuTime().toMillis()); + memoryDistribution.add(taskInfo.getStats().getPeakTotalMemoryInBytes()); } - DistributionSnapshot snapshot = cpuDistribution.snapshot(); + cpuDistributionBuilder.add(createResourceDistribution( + stageInfo.getStageId().getId(), + stageInfo.getLatestAttemptExecutionInfo().getTasks().size(), + cpuDistribution.snapshot())); - return new StageCpuDistribution( + memoryDistributionBuilder.add(createResourceDistribution( stageInfo.getStageId().getId(), - stageInfo.getTasks().size(), - snapshot.getP25(), - snapshot.getP50(), - snapshot.getP75(), - snapshot.getP90(), - snapshot.getP95(), - snapshot.getP99(), - snapshot.getMin(), - snapshot.getMax(), - (long) snapshot.getTotal(), - snapshot.getTotal() / snapshot.getCount()); + stageInfo.getLatestAttemptExecutionInfo().getTasks().size(), + memoryDistribution.snapshot())); + + for (StageInfo subStage : stageInfo.getSubStages()) { + computeCpuAndMemoryDistributions(subStage, cpuDistributionBuilder, memoryDistributionBuilder); + } } public static class JsonPlanFragment diff --git a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitorConfig.java b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitorConfig.java index 371121c31a52b..37995e73367c2 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitorConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitorConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.event; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import io.airlift.units.MaxDataSize; diff --git a/presto-main/src/main/java/com/facebook/presto/event/SplitMonitor.java b/presto-main/src/main/java/com/facebook/presto/event/SplitMonitor.java index 13c73c47935ab..6a0296aac5499 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/SplitMonitor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/SplitMonitor.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.event; +import com.facebook.airlift.log.Logger; import com.facebook.presto.eventlistener.EventListenerManager; import com.facebook.presto.execution.TaskId; import com.facebook.presto.operator.DriverStats; @@ -21,7 +22,6 @@ import com.facebook.presto.spi.eventlistener.SplitStatistics; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import io.airlift.log.Logger; import javax.annotation.Nullable; import javax.inject.Inject; @@ -77,7 +77,8 @@ private void splitCompletedEvent(TaskId taskId, DriverStats driverStats, @Nullab eventListenerManager.splitCompleted( new SplitCompletedEvent( taskId.getQueryId().toString(), - taskId.getStageId().toString(), + taskId.getStageExecutionId().getStageId().toString(), + taskId.getStageExecutionId().toString(), Integer.toString(taskId.getId()), driverStats.getCreateTime().toDate().toInstant(), Optional.ofNullable(driverStats.getStartTime()).map(startTime -> startTime.toDate().toInstant()), diff --git a/presto-main/src/main/java/com/facebook/presto/eventlistener/EventListenerManager.java b/presto-main/src/main/java/com/facebook/presto/eventlistener/EventListenerManager.java index ac39b10c76e34..a2e4e799c510b 100644 --- a/presto-main/src/main/java/com/facebook/presto/eventlistener/EventListenerManager.java +++ b/presto-main/src/main/java/com/facebook/presto/eventlistener/EventListenerManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.eventlistener; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.spi.eventlistener.EventListenerFactory; @@ -21,7 +22,6 @@ import com.facebook.presto.spi.eventlistener.SplitCompletedEvent; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; import java.io.File; import java.util.HashMap; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/BasicStageStats.java b/presto-main/src/main/java/com/facebook/presto/execution/BasicStageExecutionStats.java similarity index 95% rename from presto-main/src/main/java/com/facebook/presto/execution/BasicStageStats.java rename to presto-main/src/main/java/com/facebook/presto/execution/BasicStageExecutionStats.java index fbdbfdd81d7fc..fbba106f6c212 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/BasicStageStats.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/BasicStageExecutionStats.java @@ -29,9 +29,9 @@ import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; -public class BasicStageStats +public class BasicStageExecutionStats { - public static final BasicStageStats EMPTY_STAGE_STATS = new BasicStageStats( + public static final BasicStageExecutionStats EMPTY_STAGE_STATS = new BasicStageExecutionStats( false, 0, @@ -70,7 +70,7 @@ public class BasicStageStats private final Set blockedReasons; private final OptionalDouble progressPercentage; - public BasicStageStats( + public BasicStageExecutionStats( boolean isScheduled, int totalDrivers, @@ -185,7 +185,7 @@ public OptionalDouble getProgressPercentage() return progressPercentage; } - public static BasicStageStats aggregateBasicStageStats(Iterable stages) + public static BasicStageExecutionStats aggregateBasicStageStats(Iterable stages) { int totalDrivers = 0; int queuedDrivers = 0; @@ -207,7 +207,7 @@ public static BasicStageStats aggregateBasicStageStats(Iterable boolean fullyBlocked = true; Set blockedReasons = new HashSet<>(); - for (BasicStageStats stageStats : stages) { + for (BasicStageExecutionStats stageStats : stages) { totalDrivers += stageStats.getTotalDrivers(); queuedDrivers += stageStats.getQueuedDrivers(); runningDrivers += stageStats.getRunningDrivers(); @@ -234,7 +234,7 @@ public static BasicStageStats aggregateBasicStageStats(Iterable progressPercentage = OptionalDouble.of(min(100, (completedDrivers * 100.0) / totalDrivers)); } - return new BasicStageStats( + return new BasicStageExecutionStats( isScheduled, totalDrivers, diff --git a/presto-main/src/main/java/com/facebook/presto/execution/CallTask.java b/presto-main/src/main/java/com/facebook/presto/execution/CallTask.java index b7af80d5daba7..afeb513750ffa 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/CallTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/CallTask.java @@ -69,7 +69,7 @@ public String getName() @Override public ListenableFuture execute(Call call, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List parameters) { - if (!transactionManager.isAutoCommit(stateMachine.getSession().getRequiredTransactionId())) { + if (!transactionManager.getTransactionInfo(stateMachine.getSession().getRequiredTransactionId()).isAutoCommitContext()) { throw new PrestoException(NOT_SUPPORTED, "Procedures cannot be called within a transaction (use autocommit mode)"); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/ClusterSizeMonitor.java b/presto-main/src/main/java/com/facebook/presto/execution/ClusterSizeMonitor.java index bafe9512443e2..83bcb446f9ff9 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/ClusterSizeMonitor.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/ClusterSizeMonitor.java @@ -34,11 +34,11 @@ import java.util.concurrent.ScheduledFuture; import java.util.function.Consumer; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INSUFFICIENT_RESOURCES; import static com.facebook.presto.spi.StandardErrorCode.SERVER_STARTING_UP; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.util.concurrent.Futures.immediateFuture; -import static io.airlift.concurrent.Threads.threadsNamed; import static io.airlift.units.Duration.nanosSince; import static java.lang.String.format; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/CreateFunctionTask.java b/presto-main/src/main/java/com/facebook/presto/execution/CreateFunctionTask.java new file mode 100644 index 0000000000000..55aa1ba6e78f5 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/execution/CreateFunctionTask.java @@ -0,0 +1,97 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.execution; + +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.security.AccessControl; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.function.RoutineCharacteristics; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.facebook.presto.spi.function.SqlParameter; +import com.facebook.presto.spi.type.TypeSignature; +import com.facebook.presto.sql.analyzer.Analyzer; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.tree.CreateFunction; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.transaction.TransactionManager; +import com.google.common.util.concurrent.ListenableFuture; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.facebook.presto.sql.SqlFormatter.formatSql; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public class CreateFunctionTask + implements DataDefinitionTask +{ + private final SqlParser sqlParser; + + @Inject + public CreateFunctionTask(SqlParser sqlParser) + { + this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + } + + @Override + public String getName() + { + return "CREATE FUNCTION"; + } + + @Override + public String explain(CreateFunction statement, List parameters) + { + return "CREATE FUNCTION " + statement.getFunctionName(); + } + + @Override + public ListenableFuture execute(CreateFunction statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List parameters) + { + Analyzer analyzer = new Analyzer(stateMachine.getSession(), metadata, sqlParser, accessControl, Optional.empty(), parameters, stateMachine.getWarningCollector()); + analyzer.analyze(statement); + metadata.getFunctionManager().createFunction(createSqlInvokedFunction(statement), statement.isReplace()); + return immediateFuture(null); + } + + private SqlInvokedFunction createSqlInvokedFunction(CreateFunction statement) + { + QualifiedFunctionName functionName = QualifiedFunctionName.of(statement.getFunctionName().toString()); + List parameters = statement.getParameters().stream() + .map(parameter -> new SqlParameter(parameter.getName().toString().toLowerCase(ENGLISH), parseTypeSignature(parameter.getType()))) + .collect(toImmutableList()); + TypeSignature returnType = parseTypeSignature(statement.getReturnType()); + String description = statement.getComment().orElse(""); + RoutineCharacteristics routineCharacteristics = new RoutineCharacteristics( + RoutineCharacteristics.Language.valueOf(statement.getCharacteristics().getLanguage().name()), + RoutineCharacteristics.Determinism.valueOf(statement.getCharacteristics().getDeterminism().name()), + RoutineCharacteristics.NullCallClause.valueOf(statement.getCharacteristics().getNullCallClause().name())); + String body = formatSql(statement.getBody(), Optional.empty()); + + return new SqlInvokedFunction( + functionName, + parameters, + returnType, + description, + routineCharacteristics, + body, + Optional.empty()); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/execution/CreateViewTask.java b/presto-main/src/main/java/com/facebook/presto/execution/CreateViewTask.java index 224a1ef8b7ef1..219ea4a9fb6e8 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/CreateViewTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/CreateViewTask.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.Metadata; @@ -28,7 +29,6 @@ import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.transaction.TransactionManager; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.json.JsonCodec; import javax.inject.Inject; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/DataDefinitionExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/DataDefinitionExecution.java index 75b4e74cd871e..e10a7ed4ae32f 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/DataDefinitionExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/DataDefinitionExecution.java @@ -156,6 +156,12 @@ public BasicQueryInfo getBasicQueryInfo() .orElseGet(() -> stateMachine.getBasicQueryInfo(Optional.empty())); } + @Override + public int getRunningTaskCount() + { + return stateMachine.getCurrentRunningTaskCount(); + } + @Override public void start() { diff --git a/presto-main/src/main/java/com/facebook/presto/execution/DropFunctionTask.java b/presto-main/src/main/java/com/facebook/presto/execution/DropFunctionTask.java new file mode 100644 index 0000000000000..566bf6b2fb4e4 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/execution/DropFunctionTask.java @@ -0,0 +1,71 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.execution; + +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.security.AccessControl; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.type.TypeSignature; +import com.facebook.presto.sql.analyzer.Analyzer; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.tree.DropFunction; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.transaction.TransactionManager; +import com.google.common.util.concurrent.ListenableFuture; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Optional; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static java.util.Objects.requireNonNull; + +public class DropFunctionTask + implements DataDefinitionTask +{ + private final SqlParser sqlParser; + + @Inject + public DropFunctionTask(SqlParser sqlParser) + { + this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + } + + @Override + public String getName() + { + return "DROP FUNCTION"; + } + + @Override + public String explain(DropFunction statement, List parameters) + { + return "DROP FUNCTION " + statement.getFunctionName(); + } + + @Override + public ListenableFuture execute(DropFunction statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List parameters) + { + Analyzer analyzer = new Analyzer(stateMachine.getSession(), metadata, sqlParser, accessControl, Optional.empty(), parameters, stateMachine.getWarningCollector()); + analyzer.analyze(statement); + + metadata.getFunctionManager().dropFunction( + QualifiedFunctionName.of(statement.getFunctionName().toString()), + statement.getParameterTypes().map(types -> types.stream().map(TypeSignature::parseTypeSignature).collect(toImmutableList())), + statement.isExists()); + return immediateFuture(null); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/execution/FailedQueryExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/FailedQueryExecution.java index 417ca1578a0e9..b75e04a866a21 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/FailedQueryExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/FailedQueryExecution.java @@ -151,6 +151,12 @@ public BasicQueryInfo getBasicQueryInfo() return new BasicQueryInfo(getQueryInfo()); } + @Override + public int getRunningTaskCount() + { + return 0; + } + @Override public void start() { diff --git a/presto-main/src/main/java/com/facebook/presto/execution/MemoryRevokingScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/MemoryRevokingScheduler.java index cca4bf60287c1..a42ad738f64a5 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/MemoryRevokingScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/MemoryRevokingScheduler.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.log.Logger; import com.facebook.presto.memory.LocalMemoryManager; import com.facebook.presto.memory.MemoryPool; import com.facebook.presto.memory.MemoryPoolListener; @@ -24,7 +25,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; -import io.airlift.log.Logger; import javax.annotation.Nullable; import javax.annotation.PostConstruct; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/NodeTaskMap.java b/presto-main/src/main/java/com/facebook/presto/execution/NodeTaskMap.java index ac9c55657c120..9f8a7cb3e08e0 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/NodeTaskMap.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/NodeTaskMap.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.log.Logger; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.util.FinalizerService; import com.google.common.collect.Sets; -import io.airlift.log.Logger; import javax.annotation.concurrent.ThreadSafe; import javax.inject.Inject; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/QueryExecutionMBean.java b/presto-main/src/main/java/com/facebook/presto/execution/QueryExecutionMBean.java index 05747efc2276d..3faa63f2c906d 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/QueryExecutionMBean.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/QueryExecutionMBean.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.execution; -import io.airlift.concurrent.ThreadPoolExecutorMBean; +import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/QueryInfo.java b/presto-main/src/main/java/com/facebook/presto/execution/QueryInfo.java index a584b83b6b28e..c4a8fdaf5c6ac 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/QueryInfo.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/QueryInfo.java @@ -61,7 +61,6 @@ public class QueryInfo private final QueryStats queryStats; private final Optional setCatalog; private final Optional setSchema; - private final Optional setPath; private final Map setSessionProperties; private final Set resetSessionProperties; private final Map setRoles; @@ -96,7 +95,6 @@ public QueryInfo( @JsonProperty("queryStats") QueryStats queryStats, @JsonProperty("setCatalog") Optional setCatalog, @JsonProperty("setSchema") Optional setSchema, - @JsonProperty("setPath") Optional setPath, @JsonProperty("setSessionProperties") Map setSessionProperties, @JsonProperty("resetSessionProperties") Set resetSessionProperties, @JsonProperty("setRoles") Map setRoles, @@ -124,7 +122,6 @@ public QueryInfo( requireNonNull(queryStats, "queryStats is null"); requireNonNull(setCatalog, "setCatalog is null"); requireNonNull(setSchema, "setSchema is null"); - requireNonNull(setPath, "setPath is null"); requireNonNull(setSessionProperties, "setSessionProperties is null"); requireNonNull(resetSessionProperties, "resetSessionProperties is null"); requireNonNull(addedPreparedStatements, "addedPreparedStatemetns is null"); @@ -150,7 +147,6 @@ public QueryInfo( this.queryStats = queryStats; this.setCatalog = setCatalog; this.setSchema = setSchema; - this.setPath = setPath; this.setSessionProperties = ImmutableMap.copyOf(setSessionProperties); this.resetSessionProperties = ImmutableSet.copyOf(resetSessionProperties); this.setRoles = ImmutableMap.copyOf(setRoles); @@ -193,7 +189,6 @@ public static QueryInfo immediateFailureQueryInfo( immediateFailureQueryStats(), Optional.empty(), Optional.empty(), - Optional.empty(), ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of(), @@ -282,12 +277,6 @@ public Optional getSetSchema() return setSchema; } - @JsonProperty - public Optional getSetPath() - { - return setPath; - } - @JsonProperty public Map getSetSessionProperties() { diff --git a/presto-main/src/main/java/com/facebook/presto/execution/QueryManagerConfig.java b/presto-main/src/main/java/com/facebook/presto/execution/QueryManagerConfig.java index 29ff95dcde5a0..461a9cd4d62ed 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/QueryManagerConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/QueryManagerConfig.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.DefunctConfig; +import com.facebook.airlift.configuration.LegacyConfig; import com.facebook.presto.connector.system.GlobalSystemConnector; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.DefunctConfig; -import io.airlift.configuration.LegacyConfig; import io.airlift.units.Duration; import io.airlift.units.MinDuration; @@ -49,6 +49,8 @@ public class QueryManagerConfig private int maxQueryLength = 1_000_000; private int maxStageCount = 100; private int stageCountWarningThreshold = 50; + private int maxTotalRunningTaskCount = Integer.MAX_VALUE; + private int maxQueryRunningTaskCount = Integer.MAX_VALUE; private Duration clientTimeout = new Duration(5, TimeUnit.MINUTES); @@ -68,6 +70,8 @@ public class QueryManagerConfig private int requiredWorkers = 1; private Duration requiredWorkersMaxWait = new Duration(5, TimeUnit.MINUTES); + private int querySubmissionMaxThreads = Runtime.getRuntime().availableProcessors() * 2; + @Min(1) public int getScheduleSplitBatchSize() { @@ -234,6 +238,34 @@ public QueryManagerConfig setStageCountWarningThreshold(int stageCountWarningThr return this; } + @Min(1) + public int getMaxTotalRunningTaskCount() + { + return maxTotalRunningTaskCount; + } + + @Config("experimental.max-total-running-task-count") + @ConfigDescription("Maximal allowed running task from all queries") + public QueryManagerConfig setMaxTotalRunningTaskCount(int maxTotalRunningTaskCount) + { + this.maxTotalRunningTaskCount = maxTotalRunningTaskCount; + return this; + } + + @Min(1) + public int getMaxQueryRunningTaskCount() + { + return maxQueryRunningTaskCount; + } + + @Config("experimental.max-query-running-task-count") + @ConfigDescription("Maximal allowed running task for single query only if experimental.max-total-running-task-count is violated") + public QueryManagerConfig setMaxQueryRunningTaskCount(int maxQueryRunningTaskCount) + { + this.maxQueryRunningTaskCount = maxQueryRunningTaskCount; + return this; + } + @MinDuration("5s") @NotNull public Duration getClientTimeout() @@ -410,6 +442,19 @@ public QueryManagerConfig setRequiredWorkersMaxWait(Duration requiredWorkersMaxW return this; } + @Min(1) + public int getQuerySubmissionMaxThreads() + { + return querySubmissionMaxThreads; + } + + @Config("query-manager.query-submission-max-threads") + public QueryManagerConfig setQuerySubmissionMaxThreads(int querySubmissionMaxThreads) + { + this.querySubmissionMaxThreads = querySubmissionMaxThreads; + return this; + } + public enum ExchangeMaterializationStrategy { NONE, diff --git a/presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java b/presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java index d5e5417f49c44..e6cfde0bec4e6 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.execution.QueryExecution.QueryOutputInfo; import com.facebook.presto.execution.StateMachine.StateChangeListener; @@ -35,16 +36,17 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.transaction.TransactionId; +import com.facebook.presto.transaction.TransactionInfo; import com.facebook.presto.transaction.TransactionManager; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ticker; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import com.google.common.collect.Streams; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.log.Logger; import org.joda.time.DateTime; import javax.annotation.Nullable; @@ -62,11 +64,13 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.function.Predicate; -import static com.facebook.presto.execution.BasicStageStats.EMPTY_STAGE_STATS; +import static com.facebook.presto.execution.BasicStageExecutionStats.EMPTY_STAGE_STATS; import static com.facebook.presto.execution.QueryState.FINISHED; import static com.facebook.presto.execution.QueryState.FINISHING; import static com.facebook.presto.execution.QueryState.PLANNING; @@ -116,13 +120,15 @@ public class QueryStateMachine private final AtomicLong peakTaskUserMemory = new AtomicLong(); private final AtomicLong peakTaskTotalMemory = new AtomicLong(); + private final AtomicInteger currentRunningTaskCount = new AtomicInteger(); + private final AtomicInteger peakRunningTaskCount = new AtomicInteger(); + private final QueryStateTimer queryStateTimer; private final StateMachine queryState; private final AtomicReference setCatalog = new AtomicReference<>(); private final AtomicReference setSchema = new AtomicReference<>(); - private final AtomicReference setPath = new AtomicReference<>(); private final Map setSessionProperties = new ConcurrentHashMap<>(); private final Set resetSessionProperties = Sets.newConcurrentHashSet(); @@ -277,6 +283,28 @@ public long getPeakTaskUserMemory() return peakTaskUserMemory.get(); } + public int getCurrentRunningTaskCount() + { + return currentRunningTaskCount.get(); + } + + public int incrementCurrentRunningTaskCount() + { + int runningTaskCount = currentRunningTaskCount.incrementAndGet(); + peakRunningTaskCount.accumulateAndGet(runningTaskCount, Math::max); + return runningTaskCount; + } + + public int decrementCurrentRunningTaskCount() + { + return currentRunningTaskCount.decrementAndGet(); + } + + public int getPeakRunningTaskCount() + { + return peakRunningTaskCount.get(); + } + public WarningCollector getWarningCollector() { return warningCollector; @@ -292,7 +320,7 @@ public void updateMemoryUsage(long deltaUserMemoryInBytes, long deltaTotalMemory peakTaskTotalMemory.accumulateAndGet(taskTotalMemoryInBytes, Math::max); } - public BasicQueryInfo getBasicQueryInfo(Optional rootStage) + public BasicQueryInfo getBasicQueryInfo(Optional rootStage) { // Query state must be captured first in order to provide a // correct view of the query. For example, building this @@ -308,7 +336,7 @@ public BasicQueryInfo getBasicQueryInfo(Optional rootStage) } } - BasicStageStats stageStats = rootStage.orElse(EMPTY_STAGE_STATS); + BasicStageExecutionStats stageStats = rootStage.orElse(EMPTY_STAGE_STATS); BasicQueryStats queryStats = new BasicQueryStats( queryStateTimer.getCreateTime(), getEndTime().orElse(null), @@ -376,8 +404,8 @@ QueryInfo getQueryInfo(Optional rootStage) // Traversing all tasks is expensive, thus only construct failedTasks list when query finished. if (state.isDone()) { failedTasks = Optional.of(getAllStages(rootStage).stream() - .map(StageInfo::getTasks) - .flatMap(List::stream) + .flatMap(stageInfo -> Streams.concat(ImmutableList.of(stageInfo.getLatestAttemptExecutionInfo()).stream(), stageInfo.getPreviousAttemptsExecutionInfos().stream())) + .flatMap(execution -> execution.getTasks().stream()) .filter(taskInfo -> taskInfo.getTaskStatus().getState() == TaskState.FAILED) .map(TaskInfo::getTaskStatus) .map(TaskStatus::getTaskId) @@ -399,7 +427,6 @@ QueryInfo getQueryInfo(Optional rootStage) getQueryStats(rootStage), Optional.ofNullable(setCatalog.get()), Optional.ofNullable(setSchema.get()), - Optional.ofNullable(setPath.get()), setSessionProperties, resetSessionProperties, setRoles, @@ -463,62 +490,62 @@ private QueryStats getQueryStats(Optional rootStage) ImmutableList.Builder operatorStatsSummary = ImmutableList.builder(); boolean completeInfo = true; for (StageInfo stageInfo : getAllStages(rootStage)) { - StageStats stageStats = stageInfo.getStageStats(); - totalTasks += stageStats.getTotalTasks(); - runningTasks += stageStats.getRunningTasks(); - completedTasks += stageStats.getCompletedTasks(); - - totalDrivers += stageStats.getTotalDrivers(); - queuedDrivers += stageStats.getQueuedDrivers(); - runningDrivers += stageStats.getRunningDrivers(); - blockedDrivers += stageStats.getBlockedDrivers(); - completedDrivers += stageStats.getCompletedDrivers(); - - cumulativeUserMemory += stageStats.getCumulativeUserMemory(); - userMemoryReservation += stageStats.getUserMemoryReservation().toBytes(); - totalMemoryReservation += stageStats.getTotalMemoryReservation().toBytes(); - totalScheduledTime += stageStats.getTotalScheduledTime().roundTo(MILLISECONDS); - totalCpuTime += stageStats.getTotalCpuTime().roundTo(MILLISECONDS); - totalBlockedTime += stageStats.getTotalBlockedTime().roundTo(MILLISECONDS); - if (!stageInfo.getState().isDone()) { - fullyBlocked &= stageStats.isFullyBlocked(); - blockedReasons.addAll(stageStats.getBlockedReasons()); + StageExecutionStats stageExecutionStats = stageInfo.getLatestAttemptExecutionInfo().getStats(); + totalTasks += stageExecutionStats.getTotalTasks(); + runningTasks += stageExecutionStats.getRunningTasks(); + completedTasks += stageExecutionStats.getCompletedTasks(); + + totalDrivers += stageExecutionStats.getTotalDrivers(); + queuedDrivers += stageExecutionStats.getQueuedDrivers(); + runningDrivers += stageExecutionStats.getRunningDrivers(); + blockedDrivers += stageExecutionStats.getBlockedDrivers(); + completedDrivers += stageExecutionStats.getCompletedDrivers(); + + cumulativeUserMemory += stageExecutionStats.getCumulativeUserMemory(); + userMemoryReservation += stageExecutionStats.getUserMemoryReservation().toBytes(); + totalMemoryReservation += stageExecutionStats.getTotalMemoryReservation().toBytes(); + totalScheduledTime += stageExecutionStats.getTotalScheduledTime().roundTo(MILLISECONDS); + totalCpuTime += stageExecutionStats.getTotalCpuTime().roundTo(MILLISECONDS); + totalBlockedTime += stageExecutionStats.getTotalBlockedTime().roundTo(MILLISECONDS); + if (!stageInfo.getLatestAttemptExecutionInfo().getState().isDone()) { + fullyBlocked &= stageExecutionStats.isFullyBlocked(); + blockedReasons.addAll(stageExecutionStats.getBlockedReasons()); } if (stageInfo.getPlan().isPresent()) { PlanFragment plan = stageInfo.getPlan().get(); if (!plan.getTableScanSchedulingOrder().isEmpty()) { - rawInputDataSize += stageStats.getRawInputDataSize().toBytes(); - rawInputPositions += stageStats.getRawInputPositions(); + rawInputDataSize += stageExecutionStats.getRawInputDataSize().toBytes(); + rawInputPositions += stageExecutionStats.getRawInputPositions(); - processedInputDataSize += stageStats.getProcessedInputDataSize().toBytes(); - processedInputPositions += stageStats.getProcessedInputPositions(); + processedInputDataSize += stageExecutionStats.getProcessedInputDataSize().toBytes(); + processedInputPositions += stageExecutionStats.getProcessedInputPositions(); } if (plan.isOutputTableWriterFragment()) { - writtenOutputPositions += stageInfo.getStageStats().getOperatorSummaries().stream() + writtenOutputPositions += stageExecutionStats.getOperatorSummaries().stream() .filter(stats -> stats.getOperatorType().equals(TableWriterOperator.class.getSimpleName())) .mapToLong(OperatorStats::getInputPositions) .sum(); - writtenOutputLogicalDataSize += stageInfo.getStageStats().getOperatorSummaries().stream() + writtenOutputLogicalDataSize += stageExecutionStats.getOperatorSummaries().stream() .filter(stats -> stats.getOperatorType().equals(TableWriterOperator.class.getSimpleName())) .mapToLong(stats -> stats.getInputDataSize().toBytes()) .sum(); - writtenOutputPhysicalDataSize += stageStats.getPhysicalWrittenDataSize().toBytes(); + writtenOutputPhysicalDataSize += stageExecutionStats.getPhysicalWrittenDataSize().toBytes(); } else { - writtenIntermediatePhysicalDataSize += stageStats.getPhysicalWrittenDataSize().toBytes(); + writtenIntermediatePhysicalDataSize += stageExecutionStats.getPhysicalWrittenDataSize().toBytes(); } } - stageGcStatistics.add(stageStats.getGcInfo()); + stageGcStatistics.add(stageExecutionStats.getGcInfo()); completeInfo = completeInfo && stageInfo.isFinalStageInfo(); - operatorStatsSummary.addAll(stageInfo.getStageStats().getOperatorSummaries()); + operatorStatsSummary.addAll(stageExecutionStats.getOperatorSummaries()); } if (rootStage.isPresent()) { - StageStats outputStageStats = rootStage.get().getStageStats(); + StageExecutionStats outputStageStats = rootStage.get().getLatestAttemptExecutionInfo().getStats(); outputDataSize += outputStageStats.getOutputDataSize().toBytes(); outputPositions += outputStageStats.getOutputPositions(); } @@ -541,6 +568,7 @@ private QueryStats getQueryStats(Optional rootStage) totalTasks, runningTasks, + getPeakRunningTaskCount(), completedTasks, totalDrivers, @@ -635,17 +663,6 @@ public void setSetSchema(String schema) setSchema.set(requireNonNull(schema, "schema is null")); } - public void setSetPath(String path) - { - requireNonNull(path, "path is null"); - setPath.set(path); - } - - public String getSetPath() - { - return setPath.get(); - } - public void addSetSessionProperties(String key, String value) { setSessionProperties.put(requireNonNull(key, "key is null"), requireNonNull(value, "value is null")); @@ -753,9 +770,10 @@ public boolean transitionToFinishing() return false; } - Optional transactionId = session.getTransactionId(); - if (transactionId.isPresent() && transactionManager.transactionExists(transactionId.get()) && transactionManager.isAutoCommit(transactionId.get())) { - ListenableFuture commitFuture = transactionManager.asyncCommit(transactionId.get()); + Optional transaction = session.getTransactionId() + .flatMap(transactionManager::getOptionalTransactionInfo); + if (transaction.isPresent() && transaction.get().isAutoCommitContext()) { + ListenableFuture commitFuture = transactionManager.asyncCommit(transaction.get().getTransactionId()); Futures.addCallback(commitFuture, new FutureCallback() { @Override @@ -767,7 +785,7 @@ public void onSuccess(@Nullable Object result) @Override public void onFailure(Throwable throwable) { - transitionToFailed(throwable); + transitionToFailed(throwable, currentState -> !currentState.isDone()); } }, directExecutor()); } @@ -787,6 +805,19 @@ private void transitionToFinished() public boolean transitionToFailed(Throwable throwable) { + // When the state enters FINISHING, the only thing remaining is to commit + // the transaction. It should only be failed if the transaction commit fails. + return transitionToFailed(throwable, currentState -> currentState != FINISHING && !currentState.isDone()); + } + + private boolean transitionToFailed(Throwable throwable, Predicate predicate) + { + QueryState currentState = queryState.get(); + if (!predicate.test(currentState)) { + QUERY_STATE_LOG.debug(throwable, "Failure is ignored as the query %s is the %s state, ", queryId, currentState); + return false; + } + cleanupQueryQuietly(); queryStateTimer.endQuery(); @@ -796,15 +827,16 @@ public boolean transitionToFailed(Throwable throwable) requireNonNull(throwable, "throwable is null"); failureCause.compareAndSet(null, toFailure(throwable)); - boolean failed = queryState.setIf(QueryState.FAILED, currentState -> !currentState.isDone()); + boolean failed = queryState.setIf(QueryState.FAILED, predicate); if (failed) { QUERY_STATE_LOG.debug(throwable, "Query %s failed", queryId); - session.getTransactionId().ifPresent(transactionId -> { - if (transactionManager.isAutoCommit(transactionId)) { - transactionManager.asyncAbort(transactionId); + // if the transaction is already gone, do nothing + session.getTransactionId().flatMap(transactionManager::getOptionalTransactionInfo).ifPresent(transaction -> { + if (transaction.isAutoCommitContext()) { + transactionManager.asyncAbort(transaction.getTransactionId()); } else { - transactionManager.fail(transactionId); + transactionManager.fail(transaction.getTransactionId()); } }); } @@ -827,12 +859,13 @@ public boolean transitionToCanceled() boolean canceled = queryState.setIf(QueryState.FAILED, currentState -> !currentState.isDone()); if (canceled) { - session.getTransactionId().ifPresent(transactionId -> { - if (transactionManager.isAutoCommit(transactionId)) { - transactionManager.asyncAbort(transactionId); + // if the transaction is already gone, do nothing + session.getTransactionId().flatMap(transactionManager::getOptionalTransactionInfo).ifPresent(transaction -> { + if (transaction.isAutoCommitContext()) { + transactionManager.asyncAbort(transaction.getTransactionId()); } else { - transactionManager.fail(transactionId); + transactionManager.fail(transaction.getTransactionId()); } }); } @@ -922,8 +955,9 @@ private static boolean isScheduled(Optional rootStage) return false; } return getAllStages(rootStage).stream() - .map(StageInfo::getState) - .allMatch(state -> (state == StageState.RUNNING) || state.isDone()); + .map(StageInfo::getLatestAttemptExecutionInfo) + .map(StageExecutionInfo::getState) + .allMatch(state -> (state == StageExecutionState.RUNNING) || state.isDone()); } public Optional getFailureInfo() @@ -958,14 +992,11 @@ public void pruneQueryInfo() QueryInfo queryInfo = finalInfo.get(); Optional prunedOutputStage = queryInfo.getOutputStage().map(outputStage -> new StageInfo( outputStage.getStageId(), - outputStage.getState(), outputStage.getSelf(), Optional.empty(), // Remove the plan - outputStage.getTypes(), - outputStage.getStageStats(), - ImmutableList.of(), // Remove the tasks - ImmutableList.of(), // Remove the substages - outputStage.getFailureCause())); + pruneStageExecutionInfo(outputStage.getLatestAttemptExecutionInfo()), + ImmutableList.of(), // Remove failed attempts + ImmutableList.of())); // Remove the substages QueryInfo prunedQueryInfo = new QueryInfo( queryInfo.getQueryId(), @@ -979,7 +1010,6 @@ public void pruneQueryInfo() pruneQueryStats(queryInfo.getQueryStats()), queryInfo.getSetCatalog(), queryInfo.getSetSchema(), - queryInfo.getSetPath(), queryInfo.getSetSessionProperties(), queryInfo.getResetSessionProperties(), queryInfo.getSetRoles(), @@ -1001,6 +1031,17 @@ public void pruneQueryInfo() finalQueryInfo.compareAndSet(finalInfo, Optional.of(prunedQueryInfo)); } + private static StageExecutionInfo pruneStageExecutionInfo(StageExecutionInfo info) + { + return new StageExecutionInfo( + info.getStageExecutionId(), + info.getState(), + info.getStats(), + // Remove the tasks + ImmutableList.of(), + info.getFailureCause()); + } + private static QueryStats pruneQueryStats(QueryStats queryStats) { return new QueryStats( @@ -1018,6 +1059,7 @@ private static QueryStats pruneQueryStats(QueryStats queryStats) queryStats.getTotalTasks(), queryStats.getRunningTasks(), queryStats.getCompletedTasks(), + queryStats.getPeakRunningTasks(), queryStats.getTotalDrivers(), queryStats.getQueuedDrivers(), queryStats.getRunningDrivers(), diff --git a/presto-main/src/main/java/com/facebook/presto/execution/QueryStats.java b/presto-main/src/main/java/com/facebook/presto/execution/QueryStats.java index 5c716384ae43b..b869de1d56e66 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/QueryStats.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/QueryStats.java @@ -55,6 +55,7 @@ public class QueryStats private final int totalTasks; private final int runningTasks; + private final int peakRunningTasks; private final int completedTasks; private final int totalDrivers; @@ -114,6 +115,7 @@ public QueryStats( @JsonProperty("totalTasks") int totalTasks, @JsonProperty("runningTasks") int runningTasks, + @JsonProperty("peakRunningTasks") int peakRunningTasks, @JsonProperty("completedTasks") int completedTasks, @JsonProperty("totalDrivers") int totalDrivers, @@ -173,6 +175,8 @@ public QueryStats( this.totalTasks = totalTasks; checkArgument(runningTasks >= 0, "runningTasks is negative"); this.runningTasks = runningTasks; + checkArgument(peakRunningTasks >= 0, "peakRunningTasks is negative"); + this.peakRunningTasks = peakRunningTasks; checkArgument(completedTasks >= 0, "completedTasks is negative"); this.completedTasks = completedTasks; @@ -248,6 +252,7 @@ public static QueryStats immediateFailureQueryStats() 0, 0, 0, + 0, new DataSize(0, BYTE), new DataSize(0, BYTE), new DataSize(0, BYTE), @@ -353,6 +358,12 @@ public int getRunningTasks() return runningTasks; } + @JsonProperty + public int getPeakRunningTasks() + { + return peakRunningTasks; + } + @JsonProperty public int getCompletedTasks() { diff --git a/presto-main/src/main/java/com/facebook/presto/execution/QueryTracker.java b/presto-main/src/main/java/com/facebook/presto/execution/QueryTracker.java index 9ef467d8f78ad..11384fb6ecaf5 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/QueryTracker.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/QueryTracker.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.execution.QueryTracker.TrackedQuery; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -35,12 +35,16 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import static com.facebook.presto.SystemSessionProperties.getQueryMaxExecutionTime; import static com.facebook.presto.SystemSessionProperties.getQueryMaxRunTime; import static com.facebook.presto.spi.StandardErrorCode.ABANDONED_QUERY; import static com.facebook.presto.spi.StandardErrorCode.EXCEEDED_TIME_LIMIT; +import static com.facebook.presto.spi.StandardErrorCode.QUERY_HAS_TOO_MANY_STAGES; import static com.facebook.presto.spi.StandardErrorCode.SERVER_SHUTTING_DOWN; +import static com.facebook.presto.sql.planner.PlanFragmenter.TOO_MANY_STAGES_MESSAGE; import static com.google.common.base.Preconditions.checkState; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -51,6 +55,12 @@ public class QueryTracker private static final Logger log = Logger.get(QueryTracker.class); private final int maxQueryHistory; + private final int maxTotalRunningTaskCount; + private final int maxQueryRunningTaskCount; + + private final AtomicInteger runningTaskCount = new AtomicInteger(); + private final AtomicLong queriesKilledDueToTooManyTask = new AtomicLong(); + private final Duration minQueryExpireAge; private final ConcurrentMap queries = new ConcurrentHashMap<>(); @@ -69,6 +79,8 @@ public QueryTracker(QueryManagerConfig queryManagerConfig, ScheduledExecutorServ this.minQueryExpireAge = queryManagerConfig.getMinQueryExpireAge(); this.maxQueryHistory = queryManagerConfig.getMaxQueryHistory(); this.clientTimeout = queryManagerConfig.getClientTimeout(); + this.maxTotalRunningTaskCount = queryManagerConfig.getMaxTotalRunningTaskCount(); + this.maxQueryRunningTaskCount = queryManagerConfig.getMaxQueryRunningTaskCount(); this.queryManagementExecutor = requireNonNull(queryManagementExecutor, "queryManagementExecutor is null"); } @@ -91,6 +103,15 @@ public synchronized void start() log.error(e, "Error enforcing query timeout limits"); } + try { + if (maxTotalRunningTaskCount != Integer.MAX_VALUE && maxQueryRunningTaskCount != Integer.MAX_VALUE) { + enforceTaskLimits(); + } + } + catch (Throwable e) { + log.error(e, "Error enforcing running task limits"); + } + try { removeExpiredQueries(); } @@ -167,6 +188,16 @@ public void expireQuery(QueryId queryId) .ifPresent(expirationQueue::add); } + public long getRunningTaskCount() + { + return runningTaskCount.get(); + } + + public long getQueriesKilledDueToTooManyTask() + { + return queriesKilledDueToTooManyTask.get(); + } + /** * Enforce query max runtime/execution time limits */ @@ -189,6 +220,39 @@ private void enforceTimeLimits() } } + /** + * When cluster reaches max tasks limit and also a single query + * exceeds a threshold, kill this query + */ + private void enforceTaskLimits() + { + int totalRunningTaskCount = 0; + int highestRunningTaskCount = 0; + Optional highestRunningTaskQuery = Optional.empty(); + for (T query : queries.values()) { + if (query.isDone()) { + continue; + } + int runningTaskCount = query.getRunningTaskCount(); + totalRunningTaskCount += runningTaskCount; + if (runningTaskCount > highestRunningTaskCount) { + highestRunningTaskCount = runningTaskCount; + highestRunningTaskQuery = Optional.of(query); + } + } + + runningTaskCount.set(totalRunningTaskCount); + + if (totalRunningTaskCount > maxTotalRunningTaskCount && + highestRunningTaskCount > maxQueryRunningTaskCount && + highestRunningTaskQuery.isPresent()) { + highestRunningTaskQuery.get().fail(new PrestoException(QUERY_HAS_TOO_MANY_STAGES, format( + "Query killed because the cluster is overloaded with too many tasks (%s) and this query was running with the highest number of tasks (%s). %s Otherwise, please try again later.", + totalRunningTaskCount, highestRunningTaskCount, TOO_MANY_STAGES_MESSAGE))); + queriesKilledDueToTooManyTask.incrementAndGet(); + } + } + /** * Prune extraneous info from old queries */ @@ -292,6 +356,8 @@ public interface TrackedQuery Optional getEndTime(); + int getRunningTaskCount(); + void fail(Throwable cause); // XXX: This should be removed when the client protocol is improved, so that we don't need to hold onto so much query history diff --git a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java index d929bbe0aeac7..e048ca361df1f 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.NodeTaskMap.PartitionedSplitCountTracker; import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.plan.PlanNodeId; @@ -34,5 +35,6 @@ RemoteTask createRemoteTask(Session session, OptionalInt totalPartitions, OutputBuffers outputBuffers, PartitionedSplitCountTracker partitionedSplitCountTracker, - boolean summarizeTaskInfo); + boolean summarizeTaskInfo, + TableWriteInfo tableWriteInfo); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SetPathTask.java b/presto-main/src/main/java/com/facebook/presto/execution/SetPathTask.java deleted file mode 100644 index 1ee68ad8c36a5..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/execution/SetPathTask.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.execution; - -import com.facebook.presto.Session; -import com.facebook.presto.client.ClientCapabilities; -import com.facebook.presto.metadata.Metadata; -import com.facebook.presto.security.AccessControl; -import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.sql.SqlPath; -import com.facebook.presto.sql.SqlPathElement; -import com.facebook.presto.sql.analyzer.SemanticException; -import com.facebook.presto.sql.tree.Expression; -import com.facebook.presto.sql.tree.SetPath; -import com.facebook.presto.transaction.TransactionManager; -import com.google.common.util.concurrent.ListenableFuture; - -import java.util.List; -import java.util.Optional; - -import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; -import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; -import static com.facebook.presto.sql.analyzer.SemanticErrorCode.CATALOG_NOT_SPECIFIED; -import static com.google.common.util.concurrent.Futures.immediateFuture; -import static java.util.Locale.ENGLISH; - -public class SetPathTask - implements DataDefinitionTask -{ - @Override - public String getName() - { - return "SET PATH"; - } - - @Override - public ListenableFuture execute( - SetPath statement, - TransactionManager transactionManager, - Metadata metadata, - AccessControl accessControl, - QueryStateMachine stateMachine, - List parameters) - { - Session session = stateMachine.getSession(); - - if (!session.getClientCapabilities().contains(ClientCapabilities.PATH.toString())) { - throw new PrestoException(NOT_SUPPORTED, "SET PATH not supported by client"); - } - - // convert to IR before setting HTTP headers - ensures that the representations of all path objects outside the parser remain consistent - SqlPath sqlPath = new SqlPath(Optional.of(statement.getPathSpecification().toString())); - - for (SqlPathElement element : sqlPath.getParsedPath()) { - if (!element.getCatalog().isPresent() && !session.getCatalog().isPresent()) { - throw new SemanticException(CATALOG_NOT_SPECIFIED, statement, "Catalog must be specified for each path element when session catalog is not set"); - } - - element.getCatalog().ifPresent(catalog -> { - String catalogName = catalog.getValue().toLowerCase(ENGLISH); - if (!metadata.getCatalogHandle(session, catalogName).isPresent()) { - throw new PrestoException(NOT_FOUND, "Catalog does not exist: " + catalogName); - } - }); - } - stateMachine.setSetPath(sqlPath.toString()); - return immediateFuture(null); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java index d9ce34157960a..afaa5e442b996 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.concurrent.SetThreadName; import com.facebook.presto.Session; import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.cost.CostCalculator; @@ -62,7 +63,6 @@ import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.concurrent.SetThreadName; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -80,14 +80,14 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import static com.facebook.airlift.concurrent.MoreFutures.addExceptionCallback; +import static com.facebook.airlift.concurrent.MoreFutures.addSuccessCallback; import static com.facebook.presto.execution.buffer.OutputBuffers.BROADCAST_PARTITION_ID; import static com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers; import static com.facebook.presto.execution.scheduler.SqlQueryScheduler.createSqlQueryScheduler; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Throwables.throwIfInstanceOf; -import static io.airlift.concurrent.MoreFutures.addExceptionCallback; -import static io.airlift.concurrent.MoreFutures.addSuccessCallback; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.succinctBytes; import static java.util.Objects.requireNonNull; @@ -229,7 +229,7 @@ private SqlQueryExecution( } }); - this.remoteTaskFactory = new MemoryTrackingRemoteTaskFactory(requireNonNull(remoteTaskFactory, "remoteTaskFactory is null"), stateMachine); + this.remoteTaskFactory = new TrackingRemoteTaskFactory(requireNonNull(remoteTaskFactory, "remoteTaskFactory is null"), stateMachine); } } @@ -325,6 +325,12 @@ public BasicQueryInfo getBasicQueryInfo() .orElseGet(() -> stateMachine.getBasicQueryInfo(Optional.ofNullable(queryScheduler.get()).map(SqlQueryScheduler::getBasicStageStats))); } + @Override + public int getRunningTaskCount() + { + return stateMachine.getCurrentRunningTaskCount(); + } + @Override public void start() { @@ -504,7 +510,8 @@ private void planDistribution(PlanRoot plan) rootOutputBuffers, nodeTaskMap, executionPolicy, - schedulerStats); + schedulerStats, + metadata); queryScheduler.set(scheduler); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java index adf7664b307f8..26f206141a292 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; +import com.facebook.airlift.log.Logger; import com.facebook.presto.ExceededCpuLimitException; import com.facebook.presto.Session; import com.facebook.presto.event.QueryMonitor; @@ -36,8 +39,6 @@ import com.facebook.presto.spi.resourceGroups.QueryType; import com.facebook.presto.spi.resourceGroups.SelectionContext; import com.facebook.presto.spi.resourceGroups.SelectionCriteria; -import com.facebook.presto.sql.SqlEnvironmentConfig; -import com.facebook.presto.sql.SqlPath; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.transaction.TransactionManager; @@ -45,8 +46,6 @@ import com.google.common.collect.Ordering; import com.google.common.util.concurrent.AbstractFuture; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.concurrent.ThreadPoolExecutorMBean; -import io.airlift.log.Logger; import io.airlift.units.Duration; import org.weakref.jmx.Flatten; import org.weakref.jmx.Managed; @@ -62,6 +61,7 @@ import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -70,6 +70,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static com.facebook.presto.SystemSessionProperties.getQueryMaxCpuTime; import static com.facebook.presto.execution.QueryState.RUNNING; import static com.facebook.presto.execution.QueryStateMachine.QUERY_STATE_LOG; @@ -81,7 +82,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; -import static io.airlift.concurrent.Threads.threadsNamed; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; @@ -95,12 +95,12 @@ public class SqlQueryManager private final QueryPreparer queryPreparer; private final EmbedVersion embedVersion; - private final ExecutorService queryExecutor; + private final ExecutorService unboundedExecutorService; + private final Executor boundedExecutor; private final ThreadPoolExecutorMBean queryExecutorMBean; private final ResourceGroupManager resourceGroupManager; private final ClusterMemoryManager memoryManager; - private final Optional path; private final int maxQueryLength; private final Duration maxQueryCpuTime; @@ -134,7 +134,6 @@ public SqlQueryManager( EmbedVersion embedVersion, NodeSchedulerConfig nodeSchedulerConfig, QueryManagerConfig queryManagerConfig, - SqlEnvironmentConfig sqlEnvironmentConfig, QueryMonitor queryMonitor, ResourceGroupManager resourceGroupManager, ClusterMemoryManager memoryManager, @@ -153,8 +152,9 @@ public SqlQueryManager( this.embedVersion = requireNonNull(embedVersion, "embedVersion is null"); this.executionFactories = requireNonNull(executionFactories, "executionFactories is null"); - this.queryExecutor = newCachedThreadPool(threadsNamed("query-scheduler-%s")); - this.queryExecutorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) queryExecutor); + this.unboundedExecutorService = newCachedThreadPool(threadsNamed("query-scheduler-%s")); + this.boundedExecutor = new BoundedExecutor(unboundedExecutorService, queryManagerConfig.getQuerySubmissionMaxThreads()); + this.queryExecutorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) unboundedExecutorService); requireNonNull(nodeSchedulerConfig, "nodeSchedulerConfig is null"); requireNonNull(queryManagerConfig, "queryManagerConfig is null"); @@ -174,7 +174,6 @@ public SqlQueryManager( this.clusterSizeMonitor = requireNonNull(clusterSizeMonitor, "clusterSizeMonitor is null"); - this.path = sqlEnvironmentConfig.getPath(); this.maxQueryLength = queryManagerConfig.getMaxQueryLength(); this.maxQueryCpuTime = queryManagerConfig.getQueryMaxCpuTime(); @@ -212,7 +211,7 @@ public void stop() { queryTracker.stop(); queryManagementExecutor.shutdownNow(); - queryExecutor.shutdownNow(); + unboundedExecutorService.shutdownNow(); } @Override @@ -300,7 +299,7 @@ public QueryId createQueryId() public ListenableFuture createQuery(QueryId queryId, SessionContext sessionContext, String query) { QueryCreationFuture queryCreationFuture = new QueryCreationFuture(); - queryExecutor.submit(embedVersion.embedVersion(() -> { + boundedExecutor.execute(embedVersion.embedVersion(() -> { try { createQueryInternal(queryId, sessionContext, query, resourceGroupManager); queryCreationFuture.set(null); @@ -380,7 +379,6 @@ private void createQueryInternal(QueryId queryId, SessionContext sessionCont session = Session.builder(new SessionPropertyManager()) .setQueryId(queryId) .setIdentity(sessionContext.getIdentity()) - .setPath(new SqlPath(Optional.empty())) .build(); } QUERY_STATE_LOG.debug(e, "Query %s failed", session.getQueryId()); @@ -394,7 +392,7 @@ private void createQueryInternal(QueryId queryId, SessionContext sessionCont locationFactory.createQueryLocation(queryId), Optional.ofNullable(selectionContext).map(SelectionContext::getResourceGroupId), queryType, - queryExecutor, + unboundedExecutorService, e); try { @@ -438,7 +436,7 @@ private void createQueryInternal(QueryId queryId, SessionContext sessionCont // start the query in the background try { - resourceGroupManager.submit(preparedQuery.getStatement(), queryExecution, selectionContext, queryExecutor); + resourceGroupManager.submit(preparedQuery.getStatement(), queryExecution, selectionContext, unboundedExecutorService); } catch (Throwable e) { failQuery(queryId, e); @@ -496,6 +494,18 @@ public ThreadPoolExecutorMBean getManagementExecutor() return queryManagementExecutorMBean; } + @Managed + public long getRunningTaskCount() + { + return queryTracker.getRunningTaskCount(); + } + + @Managed + public long getQueriesKilledDueToTooManyTask() + { + return queryTracker.getQueriesKilledDueToTooManyTask(); + } + /** * Enforce memory limits at the query level */ diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManagerStats.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManagerStats.java index 2f621a8bcfafe..a8d2ee3a516fa 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManagerStats.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManagerStats.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.execution; -import io.airlift.stats.CounterStat; -import io.airlift.stats.DistributionStat; -import io.airlift.stats.TimeStat; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.DistributionStat; +import com.facebook.airlift.stats.TimeStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -47,6 +47,7 @@ public class SqlQueryManagerStats private final TimeStat queuedTime = new TimeStat(MILLISECONDS); private final DistributionStat wallInputBytesRate = new DistributionStat(); private final DistributionStat cpuInputByteRate = new DistributionStat(); + private final DistributionStat peakRunningTasksStat = new DistributionStat(); public void queryQueued() { @@ -88,6 +89,11 @@ public void queryFinished(QueryInfo info) cpuInputByteRate.add(rawInputBytes * 1000 / executionCpuMillis); } + long peakRunningTasks = info.getQueryStats().getPeakRunningTasks(); + if (peakRunningTasks > 0) { + peakRunningTasksStat.add(peakRunningTasks); + } + if (info.getErrorCode() != null) { switch (info.getErrorCode().getType()) { case USER_ERROR: @@ -245,4 +251,11 @@ public DistributionStat getCpuInputByteRate() { return cpuInputByteRate; } + + @Managed(description = "Distribution of query peak running tasks") + @Nested + public DistributionStat getPeakRunningTasksStat() + { + return peakRunningTasksStat; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java index 72be2d3920657..f37ae556e1c29 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java @@ -17,6 +17,7 @@ import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.execution.scheduler.SplitSchedulerStats; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.failureDetector.FailureDetector; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.RemoteTransactionHandle; @@ -34,7 +35,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import io.airlift.log.Logger; import io.airlift.units.Duration; import javax.annotation.concurrent.GuardedBy; @@ -59,6 +59,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; import static com.facebook.presto.SystemSessionProperties.getMaxFailedTaskPercentage; import static com.facebook.presto.failureDetector.FailureDetector.State.GONE; import static com.facebook.presto.operator.ExchangeOperator.REMOTE_CONNECTOR_ID; @@ -75,14 +76,12 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.collect.Sets.newConcurrentHashSet; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @ThreadSafe public final class SqlStageExecution { - private static final Logger log = Logger.get(SqlStageExecution.class); private static final Set RECOVERABLE_ERROR_CODES = ImmutableSet.of( TOO_MANY_REQUESTS_FAILED.toErrorCode(), PAGE_TRANSPORT_ERROR.toErrorCode(), @@ -90,7 +89,9 @@ public final class SqlStageExecution REMOTE_TASK_MISMATCH.toErrorCode(), REMOTE_TASK_ERROR.toErrorCode()); - private final StageStateMachine stateMachine; + private final Session session; + private final StageExecutionStateMachine stateMachine; + private final PlanFragment planFragment; private final RemoteTaskFactory remoteTaskFactory; private final NodeTaskMap nodeTaskMap; private final boolean summarizeTaskInfo; @@ -100,6 +101,8 @@ public final class SqlStageExecution private final Map exchangeSources; + private final TableWriteInfo tableWriteInfo; + private final Map> tasks = new ConcurrentHashMap<>(); @GuardedBy("this") @@ -134,8 +137,7 @@ public final class SqlStageExecution private Optional stageTaskRecoveryCallback = Optional.empty(); public static SqlStageExecution createSqlStageExecution( - StageId stageId, - URI location, + StageExecutionId stageExecutionId, PlanFragment fragment, RemoteTaskFactory remoteTaskFactory, Session session, @@ -143,10 +145,10 @@ public static SqlStageExecution createSqlStageExecution( NodeTaskMap nodeTaskMap, ExecutorService executor, FailureDetector failureDetector, - SplitSchedulerStats schedulerStats) + SplitSchedulerStats schedulerStats, + TableWriteInfo tableWriteInfo) { - requireNonNull(stageId, "stageId is null"); - requireNonNull(location, "location is null"); + requireNonNull(stageExecutionId, "stageId is null"); requireNonNull(fragment, "fragment is null"); requireNonNull(remoteTaskFactory, "remoteTaskFactory is null"); requireNonNull(session, "session is null"); @@ -154,44 +156,54 @@ public static SqlStageExecution createSqlStageExecution( requireNonNull(executor, "executor is null"); requireNonNull(failureDetector, "failureDetector is null"); requireNonNull(schedulerStats, "schedulerStats is null"); + requireNonNull(tableWriteInfo, "tableWriteInfo is null"); SqlStageExecution sqlStageExecution = new SqlStageExecution( - new StageStateMachine(stageId, location, session, fragment, executor, schedulerStats), + session, + new StageExecutionStateMachine(stageExecutionId, executor, schedulerStats, !fragment.getTableScanSchedulingOrder().isEmpty()), + fragment, remoteTaskFactory, nodeTaskMap, summarizeTaskInfo, executor, failureDetector, - getMaxFailedTaskPercentage(session)); + getMaxFailedTaskPercentage(session), + tableWriteInfo); sqlStageExecution.initialize(); return sqlStageExecution; } private SqlStageExecution( - StageStateMachine stateMachine, + Session session, + StageExecutionStateMachine stateMachine, + PlanFragment planFragment, RemoteTaskFactory remoteTaskFactory, NodeTaskMap nodeTaskMap, boolean summarizeTaskInfo, Executor executor, FailureDetector failureDetector, - double maxFailedTaskPercentage) + double maxFailedTaskPercentage, + TableWriteInfo tableWriteInfo) { + this.session = requireNonNull(session, "session is null"); this.stateMachine = stateMachine; + this.planFragment = requireNonNull(planFragment, "planFragment is null"); this.remoteTaskFactory = requireNonNull(remoteTaskFactory, "remoteTaskFactory is null"); this.nodeTaskMap = requireNonNull(nodeTaskMap, "nodeTaskMap is null"); this.summarizeTaskInfo = summarizeTaskInfo; this.executor = requireNonNull(executor, "executor is null"); this.failureDetector = requireNonNull(failureDetector, "failureDetector is null"); + this.tableWriteInfo = requireNonNull(tableWriteInfo); this.maxFailedTaskPercentage = maxFailedTaskPercentage; ImmutableMap.Builder fragmentToExchangeSource = ImmutableMap.builder(); - for (RemoteSourceNode remoteSourceNode : stateMachine.getFragment().getRemoteSourceNodes()) { + for (RemoteSourceNode remoteSourceNode : planFragment.getRemoteSourceNodes()) { for (PlanFragmentId planFragmentId : remoteSourceNode.getSourceFragmentIds()) { fragmentToExchangeSource.put(planFragmentId, remoteSourceNode); } } this.exchangeSources = fragmentToExchangeSource.build(); - this.totalLifespans = stateMachine.getFragment().getStageExecutionDescriptor().getTotalLifespans(); + this.totalLifespans = planFragment.getStageExecutionDescriptor().getTotalLifespans(); } // this is a separate method to ensure that the `this` reference is not leaked during construction @@ -201,12 +213,12 @@ private void initialize() completedLifespansChangeListeners.addListener(lifespans -> finishedLifespans.addAll(lifespans)); } - public StageId getStageId() + public StageExecutionId getStageExecutionId() { - return stateMachine.getStageId(); + return stateMachine.getStageExecutionId(); } - public StageState getState() + public StageExecutionState getState() { return stateMachine.getState(); } @@ -215,7 +227,7 @@ public StageState getState() * Listener is always notified asynchronously using a dedicated notification thread pool so, care should * be taken to avoid leaking {@code this} when adding a listener in a constructor. */ - public void addStateChangeListener(StateChangeListener stateChangeListener) + public void addStateChangeListener(StateChangeListener stateChangeListener) { stateMachine.addStateChangeListener(stateChangeListener); } @@ -226,7 +238,7 @@ public void addStateChangeListener(StateChangeListener stateChangeLi * be taken to avoid leaking {@code this} when adding a listener in a constructor. Additionally, it is * possible notifications are observed out of order due to the asynchronous execution. */ - public void addFinalStageInfoListener(StateChangeListener stateChangeListener) + public void addFinalStageInfoListener(StateChangeListener stateChangeListener) { stateMachine.addFinalStageInfoListener(stateChangeListener); } @@ -244,7 +256,7 @@ public synchronized void registerStageTaskRecoveryCallback(StageTaskRecoveryCall public PlanFragment getFragment() { - return stateMachine.getFragment(); + return planFragment; } public OutputBuffers getOutputBuffers() @@ -273,14 +285,14 @@ public synchronized void schedulingComplete() return; } - if (getAllTasks().stream().anyMatch(task -> getState() == StageState.RUNNING)) { + if (getAllTasks().stream().anyMatch(task -> getState() == StageExecutionState.RUNNING)) { stateMachine.transitionToRunning(); } if (finishedTasks.size() == allTasks.size()) { stateMachine.transitionToFinished(); } - for (PlanNodeId tableScanPlanNodeId : stateMachine.getFragment().getTableScanSchedulingOrder()) { + for (PlanNodeId tableScanPlanNodeId : planFragment.getTableScanSchedulingOrder()) { schedulingComplete(tableScanPlanNodeId); } } @@ -323,14 +335,14 @@ public synchronized Duration getTotalCpuTime() return new Duration(millis, TimeUnit.MILLISECONDS); } - public BasicStageStats getBasicStageStats() + public BasicStageExecutionStats getBasicStageStats() { return stateMachine.getBasicStageStats(this::getAllTaskInfo); } - public StageInfo getStageInfo() + public StageExecutionInfo getStageExecutionInfo() { - return stateMachine.getStageInfo(this::getAllTaskInfo, finishedLifespans.size(), totalLifespans); + return stateMachine.getStageExecutionInfo(this::getAllTaskInfo, finishedLifespans.size(), totalLifespans); } private Iterable getAllTaskInfo() @@ -429,7 +441,7 @@ public synchronized Optional scheduleTask(InternalNode node, int par return Optional.empty(); } checkState(!splitsScheduled.get(), "scheduleTask can not be called once splits have been scheduled"); - return Optional.of(scheduleTask(node, new TaskId(stateMachine.getStageId(), partition), ImmutableMultimap.of(), totalPartitions)); + return Optional.of(scheduleTask(node, new TaskId(stateMachine.getStageExecutionId(), partition), ImmutableMultimap.of(), totalPartitions)); } public synchronized Set scheduleSplits(InternalNode node, Multimap splits, Multimap noMoreSplitsNotification) @@ -442,7 +454,7 @@ public synchronized Set scheduleSplits(InternalNode node, Multimap

newTasks = ImmutableSet.builder(); Collection tasks = this.tasks.get(node); @@ -450,7 +462,7 @@ public synchronized Set scheduleSplits(InternalNode node, Multimap

failures) } } return stageTaskRecoveryCallback.isPresent() && - failedTasks.size() + 1 < allTasks.size() * maxFailedTaskPercentage; + failedTasks.size() < allTasks.size() * maxFailedTaskPercentage; } private synchronized void updateFinalTaskInfo(TaskInfo finalTaskInfo) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlTask.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlTask.java index d835341dc0fa4..0720da85f7a6f 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlTask.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.concurrent.SetThreadName; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.Session; import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.facebook.presto.execution.buffer.BufferResult; @@ -20,6 +23,7 @@ import com.facebook.presto.execution.buffer.OutputBuffer; import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.memory.QueryContext; import com.facebook.presto.operator.ExchangeClientSupplier; import com.facebook.presto.operator.PipelineContext; @@ -34,9 +38,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.concurrent.SetThreadName; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -372,7 +373,13 @@ public ListenableFuture getTaskInfo(TaskState callersCurrentState) return Futures.transform(futureTaskState, input -> getTaskInfo(), directExecutor()); } - public TaskInfo updateTask(Session session, Optional fragment, List sources, OutputBuffers outputBuffers, OptionalInt totalPartitions) + public TaskInfo updateTask( + Session session, + Optional fragment, + List sources, + OutputBuffers outputBuffers, + OptionalInt totalPartitions, + Optional tableWriteInfo) { try { // The LazyOutput buffer does not support write methods, so the actual @@ -391,6 +398,7 @@ public TaskInfo updateTask(Session session, Optional fragment, Lis taskExecution = taskHolder.getTaskExecution(); if (taskExecution == null) { checkState(fragment.isPresent(), "fragment must be present"); + checkState(tableWriteInfo.isPresent(), "tableWriteInfo must be present"); taskExecution = sqlTaskExecutionFactory.create( session, queryContext, @@ -399,7 +407,8 @@ public TaskInfo updateTask(Session session, Optional fragment, Lis taskExchangeClientManager, fragment.get(), sources, - totalPartitions); + totalPartitions, + tableWriteInfo.get()); taskHolderReference.compareAndSet(taskHolder, new TaskHolder(taskExecution)); needsPlan.set(false); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecution.java index 33836774e62fb..0ffd046dc53b7 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecution.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.concurrent.SetThreadName; import com.facebook.presto.event.SplitMonitor; import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.facebook.presto.execution.buffer.BufferState; @@ -37,7 +38,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.concurrent.SetThreadName; import io.airlift.units.Duration; import javax.annotation.Nullable; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecutionFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecutionFactory.java index bd22e001f64df..fe173fbea353a 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecutionFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecutionFactory.java @@ -13,10 +13,12 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.concurrent.SetThreadName; import com.facebook.presto.Session; import com.facebook.presto.event.SplitMonitor; import com.facebook.presto.execution.buffer.OutputBuffer; import com.facebook.presto.execution.executor.TaskExecutor; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.memory.QueryContext; import com.facebook.presto.operator.TaskContext; import com.facebook.presto.operator.TaskExchangeClientManager; @@ -24,7 +26,6 @@ import com.facebook.presto.sql.planner.LocalExecutionPlanner.LocalExecutionPlan; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.TypeProvider; -import io.airlift.concurrent.SetThreadName; import java.util.List; import java.util.OptionalInt; @@ -71,7 +72,8 @@ public SqlTaskExecution create( TaskExchangeClientManager taskExchangeClientManager, PlanFragment fragment, List sources, - OptionalInt totalPartitions) + OptionalInt totalPartitions, + TableWriteInfo tableWriteInfo) { TaskContext taskContext = queryContext.addTaskContext( taskStateMachine, @@ -92,7 +94,8 @@ public SqlTaskExecution create( fragment.getStageExecutionDescriptor(), fragment.getTableScanSchedulingOrder(), outputBuffer, - taskExchangeClientManager); + taskExchangeClientManager, + tableWriteInfo); } catch (Throwable e) { // planning failed diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskIoStats.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskIoStats.java index 591559cfff605..1ce081ef752ed 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskIoStats.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskIoStats.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.execution; -import io.airlift.stats.CounterStat; +import com.facebook.airlift.stats.CounterStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskManager.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskManager.java index 6b3a06bb7b860..55d1ff7d3c5e7 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskManager.java @@ -13,6 +13,11 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.node.NodeInfo; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.GcMonitor; import com.facebook.presto.Session; import com.facebook.presto.event.SplitMonitor; import com.facebook.presto.execution.StateMachine.StateChangeListener; @@ -20,6 +25,7 @@ import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId; import com.facebook.presto.execution.executor.TaskExecutor; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.memory.LocalMemoryManager; import com.facebook.presto.memory.MemoryPool; import com.facebook.presto.memory.MemoryPoolAssignment; @@ -39,11 +45,6 @@ import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.concurrent.ThreadPoolExecutorMBean; -import io.airlift.log.Logger; -import io.airlift.node.NodeInfo; -import io.airlift.stats.CounterStat; -import io.airlift.stats.GcMonitor; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -65,6 +66,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static com.facebook.presto.SystemSessionProperties.resourceOvercommit; import static com.facebook.presto.execution.SqlTask.createSqlTask; import static com.facebook.presto.memory.LocalMemoryManager.GENERAL_POOL; @@ -75,7 +77,6 @@ import static com.google.common.base.Predicates.notNull; import static com.google.common.collect.Iterables.filter; import static com.google.common.collect.Iterables.transform; -import static io.airlift.concurrent.Threads.threadsNamed; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newFixedThreadPool; @@ -348,7 +349,14 @@ public ListenableFuture getTaskStatus(TaskId taskId, TaskState curre } @Override - public TaskInfo updateTask(Session session, TaskId taskId, Optional fragment, List sources, OutputBuffers outputBuffers, OptionalInt totalPartitions) + public TaskInfo updateTask( + Session session, + TaskId taskId, + Optional fragment, + List sources, + OutputBuffers outputBuffers, + OptionalInt totalPartitions, + Optional tableWriteInfo) { requireNonNull(session, "session is null"); requireNonNull(taskId, "taskId is null"); @@ -363,7 +371,7 @@ public TaskInfo updateTask(Session session, TaskId taskId, Optional ids) + { + checkArgument(ids.size() == 3, "Expected 3 ids but got: %s", ids); + return new StageExecutionId(new StageId(new QueryId(ids.get(0)), parseInt(ids.get(1))), parseInt(ids.get(2))); + } + + public StageExecutionId(StageId stageId, int id) + { + this.stageId = requireNonNull(stageId, "stageId is null"); + this.id = id; + } + + public StageId getStageId() + { + return stageId; + } + + public int getId() + { + return id; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StageExecutionId that = (StageExecutionId) o; + return id == that.id && + Objects.equals(stageId, that.stageId); + } + + @Override + public int hashCode() + { + return Objects.hash(stageId, id); + } + + @Override + @JsonValue + public String toString() + { + return stageId + "." + id; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionInfo.java b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionInfo.java new file mode 100644 index 0000000000000..866ac13d94d08 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionInfo.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.execution; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class StageExecutionInfo +{ + private final StageExecutionId stageExecutionId; + private final StageExecutionState state; + private final StageExecutionStats stats; + private final List tasks; + private final Optional failureCause; + + @JsonCreator + public StageExecutionInfo( + @JsonProperty("stageExecutionId") StageExecutionId stageExecutionId, + @JsonProperty("state") StageExecutionState state, + @JsonProperty("stats") StageExecutionStats stats, + @JsonProperty("tasks") List tasks, + @JsonProperty("failureCause") Optional failureCause) + { + this.stageExecutionId = requireNonNull(stageExecutionId, "stageExecutionId is null"); + this.state = requireNonNull(state, "state is null"); + this.stats = requireNonNull(stats, "stats is null"); + this.tasks = ImmutableList.copyOf(requireNonNull(tasks, "tasks is null")); + this.failureCause = requireNonNull(failureCause, "failureCause is null"); + } + + @JsonProperty + public StageExecutionId getStageExecutionId() + { + return stageExecutionId; + } + + @JsonProperty + public StageExecutionState getState() + { + return state; + } + + @JsonProperty + public StageExecutionStats getStats() + { + return stats; + } + + @JsonProperty + public List getTasks() + { + return tasks; + } + + @JsonProperty + public Optional getFailureCause() + { + return failureCause; + } + + public boolean isFinal() + { + return state.isDone() && tasks.stream().allMatch(taskInfo -> taskInfo.getTaskStatus().getState().isDone()); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/execution/StageState.java b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionState.java similarity index 90% rename from presto-main/src/main/java/com/facebook/presto/execution/StageState.java rename to presto-main/src/main/java/com/facebook/presto/execution/StageExecutionState.java index 0448777f824aa..7890114c07b97 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/StageState.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionState.java @@ -19,7 +19,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableSet.toImmutableSet; -public enum StageState +public enum StageExecutionState { /** * Stage is planned but has not been scheduled yet. A stage will @@ -66,12 +66,12 @@ public enum StageState */ FAILED(true, true); - public static final Set TERMINAL_STAGE_STATES = Stream.of(StageState.values()).filter(StageState::isDone).collect(toImmutableSet()); + public static final Set TERMINAL_STAGE_STATES = Stream.of(StageExecutionState.values()).filter(StageExecutionState::isDone).collect(toImmutableSet()); private final boolean doneState; private final boolean failureState; - StageState(boolean doneState, boolean failureState) + StageExecutionState(boolean doneState, boolean failureState) { checkArgument(!failureState || doneState, "%s is a non-done failure state", name()); this.doneState = doneState; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/StageStateMachine.java b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java similarity index 74% rename from presto-main/src/main/java/com/facebook/presto/execution/StageStateMachine.java rename to presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java index 52a82abfb6617..0b0f166841b1c 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/StageStateMachine.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java @@ -13,7 +13,8 @@ */ package com.facebook.presto.execution; -import com.facebook.presto.Session; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.Distribution; import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.facebook.presto.execution.scheduler.SplitSchedulerStats; import com.facebook.presto.operator.BlockedReason; @@ -21,16 +22,12 @@ import com.facebook.presto.operator.PipelineStats; import com.facebook.presto.operator.TaskStats; import com.facebook.presto.spi.eventlistener.StageGcStatistics; -import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.util.Failures; import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; -import io.airlift.stats.Distribution; import org.joda.time.DateTime; import javax.annotation.concurrent.ThreadSafe; -import java.net.URI; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -44,17 +41,17 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; -import static com.facebook.presto.execution.StageState.ABORTED; -import static com.facebook.presto.execution.StageState.CANCELED; -import static com.facebook.presto.execution.StageState.FAILED; -import static com.facebook.presto.execution.StageState.FINISHED; -import static com.facebook.presto.execution.StageState.FINISHED_TASK_SCHEDULING; -import static com.facebook.presto.execution.StageState.PLANNED; -import static com.facebook.presto.execution.StageState.RUNNING; -import static com.facebook.presto.execution.StageState.SCHEDULED; -import static com.facebook.presto.execution.StageState.SCHEDULING; -import static com.facebook.presto.execution.StageState.SCHEDULING_SPLITS; -import static com.facebook.presto.execution.StageState.TERMINAL_STAGE_STATES; +import static com.facebook.presto.execution.StageExecutionState.ABORTED; +import static com.facebook.presto.execution.StageExecutionState.CANCELED; +import static com.facebook.presto.execution.StageExecutionState.FAILED; +import static com.facebook.presto.execution.StageExecutionState.FINISHED; +import static com.facebook.presto.execution.StageExecutionState.FINISHED_TASK_SCHEDULING; +import static com.facebook.presto.execution.StageExecutionState.PLANNED; +import static com.facebook.presto.execution.StageExecutionState.RUNNING; +import static com.facebook.presto.execution.StageExecutionState.SCHEDULED; +import static com.facebook.presto.execution.StageExecutionState.SCHEDULING; +import static com.facebook.presto.execution.StageExecutionState.SCHEDULING_SPLITS; +import static com.facebook.presto.execution.StageExecutionState.TERMINAL_STAGE_STATES; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -69,20 +66,17 @@ import static java.util.concurrent.TimeUnit.SECONDS; @ThreadSafe -public class StageStateMachine +public class StageExecutionStateMachine { - private static final Logger log = Logger.get(StageStateMachine.class); + private static final Logger log = Logger.get(StageExecutionStateMachine.class); - private final StageId stageId; - private final URI location; - private final PlanFragment fragment; - private final Session session; + private final StageExecutionId stageExecutionId; private final SplitSchedulerStats scheduledStats; + private final boolean containsTableScans; - private final StateMachine stageState; - private final StateMachine> finalStageInfo; + private final StateMachine state; + private final StateMachine> finalInfo; private final AtomicReference failureCause = new AtomicReference<>(); - private final AtomicReference> renderedPlan = new AtomicReference<>(Optional.empty()); private final AtomicReference schedulingComplete = new AtomicReference<>(); private final Distribution getSplitDistribution = new Distribution(); @@ -91,49 +85,30 @@ public class StageStateMachine private final AtomicLong currentUserMemory = new AtomicLong(); private final AtomicLong currentTotalMemory = new AtomicLong(); - public StageStateMachine( - StageId stageId, - URI location, - Session session, - PlanFragment fragment, + public StageExecutionStateMachine( + StageExecutionId stageExecutionId, ExecutorService executor, - SplitSchedulerStats schedulerStats) + SplitSchedulerStats schedulerStats, + boolean containsTableScans) { - this.stageId = requireNonNull(stageId, "stageId is null"); - this.location = requireNonNull(location, "location is null"); - this.session = requireNonNull(session, "session is null"); - this.fragment = requireNonNull(fragment, "fragment is null"); + this.stageExecutionId = requireNonNull(stageExecutionId, "stageId is null"); this.scheduledStats = requireNonNull(schedulerStats, "schedulerStats is null"); + this.containsTableScans = containsTableScans; - stageState = new StateMachine<>("stage " + stageId, executor, PLANNED, TERMINAL_STAGE_STATES); - stageState.addStateChangeListener(state -> log.debug("Stage %s is %s", stageId, state)); + state = new StateMachine<>("stage execution " + stageExecutionId, executor, PLANNED, TERMINAL_STAGE_STATES); + state.addStateChangeListener(state -> log.debug("Stage Execution %s is %s", stageExecutionId, state)); - finalStageInfo = new StateMachine<>("final stage " + stageId, executor, Optional.empty()); + finalInfo = new StateMachine<>("final stage execution " + stageExecutionId, executor, Optional.empty()); } - public StageId getStageId() + public StageExecutionId getStageExecutionId() { - return stageId; + return stageExecutionId; } - public URI getLocation() + public StageExecutionState getState() { - return location; - } - - public Session getSession() - { - return session; - } - - public StageState getState() - { - return stageState.get(); - } - - public PlanFragment getFragment() - { - return fragment; + return state.get(); } /** @@ -141,50 +116,50 @@ public PlanFragment getFragment() * be taken to avoid leaking {@code this} when adding a listener in a constructor. Additionally, it is * possible notifications are observed out of order due to the asynchronous execution. */ - public void addStateChangeListener(StateChangeListener stateChangeListener) + public void addStateChangeListener(StateChangeListener stateChangeListener) { - stageState.addStateChangeListener(stateChangeListener); + state.addStateChangeListener(stateChangeListener); } public synchronized boolean transitionToScheduling() { - return stageState.compareAndSet(PLANNED, SCHEDULING); + return state.compareAndSet(PLANNED, SCHEDULING); } public synchronized boolean transitionToFinishedTaskScheduling() { - return stageState.compareAndSet(SCHEDULING, FINISHED_TASK_SCHEDULING); + return state.compareAndSet(SCHEDULING, FINISHED_TASK_SCHEDULING); } public synchronized boolean transitionToSchedulingSplits() { - return stageState.setIf(SCHEDULING_SPLITS, currentState -> currentState == PLANNED || currentState == SCHEDULING || currentState == FINISHED_TASK_SCHEDULING); + return state.setIf(SCHEDULING_SPLITS, currentState -> currentState == PLANNED || currentState == SCHEDULING || currentState == FINISHED_TASK_SCHEDULING); } public synchronized boolean transitionToScheduled() { schedulingComplete.compareAndSet(null, DateTime.now()); - return stageState.setIf(SCHEDULED, currentState -> currentState == PLANNED || currentState == SCHEDULING || currentState == FINISHED_TASK_SCHEDULING || currentState == SCHEDULING_SPLITS); + return state.setIf(SCHEDULED, currentState -> currentState == PLANNED || currentState == SCHEDULING || currentState == FINISHED_TASK_SCHEDULING || currentState == SCHEDULING_SPLITS); } public boolean transitionToRunning() { - return stageState.setIf(RUNNING, currentState -> currentState != RUNNING && !currentState.isDone()); + return state.setIf(RUNNING, currentState -> currentState != RUNNING && !currentState.isDone()); } public boolean transitionToFinished() { - return stageState.setIf(FINISHED, currentState -> !currentState.isDone()); + return state.setIf(FINISHED, currentState -> !currentState.isDone()); } public boolean transitionToCanceled() { - return stageState.setIf(CANCELED, currentState -> !currentState.isDone()); + return state.setIf(CANCELED, currentState -> !currentState.isDone()); } public boolean transitionToAborted() { - return stageState.setIf(ABORTED, currentState -> !currentState.isDone()); + return state.setIf(ABORTED, currentState -> !currentState.isDone()); } public boolean transitionToFailed(Throwable throwable) @@ -192,12 +167,12 @@ public boolean transitionToFailed(Throwable throwable) requireNonNull(throwable, "throwable is null"); failureCause.compareAndSet(null, Failures.toFailure(throwable)); - boolean failed = stageState.setIf(FAILED, currentState -> !currentState.isDone()); + boolean failed = state.setIf(FAILED, currentState -> !currentState.isDone()); if (failed) { - log.error(throwable, "Stage %s failed", stageId); + log.error(throwable, "Stage execution %s failed", stageExecutionId); } else { - log.debug(throwable, "Failure after stage %s finished", stageId); + log.debug(throwable, "Failure after stage execution %s finished", stageExecutionId); } return failed; } @@ -208,24 +183,24 @@ public boolean transitionToFailed(Throwable throwable) * be taken to avoid leaking {@code this} when adding a listener in a constructor. Additionally, it is * possible notifications are observed out of order due to the asynchronous execution. */ - public void addFinalStageInfoListener(StateChangeListener finalStatusListener) + public void addFinalStageInfoListener(StateChangeListener finalStatusListener) { AtomicBoolean done = new AtomicBoolean(); - StateChangeListener> fireOnceStateChangeListener = finalStageInfo -> { + StateChangeListener> fireOnceStateChangeListener = finalStageInfo -> { if (finalStageInfo.isPresent() && done.compareAndSet(false, true)) { finalStatusListener.stateChanged(finalStageInfo.get()); } }; - finalStageInfo.addStateChangeListener(fireOnceStateChangeListener); + finalInfo.addStateChangeListener(fireOnceStateChangeListener); } public void setAllTasksFinal(Iterable finalTaskInfos, int totalLifespans) { requireNonNull(finalTaskInfos, "finalTaskInfos is null"); - checkState(stageState.get().isDone()); - StageInfo stageInfo = getStageInfo(() -> finalTaskInfos, totalLifespans, totalLifespans); - checkArgument(stageInfo.isFinalStageInfo(), "finalTaskInfos are not all done"); - finalStageInfo.compareAndSet(Optional.empty(), Optional.of(stageInfo)); + checkState(state.get().isDone()); + StageExecutionInfo stageInfo = getStageExecutionInfo(() -> finalTaskInfos, totalLifespans, totalLifespans); + checkArgument(stageInfo.isFinal(), "finalTaskInfos are not all done"); + finalInfo.compareAndSet(Optional.empty(), Optional.of(stageInfo)); } public long getUserMemoryReservation() @@ -245,12 +220,12 @@ public void updateMemoryUsage(long deltaUserMemoryInBytes, long deltaTotalMemory peakUserMemory.updateAndGet(currentPeakValue -> max(currentUserMemory.get(), currentPeakValue)); } - public BasicStageStats getBasicStageStats(Supplier> taskInfosSupplier) + public BasicStageExecutionStats getBasicStageStats(Supplier> taskInfosSupplier) { - Optional finalStageInfo = this.finalStageInfo.get(); + Optional finalStageInfo = this.finalInfo.get(); if (finalStageInfo.isPresent()) { return finalStageInfo.get() - .getStageStats() + .getStats() .toBasicStageStats(finalStageInfo.get().getState()); } @@ -258,7 +233,7 @@ public BasicStageStats getBasicStageStats(Supplier> taskInfos // consistent view of the stage. For example, building this // information, the stage could finish, and the task states would // never be visible. - StageState state = stageState.get(); + StageExecutionState state = this.state.get(); boolean isScheduled = (state == RUNNING) || state.isDone(); List taskInfos = ImmutableList.copyOf(taskInfosSupplier.get()); @@ -304,7 +279,7 @@ public BasicStageStats getBasicStageStats(Supplier> taskInfos blockedReasons.addAll(taskStats.getBlockedReasons()); } - if (!fragment.getTableScanSchedulingOrder().isEmpty()) { + if (containsTableScans) { rawInputDataSize += taskStats.getRawInputDataSize().toBytes(); rawInputPositions += taskStats.getRawInputPositions(); } @@ -315,7 +290,7 @@ public BasicStageStats getBasicStageStats(Supplier> taskInfos progressPercentage = OptionalDouble.of(min(100, (completedDrivers * 100.0) / totalDrivers)); } - return new BasicStageStats( + return new BasicStageExecutionStats( isScheduled, totalDrivers, @@ -339,9 +314,9 @@ public BasicStageStats getBasicStageStats(Supplier> taskInfos progressPercentage); } - public StageInfo getStageInfo(Supplier> taskInfosSupplier, int finishedLifespans, int totalLifespans) + public StageExecutionInfo getStageExecutionInfo(Supplier> taskInfosSupplier, int finishedLifespans, int totalLifespans) { - Optional finalStageInfo = this.finalStageInfo.get(); + Optional finalStageInfo = this.finalInfo.get(); if (finalStageInfo.isPresent()) { return finalStageInfo.get(); } @@ -350,7 +325,7 @@ public StageInfo getStageInfo(Supplier> taskInfosSupplier, in // consistent view of the stage. For example, building this // information, the stage could finish, and the task states would // never be visible. - StageState state = stageState.get(); + StageExecutionState state = this.state.get(); List taskInfos = ImmutableList.copyOf(taskInfosSupplier.get()); @@ -455,7 +430,7 @@ public StageInfo getStageInfo(Supplier> taskInfosSupplier, in } } - StageStats stageStats = new StageStats( + StageExecutionStats stageExecutionStats = new StageExecutionStats( schedulingComplete.get(), getSplitDistribution.snapshot(), @@ -492,7 +467,8 @@ public StageInfo getStageInfo(Supplier> taskInfosSupplier, in succinctBytes(physicalWrittenDataSize), new StageGcStatistics( - stageId.getId(), + stageExecutionId.getStageId().getId(), + stageExecutionId.getId(), totalTasks, fullGcTaskCount, minFullGcSec, @@ -506,14 +482,11 @@ public StageInfo getStageInfo(Supplier> taskInfosSupplier, in if (state == FAILED) { failureInfo = Optional.of(failureCause.get()); } - return new StageInfo(stageId, + return new StageExecutionInfo( + stageExecutionId, state, - location, - Optional.of(fragment), - fragment.getTypes(), - stageStats, + stageExecutionStats, taskInfos, - ImmutableList.of(), failureInfo); } @@ -528,8 +501,8 @@ public void recordGetSplitTime(long startNanos) public String toString() { return toStringHelper(this) - .add("stageId", stageId) - .add("stageState", stageState) + .add("stageExecutionId", stageExecutionId) + .add("state", state) .toString(); } } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/StageStats.java b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStats.java similarity index 96% rename from presto-main/src/main/java/com/facebook/presto/execution/StageStats.java rename to presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStats.java index c4a5000f9b8ac..bfd532748ed6c 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/StageStats.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStats.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.stats.Distribution.DistributionSnapshot; import com.facebook.presto.operator.BlockedReason; import com.facebook.presto.operator.OperatorStats; import com.facebook.presto.spi.eventlistener.StageGcStatistics; @@ -20,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.stats.Distribution.DistributionSnapshot; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -31,13 +31,13 @@ import java.util.OptionalDouble; import java.util.Set; -import static com.facebook.presto.execution.StageState.RUNNING; +import static com.facebook.presto.execution.StageExecutionState.RUNNING; import static com.google.common.base.Preconditions.checkArgument; import static java.lang.Math.min; import static java.util.Objects.requireNonNull; @Immutable -public class StageStats +public class StageExecutionStats { private final DateTime schedulingComplete; @@ -84,7 +84,7 @@ public class StageStats private final List operatorSummaries; @JsonCreator - public StageStats( + public StageExecutionStats( @JsonProperty("schedulingComplete") DateTime schedulingComplete, @JsonProperty("getSplitDistribution") DistributionSnapshot getSplitDistribution, @@ -372,16 +372,16 @@ public List getOperatorSummaries() return operatorSummaries; } - public BasicStageStats toBasicStageStats(StageState stageState) + public BasicStageExecutionStats toBasicStageStats(StageExecutionState stageExecutionState) { - boolean isScheduled = (stageState == RUNNING) || stageState.isDone(); + boolean isScheduled = (stageExecutionState == RUNNING) || stageExecutionState.isDone(); OptionalDouble progressPercentage = OptionalDouble.empty(); if (isScheduled && totalDrivers != 0) { progressPercentage = OptionalDouble.of(min(100, (completedDrivers * 100.0) / totalDrivers)); } - return new BasicStageStats( + return new BasicStageExecutionStats( isScheduled, totalDrivers, queuedDrivers, diff --git a/presto-main/src/main/java/com/facebook/presto/execution/StageInfo.java b/presto-main/src/main/java/com/facebook/presto/execution/StageInfo.java index 2a90a397c28aa..9900a7ea7871e 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/StageInfo.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/StageInfo.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.execution; -import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.planner.PlanFragment; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -33,36 +32,29 @@ public class StageInfo { private final StageId stageId; - private final StageState state; private final URI self; private final Optional plan; - private final List types; - private final StageStats stageStats; - private final List tasks; + + private final StageExecutionInfo latestAttemptExecutionInfo; + private final List previousAttemptsExecutionInfos; + private final List subStages; - private final Optional failureCause; @JsonCreator public StageInfo( @JsonProperty("stageId") StageId stageId, - @JsonProperty("state") StageState state, @JsonProperty("self") URI self, @JsonProperty("plan") Optional plan, - @JsonProperty("types") List types, - @JsonProperty("stageStats") StageStats stageStats, - @JsonProperty("tasks") List tasks, - @JsonProperty("subStages") List subStages, - @JsonProperty("failureCause") Optional failureCause) + @JsonProperty("latestAttemptExecutionInfo") StageExecutionInfo latestAttemptExecutionInfo, + @JsonProperty("previousAttemptsExecutionInfos") List previousAttemptsExecutionInfos, + @JsonProperty("subStages") List subStages) { this.stageId = requireNonNull(stageId, "stageId is null"); - this.state = requireNonNull(state, "state is null"); this.self = requireNonNull(self, "self is null"); this.plan = requireNonNull(plan, "plan is null"); - this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); - this.stageStats = requireNonNull(stageStats, "stageStats is null"); - this.tasks = ImmutableList.copyOf(requireNonNull(tasks, "tasks is null")); + this.latestAttemptExecutionInfo = requireNonNull(latestAttemptExecutionInfo, "latestAttemptExecutionInfo is null"); + this.previousAttemptsExecutionInfos = ImmutableList.copyOf(requireNonNull(previousAttemptsExecutionInfos, "previousAttemptsExecutionInfos is null")); this.subStages = ImmutableList.copyOf(requireNonNull(subStages, "subStages is null")); - this.failureCause = requireNonNull(failureCause, "failureCause is null"); } @JsonProperty @@ -71,12 +63,6 @@ public StageId getStageId() return stageId; } - @JsonProperty - public StageState getState() - { - return state; - } - @JsonProperty public URI getSelf() { @@ -90,21 +76,15 @@ public Optional getPlan() } @JsonProperty - public List getTypes() - { - return types; - } - - @JsonProperty - public StageStats getStageStats() + public StageExecutionInfo getLatestAttemptExecutionInfo() { - return stageStats; + return latestAttemptExecutionInfo; } @JsonProperty - public List getTasks() + public List getPreviousAttemptsExecutionInfos() { - return tasks; + return previousAttemptsExecutionInfos; } @JsonProperty @@ -113,15 +93,9 @@ public List getSubStages() return subStages; } - @JsonProperty - public Optional getFailureCause() - { - return failureCause; - } - public boolean isFinalStageInfo() { - return state.isDone() && tasks.stream().allMatch(taskInfo -> taskInfo.getTaskStatus().getState().isDone()); + return latestAttemptExecutionInfo.isFinal(); } @Override @@ -129,7 +103,7 @@ public String toString() { return toStringHelper(this) .add("stageId", stageId) - .add("state", state) + .add("state", latestAttemptExecutionInfo.getState()) .toString(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/StateMachine.java b/presto-main/src/main/java/com/facebook/presto/execution/StateMachine.java index a8e359b2955b8..b4707e0e3fdd9 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/StateMachine.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/StateMachine.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.PrestoException; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.log.Logger; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/TaskId.java b/presto-main/src/main/java/com/facebook/presto/execution/TaskId.java index bfe8a9673f0ea..805bbc054d039 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/TaskId.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/TaskId.java @@ -20,75 +20,74 @@ import java.util.List; import java.util.Objects; +import static com.facebook.presto.spi.QueryId.parseDottedId; import static com.google.common.base.Preconditions.checkArgument; import static java.lang.Integer.parseInt; +import static java.util.Objects.requireNonNull; public class TaskId { + private final StageExecutionId stageExecutionId; + private final int id; + @JsonCreator public static TaskId valueOf(String taskId) { - return new TaskId(taskId); + List parts = parseDottedId(taskId, 4, "taskId"); + return new TaskId(parts.get(0), parseInt(parts.get(1)), parseInt(parts.get(2)), parseInt(parts.get(3))); } - private final String fullId; - - public TaskId(String queryId, int stageId, int id) + public TaskId(String queryId, int stageId, int stageExecutionId, int id) { - checkArgument(id >= 0, "id is negative"); - this.fullId = queryId + "." + stageId + "." + id; + this(new StageExecutionId(new StageId(new QueryId(queryId), stageId), stageExecutionId), id); } - public TaskId(StageId stageId, int id) + public TaskId(StageExecutionId stageExecutionId, int id) { + this.stageExecutionId = requireNonNull(stageExecutionId, "stageExecutionId"); checkArgument(id >= 0, "id is negative"); - this.fullId = stageId.getQueryId().getId() + "." + stageId.getId() + "." + id; - } - - public TaskId(String fullId) - { - this.fullId = fullId; + this.id = id; } - public QueryId getQueryId() + public StageExecutionId getStageExecutionId() { - return new QueryId(QueryId.parseDottedId(fullId, 3, "taskId").get(0)); + return stageExecutionId; } - public StageId getStageId() + public int getId() { - List ids = QueryId.parseDottedId(fullId, 3, "taskId"); - return StageId.valueOf(ids.subList(0, 2)); + return id; } - public int getId() + public QueryId getQueryId() { - return parseInt(QueryId.parseDottedId(fullId, 3, "taskId").get(2)); + return stageExecutionId.getStageId().getQueryId(); } @Override @JsonValue public String toString() { - return fullId; + return stageExecutionId + "." + id; } @Override - public int hashCode() - { - return Objects.hash(fullId); - } - - @Override - public boolean equals(Object obj) + public boolean equals(Object o) { - if (this == obj) { + if (this == o) { return true; } - if (obj == null || getClass() != obj.getClass()) { + if (o == null || getClass() != o.getClass()) { return false; } - TaskId other = (TaskId) obj; - return Objects.equals(this.fullId, other.fullId); + TaskId taskId = (TaskId) o; + return id == taskId.id && + Objects.equals(stageExecutionId, taskId.stageExecutionId); + } + + @Override + public int hashCode() + { + return Objects.hash(stageExecutionId, id); } } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/TaskManagementExecutor.java b/presto-main/src/main/java/com/facebook/presto/execution/TaskManagementExecutor.java index 7895157c2efbd..876dce427deca 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/TaskManagementExecutor.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/TaskManagementExecutor.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.execution; -import io.airlift.concurrent.ThreadPoolExecutorMBean; +import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -22,7 +22,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; -import static io.airlift.concurrent.Threads.threadsNamed; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static java.util.concurrent.Executors.newScheduledThreadPool; /** diff --git a/presto-main/src/main/java/com/facebook/presto/execution/TaskManager.java b/presto-main/src/main/java/com/facebook/presto/execution/TaskManager.java index 6c08f92ad2c3d..4ec168a23f2ac 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/TaskManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/TaskManager.java @@ -18,6 +18,7 @@ import com.facebook.presto.execution.buffer.BufferResult; import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.memory.MemoryPoolAssignmentsRequest; import com.facebook.presto.sql.planner.PlanFragment; import com.google.common.util.concurrent.ListenableFuture; @@ -83,7 +84,14 @@ public interface TaskManager * Updates the task plan, sources and output buffers. If the task does not * already exist, is is created and then updated. */ - TaskInfo updateTask(Session session, TaskId taskId, Optional fragment, List sources, OutputBuffers outputBuffers, OptionalInt totalPartitions); + TaskInfo updateTask( + Session session, + TaskId taskId, + Optional fragment, + List sources, + OutputBuffers outputBuffers, + OptionalInt totalPartitions, + Optional tableWriteInfo); /** * Cancels a task. If the task does not already exist, is is created and then diff --git a/presto-main/src/main/java/com/facebook/presto/execution/TaskManagerConfig.java b/presto-main/src/main/java/com/facebook/presto/execution/TaskManagerConfig.java index 1ea66aefaefb3..c3fa0d7fcade6 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/TaskManagerConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/TaskManagerConfig.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.DefunctConfig; +import com.facebook.airlift.configuration.LegacyConfig; import com.facebook.presto.util.PowerOfTwo; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.DefunctConfig; -import io.airlift.configuration.LegacyConfig; import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import io.airlift.units.Duration; @@ -68,6 +68,7 @@ public class TaskManagerConfig private Duration infoUpdateInterval = new Duration(3, TimeUnit.SECONDS); private int writerCount = 1; + private Integer partitionedWriterCount; private int taskConcurrency = 16; private int httpResponseThreads = 100; private int httpTimeoutThreads = 3; @@ -397,6 +398,21 @@ public TaskManagerConfig setWriterCount(int writerCount) return this; } + @Min(1) + @PowerOfTwo + public Integer getPartitionedWriterCount() + { + return partitionedWriterCount; + } + + @Config("task.partitioned-writer-count") + @ConfigDescription("Number of writers per task for partitioned writes. If not set, the number set by task.writer-count will be used") + public TaskManagerConfig setPartitionedWriterCount(Integer partitionedWriterCount) + { + this.partitionedWriterCount = partitionedWriterCount; + return this; + } + @Min(1) @PowerOfTwo public int getTaskConcurrency() diff --git a/presto-main/src/main/java/com/facebook/presto/execution/TaskStateMachine.java b/presto-main/src/main/java/com/facebook/presto/execution/TaskStateMachine.java index 1b55d52d1d0f9..38cede7381882 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/TaskStateMachine.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/TaskStateMachine.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.log.Logger; import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.log.Logger; import org.joda.time.DateTime; import javax.annotation.concurrent.ThreadSafe; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/TrackingRemoteTaskFactory.java similarity index 75% rename from presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java rename to presto-main/src/main/java/com/facebook/presto/execution/TrackingRemoteTaskFactory.java index 846f15f713fb5..fd79ca7a3e650 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/MemoryTrackingRemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/TrackingRemoteTaskFactory.java @@ -17,6 +17,7 @@ import com.facebook.presto.execution.NodeTaskMap.PartitionedSplitCountTracker; import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.plan.PlanNodeId; @@ -25,15 +26,17 @@ import java.util.OptionalInt; +import static com.facebook.presto.execution.TaskState.PLANNED; +import static com.facebook.presto.execution.TaskState.RUNNING; import static java.util.Objects.requireNonNull; -public class MemoryTrackingRemoteTaskFactory +public class TrackingRemoteTaskFactory implements RemoteTaskFactory { private final RemoteTaskFactory remoteTaskFactory; private final QueryStateMachine stateMachine; - public MemoryTrackingRemoteTaskFactory(RemoteTaskFactory remoteTaskFactory, QueryStateMachine stateMachine) + public TrackingRemoteTaskFactory(RemoteTaskFactory remoteTaskFactory, QueryStateMachine stateMachine) { this.remoteTaskFactory = requireNonNull(remoteTaskFactory, "remoteTaskFactory is null"); this.stateMachine = requireNonNull(stateMachine, "stateMachine is null"); @@ -48,7 +51,8 @@ public RemoteTask createRemoteTask(Session session, OptionalInt totalPartitions, OutputBuffers outputBuffers, PartitionedSplitCountTracker partitionedSplitCountTracker, - boolean summarizeTaskInfo) + boolean summarizeTaskInfo, + TableWriteInfo tableWriteInfo) { RemoteTask task = remoteTaskFactory.createRemoteTask(session, taskId, @@ -58,22 +62,25 @@ public RemoteTask createRemoteTask(Session session, totalPartitions, outputBuffers, partitionedSplitCountTracker, - summarizeTaskInfo); + summarizeTaskInfo, + tableWriteInfo); - task.addStateChangeListener(new UpdatePeakMemory(stateMachine)); + task.addStateChangeListener(new UpdateQueryStats(stateMachine)); return task; } - private static final class UpdatePeakMemory + private static final class UpdateQueryStats implements StateChangeListener { private final QueryStateMachine stateMachine; private long previousUserMemory; private long previousSystemMemory; + private TaskState state; - public UpdatePeakMemory(QueryStateMachine stateMachine) + public UpdateQueryStats(QueryStateMachine stateMachine) { this.stateMachine = stateMachine; + this.state = PLANNED; } @Override @@ -87,6 +94,14 @@ public synchronized void stateChanged(TaskStatus newStatus) previousUserMemory = currentUserMemory; previousSystemMemory = currentSystemMemory; stateMachine.updateMemoryUsage(deltaUserMemoryInBytes, deltaTotalMemoryInBytes, currentUserMemory, currentTotalMemory); + + if (state == PLANNED && newStatus.getState() == RUNNING) { + stateMachine.incrementCurrentRunningTaskCount(); + } + else if (state == RUNNING && newStatus.getState().isDone()) { + stateMachine.decrementCurrentRunningTaskCount(); + } + state = newStatus.getState(); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/buffer/LazyOutputBuffer.java b/presto-main/src/main/java/com/facebook/presto/execution/buffer/LazyOutputBuffer.java index a0c0e48e02238..dbcb4c6d147cb 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/buffer/LazyOutputBuffer.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/buffer/LazyOutputBuffer.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution.buffer; +import com.facebook.airlift.concurrent.ExtendedSettableFuture; import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.StateMachine; import com.facebook.presto.execution.StateMachine.StateChangeListener; @@ -22,7 +23,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.concurrent.ExtendedSettableFuture; import io.airlift.units.DataSize; import javax.annotation.concurrent.GuardedBy; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/buffer/PagesSerde.java b/presto-main/src/main/java/com/facebook/presto/execution/buffer/PagesSerde.java index 74b77228f00c3..7efff296b95db 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/buffer/PagesSerde.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/buffer/PagesSerde.java @@ -28,14 +28,17 @@ import java.nio.ByteBuffer; import java.util.Optional; +import static com.facebook.presto.array.Arrays.ensureCapacity; import static com.facebook.presto.execution.buffer.PageCodecMarker.COMPRESSED; import static com.facebook.presto.execution.buffer.PageCodecMarker.ENCRYPTED; import static com.facebook.presto.execution.buffer.PagesSerdeUtil.readRawPage; import static com.facebook.presto.execution.buffer.PagesSerdeUtil.writeRawPage; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; +import static io.airlift.slice.SizeOf.sizeOf; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; +import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET; @NotThreadSafe public class PagesSerde @@ -47,6 +50,8 @@ public class PagesSerde private final Optional decompressor; private final Optional spillCipher; + private byte[] compressionBuffer; + public PagesSerde(BlockEncodingSerde blockEncodingSerde, Optional compressor, Optional decompressor, Optional spillCipher) { this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerde is null"); @@ -61,31 +66,14 @@ public SerializedPage serialize(Page page) { SliceOutput serializationBuffer = new DynamicSliceOutput(toIntExact(page.getSizeInBytes() + Integer.BYTES)); // block length is an int writeRawPage(page, serializationBuffer, blockEncodingSerde); - Slice slice = serializationBuffer.slice(); - int uncompressedSize = serializationBuffer.size(); - byte markers = PageCodecMarker.none(); - - if (compressor.isPresent()) { - ByteBuffer compressionBuffer = ByteBuffer.allocate(compressor.get().maxCompressedLength(uncompressedSize)); - compressor.get().compress(slice.toByteBuffer(), compressionBuffer); - compressionBuffer.flip(); - if ((((double) compressionBuffer.remaining()) / uncompressedSize) <= MINIMUM_COMPRESSION_RATIO) { - slice = Slices.wrappedBuffer(compressionBuffer); - markers = COMPRESSED.set(markers); - } - } - if (spillCipher.isPresent()) { - slice = Slices.wrappedBuffer(spillCipher.get().encrypt(slice.toByteBuffer())); - markers = ENCRYPTED.set(markers); - } - else { - // Encryption disabled, slice data is likely much smaller than its backing buffer - // either because of compression or dynamic sizing of the initial output slice - slice = Slices.copyOf(slice); - } + return wrapSlice(serializationBuffer.slice(), page.getPositionCount()); + } - return new SerializedPage(slice, markers, page.getPositionCount(), uncompressedSize); + public SerializedPage serialize(Slice slice, int positionCount) + { + checkArgument(slice.isCompact(), "slice is not compact"); + return wrapSlice(slice, positionCount); } public Page deserialize(SerializedPage serializedPage) @@ -114,4 +102,47 @@ public Page deserialize(SerializedPage serializedPage) return readRawPage(serializedPage.getPositionCount(), slice.getInput(), blockEncodingSerde); } + + public long getSizeInBytes() + { + return compressionBuffer == null ? 0 : compressionBuffer.length; + } + + public long getRetainedSizeInBytes() + { + return sizeOf(compressionBuffer); + } + + private SerializedPage wrapSlice(Slice slice, int positionCount) + { + int uncompressedSize = slice.length(); + byte markers = PageCodecMarker.none(); + + if (compressor.isPresent()) { + int maxCompressedSize = compressor.get().maxCompressedLength(uncompressedSize); + compressionBuffer = ensureCapacity(compressionBuffer, maxCompressedSize); + int compressedSize = compressor.get().compress( + (byte[]) slice.getBase(), + (int) (slice.getAddress() - ARRAY_BYTE_BASE_OFFSET), + uncompressedSize, + compressionBuffer, + 0, + maxCompressedSize); + + if (compressedSize / (double) uncompressedSize <= MINIMUM_COMPRESSION_RATIO) { + slice = Slices.copyOf(Slices.wrappedBuffer(compressionBuffer, 0, compressedSize)); + markers = COMPRESSED.set(markers); + } + } + + if (spillCipher.isPresent()) { + slice = Slices.wrappedBuffer(spillCipher.get().encrypt(slice.toByteBuffer())); + markers = ENCRYPTED.set(markers); + } + else if (!slice.isCompact()) { + slice = Slices.copyOf(slice); + } + + return new SerializedPage(slice, markers, positionCount, uncompressedSize); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/executor/MultilevelSplitQueue.java b/presto-main/src/main/java/com/facebook/presto/execution/executor/MultilevelSplitQueue.java index af652e848b0b7..271843355b690 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/executor/MultilevelSplitQueue.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/executor/MultilevelSplitQueue.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.execution.executor; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.execution.TaskManagerConfig; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; -import io.airlift.stats.CounterStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/executor/PrioritizedSplitRunner.java b/presto-main/src/main/java/com/facebook/presto/execution/executor/PrioritizedSplitRunner.java index 3a21c839afb9e..181df738ea62f 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/executor/PrioritizedSplitRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/executor/PrioritizedSplitRunner.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.execution.executor; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.CpuTimer; +import com.facebook.airlift.stats.TimeStat; import com.facebook.presto.execution.SplitRunner; import com.google.common.base.Ticker; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; -import io.airlift.stats.CpuTimer; -import io.airlift.stats.TimeStat; import io.airlift.units.Duration; import java.util.concurrent.TimeUnit; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/executor/TaskExecutor.java b/presto-main/src/main/java/com/facebook/presto/execution/executor/TaskExecutor.java index 5f6df55c76906..940ab1ffae09f 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/executor/TaskExecutor.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/executor/TaskExecutor.java @@ -13,6 +13,12 @@ */ package com.facebook.presto.execution.executor; +import com.facebook.airlift.concurrent.SetThreadName; +import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.TimeDistribution; +import com.facebook.airlift.stats.TimeStat; import com.facebook.presto.execution.SplitRunner; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.TaskManagerConfig; @@ -23,12 +29,6 @@ import com.google.common.base.Ticker; import com.google.common.collect.ComparisonChain; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.concurrent.SetThreadName; -import io.airlift.concurrent.ThreadPoolExecutorMBean; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; -import io.airlift.stats.TimeDistribution; -import io.airlift.stats.TimeStat; import io.airlift.units.Duration; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -60,14 +60,14 @@ import java.util.concurrent.atomic.AtomicLongArray; import java.util.function.DoubleSupplier; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static com.facebook.presto.execution.executor.MultilevelSplitQueue.computeLevel; import static com.facebook.presto.util.MoreMath.min; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Sets.newConcurrentHashSet; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.concurrent.Threads.threadsNamed; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroup.java b/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroup.java index 79e9f7033e4ff..2b33104e0e9fa 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroup.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroup.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution.resourceGroups; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.execution.ManagedQueryExecution; import com.facebook.presto.execution.resourceGroups.WeightedFairQueue.Usage; import com.facebook.presto.server.QueryStateInfo; @@ -23,7 +24,6 @@ import com.facebook.presto.spi.resourceGroups.ResourceGroupState; import com.facebook.presto.spi.resourceGroups.SchedulingPolicy; import com.google.common.collect.ImmutableList; -import io.airlift.stats.CounterStat; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.weakref.jmx.Managed; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroupManager.java b/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroupManager.java index 0ae222c9c8172..67c495ea6edc4 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroupManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroupManager.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.execution.resourceGroups; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.execution.ManagedQueryExecution; import com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup; import com.facebook.presto.server.ResourceGroupInfo; @@ -27,8 +29,6 @@ import com.facebook.presto.sql.tree.Statement; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import io.airlift.node.NodeInfo; import org.weakref.jmx.JmxException; import org.weakref.jmx.MBeanExporter; import org.weakref.jmx.Managed; @@ -53,12 +53,12 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.spi.StandardErrorCode.QUERY_REJECTED; import static com.facebook.presto.util.PropertiesUtil.loadProperties; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.isNullOrEmpty; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/AllAtOnceExecutionSchedule.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/AllAtOnceExecutionSchedule.java index 48755cd1f62c5..9d898f688beeb 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/AllAtOnceExecutionSchedule.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/AllAtOnceExecutionSchedule.java @@ -14,8 +14,9 @@ package com.facebook.presto.execution.scheduler; import com.facebook.presto.execution.SqlStageExecution; -import com.facebook.presto.execution.StageState; +import com.facebook.presto.execution.StageExecutionState; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.IndexJoinNode; @@ -25,7 +26,6 @@ import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -38,8 +38,8 @@ import java.util.Map; import java.util.Set; -import static com.facebook.presto.execution.StageState.RUNNING; -import static com.facebook.presto.execution.StageState.SCHEDULED; +import static com.facebook.presto.execution.StageExecutionState.RUNNING; +import static com.facebook.presto.execution.StageExecutionState.SCHEDULED; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; @@ -69,7 +69,7 @@ public AllAtOnceExecutionSchedule(Collection stages) public Set getStagesToSchedule() { for (Iterator iterator = schedulingStages.iterator(); iterator.hasNext(); ) { - StageState state = iterator.next().getState(); + StageExecutionState state = iterator.next().getState(); if (state == SCHEDULED || state == RUNNING || state.isDone()) { iterator.remove(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/ExecutionWriterTarget.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/ExecutionWriterTarget.java new file mode 100644 index 0000000000000..edd1a2b324b61 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/ExecutionWriterTarget.java @@ -0,0 +1,137 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.execution.scheduler; + +import com.facebook.presto.metadata.InsertTableHandle; +import com.facebook.presto.metadata.OutputTableHandle; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.TableHandle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import static java.util.Objects.requireNonNull; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "@type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = ExecutionWriterTarget.CreateHandle.class, name = "CreateHandle"), + @JsonSubTypes.Type(value = ExecutionWriterTarget.InsertHandle.class, name = "InsertHandle"), + @JsonSubTypes.Type(value = ExecutionWriterTarget.DeleteHandle.class, name = "DeleteHandle")}) +@SuppressWarnings({"EmptyClass", "ClassMayBeInterface"}) +public abstract class ExecutionWriterTarget +{ + public static class CreateHandle + extends ExecutionWriterTarget + { + private final OutputTableHandle handle; + private final SchemaTableName schemaTableName; + + @JsonCreator + public CreateHandle( + @JsonProperty("handle") OutputTableHandle handle, + @JsonProperty("schemaTableName") SchemaTableName schemaTableName) + { + this.handle = requireNonNull(handle, "handle is null"); + this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null"); + } + + @JsonProperty + public OutputTableHandle getHandle() + { + return handle; + } + + @JsonProperty + public SchemaTableName getSchemaTableName() + { + return schemaTableName; + } + + @Override + public String toString() + { + return handle.toString(); + } + } + + public static class InsertHandle + extends ExecutionWriterTarget + { + private final InsertTableHandle handle; + private final SchemaTableName schemaTableName; + + @JsonCreator + public InsertHandle( + @JsonProperty("handle") InsertTableHandle handle, + @JsonProperty("schemaTableName") SchemaTableName schemaTableName) + { + this.handle = requireNonNull(handle, "handle is null"); + this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null"); + } + + @JsonProperty + public InsertTableHandle getHandle() + { + return handle; + } + + @JsonProperty + public SchemaTableName getSchemaTableName() + { + return schemaTableName; + } + + @Override + public String toString() + { + return handle.toString(); + } + } + + public static class DeleteHandle + extends ExecutionWriterTarget + { + private final TableHandle handle; + private final SchemaTableName schemaTableName; + + @JsonCreator + public DeleteHandle( + @JsonProperty("handle") TableHandle handle, + @JsonProperty("schemaTableName") SchemaTableName schemaTableName) + { + this.handle = requireNonNull(handle, "handle is null"); + this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null"); + } + + @JsonProperty + public TableHandle getHandle() + { + return handle; + } + + @JsonProperty + public SchemaTableName getSchemaTableName() + { + return schemaTableName; + } + + @Override + public String toString() + { + return handle.toString(); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedSourcePartitionedScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedSourcePartitionedScheduler.java index 8a8fe79817859..de8ab7fc991e2 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedSourcePartitionedScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/FixedSourcePartitionedScheduler.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.airlift.log.Logger; import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.execution.SqlStageExecution; @@ -31,7 +32,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Streams; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.log.Logger; import java.util.ArrayList; import java.util.Iterator; @@ -39,16 +39,18 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; +import java.util.Queue; import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.MoreFutures.whenAnyComplete; import static com.facebook.presto.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsSourceScheduler; import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.concurrent.MoreFutures.whenAnyComplete; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; @@ -65,6 +67,8 @@ public class FixedSourcePartitionedScheduler private boolean anySourceSchedulingFinished; private final Optional groupedLifespanScheduler; + private final Queue tasksToRecover = new ConcurrentLinkedQueue<>(); + public FixedSourcePartitionedScheduler( SqlStageExecution stage, Map splitSources, @@ -161,9 +165,8 @@ private ConnectorPartitionHandle partitionHandleFor(Lifespan lifespan) return partitionHandles.get(lifespan.getId()); } - // Only schedule() and recover() are synchronized @Override - public synchronized ScheduleResult schedule() + public ScheduleResult schedule() { // schedule a task on every node in the distribution List newTasks = ImmutableList.of(); @@ -186,6 +189,13 @@ public synchronized ScheduleResult schedule() BlockedReason blockedReason = BlockedReason.NO_ACTIVE_DRIVER_GROUP; if (groupedLifespanScheduler.isPresent()) { + while (!tasksToRecover.isEmpty()) { + if (anySourceSchedulingFinished) { + throw new IllegalStateException("Recover after any source scheduling finished is not supported"); + } + groupedLifespanScheduler.get().onTaskFailed(tasksToRecover.poll(), sourceSchedulers); + } + if (groupedLifespanScheduler.get().allLifespanExecutionFinished()) { for (SourceScheduler sourceScheduler : sourceSchedulers) { sourceScheduler.notifyAllLifespansFinishedExecution(); @@ -243,15 +253,9 @@ public synchronized ScheduleResult schedule() } } - // Only schedule() and recover() are synchronized - public synchronized void recover(TaskId taskId) + public void recover(TaskId taskId) { - checkState(groupedLifespanScheduler.isPresent(), "groupedLifespanScheduler is not present for recoverable grouped execution"); - if (anySourceSchedulingFinished) { - throw new IllegalStateException("Recover after any source scheduling finished is not supported"); - } - - groupedLifespanScheduler.get().onTaskFailed(taskId.getId(), sourceSchedulers); + tasksToRecover.add(taskId.getId()); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NetworkLocationCache.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NetworkLocationCache.java index c662c8689413e..d4c75c4c72364 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NetworkLocationCache.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NetworkLocationCache.java @@ -13,19 +13,19 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.HostAddress; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import io.airlift.log.Logger; import io.airlift.units.Duration; import java.util.concurrent.ExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.execution.scheduler.NetworkLocation.ROOT_LOCATION; import static com.google.common.cache.CacheLoader.asyncReloading; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.TimeUnit.DAYS; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeMap.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeMap.java index 000221b9a4fd3..0835a7d5cf18f 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeMap.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeMap.java @@ -18,26 +18,36 @@ import com.google.common.collect.SetMultimap; import java.net.InetAddress; +import java.util.Map; import java.util.Set; public class NodeMap { + private final Map nodesByNodeId; private final SetMultimap nodesByHostAndPort; private final SetMultimap nodesByHost; private final SetMultimap workersByNetworkPath; private final Set coordinatorNodeIds; - public NodeMap(SetMultimap nodesByHostAndPort, + public NodeMap( + Map nodesByNodeId, + SetMultimap nodesByHostAndPort, SetMultimap nodesByHost, SetMultimap workersByNetworkPath, Set coordinatorNodeIds) { + this.nodesByNodeId = nodesByNodeId; this.nodesByHostAndPort = nodesByHostAndPort; this.nodesByHost = nodesByHost; this.workersByNetworkPath = workersByNetworkPath; this.coordinatorNodeIds = coordinatorNodeIds; } + public Map getNodesByNodeId() + { + return nodesByNodeId; + } + public SetMultimap getNodesByHostAndPort() { return nodesByHostAndPort; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java index 970a99461a7f7..f11520f431779 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeScheduler.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.metadata.InternalNode; @@ -21,7 +22,6 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.HostAddress; import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -29,14 +29,13 @@ import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.stats.CounterStat; +import io.airlift.units.Duration; import javax.annotation.PreDestroy; import javax.inject.Inject; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -44,17 +43,19 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.concurrent.MoreFutures.whenAnyCompleteCancelOthers; import static com.facebook.presto.execution.scheduler.NodeSchedulerConfig.NetworkTopologyType; import static com.facebook.presto.spi.NodeState.ACTIVE; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Suppliers.memoizeWithExpiration; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.util.concurrent.Futures.immediateFuture; -import static io.airlift.concurrent.MoreFutures.whenAnyCompleteCancelOthers; import static java.lang.Math.min; import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; public class NodeScheduler { @@ -68,11 +69,12 @@ public class NodeScheduler private final int maxPendingSplitsPerTask; private final NodeTaskMap nodeTaskMap; private final boolean useNetworkTopology; + private final Duration nodeMapRefreshInterval; @Inject public NodeScheduler(NetworkTopology networkTopology, InternalNodeManager nodeManager, NodeSchedulerConfig config, NodeTaskMap nodeTaskMap) { - this(new NetworkLocationCache(networkTopology), networkTopology, nodeManager, config, nodeTaskMap); + this(new NetworkLocationCache(networkTopology), networkTopology, nodeManager, config, nodeTaskMap, new Duration(5, SECONDS)); } public NodeScheduler( @@ -80,7 +82,8 @@ public NodeScheduler( NetworkTopology networkTopology, InternalNodeManager nodeManager, NodeSchedulerConfig config, - NodeTaskMap nodeTaskMap) + NodeTaskMap nodeTaskMap, + Duration nodeMapRefreshInterval) { this.networkLocationCache = networkLocationCache; this.nodeManager = nodeManager; @@ -103,6 +106,7 @@ public NodeScheduler( networkLocationSegmentNames = ImmutableList.of(); } topologicalSplitCounters = builder.build(); + this.nodeMapRefreshInterval = requireNonNull(nodeMapRefreshInterval, "nodeMapRefreshInterval is null"); } @PreDestroy @@ -121,10 +125,39 @@ public Map getTopologicalSplitCounters() } public NodeSelector createNodeSelector(ConnectorId connectorId) + { + return createNodeSelector(connectorId, Integer.MAX_VALUE); + } + + public NodeSelector createNodeSelector(ConnectorId connectorId, int maxTasksPerStage) { // this supplier is thread-safe. TODO: this logic should probably move to the scheduler since the choice of which node to run in should be // done as close to when the the split is about to be scheduled - Supplier nodeMap = Suppliers.memoizeWithExpiration(() -> { + Supplier nodeMap = nodeMapRefreshInterval.toMillis() > 0 ? + memoizeWithExpiration(createNodeMapSupplier(connectorId), nodeMapRefreshInterval.toMillis(), MILLISECONDS) : createNodeMapSupplier(connectorId); + + if (useNetworkTopology) { + return new TopologyAwareNodeSelector( + nodeManager, + nodeTaskMap, + includeCoordinator, + nodeMap, + minCandidates, + maxSplitsPerNode, + maxPendingSplitsPerTask, + topologicalSplitCounters, + networkLocationSegmentNames, + networkLocationCache); + } + else { + return new SimpleNodeSelector(nodeManager, nodeTaskMap, includeCoordinator, nodeMap, minCandidates, maxSplitsPerNode, maxPendingSplitsPerTask, maxTasksPerStage); + } + } + + private Supplier createNodeMapSupplier(ConnectorId connectorId) + { + return () -> { + ImmutableMap.Builder byNodeId = ImmutableMap.builder(); ImmutableSetMultimap.Builder byHostAndPort = ImmutableSetMultimap.builder(); ImmutableSetMultimap.Builder byHost = ImmutableSetMultimap.builder(); ImmutableSetMultimap.Builder workersByNetworkPath = ImmutableSetMultimap.builder(); @@ -142,6 +175,7 @@ public NodeSelector createNodeSelector(ConnectorId connectorId) .collect(toImmutableSet()); for (InternalNode node : nodes) { + byNodeId.put(node.getNodeIdentifier(), node); if (useNetworkTopology && (includeCoordinator || !coordinatorNodeIds.contains(node.getNodeIdentifier()))) { NetworkLocation location = networkLocationCache.get(node.getHostAndPort()); for (int i = 0; i <= location.getSegments().size(); i++) { @@ -159,37 +193,22 @@ public NodeSelector createNodeSelector(ConnectorId connectorId) } } - return new NodeMap(byHostAndPort.build(), byHost.build(), workersByNetworkPath.build(), coordinatorNodeIds); - }, 5, TimeUnit.SECONDS); - - if (useNetworkTopology) { - return new TopologyAwareNodeSelector( - nodeManager, - nodeTaskMap, - includeCoordinator, - nodeMap, - minCandidates, - maxSplitsPerNode, - maxPendingSplitsPerTask, - topologicalSplitCounters, - networkLocationSegmentNames, - networkLocationCache); - } - else { - return new SimpleNodeSelector(nodeManager, nodeTaskMap, includeCoordinator, nodeMap, minCandidates, maxSplitsPerNode, maxPendingSplitsPerTask); - } + return new NodeMap(byNodeId.build(), byHostAndPort.build(), byHost.build(), workersByNetworkPath.build(), coordinatorNodeIds); + }; } public static List selectNodes(int limit, ResettableRandomizedIterator candidates) { checkArgument(limit > 0, "limit must be at least 1"); - List selected = new ArrayList<>(min(limit, candidates.size())); - while (selected.size() < limit && candidates.hasNext()) { - selected.add(candidates.next()); + ImmutableList.Builder selectedNodes = ImmutableList.builderWithExpectedSize(min(limit, candidates.size())); + int selectedCount = 0; + while (selectedCount < limit && candidates.hasNext()) { + selectedNodes.add(candidates.next()); + selectedCount++; } - return selected; + return selectedNodes.build(); } public static ResettableRandomizedIterator randomizedNodes(NodeMap nodeMap, boolean includeCoordinator, Set excludedNodes) diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSchedulerConfig.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSchedulerConfig.java index 20b8fe40bc1a0..08044717a3ea0 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSchedulerConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSchedulerConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.execution.scheduler; -import io.airlift.configuration.Config; -import io.airlift.configuration.DefunctConfig; -import io.airlift.configuration.LegacyConfig; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.DefunctConfig; +import com.facebook.airlift.configuration.LegacyConfig; import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSchedulerExporter.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSchedulerExporter.java index 1a5d5aecae877..d76c6bd4971e6 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSchedulerExporter.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/NodeSchedulerExporter.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.execution.scheduler; -import io.airlift.stats.CounterStat; +import com.facebook.airlift.stats.CounterStat; import org.weakref.jmx.JmxException; import org.weakref.jmx.MBeanExporter; import org.weakref.jmx.ObjectNames; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/PhasedExecutionSchedule.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/PhasedExecutionSchedule.java index cd22fcf30e252..2f31f9f912d89 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/PhasedExecutionSchedule.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/PhasedExecutionSchedule.java @@ -14,8 +14,9 @@ package com.facebook.presto.execution.scheduler; import com.facebook.presto.execution.SqlStageExecution; -import com.facebook.presto.execution.StageState; +import com.facebook.presto.execution.StageExecutionState; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.IndexJoinNode; @@ -25,7 +26,6 @@ import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -47,8 +47,8 @@ import java.util.Set; import java.util.stream.Collectors; -import static com.facebook.presto.execution.StageState.RUNNING; -import static com.facebook.presto.execution.StageState.SCHEDULED; +import static com.facebook.presto.execution.StageExecutionState.RUNNING; +import static com.facebook.presto.execution.StageExecutionState.SCHEDULED; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; @@ -90,7 +90,7 @@ public Set getStagesToSchedule() private void removeCompletedStages() { for (Iterator stageIterator = activeSources.iterator(); stageIterator.hasNext(); ) { - StageState state = stageIterator.next().getState(); + StageExecutionState state = stageIterator.next().getState(); if (state == SCHEDULED || state == RUNNING || state.isDone()) { stageIterator.remove(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SimpleNodeSelector.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SimpleNodeSelector.java index 47d5b22b9f653..3f97e4bf01790 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SimpleNodeSelector.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SimpleNodeSelector.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.airlift.log.Logger; import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.metadata.InternalNode; @@ -23,13 +24,12 @@ import com.google.common.base.Suppliers; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.log.Logger; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -40,7 +40,10 @@ import static com.facebook.presto.execution.scheduler.NodeScheduler.selectNodes; import static com.facebook.presto.execution.scheduler.NodeScheduler.toWhenHasSplitQueueSpaceFuture; import static com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE; +import static com.google.common.base.Verify.verify; +import static com.google.common.collect.Sets.newHashSet; import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toList; public class SimpleNodeSelector implements NodeSelector @@ -54,6 +57,7 @@ public class SimpleNodeSelector private final int minCandidates; private final int maxSplitsPerNode; private final int maxPendingSplitsPerTask; + private final int maxTasksPerStage; public SimpleNodeSelector( InternalNodeManager nodeManager, @@ -62,7 +66,8 @@ public SimpleNodeSelector( Supplier nodeMap, int minCandidates, int maxSplitsPerNode, - int maxPendingSplitsPerTask) + int maxPendingSplitsPerTask, + int maxTasksPerStage) { this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.nodeTaskMap = requireNonNull(nodeTaskMap, "nodeTaskMap is null"); @@ -71,6 +76,7 @@ public SimpleNodeSelector( this.minCandidates = minCandidates; this.maxSplitsPerNode = maxSplitsPerNode; this.maxPendingSplitsPerTask = maxPendingSplitsPerTask; + this.maxTasksPerStage = maxTasksPerStage; } @Override @@ -105,7 +111,7 @@ public SplitPlacementResult computeAssignments(Set splits, List randomCandidates = randomizedNodes(nodeMap, includeCoordinator, ImmutableSet.of()); + ResettableRandomizedIterator randomCandidates = getRandomCandidates(maxTasksPerStage, nodeMap, existingTasks); Set blockedExactNodes = new HashSet<>(); boolean splitWaitingForAnyNode = false; for (Split split : splits) { @@ -168,6 +174,26 @@ else if (!splitWaitingForAnyNode) { return new SplitPlacementResult(blocked, assignment); } + private ResettableRandomizedIterator getRandomCandidates(int limit, NodeMap nodeMap, List existingTasks) + { + List existingNodes = existingTasks.stream() + .map(remoteTask -> nodeMap.getNodesByNodeId().get(remoteTask.getNodeId())) + // nodes may sporadically disappear from the nodeMap if the announcement is delayed + .filter(Objects::nonNull) + .collect(toList()); + + int alreadySelectedNodeCount = existingNodes.size(); + int nodeCount = nodeMap.getNodesByNodeId().size(); + + if (alreadySelectedNodeCount < limit && alreadySelectedNodeCount < nodeCount) { + List moreNodes = + selectNodes(limit - alreadySelectedNodeCount, randomizedNodes(nodeMap, includeCoordinator, newHashSet(existingNodes))); + existingNodes.addAll(moreNodes); + } + verify(existingNodes.stream().allMatch(Objects::nonNull), "existingNodes list must not contain any nulls"); + return new ResettableRandomizedIterator<>(existingNodes); + } + @Override public SplitPlacementResult computeAssignments(Set splits, List existingTasks, BucketNodeMap bucketNodeMap) { diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SourcePartitionedScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SourcePartitionedScheduler.java index 75d5620c50736..1b0dbd82b9050 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SourcePartitionedScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SourcePartitionedScheduler.java @@ -41,6 +41,9 @@ import java.util.Map.Entry; import java.util.Set; +import static com.facebook.airlift.concurrent.MoreFutures.addSuccessCallback; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.MoreFutures.whenAnyComplete; import static com.facebook.presto.execution.scheduler.ScheduleResult.BlockedReason.MIXED_SPLIT_QUEUES_FULL_AND_WAITING_FOR_SOURCE; import static com.facebook.presto.execution.scheduler.ScheduleResult.BlockedReason.NO_ACTIVE_DRIVER_GROUP; import static com.facebook.presto.execution.scheduler.ScheduleResult.BlockedReason.SPLIT_QUEUES_FULL; @@ -51,9 +54,6 @@ import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.util.concurrent.Futures.nonCancellationPropagating; -import static io.airlift.concurrent.MoreFutures.addSuccessCallback; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.MoreFutures.whenAnyComplete; import static java.util.Objects.requireNonNull; public class SourcePartitionedScheduler diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitSchedulerStats.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitSchedulerStats.java index 278f3f5ef98e0..6d0507082e8bb 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitSchedulerStats.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SplitSchedulerStats.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.execution.scheduler; -import io.airlift.stats.CounterStat; -import io.airlift.stats.DistributionStat; -import io.airlift.stats.TimeStat; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.DistributionStat; +import com.facebook.airlift.stats.TimeStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java index 213234679b832..62f7e1590da5b 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java @@ -13,8 +13,10 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.airlift.concurrent.SetThreadName; +import com.facebook.airlift.stats.TimeStat; import com.facebook.presto.Session; -import com.facebook.presto.execution.BasicStageStats; +import com.facebook.presto.execution.BasicStageExecutionStats; import com.facebook.presto.execution.LocationFactory; import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.QueryState; @@ -22,15 +24,18 @@ import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.execution.RemoteTaskFactory; import com.facebook.presto.execution.SqlStageExecution; +import com.facebook.presto.execution.StageExecutionId; +import com.facebook.presto.execution.StageExecutionInfo; +import com.facebook.presto.execution.StageExecutionState; import com.facebook.presto.execution.StageId; import com.facebook.presto.execution.StageInfo; -import com.facebook.presto.execution.StageState; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.TaskStatus; import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId; import com.facebook.presto.failureDetector.FailureDetector; import com.facebook.presto.metadata.InternalNode; +import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; @@ -45,13 +50,11 @@ import com.facebook.presto.sql.planner.plan.PlanFragmentId; import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.primitives.Ints; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import io.airlift.concurrent.SetThreadName; -import io.airlift.stats.TimeStat; import io.airlift.units.Duration; import java.net.URI; @@ -59,6 +62,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -71,21 +75,25 @@ import java.util.function.Predicate; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; +import static com.facebook.airlift.concurrent.MoreFutures.whenAnyComplete; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; import static com.facebook.presto.SystemSessionProperties.getConcurrentLifespansPerNode; import static com.facebook.presto.SystemSessionProperties.getMaxConcurrentMaterializations; import static com.facebook.presto.SystemSessionProperties.getMaxTasksPerStage; import static com.facebook.presto.SystemSessionProperties.getWriterMinSize; -import static com.facebook.presto.execution.BasicStageStats.aggregateBasicStageStats; +import static com.facebook.presto.execution.BasicStageExecutionStats.aggregateBasicStageStats; import static com.facebook.presto.execution.SqlStageExecution.createSqlStageExecution; -import static com.facebook.presto.execution.StageState.ABORTED; -import static com.facebook.presto.execution.StageState.CANCELED; -import static com.facebook.presto.execution.StageState.FAILED; -import static com.facebook.presto.execution.StageState.FINISHED; -import static com.facebook.presto.execution.StageState.PLANNED; -import static com.facebook.presto.execution.StageState.RUNNING; -import static com.facebook.presto.execution.StageState.SCHEDULED; +import static com.facebook.presto.execution.StageExecutionState.ABORTED; +import static com.facebook.presto.execution.StageExecutionState.CANCELED; +import static com.facebook.presto.execution.StageExecutionState.FAILED; +import static com.facebook.presto.execution.StageExecutionState.FINISHED; +import static com.facebook.presto.execution.StageExecutionState.PLANNED; +import static com.facebook.presto.execution.StageExecutionState.RUNNING; +import static com.facebook.presto.execution.StageExecutionState.SCHEDULED; import static com.facebook.presto.execution.buffer.OutputBuffers.createDiscardingOutputBuffers; import static com.facebook.presto.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler; +import static com.facebook.presto.execution.scheduler.TableWriteInfo.createTableWriteInfo; import static com.facebook.presto.spi.ConnectorId.isInternalSystemConnector; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE; @@ -106,9 +114,6 @@ import static com.google.common.collect.Streams.stream; import static com.google.common.graph.Traverser.forTree; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; -import static io.airlift.concurrent.MoreFutures.whenAnyComplete; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -120,14 +125,13 @@ public class SqlQueryScheduler { private final QueryStateMachine queryStateMachine; + private final LocationFactory locationFactory; private final ExecutionPolicy executionPolicy; private final SubPlan plan; private final StreamingPlanSection sectionedPlan; - private final Map stages; + private final Map stageExecutions; private final ExecutorService executor; private final StageId rootStageId; - private final Map stageSchedulers; - private final Map stageLinkages; private final SplitSchedulerStats schedulerStats; private final boolean summarizeTaskInfo; private final AtomicBoolean started = new AtomicBoolean(); @@ -151,7 +155,8 @@ public static SqlQueryScheduler createSqlQueryScheduler( OutputBuffers rootOutputBuffers, NodeTaskMap nodeTaskMap, ExecutionPolicy executionPolicy, - SplitSchedulerStats schedulerStats) + SplitSchedulerStats schedulerStats, + Metadata metadata) { SqlQueryScheduler sqlQueryScheduler = new SqlQueryScheduler( queryStateMachine, @@ -170,7 +175,8 @@ public static SqlQueryScheduler createSqlQueryScheduler( rootOutputBuffers, nodeTaskMap, executionPolicy, - schedulerStats); + schedulerStats, + metadata); sqlQueryScheduler.initialize(); return sqlQueryScheduler; } @@ -192,25 +198,23 @@ private SqlQueryScheduler( OutputBuffers rootOutputBuffers, NodeTaskMap nodeTaskMap, ExecutionPolicy executionPolicy, - SplitSchedulerStats schedulerStats) + SplitSchedulerStats schedulerStats, + Metadata metadata) { this.queryStateMachine = requireNonNull(queryStateMachine, "queryStateMachine is null"); + this.locationFactory = requireNonNull(locationFactory, "locationFactory is null"); this.plan = requireNonNull(plan, "plan is null"); this.executionPolicy = requireNonNull(executionPolicy, "schedulerPolicyFactory is null"); this.schedulerStats = requireNonNull(schedulerStats, "schedulerStats is null"); this.summarizeTaskInfo = summarizeTaskInfo; - // todo come up with a better way to build this, or eliminate this map - ImmutableMap.Builder stageSchedulers = ImmutableMap.builder(); - ImmutableMap.Builder stageLinkages = ImmutableMap.builder(); - OutputBufferId rootBufferId = getOnlyElement(rootOutputBuffers.getBuffers().keySet()); sectionedPlan = extractStreamingSections(plan); - List stages = createStages( + List stageExecutions = createStageExecutions( (fragmentId, tasks, noMoreExchangeLocations) -> updateQueryOutputLocations(queryStateMachine, rootBufferId, tasks, noMoreExchangeLocations), - locationFactory, sectionedPlan, Optional.of(new int[1]), + metadata, rootOutputBuffers, nodeScheduler, remoteTaskFactory, @@ -221,17 +225,12 @@ private SqlQueryScheduler( queryExecutor, schedulerExecutor, failureDetector, - nodeTaskMap, - stageSchedulers, - stageLinkages); + nodeTaskMap); - this.rootStageId = stages.get(0).getStageId(); + this.rootStageId = Iterables.getLast(stageExecutions).getStageExecution().getStageExecutionId().getStageId(); - this.stages = stages.stream() - .collect(toImmutableMap(SqlStageExecution::getStageId, identity())); - - this.stageSchedulers = stageSchedulers.build(); - this.stageLinkages = stageLinkages.build(); + this.stageExecutions = stageExecutions.stream() + .collect(toImmutableMap(execution -> execution.getStageExecution().getStageExecutionId().getStageId(), identity())); this.executor = queryExecutor; this.maxConcurrentMaterializations = getMaxConcurrentMaterializations(session); @@ -240,7 +239,7 @@ private SqlQueryScheduler( // this is a separate method to ensure that the `this` reference is not leaked during construction private void initialize() { - SqlStageExecution rootStage = stages.get(rootStageId); + SqlStageExecution rootStage = stageExecutions.get(rootStageId).getStageExecution(); rootStage.addStateChangeListener(state -> { if (state == FINISHED) { queryStateMachine.transitionToFinishing(); @@ -251,31 +250,31 @@ else if (state == CANCELED) { } }); - for (SqlStageExecution stage : stages.values()) { - stage.addStateChangeListener(state -> { + for (StageExecutionAndScheduler stageExecutionInfo : stageExecutions.values()) { + SqlStageExecution stageExecution = stageExecutionInfo.getStageExecution(); + stageExecution.addStateChangeListener(state -> { if (queryStateMachine.isDone()) { return; } if (state == FAILED) { - queryStateMachine.transitionToFailed(stage.getStageInfo().getFailureCause().get().toException()); + queryStateMachine.transitionToFailed(stageExecution.getStageExecutionInfo().getFailureCause().get().toException()); } else if (state == ABORTED) { // this should never happen, since abort can only be triggered in query clean up after the query is finished queryStateMachine.transitionToFailed(new PrestoException(GENERIC_INTERNAL_ERROR, "Query stage was aborted")); } else if (state == FINISHED) { - List nextStagesToSchedule = getStagesReadyForExecution(); - if (!nextStagesToSchedule.isEmpty()) { - startScheduling(nextStagesToSchedule); - } + // checks if there's any new sections available for execution and starts the scheduling if any + startScheduling(); } else if (queryStateMachine.getQueryState() == QueryState.STARTING) { // if the stage has at least one task, we are running - if (stage.hasTasks()) { + if (stageExecution.hasTasks()) { queryStateMachine.transitionToRunning(); } } }); + stageExecution.addFinalStageInfoListener(status -> queryStateMachine.updateQueryInfo(Optional.of(getStageInfo()))); } // when query is done or any time a stage completes, attempt to transition query to "final query info ready" @@ -284,9 +283,6 @@ else if (queryStateMachine.getQueryState() == QueryState.STARTING) { queryStateMachine.updateQueryInfo(Optional.of(getStageInfo())); } }); - for (SqlStageExecution stage : stages.values()) { - stage.addFinalStageInfoListener(status -> queryStateMachine.updateQueryInfo(Optional.of(getStageInfo()))); - } } private static void updateQueryOutputLocations(QueryStateMachine queryStateMachine, OutputBufferId rootBufferId, Set tasks, boolean noMoreExchangeLocations) @@ -304,11 +300,14 @@ private static URI getBufferLocation(RemoteTask remoteTask, OutputBufferId rootB return uriBuilderFrom(location).appendPath("results").appendPath(rootBufferId.toString()).build(); } - private List createStages( + /** + * returns a List of SqlStageExecutionInfos in a postorder representation of the tree + */ + private List createStageExecutions( ExchangeLocationsConsumer locationsConsumer, - LocationFactory locationFactory, StreamingPlanSection section, Optional bucketToPartition, + Metadata metadata, OutputBuffers outputBuffers, NodeScheduler nodeScheduler, RemoteTaskFactory remoteTaskFactory, @@ -319,17 +318,34 @@ private List createStages( ExecutorService queryExecutor, ScheduledExecutorService schedulerExecutor, FailureDetector failureDetector, - NodeTaskMap nodeTaskMap, - ImmutableMap.Builder stageSchedulers, - ImmutableMap.Builder stageLinkages) + NodeTaskMap nodeTaskMap) { - ImmutableList.Builder stages = ImmutableList.builder(); + ImmutableList.Builder stages = ImmutableList.builder(); + + for (StreamingPlanSection childSection : section.getChildren()) { + stages.addAll(createStageExecutions( + discardingLocationConsumer(), + childSection, + Optional.empty(), + metadata, + createDiscardingOutputBuffers(), + nodeScheduler, + remoteTaskFactory, + splitSourceFactory, + session, + splitBatchSize, + nodePartitioningManager, + queryExecutor, + schedulerExecutor, + failureDetector, + nodeTaskMap)); + } // Only fetch a distribution once per section to ensure all stages see the same machine assignments Map partitioningCache = new HashMap<>(); - List sectionStages = createStreamingLinkedStages( + TableWriteInfo tableWriteInfo = createTableWriteInfo(section.getPlan(), metadata, session); + List sectionStages = createStreamingLinkedStageExecutions( locationsConsumer, - locationFactory, section.getPlan().withBucketToPartition(bucketToPartition), nodeScheduler, remoteTaskFactory, @@ -342,39 +358,21 @@ private List createStages( schedulerExecutor, failureDetector, nodeTaskMap, - stageSchedulers, - stageLinkages, + tableWriteInfo, Optional.empty()); - sectionStages.get(0).setOutputBuffers(outputBuffers); + Iterables.getLast(sectionStages) + .getStageExecution() + .setOutputBuffers(outputBuffers); stages.addAll(sectionStages); - for (StreamingPlanSection childSection : section.getChildren()) { - stages.addAll(createStages( - discardingLocationConsumer(), - locationFactory, - childSection, - Optional.empty(), - createDiscardingOutputBuffers(), - nodeScheduler, - remoteTaskFactory, - splitSourceFactory, - session, - splitBatchSize, - nodePartitioningManager, - queryExecutor, - schedulerExecutor, - failureDetector, - nodeTaskMap, - stageSchedulers, - stageLinkages)); - } - return stages.build(); } - private List createStreamingLinkedStages( + /** + * returns a List of SqlStageExecutionInfos in a postorder representation of the tree + */ + private List createStreamingLinkedStageExecutions( ExchangeLocationsConsumer parent, - LocationFactory locationFactory, StreamingSubPlan plan, NodeScheduler nodeScheduler, RemoteTaskFactory remoteTaskFactory, @@ -387,17 +385,15 @@ private List createStreamingLinkedStages( ScheduledExecutorService schedulerExecutor, FailureDetector failureDetector, NodeTaskMap nodeTaskMap, - ImmutableMap.Builder stageSchedulers, - ImmutableMap.Builder stageLinkages, + TableWriteInfo tableWriteInfo, Optional parentStageExecution) { - ImmutableList.Builder stages = ImmutableList.builder(); + ImmutableList.Builder stageExecutionInfos = ImmutableList.builder(); PlanFragmentId fragmentId = plan.getFragment().getId(); StageId stageId = getStageId(fragmentId); - SqlStageExecution stage = createSqlStageExecution( - stageId, - locationFactory.createStageLocation(stageId), + SqlStageExecution stageExecution = createSqlStageExecution( + new StageExecutionId(stageId, 0), plan.getFragment(), remoteTaskFactory, session, @@ -405,15 +401,80 @@ private List createStreamingLinkedStages( nodeTaskMap, queryExecutor, failureDetector, - schedulerStats); - - stages.add(stage); + schedulerStats, + tableWriteInfo); - Optional bucketToPartition; PartitioningHandle partitioningHandle = plan.getFragment().getPartitioning(); + Map splitSources = splitSourceFactory.createSplitSources(plan.getFragment(), session, tableWriteInfo); + List remoteSourceNodes = plan.getFragment().getRemoteSourceNodes(); + Optional bucketToPartition = getBucketToPartition(partitioningHandle, partitioningCache, splitSources, remoteSourceNodes); + + // create child stages + ImmutableSet.Builder childStagesBuilder = ImmutableSet.builder(); + for (StreamingSubPlan stagePlan : plan.getChildren()) { + List subTree = createStreamingLinkedStageExecutions( + stageExecution::addExchangeLocations, + stagePlan.withBucketToPartition(bucketToPartition), + nodeScheduler, + remoteTaskFactory, + splitSourceFactory, + session, + splitBatchSize, + partitioningCache, + nodePartitioningManager, + queryExecutor, + schedulerExecutor, + failureDetector, + nodeTaskMap, + tableWriteInfo, + Optional.of(stageExecution)); + stageExecutionInfos.addAll(subTree); + childStagesBuilder.add(Iterables.getLast(subTree).getStageExecution()); + } + Set childStageExecutions = childStagesBuilder.build(); + stageExecution.addStateChangeListener(newState -> { + if (newState.isDone()) { + childStageExecutions.forEach(SqlStageExecution::cancel); + } + }); + + StageScheduler stageScheduler = createStageScheduler( + plan, + nodeScheduler, + session, + splitBatchSize, + partitioningCache, + nodePartitioningManager, + schedulerExecutor, + parentStageExecution, + stageId, + stageExecution, + partitioningHandle, + splitSources, + childStageExecutions); + StageLinkage stageLinkage = new StageLinkage(fragmentId, parent, childStageExecutions); + stageExecutionInfos.add(new StageExecutionAndScheduler(stageExecution, stageLinkage, stageScheduler)); + + return stageExecutionInfos.build(); + } + + private StageScheduler createStageScheduler( + StreamingSubPlan plan, + NodeScheduler nodeScheduler, + Session session, + int splitBatchSize, + Function partitioningCache, + NodePartitioningManager nodePartitioningManager, + ScheduledExecutorService schedulerExecutor, + Optional parentStageExecution, + StageId stageId, SqlStageExecution stageExecution, + PartitioningHandle partitioningHandle, + Map splitSources, + Set childStageExecutions) + { + int maxTasksPerStage = getMaxTasksPerStage(session); if (partitioningHandle.equals(SOURCE_DISTRIBUTION)) { // TODO: defer opening split sources when stage scheduling starts - Map splitSources = splitSourceFactory.createSplitSources(plan.getFragment(), session); // nodes are selected dynamically based on the constraints of the splits and the system load Entry entry = getOnlyElement(splitSources.entrySet()); PlanNodeId planNodeId = entry.getKey(); @@ -422,19 +483,37 @@ private List createStreamingLinkedStages( if (isInternalSystemConnector(connectorId)) { connectorId = null; } - NodeSelector nodeSelector = nodeScheduler.createNodeSelector(connectorId); - SplitPlacementPolicy placementPolicy = new DynamicSplitPlacementPolicy(nodeSelector, stage::getAllTasks); + + NodeSelector nodeSelector = nodeScheduler.createNodeSelector(connectorId, maxTasksPerStage); + SplitPlacementPolicy placementPolicy = new DynamicSplitPlacementPolicy(nodeSelector, stageExecution::getAllTasks); checkArgument(!plan.getFragment().getStageExecutionDescriptor().isStageGroupedExecution()); - stageSchedulers.put(stageId, newSourcePartitionedSchedulerAsStageScheduler(stage, planNodeId, splitSource, placementPolicy, splitBatchSize)); - bucketToPartition = Optional.of(new int[1]); + return newSourcePartitionedSchedulerAsStageScheduler(stageExecution, planNodeId, splitSource, placementPolicy, splitBatchSize); } else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { - bucketToPartition = Optional.of(new int[1]); + Supplier> sourceTasksProvider = () -> childStageExecutions.stream() + .map(SqlStageExecution::getAllTasks) + .flatMap(Collection::stream) + .map(RemoteTask::getTaskStatus) + .collect(toList()); + + Supplier> writerTasksProvider = () -> stageExecution.getAllTasks().stream() + .map(RemoteTask::getTaskStatus) + .collect(toList()); + + ScaledWriterScheduler scheduler = new ScaledWriterScheduler( + stageExecution, + sourceTasksProvider, + writerTasksProvider, + nodeScheduler.createNodeSelector(null), + schedulerExecutor, + getWriterMinSize(session)); + whenAllStages(childStageExecutions, StageExecutionState::isDone) + .addListener(scheduler::finish, directExecutor()); + return scheduler; } else { // TODO: defer opening split sources when stage scheduling starts - Map splitSources = splitSourceFactory.createSplitSources(plan.getFragment(), session); if (!splitSources.isEmpty()) { // contains local source List schedulingOrder = plan.getFragment().getTableScanSchedulingOrder(); @@ -465,9 +544,8 @@ else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { .collect(toImmutableList()); } else { - stageNodeList = new ArrayList<>(nodeScheduler.createNodeSelector(connectorId).selectRandomNodes(getMaxTasksPerStage(session))); + stageNodeList = new ArrayList<>(nodeScheduler.createNodeSelector(connectorId).selectRandomNodes(maxTasksPerStage)); } - bucketToPartition = Optional.empty(); } else { // cannot use dynamic lifespan schedule @@ -480,11 +558,10 @@ else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { } stageNodeList = nodePartitionMap.getPartitionToNode(); bucketNodeMap = nodePartitionMap.asBucketNodeMap(); - bucketToPartition = Optional.of(nodePartitionMap.getBucketToPartition()); } - FixedSourcePartitionedScheduler stageScheduler = new FixedSourcePartitionedScheduler( - stage, + FixedSourcePartitionedScheduler fixedSourcePartitionedScheduler = new FixedSourcePartitionedScheduler( + stageExecution, splitSources, plan.getFragment().getStageExecutionDescriptor(), schedulingOrder, @@ -494,93 +571,62 @@ else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { getConcurrentLifespansPerNode(session), nodeScheduler.createNodeSelector(connectorId), connectorPartitionHandles); - stageSchedulers.put(stageId, stageScheduler); if (plan.getFragment().getStageExecutionDescriptor().isRecoverableGroupedExecution()) { - stage.registerStageTaskRecoveryCallback(taskId -> { - checkArgument(taskId.getStageId().equals(stageId), "The task did not execute this stage"); + stageExecution.registerStageTaskRecoveryCallback(taskId -> { + checkArgument(taskId.getStageExecutionId().getStageId().equals(stageId), "The task did not execute this stage"); checkArgument(parentStageExecution.isPresent(), "Parent stage execution must exist"); checkArgument(parentStageExecution.get().getAllTasks().size() == 1, "Parent stage should only have one task for recoverable grouped execution"); parentStageExecution.get().removeRemoteSourceIfSingleTaskStage(taskId); - stageScheduler.recover(taskId); + fixedSourcePartitionedScheduler.recover(taskId); }); } + return fixedSourcePartitionedScheduler; } + else { // all sources are remote NodePartitionMap nodePartitionMap = partitioningCache.apply(plan.getFragment().getPartitioning()); List partitionToNode = nodePartitionMap.getPartitionToNode(); // todo this should asynchronously wait a standard timeout period before failing checkCondition(!partitionToNode.isEmpty(), NO_NODES_AVAILABLE, "No worker nodes available"); - stageSchedulers.put(stageId, new FixedCountScheduler(stage, partitionToNode)); - bucketToPartition = Optional.of(nodePartitionMap.getBucketToPartition()); + return new FixedCountScheduler(stageExecution, partitionToNode); } } + } - ImmutableSet.Builder childStagesBuilder = ImmutableSet.builder(); - for (StreamingSubPlan stagePlan : plan.getChildren()) { - List subTree = createStreamingLinkedStages( - stage::addExchangeLocations, - locationFactory, - stagePlan.withBucketToPartition(bucketToPartition), - nodeScheduler, - remoteTaskFactory, - splitSourceFactory, - session, - splitBatchSize, - partitioningCache, - nodePartitioningManager, - queryExecutor, - schedulerExecutor, - failureDetector, - nodeTaskMap, - stageSchedulers, - stageLinkages, - Optional.of(stage)); - stages.addAll(subTree); - - SqlStageExecution childStage = subTree.get(0); - childStagesBuilder.add(childStage); + private Optional getBucketToPartition( + PartitioningHandle partitioningHandle, + Function partitioningCache, + Map splitSources, + List remoteSourceNodes) + { + if (partitioningHandle.equals(SOURCE_DISTRIBUTION) || partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { + return Optional.of(new int[1]); } - Set childStages = childStagesBuilder.build(); - stage.addStateChangeListener(newState -> { - if (newState.isDone()) { - childStages.forEach(SqlStageExecution::cancel); + else if (!splitSources.isEmpty()) { + if (remoteSourceNodes.stream().allMatch(node -> node.getExchangeType() == REPLICATE)) { + return Optional.empty(); + } + else { + // remote source requires nodePartitionMap + NodePartitionMap nodePartitionMap = partitioningCache.apply(partitioningHandle); + return Optional.of(nodePartitionMap.getBucketToPartition()); } - }); - - stageLinkages.put(stageId, new StageLinkage(fragmentId, parent, childStages)); - - if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { - Supplier> sourceTasksProvider = () -> childStages.stream() - .map(SqlStageExecution::getAllTasks) - .flatMap(Collection::stream) - .map(RemoteTask::getTaskStatus) - .collect(toList()); - - Supplier> writerTasksProvider = () -> stage.getAllTasks().stream() - .map(RemoteTask::getTaskStatus) - .collect(toList()); - - ScaledWriterScheduler scheduler = new ScaledWriterScheduler( - stage, - sourceTasksProvider, - writerTasksProvider, - nodeScheduler.createNodeSelector(null), - schedulerExecutor, - getWriterMinSize(session)); - whenAllStages(childStages, StageState::isDone) - .addListener(scheduler::finish, directExecutor()); - stageSchedulers.put(stageId, scheduler); } - - return stages.build(); + else { + NodePartitionMap nodePartitionMap = partitioningCache.apply(partitioningHandle); + List partitionToNode = nodePartitionMap.getPartitionToNode(); + // todo this should asynchronously wait a standard timeout period before failing + checkCondition(!partitionToNode.isEmpty(), NO_NODES_AVAILABLE, "No worker nodes available"); + return Optional.of(nodePartitionMap.getBucketToPartition()); + } } - public BasicStageStats getBasicStageStats() + public BasicStageExecutionStats getBasicStageStats() { - List stageStats = stages.values().stream() - .map(SqlStageExecution::getBasicStageStats) + List stageStats = stageExecutions.values().stream() + .map(stageExecutionInfo -> stageExecutionInfo.getStageExecution().getBasicStageStats()) .collect(toImmutableList()); return aggregateBasicStageStats(stageStats); @@ -588,52 +634,47 @@ public BasicStageStats getBasicStageStats() public StageInfo getStageInfo() { - Map stageInfos = stages.values().stream() - .map(SqlStageExecution::getStageInfo) - .collect(toImmutableMap(StageInfo::getStageId, identity())); + Map stageInfos = stageExecutions.values().stream() + .map(stageExecutionInfo -> stageExecutionInfo.getStageExecution().getStageExecutionInfo()) + .collect(toImmutableMap(execution -> execution.getStageExecutionId().getStageId(), identity())); return buildStageInfo(plan, stageInfos); } - private StageInfo buildStageInfo(SubPlan subPlan, Map stageInfos) + private StageInfo buildStageInfo(SubPlan subPlan, Map stageExecutionInfos) { - StageInfo stageInfo = stageInfos.get(getStageId(subPlan.getFragment().getId())); - checkArgument(stageInfo != null, "No stageInfo for %s", stageInfo); - if (subPlan.getChildren().isEmpty()) { - return stageInfo; - } + StageId stageId = getStageId(subPlan.getFragment().getId()); + StageExecutionInfo stageExecutionInfo = stageExecutionInfos.get(stageId); + checkArgument(stageExecutionInfo != null, "No stageExecutionInfo for %s", stageId); return new StageInfo( - stageInfo.getStageId(), - stageInfo.getState(), - stageInfo.getSelf(), - stageInfo.getPlan(), - stageInfo.getTypes(), - stageInfo.getStageStats(), - stageInfo.getTasks(), + stageId, + locationFactory.createStageLocation(stageId), + Optional.of(subPlan.getFragment()), + stageExecutionInfo, + ImmutableList.of(), subPlan.getChildren().stream() - .map(plan -> buildStageInfo(plan, stageInfos)) - .collect(toImmutableList()), - stageInfo.getFailureCause()); + .map(plan -> buildStageInfo(plan, stageExecutionInfos)) + .collect(toImmutableList())); } public long getUserMemoryReservation() { - return stages.values().stream() - .mapToLong(SqlStageExecution::getUserMemoryReservation) + return stageExecutions.values().stream() + .mapToLong(stageExecutionInfo -> stageExecutionInfo.getStageExecution().getUserMemoryReservation()) .sum(); } public long getTotalMemoryReservation() { - return stages.values().stream() - .mapToLong(SqlStageExecution::getTotalMemoryReservation) + return stageExecutions.values().stream() + .mapToLong(stageExecutionInfo -> stageExecutionInfo.getStageExecution().getTotalMemoryReservation()) .sum(); } public Duration getTotalCpuTime() { - long millis = stages.values().stream() - .mapToLong(stage -> stage.getTotalCpuTime().toMillis()) + long millis = stageExecutions.values().stream() + .mapToLong(stage -> stage.getStageExecution().getTotalCpuTime().toMillis()) .sum(); return new Duration(millis, MILLISECONDS); } @@ -641,148 +682,144 @@ public Duration getTotalCpuTime() public void start() { if (started.compareAndSet(false, true)) { - startScheduling(getStagesReadyForExecution()); + startScheduling(); } } - private List getStagesReadyForExecution() + private void startScheduling() { - long runningPlanSections = - stream(forTree(StreamingPlanSection::getChildren).depthFirstPreOrder(sectionedPlan)) - .map(section -> getStageExecution(section.getPlan().getFragment().getId()).getState()) - .filter(state -> !state.isDone() && state != PLANNED) - .count(); - return stream(forTree(StreamingPlanSection::getChildren).depthFirstPreOrder(sectionedPlan)) - // get all sections ready for execution - .filter(this::isReadyForExecution) - .limit(maxConcurrentMaterializations - runningPlanSections) - // get all stages in the sections - .flatMap(section -> stream(forTree(StreamingSubPlan::getChildren).depthFirstPreOrder(section.getPlan()))) - .map(StreamingSubPlan::getFragment) - .map(PlanFragment::getId) - .map(this::getStageExecution) - .collect(toImmutableList()); - } - - private boolean isReadyForExecution(StreamingPlanSection section) - { - SqlStageExecution stage = getStageExecution(section.getPlan().getFragment().getId()); - if (stage.getState() != PLANNED) { - // already scheduled - return false; - } - for (StreamingPlanSection child : section.getChildren()) { - SqlStageExecution childRootStage = getStageExecution(child.getPlan().getFragment().getId()); - if (childRootStage.getState() != FINISHED) { - return false; - } - } - return true; - } - - private SqlStageExecution getStageExecution(PlanFragmentId planFragmentId) - { - return stages.get(getStageId(planFragmentId)); - } - - private StageId getStageId(PlanFragmentId fragmentId) - { - return new StageId(queryStateMachine.getQueryId(), fragmentId.getId()); - } - - private void startScheduling(Collection stages) - { - requireNonNull(stages); + requireNonNull(stageExecutions); // still scheduling the previous batch of stages if (scheduling.get()) { return; } - executor.submit(() -> schedule(stages)); + executor.submit(this::schedule); } - private void schedule(Collection stages) + private void schedule() { if (!scheduling.compareAndSet(false, true)) { // still scheduling the previous batch of stages return; } + List scheduledStageExecutions = new ArrayList<>(); + try (SetThreadName ignored = new SetThreadName("Query-%s", queryStateMachine.getQueryId())) { Set completedStages = new HashSet<>(); - ExecutionSchedule executionSchedule = executionPolicy.createExecutionSchedule(stages); - while (!executionSchedule.isFinished()) { - List> blockedStages = new ArrayList<>(); - for (SqlStageExecution stage : executionSchedule.getStagesToSchedule()) { - stage.beginScheduling(); - - // perform some scheduling work - ScheduleResult result = stageSchedulers.get(stage.getStageId()) - .schedule(); - - // modify parent and children based on the results of the scheduling - if (result.isFinished()) { - stage.schedulingComplete(); - } - else if (!result.getBlocked().isDone()) { - blockedStages.add(result.getBlocked()); - } - stageLinkages.get(stage.getStageId()) - .processScheduleResults(stage.getState(), result.getNewTasks()); - schedulerStats.getSplitsScheduledPerIteration().add(result.getSplitsScheduled()); - if (result.getBlockedReason().isPresent()) { - switch (result.getBlockedReason().get()) { - case WRITER_SCALING: - // no-op - break; - case WAITING_FOR_SOURCE: - schedulerStats.getWaitingForSource().update(1); - break; - case SPLIT_QUEUES_FULL: - schedulerStats.getSplitQueuesFull().update(1); - break; - case MIXED_SPLIT_QUEUES_FULL_AND_WAITING_FOR_SOURCE: - schedulerStats.getMixedSplitQueuesFullAndWaitingForSource().update(1); - break; - case NO_ACTIVE_DRIVER_GROUP: - schedulerStats.getNoActiveDriverGroup().update(1); - break; - default: - throw new UnsupportedOperationException("Unknown blocked reason: " + result.getBlockedReason().get()); + + List sectionExecutionSchedules = new LinkedList<>(); + + while (!Thread.currentThread().isInterrupted()) { + // remove finished section + sectionExecutionSchedules.removeIf(ExecutionSchedule::isFinished); + + // try to pull more section that are ready to be run + List sectionsReadyForExecution = getSectionsReadyForExecution(); + + // all finished + if (sectionsReadyForExecution.isEmpty() && sectionExecutionSchedules.isEmpty()) { + break; + } + + List> sectionStageExecutions = getStageExecutions(sectionsReadyForExecution); + sectionStageExecutions.forEach(scheduledStageExecutions::addAll); + sectionStageExecutions.stream() + .map(executionInfos -> executionInfos.stream() + .map(StageExecutionAndScheduler::getStageExecution) + .collect(toImmutableList())) + .map(executionPolicy::createExecutionSchedule) + .forEach(sectionExecutionSchedules::add); + + while (sectionExecutionSchedules.stream().noneMatch(ExecutionSchedule::isFinished)) { + List> blockedStages = new ArrayList<>(); + + List executionsToSchedule = sectionExecutionSchedules.stream() + .flatMap(schedule -> schedule.getStagesToSchedule().stream()) + .collect(toImmutableList()); + + for (SqlStageExecution stageExecution : executionsToSchedule) { + StageId stageId = stageExecution.getStageExecutionId().getStageId(); + stageExecution.beginScheduling(); + + // perform some scheduling work + ScheduleResult result = stageExecutions.get(stageId).getStageScheduler() + .schedule(); + + // modify parent and children based on the results of the scheduling + if (result.isFinished()) { + stageExecution.schedulingComplete(); + } + else if (!result.getBlocked().isDone()) { + blockedStages.add(result.getBlocked()); + } + stageExecutions.get(stageId).getStageLinkage() + .processScheduleResults(stageExecution.getState(), result.getNewTasks()); + schedulerStats.getSplitsScheduledPerIteration().add(result.getSplitsScheduled()); + if (result.getBlockedReason().isPresent()) { + switch (result.getBlockedReason().get()) { + case WRITER_SCALING: + // no-op + break; + case WAITING_FOR_SOURCE: + schedulerStats.getWaitingForSource().update(1); + break; + case SPLIT_QUEUES_FULL: + schedulerStats.getSplitQueuesFull().update(1); + break; + case MIXED_SPLIT_QUEUES_FULL_AND_WAITING_FOR_SOURCE: + schedulerStats.getMixedSplitQueuesFullAndWaitingForSource().update(1); + break; + case NO_ACTIVE_DRIVER_GROUP: + schedulerStats.getNoActiveDriverGroup().update(1); + break; + default: + throw new UnsupportedOperationException("Unknown blocked reason: " + result.getBlockedReason().get()); + } } } - } - // make sure to update stage linkage at least once per loop to catch async state changes (e.g., partial cancel) - for (SqlStageExecution stage : stages) { - if (!completedStages.contains(stage.getStageId()) && stage.getState().isDone()) { - stageLinkages.get(stage.getStageId()) - .processScheduleResults(stage.getState(), ImmutableSet.of()); - completedStages.add(stage.getStageId()); + // make sure to update stage linkage at least once per loop to catch async state changes (e.g., partial cancel) + boolean stageFinishedExecution = false; + for (StageExecutionAndScheduler stageExecutionInfo : scheduledStageExecutions) { + SqlStageExecution stageExecution = stageExecutionInfo.getStageExecution(); + StageId stageId = stageExecution.getStageExecutionId().getStageId(); + if (!completedStages.contains(stageId) && stageExecution.getState().isDone()) { + stageExecutionInfo.getStageLinkage() + .processScheduleResults(stageExecution.getState(), ImmutableSet.of()); + completedStages.add(stageId); + stageFinishedExecution = true; + } } - } - // wait for a state change and then schedule again - if (!blockedStages.isEmpty()) { - try (TimeStat.BlockTimer timer = schedulerStats.getSleepTime().time()) { - tryGetFutureValue(whenAnyComplete(blockedStages), 1, SECONDS); + // if any stage has just finished execution try to pull more sections for scheduling + if (stageFinishedExecution) { + break; } - for (ListenableFuture blockedStage : blockedStages) { - blockedStage.cancel(true); + + // wait for a state change and then schedule again + if (!blockedStages.isEmpty()) { + try (TimeStat.BlockTimer timer = schedulerStats.getSleepTime().time()) { + tryGetFutureValue(whenAnyComplete(blockedStages), 1, SECONDS); + } + for (ListenableFuture blockedStage : blockedStages) { + blockedStage.cancel(true); + } } } } - for (SqlStageExecution stage : stages) { - StageState state = stage.getState(); + for (StageExecutionAndScheduler stageExecutionInfo : scheduledStageExecutions) { + StageExecutionState state = stageExecutionInfo.getStageExecution().getState(); if (state != SCHEDULED && state != RUNNING && !state.isDone()) { - throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Scheduling is complete, but stage %s is in state %s", stage.getStageId(), state)); + throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Scheduling is complete, but stage execution %s is in state %s", stageExecutionInfo.getStageExecution().getStageExecutionId(), state)); } } scheduling.set(false); - List nextStagesToSchedule = getStagesReadyForExecution(); - if (!nextStagesToSchedule.isEmpty()) { - startScheduling(nextStagesToSchedule); + + if (!getSectionsReadyForExecution().isEmpty()) { + startScheduling(); } } catch (Throwable t) { @@ -792,9 +829,9 @@ else if (!result.getBlocked().isDone()) { } finally { RuntimeException closeError = new RuntimeException(); - for (SqlStageExecution stage : stages) { + for (StageExecutionAndScheduler stageExecutionInfo : scheduledStageExecutions) { try { - stageSchedulers.get(stage.getStageId()).close(); + stageExecutionInfo.getStageScheduler().close(); } catch (Throwable t) { queryStateMachine.transitionToFailed(t); @@ -810,11 +847,68 @@ else if (!result.getBlocked().isDone()) { } } + private List getSectionsReadyForExecution() + { + long runningPlanSections = + stream(forTree(StreamingPlanSection::getChildren).depthFirstPreOrder(sectionedPlan)) + .map(section -> getStageExecution(section.getPlan().getFragment().getId()).getState()) + .filter(state -> !state.isDone() && state != PLANNED) + .count(); + return stream(forTree(StreamingPlanSection::getChildren).depthFirstPreOrder(sectionedPlan)) + // get all sections ready for execution + .filter(this::isReadyForExecution) + .limit(maxConcurrentMaterializations - runningPlanSections) + .collect(toImmutableList()); + } + + private boolean isReadyForExecution(StreamingPlanSection section) + { + SqlStageExecution stageExecution = getStageExecution(section.getPlan().getFragment().getId()); + if (stageExecution.getState() != PLANNED) { + // already scheduled + return false; + } + for (StreamingPlanSection child : section.getChildren()) { + SqlStageExecution rootStageExecution = getStageExecution(child.getPlan().getFragment().getId()); + if (rootStageExecution.getState() != FINISHED) { + return false; + } + } + return true; + } + + private List> getStageExecutions(List sections) + { + return sections.stream() + .map(section -> stream(forTree(StreamingSubPlan::getChildren).depthFirstPreOrder(section.getPlan())).collect(toImmutableList())) + .map(plans -> plans.stream() + .map(StreamingSubPlan::getFragment) + .map(PlanFragment::getId) + .map(this::getStageExecutionInfo) + .collect(toImmutableList())) + .collect(toImmutableList()); + } + + private SqlStageExecution getStageExecution(PlanFragmentId planFragmentId) + { + return stageExecutions.get(getStageId(planFragmentId)).getStageExecution(); + } + + private StageExecutionAndScheduler getStageExecutionInfo(PlanFragmentId planFragmentId) + { + return stageExecutions.get(getStageId(planFragmentId)); + } + + private StageId getStageId(PlanFragmentId fragmentId) + { + return new StageId(queryStateMachine.getQueryId(), fragmentId.getId()); + } + public void cancelStage(StageId stageId) { try (SetThreadName ignored = new SetThreadName("Query-%s", queryStateMachine.getQueryId())) { - SqlStageExecution sqlStageExecution = stages.get(stageId); - SqlStageExecution stage = requireNonNull(sqlStageExecution, () -> format("Stage %s does not exist", stageId)); + SqlStageExecution execution = stageExecutions.get(stageId).getStageExecution(); + SqlStageExecution stage = requireNonNull(execution, () -> format("Stage %s does not exist", stageId)); stage.cancel(); } } @@ -822,21 +916,21 @@ public void cancelStage(StageId stageId) public void abort() { try (SetThreadName ignored = new SetThreadName("Query-%s", queryStateMachine.getQueryId())) { - stages.values().forEach(SqlStageExecution::abort); + stageExecutions.values().forEach(stageExecutionInfo -> stageExecutionInfo.getStageExecution().abort()); } } - private static ListenableFuture whenAllStages(Collection stages, Predicate predicate) + private static ListenableFuture whenAllStages(Collection stageExecutions, Predicate predicate) { - checkArgument(!stages.isEmpty(), "stages is empty"); - Set stageIds = newConcurrentHashSet(stages.stream() - .map(SqlStageExecution::getStageId) + checkArgument(!stageExecutions.isEmpty(), "stageExecutions is empty"); + Set stageIds = newConcurrentHashSet(stageExecutions.stream() + .map(SqlStageExecution::getStageExecutionId) .collect(toSet())); SettableFuture future = SettableFuture.create(); - for (SqlStageExecution stage : stages) { + for (SqlStageExecution stage : stageExecutions) { stage.addStateChangeListener(state -> { - if (predicate.test(state) && stageIds.remove(stage.getStageId()) && stageIds.isEmpty()) { + if (predicate.test(state) && stageIds.remove(stage.getStageExecutionId()) && stageIds.isEmpty()) { future.set(null); } }); @@ -889,7 +983,6 @@ private static class StageLinkage private final PlanFragmentId currentStageFragmentId; private final ExchangeLocationsConsumer parent; private final Set childOutputBufferManagers; - private final Set childStageIds; public StageLinkage(PlanFragmentId fragmentId, ExchangeLocationsConsumer parent, Set children) { @@ -910,18 +1003,9 @@ else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { } }) .collect(toImmutableSet()); - - this.childStageIds = children.stream() - .map(SqlStageExecution::getStageId) - .collect(toImmutableSet()); } - public Set getChildStageIds() - { - return childStageIds; - } - - public void processScheduleResults(StageState newState, Set newTasks) + public void processScheduleResults(StageExecutionState newState, Set newTasks) { boolean noMoreTasks = false; switch (newState) { @@ -960,7 +1044,7 @@ public void processScheduleResults(StageState newState, Set newTasks } } - private static class StreamingPlanSection + public static class StreamingPlanSection { private final StreamingSubPlan plan; // materialized exchange children @@ -986,7 +1070,7 @@ public List getChildren() /** * StreamingSubPlan is similar to SubPlan but only contains streaming children */ - private static class StreamingSubPlan + public static class StreamingSubPlan { private final PlanFragment fragment; // streaming children @@ -1013,4 +1097,33 @@ public StreamingSubPlan withBucketToPartition(Optional bucketToPartition) return new StreamingSubPlan(fragment.withBucketToPartition(bucketToPartition), children); } } + + private static class StageExecutionAndScheduler + { + private final SqlStageExecution stageExecution; + private final StageLinkage stageLinkage; + private final StageScheduler stageScheduler; + + private StageExecutionAndScheduler(SqlStageExecution stageExecution, StageLinkage stageLinkage, StageScheduler stageScheduler) + { + this.stageExecution = requireNonNull(stageExecution, "stageExecution is null"); + this.stageLinkage = requireNonNull(stageLinkage, "stageLinkage is null"); + this.stageScheduler = requireNonNull(stageScheduler, "stageScheduler is null"); + } + + public SqlStageExecution getStageExecution() + { + return stageExecution; + } + + public StageLinkage getStageLinkage() + { + return stageLinkage; + } + + public StageScheduler getStageScheduler() + { + return stageScheduler; + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TableWriteInfo.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TableWriteInfo.java new file mode 100644 index 0000000000000..7b73ad8c3b302 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TableWriteInfo.java @@ -0,0 +1,222 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.execution.scheduler; + +import com.facebook.presto.Session; +import com.facebook.presto.execution.scheduler.SqlQueryScheduler.StreamingSubPlan; +import com.facebook.presto.metadata.AnalyzeTableHandle; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.TableLayoutResult; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.Constraint; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher; +import com.facebook.presto.sql.planner.plan.DeleteNode; +import com.facebook.presto.sql.planner.plan.JoinNode; +import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; +import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterNode; +import com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.VerifyException; +import com.google.common.collect.ImmutableSet; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.Iterables.getOnlyElement; +import static com.google.common.collect.Streams.stream; +import static com.google.common.graph.Traverser.forTree; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class TableWriteInfo +{ + private final Optional writerTarget; + private final Optional analyzeTableHandle; + private final Optional deleteScanInfo; + + @JsonCreator + public TableWriteInfo( + @JsonProperty("writerTarget") Optional writerTarget, + @JsonProperty("analyzeTableHandle") Optional analyzeTableHandle, + @JsonProperty("deleteScanInfo") Optional deleteScanInfo) + { + this.writerTarget = requireNonNull(writerTarget, "writerTarget is null"); + this.analyzeTableHandle = requireNonNull(analyzeTableHandle, "analyzeTableHandle is null"); + this.deleteScanInfo = requireNonNull(deleteScanInfo, "deleteScanInfo is null"); + checkArgument(!analyzeTableHandle.isPresent() || !writerTarget.isPresent() && !deleteScanInfo.isPresent(), "analyzeTableHandle is present, so no other fields should be present"); + checkArgument(!deleteScanInfo.isPresent() || writerTarget.isPresent(), "deleteScanInfo is present, but writerTarget is not present"); + } + + public static TableWriteInfo createTableWriteInfo(StreamingSubPlan plan, Metadata metadata, Session session) + { + Optional writerTarget = createWriterTarget(plan, metadata, session); + Optional analyzeTableHandle = createAnalyzeTableHandle(plan, metadata, session); + Optional deleteScanInfo = createDeleteScanInfo(plan, writerTarget, metadata, session); + return new TableWriteInfo(writerTarget, analyzeTableHandle, deleteScanInfo); + } + + private static Optional createWriterTarget(StreamingSubPlan plan, Metadata metadata, Session session) + { + Optional tableFinishNode = findSinglePlanNode(plan, TableFinishNode.class); + if (tableFinishNode.isPresent()) { + WriterTarget target = tableFinishNode.get().getTarget().orElseThrow(() -> new VerifyException("target is absent")); + if (target instanceof TableWriterNode.CreateName) { + TableWriterNode.CreateName create = (TableWriterNode.CreateName) target; + return Optional.of(new ExecutionWriterTarget.CreateHandle(metadata.beginCreateTable(session, create.getConnectorId().getCatalogName(), create.getTableMetadata(), create.getLayout()), create.getSchemaTableName())); + } + if (target instanceof TableWriterNode.InsertReference) { + TableWriterNode.InsertReference insert = (TableWriterNode.InsertReference) target; + return Optional.of(new ExecutionWriterTarget.InsertHandle(metadata.beginInsert(session, insert.getHandle()), insert.getSchemaTableName())); + } + if (target instanceof TableWriterNode.DeleteHandle) { + TableWriterNode.DeleteHandle delete = (TableWriterNode.DeleteHandle) target; + return Optional.of(new ExecutionWriterTarget.DeleteHandle(metadata.beginDelete(session, delete.getHandle()), delete.getSchemaTableName())); + } + throw new IllegalArgumentException("Unhandled target type: " + target.getClass().getSimpleName()); + } + + return Optional.empty(); + } + + private static Optional createAnalyzeTableHandle(StreamingSubPlan plan, Metadata metadata, Session session) + { + Optional node = findSinglePlanNode(plan, StatisticsWriterNode.class); + if (node.isPresent()) { + return Optional.of(metadata.beginStatisticsCollection(session, node.get().getTableHandle())); + } + return Optional.empty(); + } + + private static Optional createDeleteScanInfo(StreamingSubPlan plan, Optional writerTarget, Metadata metadata, Session session) + { + if (writerTarget.isPresent() && writerTarget.get() instanceof ExecutionWriterTarget.DeleteHandle) { + TableHandle tableHandle = ((ExecutionWriterTarget.DeleteHandle) writerTarget.get()).getHandle(); + DeleteNode delete = getOnlyElement(findPlanNodes(plan, DeleteNode.class)); + TableScanNode tableScan = getDeleteTableScan(delete); + TupleDomain originalEnforcedConstraint = tableScan.getEnforcedConstraint(); + TableLayoutResult layoutResult = metadata.getLayout( + session, + tableHandle, + new Constraint<>(originalEnforcedConstraint), + Optional.of(ImmutableSet.copyOf(tableScan.getAssignments().values()))); + + return Optional.of(new DeleteScanInfo(tableScan.getId(), layoutResult.getLayout().getNewTableHandle())); + } + return Optional.empty(); + } + + private static Optional findSinglePlanNode(StreamingSubPlan plan, Class clazz) + { + List allMatches = findPlanNodes(plan, clazz); + switch (allMatches.size()) { + case 0: + return Optional.empty(); + case 1: + return Optional.of(getOnlyElement(allMatches)); + default: + throw new IllegalArgumentException(format("Multiple matches found for class %s", clazz)); + } + } + + private static List findPlanNodes(StreamingSubPlan plan, Class clazz) + { + return stream(forTree(StreamingSubPlan::getChildren).depthFirstPreOrder(plan)) + .map(subPlan -> PlanNodeSearcher.searchFrom(subPlan.getFragment().getRoot()) + .where(clazz::isInstance) + .findAll()) + .flatMap(Collection::stream) + .collect(toImmutableList()); + } + + private static TableScanNode getDeleteTableScan(PlanNode node) + { + if (node instanceof TableScanNode) { + return (TableScanNode) node; + } + if (node instanceof DeleteNode) { + return getDeleteTableScan(((DeleteNode) node).getSource()); + } + if (node instanceof FilterNode) { + return getDeleteTableScan(((FilterNode) node).getSource()); + } + if (node instanceof ProjectNode) { + return getDeleteTableScan(((ProjectNode) node).getSource()); + } + if (node instanceof SemiJoinNode) { + return getDeleteTableScan(((SemiJoinNode) node).getSource()); + } + if (node instanceof JoinNode) { + JoinNode joinNode = (JoinNode) node; + return getDeleteTableScan(joinNode.getLeft()); + } + throw new IllegalArgumentException("Invalid descendant for DeleteNode: " + node.getClass().getName()); + } + + @JsonProperty + public Optional getWriterTarget() + { + return writerTarget; + } + + @JsonProperty + public Optional getAnalyzeTableHandle() + { + return analyzeTableHandle; + } + + @JsonProperty + public Optional getDeleteScanInfo() + { + return deleteScanInfo; + } + + public static class DeleteScanInfo + { + private final PlanNodeId id; + private final TableHandle tableHandle; + + @JsonCreator + public DeleteScanInfo(@JsonProperty("id") PlanNodeId id, @JsonProperty("tableHandle") TableHandle tableHandle) + { + this.id = id; + this.tableHandle = tableHandle; + } + + @JsonProperty + public PlanNodeId getId() + { + return id; + } + + @JsonProperty + public TableHandle getTableHandle() + { + return tableHandle; + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TopologyAwareNodeSelector.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TopologyAwareNodeSelector.java index 0b4a1fd1b2122..48c0bdd1f90ff 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TopologyAwareNodeSelector.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TopologyAwareNodeSelector.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.execution.scheduler; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.metadata.InternalNode; @@ -26,8 +28,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; import javax.annotation.Nullable; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/warnings/WarningCollectorConfig.java b/presto-main/src/main/java/com/facebook/presto/execution/warnings/WarningCollectorConfig.java index 26fc5f97d7f3b..24ae10a7bdd1f 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/warnings/WarningCollectorConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/warnings/WarningCollectorConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.execution.warnings; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import static com.google.common.base.Preconditions.checkArgument; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/warnings/WarningCollectorModule.java b/presto-main/src/main/java/com/facebook/presto/execution/warnings/WarningCollectorModule.java index 6a431c2897672..c1298eb6caffc 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/warnings/WarningCollectorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/warnings/WarningCollectorModule.java @@ -18,7 +18,7 @@ import com.google.inject.Provides; import com.google.inject.Singleton; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; public class WarningCollectorModule diff --git a/presto-main/src/main/java/com/facebook/presto/failureDetector/FailureDetector.java b/presto-main/src/main/java/com/facebook/presto/failureDetector/FailureDetector.java index c8e27acfbaaa5..7bfdb58753ef9 100644 --- a/presto-main/src/main/java/com/facebook/presto/failureDetector/FailureDetector.java +++ b/presto-main/src/main/java/com/facebook/presto/failureDetector/FailureDetector.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.failureDetector; +import com.facebook.airlift.discovery.client.ServiceDescriptor; import com.facebook.presto.spi.HostAddress; -import io.airlift.discovery.client.ServiceDescriptor; import java.util.Set; diff --git a/presto-main/src/main/java/com/facebook/presto/failureDetector/FailureDetectorConfig.java b/presto-main/src/main/java/com/facebook/presto/failureDetector/FailureDetectorConfig.java index b07471e1d8c87..1e98d9f7fe2d7 100644 --- a/presto-main/src/main/java/com/facebook/presto/failureDetector/FailureDetectorConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/failureDetector/FailureDetectorConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.failureDetector; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import io.airlift.units.Duration; import io.airlift.units.MinDuration; diff --git a/presto-main/src/main/java/com/facebook/presto/failureDetector/FailureDetectorModule.java b/presto-main/src/main/java/com/facebook/presto/failureDetector/FailureDetectorModule.java index 15486109f4ee2..72dc65b3c0bf1 100644 --- a/presto-main/src/main/java/com/facebook/presto/failureDetector/FailureDetectorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/failureDetector/FailureDetectorModule.java @@ -18,8 +18,8 @@ import com.google.inject.Scopes; import org.weakref.jmx.guice.ExportBinder; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.http.client.HttpClientBinder.httpClientBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; public class FailureDetectorModule implements Module diff --git a/presto-main/src/main/java/com/facebook/presto/failureDetector/HeartbeatFailureDetector.java b/presto-main/src/main/java/com/facebook/presto/failureDetector/HeartbeatFailureDetector.java index f641864c1cffa..18db00175d1b0 100644 --- a/presto-main/src/main/java/com/facebook/presto/failureDetector/HeartbeatFailureDetector.java +++ b/presto-main/src/main/java/com/facebook/presto/failureDetector/HeartbeatFailureDetector.java @@ -13,6 +13,18 @@ */ package com.facebook.presto.failureDetector; +import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; +import com.facebook.airlift.discovery.client.ServiceDescriptor; +import com.facebook.airlift.discovery.client.ServiceSelector; +import com.facebook.airlift.discovery.client.ServiceType; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.ResponseHandler; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.node.NodeInfo; +import com.facebook.airlift.stats.DecayCounter; +import com.facebook.airlift.stats.ExponentialDecay; import com.facebook.presto.client.FailureInfo; import com.facebook.presto.server.InternalCommunicationConfig; import com.facebook.presto.spi.HostAddress; @@ -21,18 +33,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; -import io.airlift.concurrent.ThreadPoolExecutorMBean; -import io.airlift.discovery.client.ServiceDescriptor; -import io.airlift.discovery.client.ServiceSelector; -import io.airlift.discovery.client.ServiceType; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.ResponseHandler; -import io.airlift.log.Logger; -import io.airlift.node.NodeInfo; -import io.airlift.stats.DecayCounter; -import io.airlift.stats.ExponentialDecay; import io.airlift.units.Duration; import org.joda.time.DateTime; import org.weakref.jmx.Managed; @@ -62,6 +62,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.http.client.Request.Builder.prepareHead; import static com.facebook.presto.failureDetector.FailureDetector.State.ALIVE; import static com.facebook.presto.failureDetector.FailureDetector.State.GONE; import static com.facebook.presto.failureDetector.FailureDetector.State.UNKNOWN; @@ -70,8 +72,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.http.client.Request.Builder.prepareHead; import static java.util.Objects.requireNonNull; public class HeartbeatFailureDetector diff --git a/presto-main/src/main/java/com/facebook/presto/failureDetector/NoOpFailureDetector.java b/presto-main/src/main/java/com/facebook/presto/failureDetector/NoOpFailureDetector.java index ca996d480a446..b2d1b8aff2ce4 100644 --- a/presto-main/src/main/java/com/facebook/presto/failureDetector/NoOpFailureDetector.java +++ b/presto-main/src/main/java/com/facebook/presto/failureDetector/NoOpFailureDetector.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.failureDetector; +import com.facebook.airlift.discovery.client.ServiceDescriptor; import com.facebook.presto.spi.HostAddress; import com.google.common.collect.ImmutableSet; -import io.airlift.discovery.client.ServiceDescriptor; import java.util.Set; diff --git a/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryLeakDetector.java b/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryLeakDetector.java index 972b2245959bd..5aabefff8290b 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryLeakDetector.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryLeakDetector.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.memory; +import com.facebook.airlift.log.Logger; import com.facebook.presto.server.BasicQueryInfo; import com.facebook.presto.spi.QueryId; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; -import io.airlift.log.Logger; import org.joda.time.DateTime; import javax.annotation.concurrent.GuardedBy; diff --git a/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java b/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java index 9ff89fa220a69..ce7e43d62bf60 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/ClusterMemoryManager.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.memory; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.facebook.presto.execution.LocationFactory; import com.facebook.presto.execution.QueryExecution; import com.facebook.presto.execution.QueryIdGenerator; @@ -37,9 +40,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Streams; import com.google.common.io.Closer; -import io.airlift.http.client.HttpClient; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.weakref.jmx.JmxException; @@ -52,6 +52,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -64,6 +65,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.function.Supplier; +import java.util.stream.Stream; import static com.facebook.presto.ExceededMemoryLimitException.exceededGlobalTotalLimit; import static com.facebook.presto.ExceededMemoryLimitException.exceededGlobalUserLimit; @@ -86,6 +88,8 @@ import static io.airlift.units.Duration.nanosSince; import static java.lang.Math.min; import static java.lang.String.format; +import static java.util.AbstractMap.SimpleEntry; +import static java.util.Comparator.comparingLong; import static java.util.Objects.requireNonNull; import static org.weakref.jmx.ObjectNames.generatedNameOf; @@ -366,18 +370,27 @@ private void logQueryKill(QueryId killedQueryId, List nodes) } StringBuilder nodeDescription = new StringBuilder(); nodeDescription.append("Query Kill Decision: Killed ").append(killedQueryId).append("\n"); - for (MemoryInfo node : nodes) { - MemoryPoolInfo memoryPoolInfo = node.getPools().get(GENERAL_POOL); - if (memoryPoolInfo == null) { - continue; - } - nodeDescription.append("Query Kill Scenario: "); - nodeDescription.append("MaxBytes ").append(memoryPoolInfo.getMaxBytes()).append(' '); - nodeDescription.append("FreeBytes ").append(memoryPoolInfo.getFreeBytes() + memoryPoolInfo.getReservedRevocableBytes()).append(' '); - nodeDescription.append("Queries "); - Joiner.on(",").withKeyValueSeparator("=").appendTo(nodeDescription, memoryPoolInfo.getQueryMemoryReservations()); - nodeDescription.append('\n'); - } + Comparator> nodeMemoryComparator = comparingLong(Entry::getValue); + nodes.stream() + .filter(node -> node.getPools().get(GENERAL_POOL) != null) + .map(node -> + new SimpleEntry( + node.getPools().get(GENERAL_POOL), + node.getPools().get(GENERAL_POOL).getQueryMemoryReservations().values().stream().mapToLong(l -> l).sum())) + .sorted(nodeMemoryComparator.reversed()) + .map(Entry::getKey) + .forEachOrdered(memoryPoolInfo -> { + nodeDescription.append("Query Kill Scenario: "); + nodeDescription.append("MaxBytes ").append(memoryPoolInfo.getMaxBytes()).append(' '); + nodeDescription.append("FreeBytes ").append(memoryPoolInfo.getFreeBytes() + memoryPoolInfo.getReservedRevocableBytes()).append(' '); + nodeDescription.append("Queries "); + Comparator> queryMemoryComparator = comparingLong(Entry::getValue); + Stream> sortedMemoryReservations = + memoryPoolInfo.getQueryMemoryReservations().entrySet().stream() + .sorted(queryMemoryComparator.reversed()); + Joiner.on(",").withKeyValueSeparator("=").appendTo(nodeDescription, (Iterable>) sortedMemoryReservations::iterator); + nodeDescription.append('\n'); + }); log.info(nodeDescription.toString()); } diff --git a/presto-main/src/main/java/com/facebook/presto/memory/MemoryManagerConfig.java b/presto-main/src/main/java/com/facebook/presto/memory/MemoryManagerConfig.java index 177c72f550387..45f5db6e71437 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/MemoryManagerConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/MemoryManagerConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.memory; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.DefunctConfig; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.DefunctConfig; import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.airlift.units.MinDuration; diff --git a/presto-main/src/main/java/com/facebook/presto/memory/MemoryPool.java b/presto-main/src/main/java/com/facebook/presto/memory/MemoryPool.java index 4cdfec4451f53..73f53f9b3bc11 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/MemoryPool.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/MemoryPool.java @@ -18,7 +18,6 @@ import com.facebook.presto.spi.memory.MemoryPoolId; import com.facebook.presto.spi.memory.MemoryPoolInfo; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.AbstractFuture; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.DataSize; @@ -34,11 +33,14 @@ import java.util.Map.Entry; import java.util.concurrent.CopyOnWriteArrayList; +import static com.facebook.presto.memory.context.AbstractAggregatedMemoryContext.FORCE_FREE_TAG; import static com.facebook.presto.operator.Operator.NOT_BLOCKED; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; public class MemoryPool { @@ -351,6 +353,19 @@ private synchronized void updateTaggedMemoryAllocations(QueryId queryId, String @VisibleForTesting synchronized Map> getTaggedMemoryAllocations() { - return ImmutableMap.copyOf(taggedMemoryAllocations); + return taggedMemoryAllocations.keySet().stream() + .collect(toImmutableMap(identity(), this::getTaggedMemoryAllocations)); + } + + @VisibleForTesting + synchronized Map getTaggedMemoryAllocations(QueryId targetQueryId) + { + if (taggedMemoryAllocations.get(targetQueryId) == null) { + return null; + } + return taggedMemoryAllocations.get(targetQueryId) + .entrySet().stream() + .filter(entry -> !entry.getKey().equals(FORCE_FREE_TAG)) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); } } diff --git a/presto-main/src/main/java/com/facebook/presto/memory/NodeMemoryConfig.java b/presto-main/src/main/java/com/facebook/presto/memory/NodeMemoryConfig.java index c99e91ab813f1..107f802b730c1 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/NodeMemoryConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/NodeMemoryConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.memory; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import io.airlift.units.DataSize; import javax.validation.constraints.NotNull; diff --git a/presto-main/src/main/java/com/facebook/presto/memory/QueryContext.java b/presto-main/src/main/java/com/facebook/presto/memory/QueryContext.java index adacaf2ddb198..3f74b58fccbb9 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/QueryContext.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/QueryContext.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.memory; +import com.facebook.airlift.stats.GcMonitor; import com.facebook.presto.Session; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.TaskStateMachine; @@ -23,7 +24,6 @@ import com.facebook.presto.spiller.SpillSpaceTracker; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.stats.GcMonitor; import io.airlift.units.DataSize; import javax.annotation.concurrent.GuardedBy; @@ -246,6 +246,16 @@ public synchronized MemoryPool getMemoryPool() return memoryPool; } + public long getMaxUserMemory() + { + return maxUserMemory; + } + + public long getMaxTotalMemory() + { + return maxTotalMemory; + } + public TaskContext addTaskContext( TaskStateMachine taskStateMachine, Session session, @@ -290,6 +300,11 @@ public TaskContext getTaskContextByTaskId(TaskId taskId) return taskContext; } + public QueryId getQueryId() + { + return queryId; + } + private static class QueryMemoryReservationHandler implements MemoryReservationHandler { @@ -341,7 +356,7 @@ private void enforceTotalMemoryLimit(long allocated, long delta, long maxMemory) @GuardedBy("this") private String getAdditionalFailureInfo(long allocated, long delta) { - Map queryAllocations = memoryPool.getTaggedMemoryAllocations().get(queryId); + Map queryAllocations = memoryPool.getTaggedMemoryAllocations(queryId); String additionalInfo = format("Allocated: %s, Delta: %s", succinctBytes(allocated), succinctBytes(delta)); diff --git a/presto-main/src/main/java/com/facebook/presto/memory/RemoteNodeMemory.java b/presto-main/src/main/java/com/facebook/presto/memory/RemoteNodeMemory.java index df98ce5d8a31e..89f1215c1aefe 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/RemoteNodeMemory.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/RemoteNodeMemory.java @@ -13,18 +13,18 @@ */ package com.facebook.presto.memory; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpClient.HttpResponseFuture; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.ResponseHandler; +import com.facebook.airlift.http.client.StaticBodyGenerator; +import com.facebook.airlift.log.Logger; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.server.smile.BaseResponse; import com.facebook.presto.server.smile.Codec; import com.facebook.presto.server.smile.SmileCodec; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpClient.HttpResponseFuture; -import io.airlift.http.client.Request; -import io.airlift.http.client.ResponseHandler; -import io.airlift.http.client.StaticBodyGenerator; -import io.airlift.log.Logger; import io.airlift.units.Duration; import javax.annotation.Nullable; @@ -36,15 +36,15 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import static com.facebook.airlift.http.client.HttpStatus.OK; +import static com.facebook.airlift.http.client.JsonBodyGenerator.jsonBodyGenerator; +import static com.facebook.airlift.http.client.Request.Builder.preparePost; import static com.facebook.presto.server.RequestHelpers.setContentTypeHeaders; import static com.facebook.presto.server.smile.AdaptingJsonResponseHandler.createAdaptingJsonResponseHandler; import static com.facebook.presto.server.smile.FullSmileResponseHandler.createFullSmileResponseHandler; import static com.facebook.presto.server.smile.JsonCodecWrapper.unwrapJsonCodec; import static com.facebook.presto.server.smile.SmileBodyGenerator.smileBodyGenerator; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.http.client.HttpStatus.OK; -import static io.airlift.http.client.JsonBodyGenerator.jsonBodyGenerator; -import static io.airlift.http.client.Request.Builder.preparePost; import static io.airlift.units.Duration.nanosSince; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-main/src/main/java/com/facebook/presto/memory/ReservedSystemMemoryConfig.java b/presto-main/src/main/java/com/facebook/presto/memory/ReservedSystemMemoryConfig.java index 11e05c67d190a..c868373c04506 100644 --- a/presto-main/src/main/java/com/facebook/presto/memory/ReservedSystemMemoryConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/memory/ReservedSystemMemoryConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.memory; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import io.airlift.units.DataSize; import javax.validation.constraints.NotNull; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunction.java b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunction.java new file mode 100644 index 0000000000000..da402d101d808 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunction.java @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.function.SqlFunction; + +public abstract class BuiltInFunction + implements SqlFunction +{ + @Override + public boolean isCalledOnNullInput() + { + return false; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java similarity index 82% rename from presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java rename to presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java index ce5e7fa82d3e3..c0db340eb834f 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java @@ -23,14 +23,13 @@ import static java.util.Objects.requireNonNull; -public class StaticFunctionHandle +public class BuiltInFunctionHandle implements FunctionHandle { - private static final CatalogSchemaName STATIC_FUNCTION_NAMESPACE_CATALOG_SCHEMA_NAME = new CatalogSchemaName("static", "system"); private final Signature signature; @JsonCreator - public StaticFunctionHandle(@JsonProperty("signature") Signature signature) + public BuiltInFunctionHandle(@JsonProperty("signature") Signature signature) { this.signature = requireNonNull(signature, "signature is null"); checkArgument(signature.getTypeVariableConstraints().isEmpty(), "%s has unbound type parameters", signature); @@ -42,6 +41,12 @@ public Signature getSignature() return signature; } + @Override + public CatalogSchemaName getFunctionNamespace() + { + return signature.getName().getFunctionNamespace(); + } + @Override public boolean equals(Object o) { @@ -51,16 +56,10 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - StaticFunctionHandle that = (StaticFunctionHandle) o; + BuiltInFunctionHandle that = (BuiltInFunctionHandle) o; return Objects.equals(signature, that.signature); } - @Override - public CatalogSchemaName getCatalogSchemaName() - { - return STATIC_FUNCTION_NAMESPACE_CATALOG_SCHEMA_NAME; - } - @Override public int hashCode() { diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionNamespaceHandleResolver.java similarity index 83% rename from presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java rename to presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionNamespaceHandleResolver.java index 19792a7eccaa1..784c4f385bf71 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceHandleResolver.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionNamespaceHandleResolver.java @@ -14,13 +14,14 @@ package com.facebook.presto.metadata; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionHandleResolver; -public class StaticFunctionNamespaceHandleResolver +public class BuiltInFunctionNamespaceHandleResolver implements FunctionHandleResolver { @Override public Class getFunctionHandleClass() { - return StaticFunctionHandle.class; + return BuiltInFunctionHandle.class; } } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionNamespaceManager.java similarity index 72% rename from presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java rename to presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionNamespaceManager.java index 77d6b09e45861..07737b9c97640 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespace.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionNamespaceManager.java @@ -60,9 +60,13 @@ import com.facebook.presto.operator.aggregation.SumDataSizeForStats; import com.facebook.presto.operator.aggregation.VarianceAggregation; import com.facebook.presto.operator.aggregation.arrayagg.ArrayAggregationFunction; +import com.facebook.presto.operator.aggregation.differentialentropy.DifferentialEntropyAggregation; import com.facebook.presto.operator.aggregation.histogram.Histogram; import com.facebook.presto.operator.aggregation.multimapagg.MultimapAggregationFunction; +import com.facebook.presto.operator.scalar.ArrayAllMatchFunction; +import com.facebook.presto.operator.scalar.ArrayAnyMatchFunction; import com.facebook.presto.operator.scalar.ArrayCardinalityFunction; +import com.facebook.presto.operator.scalar.ArrayCombinationsFunction; import com.facebook.presto.operator.scalar.ArrayContains; import com.facebook.presto.operator.scalar.ArrayDistinctFromOperator; import com.facebook.presto.operator.scalar.ArrayDistinctFunction; @@ -81,6 +85,7 @@ import com.facebook.presto.operator.scalar.ArrayMaxFunction; import com.facebook.presto.operator.scalar.ArrayMinFunction; import com.facebook.presto.operator.scalar.ArrayNgramsFunction; +import com.facebook.presto.operator.scalar.ArrayNoneMatchFunction; import com.facebook.presto.operator.scalar.ArrayNotEqualOperator; import com.facebook.presto.operator.scalar.ArrayPositionFunction; import com.facebook.presto.operator.scalar.ArrayRemoveFunction; @@ -92,6 +97,7 @@ import com.facebook.presto.operator.scalar.ArrayUnionFunction; import com.facebook.presto.operator.scalar.ArraysOverlapFunction; import com.facebook.presto.operator.scalar.BitwiseFunctions; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.operator.scalar.CharacterStringCasts; import com.facebook.presto.operator.scalar.ColorFunctions; import com.facebook.presto.operator.scalar.CombineHashFunction; @@ -101,6 +107,7 @@ import com.facebook.presto.operator.scalar.FailureFunction; import com.facebook.presto.operator.scalar.HmacFunctions; import com.facebook.presto.operator.scalar.HyperLogLogFunctions; +import com.facebook.presto.operator.scalar.IpPrefixFunctions; import com.facebook.presto.operator.scalar.JoniRegexpCasts; import com.facebook.presto.operator.scalar.JoniRegexpFunctions; import com.facebook.presto.operator.scalar.JoniRegexpReplaceLambdaFunction; @@ -123,7 +130,6 @@ import com.facebook.presto.operator.scalar.Re2JRegexpFunctions; import com.facebook.presto.operator.scalar.Re2JRegexpReplaceLambdaFunction; import com.facebook.presto.operator.scalar.RepeatFunction; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; import com.facebook.presto.operator.scalar.SequenceFunction; import com.facebook.presto.operator.scalar.SessionFunctions; import com.facebook.presto.operator.scalar.SplitToMapFunction; @@ -148,20 +154,25 @@ import com.facebook.presto.operator.window.RowNumberFunction; import com.facebook.presto.operator.window.SqlWindowFunction; import com.facebook.presto.operator.window.WindowFunctionSupplier; +import com.facebook.presto.spi.CatalogSchemaName; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.function.FunctionHandle; -import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionNamespaceManager; +import com.facebook.presto.spi.function.FunctionNamespaceTransactionHandle; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.function.ScalarFunctionImplementation; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunction; +import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.TypeSignatureProvider; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.type.BigintOperators; import com.facebook.presto.type.BooleanOperators; import com.facebook.presto.type.CharOperators; @@ -175,6 +186,7 @@ import com.facebook.presto.type.IntervalDayTimeOperators; import com.facebook.presto.type.IntervalYearMonthOperators; import com.facebook.presto.type.IpAddressOperators; +import com.facebook.presto.type.IpPrefixOperators; import com.facebook.presto.type.LikeFunctions; import com.facebook.presto.type.QuantileDigestOperators; import com.facebook.presto.type.RealOperators; @@ -191,8 +203,6 @@ import com.facebook.presto.type.setdigest.MergeSetDigestAggregation; import com.facebook.presto.type.setdigest.SetDigestFunctions; import com.facebook.presto.type.setdigest.SetDigestOperators; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -200,7 +210,6 @@ import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; -import com.google.common.collect.Ordering; import com.google.common.util.concurrent.UncheckedExecutionException; import io.airlift.slice.Slice; @@ -208,17 +217,11 @@ import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import static com.facebook.presto.metadata.CastType.toOperatorType; -import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; import static com.facebook.presto.metadata.SignatureBinder.applyBoundVariables; import static com.facebook.presto.operator.aggregation.ArbitraryAggregationFunction.ARBITRARY_AGGREGATION; import static com.facebook.presto.operator.aggregation.ChecksumAggregationFunction.CHECKSUM_AGGREGATION; @@ -251,6 +254,8 @@ import static com.facebook.presto.operator.scalar.ArrayToElementConcatFunction.ARRAY_TO_ELEMENT_CONCAT_FUNCTION; import static com.facebook.presto.operator.scalar.ArrayToJsonCast.ARRAY_TO_JSON; import static com.facebook.presto.operator.scalar.ArrayTransformFunction.ARRAY_TRANSFORM_FUNCTION; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.operator.scalar.CastFromUnknownOperator.CAST_FROM_UNKNOWN; import static com.facebook.presto.operator.scalar.ConcatFunction.VARBINARY_CONCAT; import static com.facebook.presto.operator.scalar.ConcatFunction.VARCHAR_CONCAT; @@ -288,15 +293,13 @@ import static com.facebook.presto.operator.scalar.RowNotEqualOperator.ROW_NOT_EQUAL; import static com.facebook.presto.operator.scalar.RowToJsonCast.ROW_TO_JSON; import static com.facebook.presto.operator.scalar.RowToRowCast.ROW_TO_ROW_CAST; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.operator.scalar.TryCastFunction.TRY_CAST; import static com.facebook.presto.operator.scalar.ZipFunction.ZIP_FUNCTIONS; import static com.facebook.presto.operator.scalar.ZipWithFunction.ZIP_WITH_FUNCTION; import static com.facebook.presto.operator.window.AggregateWindowFunction.supplier; -import static com.facebook.presto.spi.StandardErrorCode.AMBIGUOUS_FUNCTION_CALL; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_MISSING; -import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_NOT_FOUND; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR; +import static com.facebook.presto.spi.function.FunctionImplementationType.BUILTIN; import static com.facebook.presto.spi.function.FunctionKind.AGGREGATE; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.FunctionKind.WINDOW; @@ -304,7 +307,6 @@ import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypeSignatures; import static com.facebook.presto.sql.planner.LiteralEncoder.MAGIC_LITERAL_FUNCTION_PREFIX; -import static com.facebook.presto.sql.planner.LiteralEncoder.getMagicLiteralFunctionSignature; import static com.facebook.presto.type.DecimalCasts.BIGINT_TO_DECIMAL_CAST; import static com.facebook.presto.type.DecimalCasts.BOOLEAN_TO_DECIMAL_CAST; import static com.facebook.presto.type.DecimalCasts.DECIMAL_TO_BIGINT_CAST; @@ -347,33 +349,29 @@ import static com.facebook.presto.type.DecimalSaturatedFloorCasts.TINYINT_TO_DECIMAL_SATURATED_FLOOR_CAST; import static com.facebook.presto.type.DecimalToDecimalCasts.DECIMAL_TO_DECIMAL_CAST; import static com.facebook.presto.type.TypeUtils.resolveTypes; -import static com.facebook.presto.type.UnknownType.UNKNOWN; -import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Throwables.throwIfInstanceOf; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static com.google.common.collect.Iterables.getOnlyElement; import static java.lang.String.format; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.HOURS; @ThreadSafe -class StaticFunctionNamespace - implements FunctionNamespace +public class BuiltInFunctionNamespaceManager + implements FunctionNamespaceManager { + public static final CatalogSchemaName DEFAULT_NAMESPACE = new CatalogSchemaName("presto", "default"); + public static final String ID = "builtin"; + private final TypeManager typeManager; private final LoadingCache specializedFunctionKeyCache; - private final LoadingCache specializedScalarCache; + private final LoadingCache specializedScalarCache; private final LoadingCache specializedAggregationCache; private final LoadingCache specializedWindowCache; private final MagicLiteralFunction magicLiteralFunction; private volatile FunctionMap functions = new FunctionMap(); - public StaticFunctionNamespace( + public BuiltInFunctionNamespaceManager( TypeManager typeManager, BlockEncodingSerde blockEncodingSerde, FeaturesConfig featuresConfig, @@ -455,6 +453,7 @@ public StaticFunctionNamespace( .function(REAL_AVERAGE_AGGREGATION) .aggregates(IntervalDayToSecondAverageAggregation.class) .aggregates(IntervalYearToMonthAverageAggregation.class) + .aggregates(DifferentialEntropyAggregation.class) .aggregates(EntropyAggregation.class) .aggregates(GeometricMeanAggregations.class) .aggregates(RealGeometricMeanAggregations.class) @@ -542,6 +541,9 @@ public StaticFunctionNamespace( .scalars(QuantileDigestOperators.class) .scalars(IpAddressOperators.class) .scalar(IpAddressOperators.IpAddressDistinctFromOperator.class) + .scalars(IpPrefixFunctions.class) + .scalars(IpPrefixOperators.class) + .scalar(IpPrefixOperators.IpPrefixDistinctFromOperator.class) .scalars(LikeFunctions.class) .scalars(ArrayFunctions.class) .scalars(HmacFunctions.class) @@ -586,7 +588,11 @@ public StaticFunctionNamespace( .scalar(ArrayExceptFunction.class) .scalar(ArraySliceFunction.class) .scalar(ArrayIndeterminateOperator.class) + .scalar(ArrayCombinationsFunction.class) .scalar(ArrayNgramsFunction.class) + .scalar(ArrayAllMatchFunction.class) + .scalar(ArrayAnyMatchFunction.class) + .scalar(ArrayNoneMatchFunction.class) .scalar(MapDistinctFromOperator.class) .scalar(MapEqualOperator.class) .scalar(MapEntriesFunction.class) @@ -672,11 +678,10 @@ public StaticFunctionNamespace( builder.scalar(LegacyLogFunction.class); } - addFunctions(builder.getFunctions()); + registerBuiltInFunctions(builder.getFunctions()); } - @Override - public final synchronized void addFunctions(List functions) + public synchronized void registerBuiltInFunctions(List functions) { for (SqlFunction function : functions) { for (SqlFunction existingFunction : this.functions.list()) { @@ -687,307 +692,97 @@ public final synchronized void addFunctions(List function } @Override - public List listFunctions() + public void createFunction(SqlInvokedFunction function, boolean replace) { - return functions.list().stream() - .filter(function -> !function.isHidden()) - .collect(toImmutableList()); + throw new PrestoException(GENERIC_USER_ERROR, format("Cannot create function in built-in function namespace: %s", function.getSignature().getName())); } @Override - public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) + public void dropFunction(QualifiedFunctionName functionName, Optional> parameterTypes, boolean exists) { - checkArgument(functionHandle instanceof StaticFunctionHandle, "Expect StaticFunctionHandle"); - Signature signature = ((StaticFunctionHandle) functionHandle).getSignature(); - SpecializedFunctionKey functionKey; - try { - functionKey = specializedFunctionKeyCache.getUnchecked(signature); - } - catch (UncheckedExecutionException e) { - throwIfInstanceOf(e.getCause(), PrestoException.class); - throw e; - } - SqlFunction function = functionKey.getFunction(); - Optional operatorType = tryGetOperatorType(signature.getName()); - if (operatorType.isPresent()) { - return new FunctionMetadata( - operatorType.get(), - signature.getArgumentTypes(), - signature.getReturnType(), - signature.getKind(), - function.isDeterministic(), - function.isCalledOnNullInput()); - } - else { - return new FunctionMetadata( - signature.getName(), - signature.getArgumentTypes(), - signature.getReturnType(), - signature.getKind(), - function.isDeterministic(), - function.isCalledOnNullInput()); - } + throw new PrestoException(GENERIC_USER_ERROR, format("Cannot drop function in built-in function namespace: %s", functionName)); } - public FunctionHandle lookupFunction(QualifiedName name, List parameterTypes) + public String getName() { - Collection allCandidates = functions.get(name); - List exactCandidates = allCandidates.stream() - .filter(function -> function.getSignature().getTypeVariableConstraints().isEmpty()) - .collect(Collectors.toList()); - - Optional match = matchFunctionExact(exactCandidates, parameterTypes); - if (match.isPresent()) { - return new StaticFunctionHandle(match.get()); - } - - List genericCandidates = allCandidates.stream() - .filter(function -> !function.getSignature().getTypeVariableConstraints().isEmpty()) - .collect(Collectors.toList()); - - match = matchFunctionExact(genericCandidates, parameterTypes); - if (match.isPresent()) { - return new StaticFunctionHandle(match.get()); - } - - throw new PrestoException(FUNCTION_NOT_FOUND, constructFunctionNotFoundErrorMessage(name, parameterTypes, allCandidates)); + return ID; } @Override - public FunctionHandle resolveFunction(QualifiedName name, List parameterTypes) - { - try { - return lookupFunction(name, parameterTypes); - } - catch (PrestoException e) { - if (e.getErrorCode().getCode() != FUNCTION_NOT_FOUND.toErrorCode().getCode()) { - throw e; - } - } - - Collection allCandidates = functions.get(name); - Optional match = matchFunctionWithCoercion(allCandidates, parameterTypes); - if (match.isPresent()) { - return new StaticFunctionHandle(match.get()); - } - - if (name.getSuffix().startsWith(MAGIC_LITERAL_FUNCTION_PREFIX)) { - // extract type from function name - String typeName = name.getSuffix().substring(MAGIC_LITERAL_FUNCTION_PREFIX.length()); - - // lookup the type - Type type = typeManager.getType(parseTypeSignature(typeName)); - - // verify we have one parameter of the proper type - checkArgument(parameterTypes.size() == 1, "Expected one argument to literal function, but got %s", parameterTypes); - - return new StaticFunctionHandle(getMagicLiteralFunctionSignature(type)); - } - - throw new PrestoException(FUNCTION_NOT_FOUND, constructFunctionNotFoundErrorMessage(name, parameterTypes, allCandidates)); - } - - private String constructFunctionNotFoundErrorMessage(QualifiedName name, List parameterTypes, Collection candidates) + public FunctionNamespaceTransactionHandle beginTransaction() { - List expectedParameters = new ArrayList<>(); - for (SqlFunction function : candidates) { - expectedParameters.add(format("%s(%s) %s", - name, - Joiner.on(", ").join(function.getSignature().getArgumentTypes()), - Joiner.on(", ").join(function.getSignature().getTypeVariableConstraints()))); - } - String parameters = Joiner.on(", ").join(parameterTypes); - String message = format("Function %s not registered", name); - if (!expectedParameters.isEmpty()) { - String expected = Joiner.on(", ").join(expectedParameters); - message = format("Unexpected parameters (%s) for function %s. Expected: %s", parameters, name, expected); - } - return message; + return new EmptyTransactionHandle(); } - private Optional matchFunctionExact(List candidates, List actualParameters) + @Override + public void commit(FunctionNamespaceTransactionHandle transactionHandle) { - return matchFunction(candidates, actualParameters, false); } - private Optional matchFunctionWithCoercion(Collection candidates, List actualParameters) + @Override + public void abort(FunctionNamespaceTransactionHandle transactionHandle) { - return matchFunction(candidates, actualParameters, true); } - private Optional matchFunction(Collection candidates, List parameters, boolean coercionAllowed) + @Override + public Collection listFunctions() { - List applicableFunctions = identifyApplicableFunctions(candidates, parameters, coercionAllowed); - if (applicableFunctions.isEmpty()) { - return Optional.empty(); - } - - if (coercionAllowed) { - applicableFunctions = selectMostSpecificFunctions(applicableFunctions, parameters); - checkState(!applicableFunctions.isEmpty(), "at least single function must be left"); - } - - if (applicableFunctions.size() == 1) { - return Optional.of(getOnlyElement(applicableFunctions).getBoundSignature()); - } - - StringBuilder errorMessageBuilder = new StringBuilder(); - errorMessageBuilder.append("Could not choose a best candidate operator. Explicit type casts must be added.\n"); - errorMessageBuilder.append("Candidates are:\n"); - for (ApplicableFunction function : applicableFunctions) { - errorMessageBuilder.append("\t * "); - errorMessageBuilder.append(function.getBoundSignature().toString()); - errorMessageBuilder.append("\n"); - } - throw new PrestoException(AMBIGUOUS_FUNCTION_CALL, errorMessageBuilder.toString()); + return functions.list(); } - private List identifyApplicableFunctions(Collection candidates, List actualParameters, boolean allowCoercion) + @Override + public Collection getFunctions(Optional transactionHandle, QualifiedFunctionName functionName) { - ImmutableList.Builder applicableFunctions = ImmutableList.builder(); - for (SqlFunction function : candidates) { - Signature declaredSignature = function.getSignature(); - Optional boundSignature = new SignatureBinder(typeManager, declaredSignature, allowCoercion) - .bind(actualParameters); - if (boundSignature.isPresent()) { - applicableFunctions.add(new ApplicableFunction(declaredSignature, boundSignature.get())); - } - } - return applicableFunctions.build(); + return functions.get(functionName); } - private List selectMostSpecificFunctions(List applicableFunctions, List parameters) + @Override + public FunctionHandle getFunctionHandle(Optional transactionHandle, Signature signature) { - checkArgument(!applicableFunctions.isEmpty()); - - List mostSpecificFunctions = selectMostSpecificFunctions(applicableFunctions); - if (mostSpecificFunctions.size() <= 1) { - return mostSpecificFunctions; - } - - Optional> optionalParameterTypes = toTypes(parameters, typeManager); - if (!optionalParameterTypes.isPresent()) { - // give up and return all remaining matches - return mostSpecificFunctions; - } - - List parameterTypes = optionalParameterTypes.get(); - if (!someParameterIsUnknown(parameterTypes)) { - // give up and return all remaining matches - return mostSpecificFunctions; - } - - // look for functions that only cast the unknown arguments - List unknownOnlyCastFunctions = getUnknownOnlyCastFunctions(applicableFunctions, parameterTypes); - if (!unknownOnlyCastFunctions.isEmpty()) { - mostSpecificFunctions = unknownOnlyCastFunctions; - if (mostSpecificFunctions.size() == 1) { - return mostSpecificFunctions; - } - } - - // If the return type for all the selected function is the same, and the parameters are declared as RETURN_NULL_ON_NULL - // all the functions are semantically the same. We can return just any of those. - if (returnTypeIsTheSame(mostSpecificFunctions) && allReturnNullOnGivenInputTypes(mostSpecificFunctions, parameterTypes)) { - // make it deterministic - ApplicableFunction selectedFunction = Ordering.usingToString() - .reverse() - .sortedCopy(mostSpecificFunctions) - .get(0); - return ImmutableList.of(selectedFunction); - } - - return mostSpecificFunctions; + return new BuiltInFunctionHandle(signature); } - private List selectMostSpecificFunctions(List candidates) + @Override + public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) { - List representatives = new ArrayList<>(); - - for (ApplicableFunction current : candidates) { - boolean found = false; - for (int i = 0; i < representatives.size(); i++) { - ApplicableFunction representative = representatives.get(i); - if (isMoreSpecificThan(current, representative)) { - representatives.set(i, current); - } - if (isMoreSpecificThan(current, representative) || isMoreSpecificThan(representative, current)) { - found = true; - break; - } - } - - if (!found) { - representatives.add(current); - } + checkArgument(functionHandle instanceof BuiltInFunctionHandle, "Expect BuiltInFunctionHandle"); + Signature signature = ((BuiltInFunctionHandle) functionHandle).getSignature(); + SpecializedFunctionKey functionKey; + try { + functionKey = specializedFunctionKeyCache.getUnchecked(signature); } - - return representatives; - } - - private static boolean someParameterIsUnknown(List parameters) - { - return parameters.stream().anyMatch(type -> type.equals(UNKNOWN)); - } - - private List getUnknownOnlyCastFunctions(List applicableFunction, List actualParameters) - { - return applicableFunction.stream() - .filter((function) -> onlyCastsUnknown(function, actualParameters)) - .collect(toImmutableList()); - } - - private boolean onlyCastsUnknown(ApplicableFunction applicableFunction, List actualParameters) - { - List boundTypes = resolveTypes(applicableFunction.getBoundSignature().getArgumentTypes(), typeManager); - checkState(actualParameters.size() == boundTypes.size(), "type lists are of different lengths"); - for (int i = 0; i < actualParameters.size(); i++) { - if (!boundTypes.get(i).equals(actualParameters.get(i)) && actualParameters.get(i) != UNKNOWN) { - return false; - } + catch (UncheckedExecutionException e) { + throwIfInstanceOf(e.getCause(), PrestoException.class); + throw e; } - return true; - } - - private boolean returnTypeIsTheSame(List applicableFunctions) - { - Set returnTypes = applicableFunctions.stream() - .map(function -> typeManager.getType(function.getBoundSignature().getReturnType())) - .collect(Collectors.toSet()); - return returnTypes.size() == 1; - } - - private boolean allReturnNullOnGivenInputTypes(List applicableFunctions, List parameters) - { - return applicableFunctions.stream().allMatch(x -> returnsNullOnGivenInputTypes(x, parameters)); - } - - private boolean returnsNullOnGivenInputTypes(ApplicableFunction applicableFunction, List parameterTypes) - { - Signature boundSignature = applicableFunction.getBoundSignature(); - FunctionKind functionKind = boundSignature.getKind(); - // Window and Aggregation functions have fixed semantic where NULL values are always skipped - if (functionKind != SCALAR) { - return true; + BuiltInFunction function = functionKey.getFunction(); + Optional operatorType = tryGetOperatorType(signature.getName()); + if (operatorType.isPresent()) { + return new FunctionMetadata( + operatorType.get(), + signature.getArgumentTypes(), + signature.getReturnType(), + signature.getKind(), + BUILTIN, + function.isDeterministic(), + function.isCalledOnNullInput()); } - - for (int i = 0; i < parameterTypes.size(); i++) { - Type parameterType = parameterTypes.get(i); - if (parameterType.equals(UNKNOWN)) { - // TODO: This still doesn't feel right. Need to understand function resolution logic better to know what's the right way. - StaticFunctionHandle functionHandle = new StaticFunctionHandle(boundSignature); - if (getFunctionMetadata(functionHandle).isCalledOnNullInput()) { - return false; - } - } + else { + return new FunctionMetadata( + signature.getName(), + signature.getArgumentTypes(), + signature.getReturnType(), + signature.getKind(), + BUILTIN, + function.isDeterministic(), + function.isCalledOnNullInput()); } - return true; } public WindowFunctionSupplier getWindowFunctionImplementation(FunctionHandle functionHandle) { - checkArgument(functionHandle instanceof StaticFunctionHandle, "Expect StaticFunctionHandle"); - Signature signature = ((StaticFunctionHandle) functionHandle).getSignature(); + checkArgument(functionHandle instanceof BuiltInFunctionHandle, "Expect BuiltInFunctionHandle"); + Signature signature = ((BuiltInFunctionHandle) functionHandle).getSignature(); checkArgument(signature.getKind() == WINDOW || signature.getKind() == AGGREGATE, "%s is not a window function", signature); checkArgument(signature.getTypeVariableConstraints().isEmpty(), "%s has unbound type parameters", signature); @@ -1002,8 +797,8 @@ public WindowFunctionSupplier getWindowFunctionImplementation(FunctionHandle fun public InternalAggregationFunction getAggregateFunctionImplementation(FunctionHandle functionHandle) { - checkArgument(functionHandle instanceof StaticFunctionHandle, "Expect StaticFunctionHandle"); - Signature signature = ((StaticFunctionHandle) functionHandle).getSignature(); + checkArgument(functionHandle instanceof BuiltInFunctionHandle, "Expect BuiltInFunctionHandle"); + Signature signature = ((BuiltInFunctionHandle) functionHandle).getSignature(); checkArgument(signature.getKind() == AGGREGATE, "%s is not an aggregate function", signature); checkArgument(signature.getTypeVariableConstraints().isEmpty(), "%s has unbound type parameters", signature); @@ -1016,13 +811,14 @@ public InternalAggregationFunction getAggregateFunctionImplementation(FunctionHa } } + @Override public ScalarFunctionImplementation getScalarFunctionImplementation(FunctionHandle functionHandle) { - checkArgument(functionHandle instanceof StaticFunctionHandle, "Expect StaticFunctionHandle"); - return getScalarFunctionImplementation(((StaticFunctionHandle) functionHandle).getSignature()); + checkArgument(functionHandle instanceof BuiltInFunctionHandle, "Expect BuiltInFunctionHandle"); + return getScalarFunctionImplementation(((BuiltInFunctionHandle) functionHandle).getSignature()); } - public ScalarFunctionImplementation getScalarFunctionImplementation(Signature signature) + public BuiltInScalarFunctionImplementation getScalarFunctionImplementation(Signature signature) { checkArgument(signature.getKind() == SCALAR, "%s is not a scalar function", signature); checkArgument(signature.getTypeVariableConstraints().isEmpty(), "%s has unbound type parameters", signature); @@ -1049,11 +845,11 @@ private SpecializedFunctionKey getSpecializedFunctionKey(Signature signature) private SpecializedFunctionKey doGetSpecializedFunctionKey(Signature signature) { - Iterable candidates = functions.get(QualifiedName.of(signature.getName())); + Iterable candidates = getFunctions(null, signature.getName()); // search for exact match Type returnType = typeManager.getType(signature.getReturnType()); List argumentTypeSignatureProviders = fromTypeSignatures(signature.getArgumentTypes()); - for (SqlFunction candidate : candidates) { + for (BuiltInFunction candidate : candidates) { Optional boundVariables = new SignatureBinder(typeManager, candidate.getSignature(), false) .bindVariables(argumentTypeSignatureProviders, returnType); if (boundVariables.isPresent()) { @@ -1064,7 +860,7 @@ private SpecializedFunctionKey doGetSpecializedFunctionKey(Signature signature) // TODO: hack because there could be "type only" coercions (which aren't necessarily included as implicit casts), // so do a second pass allowing "type only" coercions List argumentTypes = resolveTypes(signature.getArgumentTypes(), typeManager); - for (SqlFunction candidate : candidates) { + for (BuiltInFunction candidate : candidates) { SignatureBinder binder = new SignatureBinder(typeManager, candidate.getSignature(), true); Optional boundVariables = binder.bindVariables(argumentTypeSignatureProviders, returnType); if (!boundVariables.isPresent()) { @@ -1091,10 +887,10 @@ private SpecializedFunctionKey doGetSpecializedFunctionKey(Signature signature) } // TODO: this is a hack and should be removed - if (signature.getName().startsWith(MAGIC_LITERAL_FUNCTION_PREFIX)) { + if (signature.getNameSuffix().startsWith(MAGIC_LITERAL_FUNCTION_PREFIX)) { List parameterTypes = signature.getArgumentTypes(); // extract type from function name - String typeName = signature.getName().substring(MAGIC_LITERAL_FUNCTION_PREFIX.length()); + String typeName = signature.getNameSuffix().substring(MAGIC_LITERAL_FUNCTION_PREFIX.length()); // lookup the type Type type = typeManager.getType(parseTypeSignature(typeName)); @@ -1116,96 +912,30 @@ private SpecializedFunctionKey doGetSpecializedFunctionKey(Signature signature) throw new PrestoException(FUNCTION_IMPLEMENTATION_MISSING, format("%s not found", signature)); } - @VisibleForTesting - public List listOperators() - { - Set operatorNames = Arrays.asList(OperatorType.values()).stream() - .map(OperatorSignatureUtils::mangleOperatorName) - .collect(toImmutableSet()); - - return functions.list().stream() - .filter(function -> operatorNames.contains(function.getSignature().getName())) - .collect(toImmutableList()); - } - - public FunctionHandle resolveOperator(OperatorType operatorType, List argumentTypes) - throws OperatorNotFoundException + private static class EmptyTransactionHandle + implements FunctionNamespaceTransactionHandle { - try { - return resolveFunction(QualifiedName.of(mangleOperatorName(operatorType)), argumentTypes); - } - catch (PrestoException e) { - if (e.getErrorCode().getCode() == FUNCTION_NOT_FOUND.toErrorCode().getCode()) { - throw new OperatorNotFoundException( - operatorType, - argumentTypes.stream() - .map(TypeSignatureProvider::getTypeSignature) - .collect(toImmutableList())); - } - else { - throw e; - } - } - } - - public FunctionHandle lookupCast(CastType castType, TypeSignature fromType, TypeSignature toType) - { - Signature signature = new Signature(castType.getCastName(), SCALAR, emptyList(), emptyList(), toType, singletonList(fromType), false); - - try { - getScalarFunctionImplementation(signature); - } - catch (PrestoException e) { - if (castType.isOperatorType() && e.getErrorCode().getCode() == FUNCTION_IMPLEMENTATION_MISSING.toErrorCode().getCode()) { - throw new OperatorNotFoundException(toOperatorType(castType), ImmutableList.of(fromType), toType); - } - throw e; - } - return new StaticFunctionHandle(signature); - } - - private static Optional> toTypes(List typeSignatureProviders, TypeManager typeManager) - { - ImmutableList.Builder resultBuilder = ImmutableList.builder(); - for (TypeSignatureProvider typeSignatureProvider : typeSignatureProviders) { - if (typeSignatureProvider.hasDependency()) { - return Optional.empty(); - } - resultBuilder.add(typeManager.getType(typeSignatureProvider.getTypeSignature())); - } - return Optional.of(resultBuilder.build()); - } - - /** - * One method is more specific than another if invocation handled by the first method could be passed on to the other one - */ - private boolean isMoreSpecificThan(ApplicableFunction left, ApplicableFunction right) - { - List resolvedTypes = fromTypeSignatures(left.getBoundSignature().getArgumentTypes()); - Optional boundVariables = new SignatureBinder(typeManager, right.getDeclaredSignature(), true) - .bindVariables(resolvedTypes); - return boundVariables.isPresent(); } private static class FunctionMap { - private final Multimap functions; + private final Multimap functions; public FunctionMap() { functions = ImmutableListMultimap.of(); } - public FunctionMap(FunctionMap map, Iterable functions) + public FunctionMap(FunctionMap map, Iterable functions) { - this.functions = ImmutableListMultimap.builder() + this.functions = ImmutableListMultimap.builder() .putAll(map.functions) - .putAll(Multimaps.index(functions, function -> QualifiedName.of(function.getSignature().getName()))) + .putAll(Multimaps.index(functions, function -> function.getSignature().getName())) .build(); // Make sure all functions with the same name are aggregations or none of them are - for (Map.Entry> entry : this.functions.asMap().entrySet()) { - Collection values = entry.getValue(); + for (Map.Entry> entry : this.functions.asMap().entrySet()) { + Collection values = entry.getValue(); long aggregations = values.stream() .map(function -> function.getSignature().getKind()) .filter(kind -> kind == AGGREGATE) @@ -1214,48 +944,17 @@ public FunctionMap(FunctionMap map, Iterable functions) } } - public List list() + public List list() { return ImmutableList.copyOf(functions.values()); } - public Collection get(QualifiedName name) + public Collection get(QualifiedFunctionName name) { return functions.get(name); } } - private static class ApplicableFunction - { - private final Signature declaredSignature; - private final Signature boundSignature; - - private ApplicableFunction(Signature declaredSignature, Signature boundSignature) - { - this.declaredSignature = declaredSignature; - this.boundSignature = boundSignature; - } - - public Signature getDeclaredSignature() - { - return declaredSignature; - } - - public Signature getBoundSignature() - { - return boundSignature; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("declaredSignature", declaredSignature) - .add("boundSignature", boundSignature) - .toString(); - } - } - private static class MagicLiteralFunction extends SqlScalarFunction { @@ -1263,7 +962,7 @@ private static class MagicLiteralFunction MagicLiteralFunction(BlockEncodingSerde blockEncodingSerde) { - super(new Signature(MAGIC_LITERAL_FUNCTION_PREFIX, SCALAR, TypeSignature.parseTypeSignature("R"), TypeSignature.parseTypeSignature("T"))); + super(new Signature(QualifiedFunctionName.of(DEFAULT_NAMESPACE, MAGIC_LITERAL_FUNCTION_PREFIX), SCALAR, TypeSignature.parseTypeSignature("R"), TypeSignature.parseTypeSignature("T"))); this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerde is null"); } @@ -1286,7 +985,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type parameterType = boundVariables.getTypeVariable("T"); Type type = boundVariables.getTypeVariable("R"); @@ -1308,7 +1007,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in parameterType.getJavaType(), type.getJavaType()); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/CastType.java b/presto-main/src/main/java/com/facebook/presto/metadata/CastType.java index 908a16fcbe13a..6e53d79797ede 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/CastType.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/CastType.java @@ -14,8 +14,9 @@ package com.facebook.presto.metadata; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; -import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.operator.scalar.JsonStringToArrayCast.JSON_STRING_TO_ARRAY_NAME; import static com.facebook.presto.operator.scalar.JsonStringToMapCast.JSON_STRING_TO_MAP_NAME; import static com.facebook.presto.operator.scalar.JsonStringToRowCast.JSON_STRING_TO_ROW_NAME; @@ -24,23 +25,23 @@ public enum CastType { - CAST(mangleOperatorName(OperatorType.CAST.name()), true), - SATURATED_FLOOR_CAST(mangleOperatorName(OperatorType.SATURATED_FLOOR_CAST.name()), true), - TRY_CAST(TRY_CAST_NAME, false), - JSON_TO_ARRAY_CAST(JSON_STRING_TO_ARRAY_NAME, false), - JSON_TO_MAP_CAST(JSON_STRING_TO_MAP_NAME, false), - JSON_TO_ROW_CAST(JSON_STRING_TO_ROW_NAME, false); + CAST(OperatorType.CAST.getFunctionName(), true), + SATURATED_FLOOR_CAST(OperatorType.SATURATED_FLOOR_CAST.getFunctionName(), true), + TRY_CAST(QualifiedFunctionName.of(DEFAULT_NAMESPACE, TRY_CAST_NAME), false), + JSON_TO_ARRAY_CAST(QualifiedFunctionName.of(DEFAULT_NAMESPACE, JSON_STRING_TO_ARRAY_NAME), false), + JSON_TO_MAP_CAST(QualifiedFunctionName.of(DEFAULT_NAMESPACE, JSON_STRING_TO_MAP_NAME), false), + JSON_TO_ROW_CAST(QualifiedFunctionName.of(DEFAULT_NAMESPACE, JSON_STRING_TO_ROW_NAME), false); - private final String castName; + private final QualifiedFunctionName castName; private final boolean isOperatorType; - CastType(String castName, boolean isOperatorType) + CastType(QualifiedFunctionName castName, boolean isOperatorType) { this.castName = castName; this.isOperatorType = isOperatorType; } - public String getCastName() + public QualifiedFunctionName getCastName() { return castName; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java index 8b5766013c502..272445019580f 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java @@ -13,6 +13,12 @@ */ package com.facebook.presto.metadata; +import com.facebook.airlift.discovery.client.ServiceDescriptor; +import com.facebook.airlift.discovery.client.ServiceSelector; +import com.facebook.airlift.discovery.client.ServiceType; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.client.NodeVersion; import com.facebook.presto.connector.system.GlobalSystemConnector; import com.facebook.presto.failureDetector.FailureDetector; @@ -26,12 +32,6 @@ import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; import com.google.common.collect.Sets.SetView; -import io.airlift.discovery.client.ServiceDescriptor; -import io.airlift.discovery.client.ServiceSelector; -import io.airlift.discovery.client.ServiceType; -import io.airlift.http.client.HttpClient; -import io.airlift.log.Logger; -import io.airlift.node.NodeInfo; import org.weakref.jmx.Managed; import javax.annotation.PostConstruct; @@ -52,14 +52,14 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; import static com.facebook.presto.spi.NodeState.ACTIVE; import static com.facebook.presto.spi.NodeState.INACTIVE; import static com.facebook.presto.spi.NodeState.SHUTTING_DOWN; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Sets.difference; -import static io.airlift.concurrent.Threads.threadsNamed; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionExtractor.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionExtractor.java index 19097d9cc8bb0..5c40e63704318 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionExtractor.java @@ -29,7 +29,7 @@ public final class FunctionExtractor { private FunctionExtractor() {} - public static List extractFunctions(Collection> classes) + public static List extractFunctions(Collection> classes) { return classes.stream() .map(FunctionExtractor::extractFunctions) @@ -37,7 +37,7 @@ public static List extractFunctions(Collection> .collect(toImmutableList()); } - public static List extractFunctions(Class clazz) + public static List extractFunctions(Class clazz) { if (WindowFunction.class.isAssignableFrom(clazz)) { @SuppressWarnings("unchecked") diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionInvokerProvider.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionInvokerProvider.java index d2750720a1420..cf208410300ed 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionInvokerProvider.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionInvokerProvider.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.metadata; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ScalarImplementationChoice; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ScalarImplementationChoice; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.InvocationConvention; @@ -27,11 +27,11 @@ import java.util.List; import java.util.Optional; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentType.FUNCTION_TYPE; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentType.FUNCTION_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_NOT_FOUND; import static com.google.common.base.Preconditions.checkState; import static java.lang.String.format; @@ -47,8 +47,8 @@ public FunctionInvokerProvider(FunctionManager functionManager) public FunctionInvoker createFunctionInvoker(FunctionHandle functionHandle, Optional invocationConvention) { - ScalarFunctionImplementation scalarFunctionImplementation = functionManager.getScalarFunctionImplementation(functionHandle); - for (ScalarImplementationChoice choice : scalarFunctionImplementation.getAllChoices()) { + BuiltInScalarFunctionImplementation builtInScalarFunctionImplementation = functionManager.getBuiltInScalarFunctionImplementation(functionHandle); + for (ScalarImplementationChoice choice : builtInScalarFunctionImplementation.getAllChoices()) { if (checkChoice(choice.getArgumentProperties(), choice.isNullable(), choice.hasSession(), invocationConvention)) { return new FunctionInvoker(choice.getMethodHandle()); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java index 35116b4d42c5c..d250be11f33b0 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java @@ -25,7 +25,7 @@ public class FunctionListBuilder { - private final List functions = new ArrayList<>(); + private final List functions = new ArrayList<>(); public FunctionListBuilder window(Class clazz) { @@ -57,22 +57,22 @@ public FunctionListBuilder scalars(Class clazz) return this; } - public FunctionListBuilder functions(SqlFunction... sqlFunctions) + public FunctionListBuilder functions(BuiltInFunction... sqlFunctions) { - for (SqlFunction sqlFunction : sqlFunctions) { + for (BuiltInFunction sqlFunction : sqlFunctions) { function(sqlFunction); } return this; } - public FunctionListBuilder function(SqlFunction sqlFunction) + public FunctionListBuilder function(BuiltInFunction sqlFunction) { requireNonNull(sqlFunction, "parametricFunction is null"); functions.add(sqlFunction); return this; } - public List getFunctions() + public List getFunctions() { return ImmutableList.copyOf(functions); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java index a904b854870a4..63fd6a0b8c4ea 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java @@ -13,40 +13,144 @@ */ package com.facebook.presto.metadata; -import com.facebook.presto.Session; import com.facebook.presto.operator.aggregation.InternalAggregationFunction; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.operator.window.WindowFunctionSupplier; +import com.facebook.presto.spi.CatalogSchemaName; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.BlockEncodingSerde; +import com.facebook.presto.spi.function.CatalogSchemaPrefix; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.FunctionNamespaceManager; +import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory; +import com.facebook.presto.spi.function.FunctionNamespaceTransactionHandle; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.function.ScalarFunctionImplementation; +import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunction; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.TypeSignatureProvider; import com.facebook.presto.sql.tree.QualifiedName; +import com.facebook.presto.transaction.TransactionId; +import com.facebook.presto.transaction.TransactionManager; import com.facebook.presto.type.TypeRegistry; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Ordering; import javax.annotation.concurrent.ThreadSafe; +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.CastType.toOperatorType; +import static com.facebook.presto.spi.StandardErrorCode.AMBIGUOUS_FUNCTION_CALL; +import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_MISSING; +import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_NOT_FOUND; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR; +import static com.facebook.presto.spi.function.FunctionKind.SCALAR; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypeSignatures; +import static com.facebook.presto.sql.planner.LiteralEncoder.MAGIC_LITERAL_FUNCTION_PREFIX; +import static com.facebook.presto.sql.planner.LiteralEncoder.getMagicLiteralFunctionSignature; +import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; +import static com.facebook.presto.type.TypeUtils.resolveTypes; +import static com.facebook.presto.type.UnknownType.UNKNOWN; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.common.collect.Iterables.getOnlyElement; +import static java.lang.String.format; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static java.util.Objects.requireNonNull; @ThreadSafe public class FunctionManager implements FunctionMetadataManager { - private final StaticFunctionNamespace staticFunctionNamespace; + private final TypeManager typeManager; + private final TransactionManager transactionManager; + private final BuiltInFunctionNamespaceManager builtInFunctionNamespaceManager; private final FunctionInvokerProvider functionInvokerProvider; + private final Map functionNamespaceManagerFactories = new ConcurrentHashMap<>(); + private final HandleResolver handleResolver; + private final Map functionNamespaces = new ConcurrentHashMap<>(); + private final Map> functionNamespaceManagers = new ConcurrentHashMap<>(); - public FunctionManager(TypeManager typeManager, BlockEncodingSerde blockEncodingSerde, FeaturesConfig featuresConfig) + private final boolean listNonBuiltInFunctions; + + @Inject + public FunctionManager( + TypeManager typeManager, + TransactionManager transactionManager, + BlockEncodingSerde blockEncodingSerde, + FeaturesConfig featuresConfig, + HandleResolver handleResolver) { - this.staticFunctionNamespace = new StaticFunctionNamespace(typeManager, blockEncodingSerde, featuresConfig, this); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); + this.builtInFunctionNamespaceManager = new BuiltInFunctionNamespaceManager(typeManager, blockEncodingSerde, featuresConfig, this); + this.functionNamespaces.put(DEFAULT_NAMESPACE.asCatalogSchemaPrefix(), BuiltInFunctionNamespaceManager.ID); + this.functionNamespaceManagers.put(BuiltInFunctionNamespaceManager.ID, builtInFunctionNamespaceManager); this.functionInvokerProvider = new FunctionInvokerProvider(this); + this.handleResolver = handleResolver; if (typeManager instanceof TypeRegistry) { ((TypeRegistry) typeManager).setFunctionManager(this); } + // TODO: Provide a more encapsulated way for TransactionManager to register FunctionNamespaceManager + transactionManager.registerFunctionNamespaceManager(BuiltInFunctionNamespaceManager.ID, builtInFunctionNamespaceManager); + this.listNonBuiltInFunctions = featuresConfig.isListNonBuiltInFunctions(); + } + + @VisibleForTesting + public FunctionManager(TypeManager typeManager, BlockEncodingSerde blockEncodingSerde, FeaturesConfig featuresConfig) + { + // TODO: Convert this constructor to a function in the testing package + this(typeManager, createTestTransactionManager(), blockEncodingSerde, featuresConfig, new HandleResolver()); + } + + public void loadFunctionNamespaceManager( + String functionNamespaceManagerName, + String functionNamespaceManagerId, + List catalogSchemaPrefixes, + Map properties) + { + requireNonNull(functionNamespaceManagerName, "functionNamespaceManagerName is null"); + FunctionNamespaceManagerFactory factory = functionNamespaceManagerFactories.get(functionNamespaceManagerName); + checkState(factory != null, "No factory for function namespace manager %s", functionNamespaceManagerName); + FunctionNamespaceManager functionNamespaceManager = factory.create(properties); + + transactionManager.registerFunctionNamespaceManager(functionNamespaceManagerId, functionNamespaceManager); + if (functionNamespaceManagers.putIfAbsent(functionNamespaceManagerId, functionNamespaceManager) != null) { + throw new IllegalArgumentException(format("Function namespace manager [%s] is already registered", functionNamespaceManagerId)); + } + for (String catalogSchemaPrefix : catalogSchemaPrefixes) { + if (functionNamespaces.putIfAbsent(CatalogSchemaPrefix.of(catalogSchemaPrefix), functionNamespaceManagerId) != null) { + throw new IllegalArgumentException(format("Function namespace [%s] is already registered to function namespace manager [%s]", catalogSchemaPrefix, functionNamespaceManagerId)); + } + } } public FunctionInvokerProvider getFunctionInvokerProvider() @@ -54,70 +158,525 @@ public FunctionInvokerProvider getFunctionInvokerProvider() return functionInvokerProvider; } - public void addFunctions(List functions) + public void addFunctionNamespaceFactory(FunctionNamespaceManagerFactory factory) + { + if (functionNamespaceManagerFactories.putIfAbsent(factory.getName(), factory) != null) { + throw new IllegalArgumentException(format("Resource group configuration manager '%s' is already registered", factory.getName())); + } + handleResolver.addFunctionNamespace(factory.getName(), factory.getHandleResolver()); + } + + public void registerBuiltInFunctions(List functions) { - staticFunctionNamespace.addFunctions(functions); + builtInFunctionNamespaceManager.registerBuiltInFunctions(functions); } public List listFunctions() { - return staticFunctionNamespace.listFunctions(); + Collection> managers = listNonBuiltInFunctions ? + functionNamespaceManagers.values() : + ImmutableSet.of(builtInFunctionNamespaceManager); + + return managers.stream() + .flatMap(manager -> manager.listFunctions().stream()) + .filter(function -> !function.isHidden()) + .collect(toImmutableList()); + } + + public void createFunction(SqlInvokedFunction function, boolean replace) + { + Optional> functionNamespaceManager = getServingFunctionNamespaceManager(function.getSignature().getName().getFunctionNamespace()); + if (!functionNamespaceManager.isPresent()) { + throw new PrestoException(GENERIC_USER_ERROR, format("Cannot create function in function namespace: %s", function.getFunctionId().getFunctionName().getFunctionNamespace())); + } + functionNamespaceManager.get().createFunction(function, replace); + } + + public void dropFunction(QualifiedFunctionName functionName, Optional> parameterTypes, boolean exists) + { + Optional> functionNamespaceManager = getServingFunctionNamespaceManager(functionName.getFunctionNamespace()); + if (functionNamespaceManager.isPresent()) { + functionNamespaceManager.get().dropFunction(functionName, parameterTypes, exists); + } + else if (!exists) { + throw new PrestoException(FUNCTION_NOT_FOUND, format("Function not found: %s", functionName.getFunctionNamespace())); + } } /** - * Resolves a function using the SQL path, and implicit type coercions. + * Resolves a function using implicit type coercions. We enforce explicit naming for dynamic function namespaces. + * All unqualified function names will only be resolved against the built-in static function namespace. While it is + * possible to define an ordering (through SQL path or other means) and convention (best match / first match), in + * reality when complicated namespaces are involved such implicit resolution might hide errors and cause confusion. * * @throws PrestoException if there are no matches or multiple matches */ - public FunctionHandle resolveFunction(Session session, QualifiedName name, List parameterTypes) + public FunctionHandle resolveFunction(Optional transactionId, QualifiedName name, List parameterTypes) { - // TODO Actually use session - // Session will be used to provide information about the order of function namespaces to through resolving the function. - // This is likely to be in terms of SQL path. Currently we still don't have support multiple function namespaces, nor - // SQL path. As a result, session is not used here. We still add this to distinguish the two versions of resolveFunction - // while the refactoring is on-going. - return staticFunctionNamespace.resolveFunction(name, parameterTypes); + QualifiedFunctionName functionName; + if (!name.getPrefix().isPresent()) { + functionName = QualifiedFunctionName.of(DEFAULT_NAMESPACE, name.getSuffix()); + } + else { + if (name.getOriginalParts().size() != 3) { + throw new PrestoException(FUNCTION_NOT_FOUND, format("Non-builtin functions must be reference by three parts: catalog.schema.function_name, found: %s", name.toString())); + } + functionName = QualifiedFunctionName.of(new CatalogSchemaName(name.getOriginalParts().get(0), name.getOriginalParts().get(1)), name.getOriginalParts().get(2)); + } + + return resolveFunction(transactionId, functionName, parameterTypes); + } + + public FunctionHandle resolveFunction(Optional transactionId, QualifiedFunctionName functionName, List parameterTypes) + { + Optional functionNamespaceManagerId = getServingFunctionNamespaceManagerId(functionName.getFunctionNamespace()); + if (!functionNamespaceManagerId.isPresent()) { + throw new PrestoException(FUNCTION_NOT_FOUND, constructFunctionNotFoundErrorMessage(functionName, parameterTypes, ImmutableList.of())); + } + + Optional transactionHandle = transactionId + .map(id -> transactionManager.getFunctionNamespaceTransaction(id, functionNamespaceManagerId.get())); + FunctionNamespaceManager functionNamespaceManager = functionNamespaceManagers.get(functionNamespaceManagerId.get()); + Collection candidates = functionNamespaceManager.getFunctions(transactionHandle, functionName); + + try { + return lookupFunction(functionNamespaceManager, transactionHandle, functionName, parameterTypes, candidates); + } + catch (PrestoException e) { + if (e.getErrorCode().getCode() != FUNCTION_NOT_FOUND.toErrorCode().getCode()) { + throw e; + } + } + + Optional match = matchFunctionWithCoercion(candidates, parameterTypes); + if (match.isPresent()) { + return functionNamespaceManager.getFunctionHandle(transactionHandle, match.get()); + } + + if (functionName.getFunctionName().startsWith(MAGIC_LITERAL_FUNCTION_PREFIX)) { + // extract type from function functionName + String typeName = functionName.getFunctionName().substring(MAGIC_LITERAL_FUNCTION_PREFIX.length()); + + // lookup the type + Type type = typeManager.getType(parseTypeSignature(typeName)); + + // verify we have one parameter of the proper type + checkArgument(parameterTypes.size() == 1, "Expected one argument to literal function, but got %s", parameterTypes); + + return new BuiltInFunctionHandle(getMagicLiteralFunctionSignature(type)); + } + + throw new PrestoException(FUNCTION_NOT_FOUND, constructFunctionNotFoundErrorMessage(functionName, parameterTypes, candidates)); } @Override public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) { - return staticFunctionNamespace.getFunctionMetadata(functionHandle); + Optional> functionNamespaceManager = getServingFunctionNamespaceManager(functionHandle.getFunctionNamespace()); + checkArgument(functionNamespaceManager.isPresent(), "Cannot find function namespace for '%s'", functionHandle.getFunctionNamespace()); + return functionNamespaceManager.get().getFunctionMetadata(functionHandle); + } + + public ScalarFunctionImplementation getScalarFunctionImplementation(FunctionHandle functionHandle) + { + Optional> functionNamespaceManager = getServingFunctionNamespaceManager(functionHandle.getFunctionNamespace()); + checkArgument(functionNamespaceManager.isPresent(), "Cannot find function namespace for '%s'", functionHandle.getFunctionNamespace()); + return functionNamespaceManager.get().getScalarFunctionImplementation(functionHandle); } public WindowFunctionSupplier getWindowFunctionImplementation(FunctionHandle functionHandle) { - return staticFunctionNamespace.getWindowFunctionImplementation(functionHandle); + return builtInFunctionNamespaceManager.getWindowFunctionImplementation(functionHandle); } public InternalAggregationFunction getAggregateFunctionImplementation(FunctionHandle functionHandle) { - return staticFunctionNamespace.getAggregateFunctionImplementation(functionHandle); + return builtInFunctionNamespaceManager.getAggregateFunctionImplementation(functionHandle); } - public ScalarFunctionImplementation getScalarFunctionImplementation(FunctionHandle functionHandle) + public BuiltInScalarFunctionImplementation getBuiltInScalarFunctionImplementation(FunctionHandle functionHandle) { - return staticFunctionNamespace.getScalarFunctionImplementation(functionHandle); + return (BuiltInScalarFunctionImplementation) builtInFunctionNamespaceManager.getScalarFunctionImplementation(functionHandle); + } + + @VisibleForTesting + public List listOperators() + { + Set operatorNames = Arrays.asList(OperatorType.values()).stream() + .map(OperatorType::getFunctionName) + .collect(toImmutableSet()); + + return builtInFunctionNamespaceManager.listFunctions().stream() + .filter(function -> operatorNames.contains(function.getSignature().getName())) + .collect(toImmutableList()); } public FunctionHandle resolveOperator(OperatorType operatorType, List argumentTypes) { - return staticFunctionNamespace.resolveOperator(operatorType, argumentTypes); + try { + return resolveFunction(Optional.empty(), operatorType.getFunctionName(), argumentTypes); + } + catch (PrestoException e) { + if (e.getErrorCode().getCode() == FUNCTION_NOT_FOUND.toErrorCode().getCode()) { + throw new OperatorNotFoundException( + operatorType, + argumentTypes.stream() + .map(TypeSignatureProvider::getTypeSignature) + .collect(toImmutableList())); + } + else { + throw e; + } + } } /** - * Lookup up a function with name and fully bound types. This can only be used for builtin functions. {@link #resolveFunction(Session, QualifiedName, List)} + * Lookup up a function with name and fully bound types. This can only be used for builtin functions. {@link #resolveFunction(Optional, QualifiedName, List)} * should be used for dynamically registered functions. * * @throws PrestoException if function could not be found */ public FunctionHandle lookupFunction(String name, List parameterTypes) { - return staticFunctionNamespace.lookupFunction(QualifiedName.of(name), parameterTypes); + QualifiedFunctionName functionName = QualifiedFunctionName.of(DEFAULT_NAMESPACE, name); + Collection candidates = builtInFunctionNamespaceManager.getFunctions(Optional.empty(), functionName); + return lookupFunction(builtInFunctionNamespaceManager, Optional.empty(), functionName, parameterTypes, candidates); } public FunctionHandle lookupCast(CastType castType, TypeSignature fromType, TypeSignature toType) { - return staticFunctionNamespace.lookupCast(castType, fromType, toType); + Signature signature = new Signature(castType.getCastName(), SCALAR, emptyList(), emptyList(), toType, singletonList(fromType), false); + + try { + builtInFunctionNamespaceManager.getScalarFunctionImplementation(signature); + } + catch (PrestoException e) { + if (castType.isOperatorType() && e.getErrorCode().getCode() == FUNCTION_IMPLEMENTATION_MISSING.toErrorCode().getCode()) { + throw new OperatorNotFoundException(toOperatorType(castType), ImmutableList.of(fromType), toType); + } + throw e; + } + return builtInFunctionNamespaceManager.getFunctionHandle(Optional.empty(), signature); + } + + private FunctionHandle lookupFunction( + FunctionNamespaceManager functionNamespaceManager, + Optional transactionHandle, + QualifiedFunctionName functionName, + List parameterTypes, + Collection candidates) + { + List exactCandidates = candidates.stream() + .filter(function -> function.getSignature().getTypeVariableConstraints().isEmpty()) + .collect(Collectors.toList()); + + Optional match = matchFunctionExact(exactCandidates, parameterTypes); + if (match.isPresent()) { + return functionNamespaceManager.getFunctionHandle(transactionHandle, match.get()); + } + + List genericCandidates = candidates.stream() + .filter(function -> !function.getSignature().getTypeVariableConstraints().isEmpty()) + .collect(Collectors.toList()); + + match = matchFunctionExact(genericCandidates, parameterTypes); + if (match.isPresent()) { + return functionNamespaceManager.getFunctionHandle(transactionHandle, match.get()); + } + + throw new PrestoException(FUNCTION_NOT_FOUND, constructFunctionNotFoundErrorMessage(functionName, parameterTypes, candidates)); + } + + private Optional> getServingFunctionNamespaceManager(CatalogSchemaName functionNamespace) + { + return getServingFunctionNamespaceManagerId(functionNamespace).map(functionNamespaceManagers::get); + } + + private Optional getServingFunctionNamespaceManagerId(CatalogSchemaName functionNamespace) + { + if (functionNamespace.equals(DEFAULT_NAMESPACE)) { + return Optional.of(BuiltInFunctionNamespaceManager.ID); + } + + CatalogSchemaPrefix bestMatch = null; + String servingFunctionNamespaceManagerId = null; + + for (Map.Entry entry : functionNamespaces.entrySet()) { + CatalogSchemaPrefix prefix = entry.getKey(); + if (prefix.includes(functionNamespace) && (bestMatch == null || bestMatch.includes(prefix))) { + bestMatch = prefix; + servingFunctionNamespaceManagerId = entry.getValue(); + } + } + return Optional.ofNullable(servingFunctionNamespaceManagerId); + } + + private String constructFunctionNotFoundErrorMessage(QualifiedFunctionName functionName, List parameterTypes, Collection candidates) + { + String name = toConciseFunctionName(functionName); + List expectedParameters = new ArrayList<>(); + for (SqlFunction function : candidates) { + expectedParameters.add(format("%s(%s) %s", + name, + Joiner.on(", ").join(function.getSignature().getArgumentTypes()), + Joiner.on(", ").join(function.getSignature().getTypeVariableConstraints()))); + } + String parameters = Joiner.on(", ").join(parameterTypes); + String message = format("Function %s not registered", name); + if (!expectedParameters.isEmpty()) { + String expected = Joiner.on(", ").join(expectedParameters); + message = format("Unexpected parameters (%s) for function %s. Expected: %s", parameters, name, expected); + } + return message; + } + + private String toConciseFunctionName(QualifiedFunctionName functionName) + { + if (functionName.getFunctionNamespace().equals(DEFAULT_NAMESPACE)) { + return functionName.getFunctionName(); + } + return functionName.toString(); + } + + private Optional matchFunctionExact(List candidates, List actualParameters) + { + return matchFunction(candidates, actualParameters, false); + } + + private Optional matchFunctionWithCoercion(Collection candidates, List actualParameters) + { + return matchFunction(candidates, actualParameters, true); + } + + private Optional matchFunction(Collection candidates, List parameters, boolean coercionAllowed) + { + List applicableFunctions = identifyApplicableFunctions(candidates, parameters, coercionAllowed); + if (applicableFunctions.isEmpty()) { + return Optional.empty(); + } + + if (coercionAllowed) { + applicableFunctions = selectMostSpecificFunctions(applicableFunctions, parameters); + checkState(!applicableFunctions.isEmpty(), "at least single function must be left"); + } + + if (applicableFunctions.size() == 1) { + return Optional.of(getOnlyElement(applicableFunctions).getBoundSignature()); + } + + StringBuilder errorMessageBuilder = new StringBuilder(); + errorMessageBuilder.append("Could not choose a best candidate operator. Explicit type casts must be added.\n"); + errorMessageBuilder.append("Candidates are:\n"); + for (ApplicableFunction function : applicableFunctions) { + errorMessageBuilder.append("\t * "); + errorMessageBuilder.append(function.getBoundSignature().toString()); + errorMessageBuilder.append("\n"); + } + throw new PrestoException(AMBIGUOUS_FUNCTION_CALL, errorMessageBuilder.toString()); + } + + private List identifyApplicableFunctions(Collection candidates, List actualParameters, boolean allowCoercion) + { + ImmutableList.Builder applicableFunctions = ImmutableList.builder(); + for (SqlFunction function : candidates) { + Signature declaredSignature = function.getSignature(); + Optional boundSignature = new SignatureBinder(typeManager, declaredSignature, allowCoercion) + .bind(actualParameters); + if (boundSignature.isPresent()) { + applicableFunctions.add(new ApplicableFunction(declaredSignature, boundSignature.get(), function.isCalledOnNullInput())); + } + } + return applicableFunctions.build(); + } + + private List selectMostSpecificFunctions(List applicableFunctions, List parameters) + { + checkArgument(!applicableFunctions.isEmpty()); + + List mostSpecificFunctions = selectMostSpecificFunctions(applicableFunctions); + if (mostSpecificFunctions.size() <= 1) { + return mostSpecificFunctions; + } + + Optional> optionalParameterTypes = toTypes(parameters, typeManager); + if (!optionalParameterTypes.isPresent()) { + // give up and return all remaining matches + return mostSpecificFunctions; + } + + List parameterTypes = optionalParameterTypes.get(); + if (!someParameterIsUnknown(parameterTypes)) { + // give up and return all remaining matches + return mostSpecificFunctions; + } + + // look for functions that only cast the unknown arguments + List unknownOnlyCastFunctions = getUnknownOnlyCastFunctions(applicableFunctions, parameterTypes); + if (!unknownOnlyCastFunctions.isEmpty()) { + mostSpecificFunctions = unknownOnlyCastFunctions; + if (mostSpecificFunctions.size() == 1) { + return mostSpecificFunctions; + } + } + + // If the return type for all the selected function is the same, and the parameters are declared as RETURN_NULL_ON_NULL + // all the functions are semantically the same. We can return just any of those. + if (returnTypeIsTheSame(mostSpecificFunctions) && allReturnNullOnGivenInputTypes(mostSpecificFunctions, parameterTypes)) { + // make it deterministic + ApplicableFunction selectedFunction = Ordering.usingToString() + .reverse() + .sortedCopy(mostSpecificFunctions) + .get(0); + return ImmutableList.of(selectedFunction); + } + + return mostSpecificFunctions; + } + + private List selectMostSpecificFunctions(List candidates) + { + List representatives = new ArrayList<>(); + + for (ApplicableFunction current : candidates) { + boolean found = false; + for (int i = 0; i < representatives.size(); i++) { + ApplicableFunction representative = representatives.get(i); + if (isMoreSpecificThan(current, representative)) { + representatives.set(i, current); + } + if (isMoreSpecificThan(current, representative) || isMoreSpecificThan(representative, current)) { + found = true; + break; + } + } + + if (!found) { + representatives.add(current); + } + } + + return representatives; + } + + private static boolean someParameterIsUnknown(List parameters) + { + return parameters.stream().anyMatch(type -> type.equals(UNKNOWN)); + } + + private List getUnknownOnlyCastFunctions(List applicableFunction, List actualParameters) + { + return applicableFunction.stream() + .filter((function) -> onlyCastsUnknown(function, actualParameters)) + .collect(toImmutableList()); + } + + private boolean onlyCastsUnknown(ApplicableFunction applicableFunction, List actualParameters) + { + List boundTypes = resolveTypes(applicableFunction.getBoundSignature().getArgumentTypes(), typeManager); + checkState(actualParameters.size() == boundTypes.size(), "type lists are of different lengths"); + for (int i = 0; i < actualParameters.size(); i++) { + if (!boundTypes.get(i).equals(actualParameters.get(i)) && actualParameters.get(i) != UNKNOWN) { + return false; + } + } + return true; + } + + private boolean returnTypeIsTheSame(List applicableFunctions) + { + Set returnTypes = applicableFunctions.stream() + .map(function -> typeManager.getType(function.getBoundSignature().getReturnType())) + .collect(Collectors.toSet()); + return returnTypes.size() == 1; + } + + private boolean allReturnNullOnGivenInputTypes(List applicableFunctions, List parameters) + { + return applicableFunctions.stream().allMatch(x -> returnsNullOnGivenInputTypes(x, parameters)); + } + + private boolean returnsNullOnGivenInputTypes(ApplicableFunction applicableFunction, List parameterTypes) + { + Signature boundSignature = applicableFunction.getBoundSignature(); + FunctionKind functionKind = boundSignature.getKind(); + // Window and Aggregation functions have fixed semantic where NULL values are always skipped + if (functionKind != SCALAR) { + return true; + } + + for (int i = 0; i < parameterTypes.size(); i++) { + Type parameterType = parameterTypes.get(i); + if (parameterType.equals(UNKNOWN)) { + // The original implementation checks only whether the particular argument has @SqlNullable. + // However, RETURNS NULL ON NULL INPUT / CALLED ON NULL INPUT is a function level metadata according + // to SQL spec. So there is a loss of precision here. + if (applicableFunction.isCalledOnNullInput()) { + return false; + } + } + } + return true; + } + + private static Optional> toTypes(List typeSignatureProviders, TypeManager typeManager) + { + ImmutableList.Builder resultBuilder = ImmutableList.builder(); + for (TypeSignatureProvider typeSignatureProvider : typeSignatureProviders) { + if (typeSignatureProvider.hasDependency()) { + return Optional.empty(); + } + resultBuilder.add(typeManager.getType(typeSignatureProvider.getTypeSignature())); + } + return Optional.of(resultBuilder.build()); + } + + /** + * One method is more specific than another if invocation handled by the first method could be passed on to the other one + */ + private boolean isMoreSpecificThan(ApplicableFunction left, ApplicableFunction right) + { + List resolvedTypes = fromTypeSignatures(left.getBoundSignature().getArgumentTypes()); + Optional boundVariables = new SignatureBinder(typeManager, right.getDeclaredSignature(), true) + .bindVariables(resolvedTypes); + return boundVariables.isPresent(); + } + + private static class ApplicableFunction + { + private final Signature declaredSignature; + private final Signature boundSignature; + private final boolean calledOnNullInput; + + private ApplicableFunction(Signature declaredSignature, Signature boundSignature, boolean calledOnNullInput) + { + this.declaredSignature = declaredSignature; + this.boundSignature = boundSignature; + this.calledOnNullInput = calledOnNullInput; + } + + public Signature getDeclaredSignature() + { + return declaredSignature; + } + + public Signature getBoundSignature() + { + return boundSignature; + } + + public boolean isCalledOnNullInput() + { + return calledOnNullInput; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("declaredSignature", declaredSignature) + .add("boundSignature", boundSignature) + .add("calledOnNullInput", calledOnNullInput) + .toString(); + } } } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java b/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java index 62238515de1ec..484de0c61cd38 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java @@ -18,7 +18,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; public class HandleJsonModule implements Module diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java b/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java index cea0264385aca..9640f26df5205 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/HandleResolver.java @@ -26,6 +26,7 @@ import com.facebook.presto.spi.connector.ConnectorPartitioningHandle; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionHandleResolver; import com.facebook.presto.split.EmptySplitHandleResolver; import javax.inject.Inject; @@ -56,7 +57,7 @@ public HandleResolver() handleResolvers.put("$info_schema", new MaterializedHandleResolver(new InformationSchemaHandleResolver())); handleResolvers.put("$empty", new MaterializedHandleResolver(new EmptySplitHandleResolver())); - functionHandleResolvers.put("$static", new MaterializedFunctionHandleResolver(new StaticFunctionNamespaceHandleResolver())); + functionHandleResolvers.put("$static", new MaterializedFunctionHandleResolver(new BuiltInFunctionNamespaceHandleResolver())); } public void addConnectorName(String name, ConnectorHandleResolver resolver) @@ -68,7 +69,7 @@ public void addConnectorName(String name, ConnectorHandleResolver resolver) "Connector '%s' is already assigned to resolver: %s", name, existingResolver); } - public void addFunctionNamepsace(String name, FunctionHandleResolver resolver) + public void addFunctionNamespace(String name, FunctionHandleResolver resolver) { requireNonNull(name, "name is null"); requireNonNull(resolver, "resolver is null"); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java b/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java index 8aac2f0d7a993..e81b5f90a9d4c 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java @@ -29,6 +29,7 @@ import com.facebook.presto.spi.connector.ConnectorCapabilities; import com.facebook.presto.spi.connector.ConnectorOutputMetadata; import com.facebook.presto.spi.connector.ConnectorPartitioningHandle; +import com.facebook.presto.spi.function.SqlFunction; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.security.GrantInfo; @@ -42,6 +43,7 @@ import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.sql.planner.PartitioningHandle; +import com.google.common.util.concurrent.ListenableFuture; import io.airlift.slice.Slice; import java.util.Collection; @@ -59,7 +61,7 @@ public interface Metadata List listFunctions(); - void addFunctions(List functions); + void registerBuiltInFunctions(List functions); boolean schemaExists(Session session, CatalogSchemaName schema); @@ -143,9 +145,9 @@ public interface Metadata TableMetadata getTableMetadata(Session session, TableHandle tableHandle); /** - * Return statistics for specified table for given filtering contraint. + * Return statistics for specified table for given columns and filtering constraint. */ - TableStatistics getTableStatistics(Session session, TableHandle tableHandle, Constraint constraint); + TableStatistics getTableStatistics(Session session, TableHandle tableHandle, List columnHandles, Constraint constraint); /** * Get the names that match the specified table prefix (never null). @@ -417,13 +419,13 @@ public interface Metadata * Commits partition for table creation. */ @Experimental - void commitPartition(Session session, OutputTableHandle tableHandle, Collection fragments); + ListenableFuture commitPartitionAsync(Session session, OutputTableHandle tableHandle, Collection fragments); /** * Commits partition for table insertion. */ @Experimental - void commitPartition(Session session, InsertTableHandle tableHandle, Collection fragments); + ListenableFuture commitPartitionAsync(Session session, InsertTableHandle tableHandle, Collection fragments); FunctionManager getFunctionManager(); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java index 014142e05d770..a9dfb6dd70756 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.metadata; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.Session; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.spi.CatalogSchemaName; @@ -44,6 +47,7 @@ import com.facebook.presto.spi.connector.ConnectorPartitioningMetadata; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.SqlFunction; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.security.GrantInfo; @@ -69,9 +73,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; +import com.google.common.util.concurrent.ListenableFuture; import io.airlift.slice.Slice; import javax.inject.Inject; @@ -91,6 +93,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import static com.facebook.airlift.concurrent.MoreFutures.toListenableFuture; +import static com.facebook.presto.expressions.LogicalRowExpressions.FALSE_CONSTANT; import static com.facebook.presto.metadata.QualifiedObjectName.convertFromSchemaTableName; import static com.facebook.presto.metadata.TableLayout.fromConnectorLayout; import static com.facebook.presto.metadata.ViewDefinition.ViewColumn; @@ -107,7 +111,6 @@ import static com.facebook.presto.spi.function.OperatorType.LESS_THAN; import static com.facebook.presto.spi.function.OperatorType.LESS_THAN_OR_EQUAL; import static com.facebook.presto.spi.function.OperatorType.NOT_EQUAL; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.FALSE_CONSTANT; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; import static com.google.common.base.Preconditions.checkArgument; @@ -133,7 +136,9 @@ public class MetadataManager private final ConcurrentMap> catalogsByQueryId = new ConcurrentHashMap<>(); - public MetadataManager(FeaturesConfig featuresConfig, + @VisibleForTesting + public MetadataManager( + FeaturesConfig featuresConfig, TypeManager typeManager, BlockEncodingSerde blockEncodingSerde, SessionPropertyManager sessionPropertyManager, @@ -143,8 +148,7 @@ public MetadataManager(FeaturesConfig featuresConfig, AnalyzePropertyManager analyzePropertyManager, TransactionManager transactionManager) { - this(featuresConfig, - typeManager, + this(typeManager, createTestingViewCodec(), blockEncodingSerde, sessionPropertyManager, @@ -152,11 +156,12 @@ public MetadataManager(FeaturesConfig featuresConfig, tablePropertyManager, columnPropertyManager, analyzePropertyManager, - transactionManager); + transactionManager, + new FunctionManager(typeManager, transactionManager, blockEncodingSerde, featuresConfig, new HandleResolver())); } @Inject - public MetadataManager(FeaturesConfig featuresConfig, + public MetadataManager( TypeManager typeManager, JsonCodec viewCodec, BlockEncodingSerde blockEncodingSerde, @@ -165,9 +170,9 @@ public MetadataManager(FeaturesConfig featuresConfig, TablePropertyManager tablePropertyManager, ColumnPropertyManager columnPropertyManager, AnalyzePropertyManager analyzePropertyManager, - TransactionManager transactionManager) + TransactionManager transactionManager, + FunctionManager functionManager) { - functions = new FunctionManager(typeManager, blockEncodingSerde, featuresConfig); procedures = new ProcedureRegistry(typeManager); this.typeManager = requireNonNull(typeManager, "types is null"); this.viewCodec = requireNonNull(viewCodec, "viewCodec is null"); @@ -178,6 +183,7 @@ public MetadataManager(FeaturesConfig featuresConfig, this.columnPropertyManager = requireNonNull(columnPropertyManager, "columnPropertyManager is null"); this.analyzePropertyManager = requireNonNull(analyzePropertyManager, "analyzePropertyManager is null"); this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); + this.functions = requireNonNull(functionManager, "functionManager is null"); verifyComparableOrderableContract(); } @@ -267,10 +273,9 @@ public List listFunctions() } @Override - public void addFunctions(List functionInfos) + public void registerBuiltInFunctions(List functionInfos) { - // TODO: transactional when FunctionManager is made transactional - functions.addFunctions(functionInfos); + functions.registerBuiltInFunctions(functionInfos); } @Override @@ -513,11 +518,11 @@ public TableMetadata getTableMetadata(Session session, TableHandle tableHandle) } @Override - public TableStatistics getTableStatistics(Session session, TableHandle tableHandle, Constraint constraint) + public TableStatistics getTableStatistics(Session session, TableHandle tableHandle, List columnHandles, Constraint constraint) { ConnectorId connectorId = tableHandle.getConnectorId(); ConnectorMetadata metadata = getMetadata(session, connectorId); - return metadata.getTableStatistics(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), constraint); + return metadata.getTableStatistics(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), tableHandle.getLayout(), columnHandles, constraint); } @Override @@ -868,7 +873,8 @@ public boolean supportsMetadataDelete(Session session, TableHandle tableHandle) ConnectorMetadata metadata = getMetadata(session, connectorId); return metadata.supportsMetadataDelete( session.toConnectorSession(connectorId), - tableHandle.getConnectorHandle()); + tableHandle.getConnectorHandle(), + tableHandle.getLayout()); } @Override @@ -1153,25 +1159,25 @@ public List listTablePrivileges(Session session, QualifiedTablePrefix } @Override - public void commitPartition(Session session, OutputTableHandle tableHandle, Collection fragments) + public ListenableFuture commitPartitionAsync(Session session, OutputTableHandle tableHandle, Collection fragments) { ConnectorId connectorId = tableHandle.getConnectorId(); CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); ConnectorMetadata metadata = catalogMetadata.getMetadata(); ConnectorSession connectorSession = session.toConnectorSession(connectorId); - metadata.commitPartition(connectorSession, tableHandle.getConnectorHandle(), fragments); + return toListenableFuture(metadata.commitPartitionAsync(connectorSession, tableHandle.getConnectorHandle(), fragments)); } @Override - public void commitPartition(Session session, InsertTableHandle tableHandle, Collection fragments) + public ListenableFuture commitPartitionAsync(Session session, InsertTableHandle tableHandle, Collection fragments) { ConnectorId connectorId = tableHandle.getConnectorId(); CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); ConnectorMetadata metadata = catalogMetadata.getMetadata(); ConnectorSession connectorSession = session.toConnectorSession(connectorId); - metadata.commitPartition(connectorSession, tableHandle.getConnectorHandle(), fragments); + return toListenableFuture(metadata.commitPartitionAsync(connectorSession, tableHandle.getConnectorHandle(), fragments)); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/OperatorSignatureUtils.java b/presto-main/src/main/java/com/facebook/presto/metadata/OperatorSignatureUtils.java deleted file mode 100644 index 035a9d46a68dc..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/metadata/OperatorSignatureUtils.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.metadata; - -import com.facebook.presto.spi.function.OperatorType; -import com.google.common.annotations.VisibleForTesting; - -import static com.google.common.base.Preconditions.checkArgument; - -public final class OperatorSignatureUtils -{ - private static final String OPERATOR_PREFIX = "$operator$"; - - private OperatorSignatureUtils() {} - - public static String mangleOperatorName(OperatorType operatorType) - { - return mangleOperatorName(operatorType.name()); - } - - public static String mangleOperatorName(String operatorName) - { - return OPERATOR_PREFIX + operatorName; - } - - @VisibleForTesting - public static OperatorType unmangleOperator(String mangledName) - { - checkArgument(mangledName.startsWith(OPERATOR_PREFIX), "%s is not a mangled operator name", mangledName); - return OperatorType.valueOf(mangledName.substring(OPERATOR_PREFIX.length())); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunction.java b/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunction.java index 80741ea1611f8..c015f044c0279 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunction.java @@ -16,11 +16,11 @@ import com.facebook.presto.metadata.PolymorphicScalarFunctionBuilder.MethodAndNativeContainerTypes; import com.facebook.presto.metadata.PolymorphicScalarFunctionBuilder.MethodsGroup; import com.facebook.presto.metadata.PolymorphicScalarFunctionBuilder.SpecializeContext; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ReturnPlaceConvention; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ScalarImplementationChoice; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ReturnPlaceConvention; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ScalarImplementationChoice; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -35,8 +35,8 @@ import java.util.Optional; import static com.facebook.presto.metadata.SignatureBinder.applyBoundVariables; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; import static com.google.common.base.Preconditions.checkState; import static java.util.Collections.emptyList; import static java.util.Objects.requireNonNull; @@ -92,7 +92,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { ImmutableList.Builder implementationChoices = ImmutableList.builder(); @@ -100,7 +100,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in implementationChoices.add(getScalarFunctionImplementationChoice(boundVariables, typeManager, functionManager, choice)); } - return new ScalarFunctionImplementation(implementationChoices.build()); + return new BuiltInScalarFunctionImplementation(implementationChoices.build()); } private ScalarImplementationChoice getScalarFunctionImplementationChoice( diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunctionBuilder.java b/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunctionBuilder.java index 9a88df70c7cb1..75265f9a1c691 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunctionBuilder.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunctionBuilder.java @@ -14,8 +14,8 @@ package com.facebook.presto.metadata; import com.facebook.presto.metadata.PolymorphicScalarFunction.PolymorphicScalarFunctionChoice; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ReturnPlaceConvention; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ReturnPlaceConvention; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; @@ -29,10 +29,9 @@ import java.util.Optional; import java.util.function.Function; -import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -136,7 +135,7 @@ public static Function> constant(T value) private static boolean isOperator(Signature signature) { for (OperatorType operator : OperatorType.values()) { - if (signature.getName().equals(mangleOperatorName(operator))) { + if (signature.getName().equals(operator.getFunctionName())) { return true; } } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/RemoteNodeState.java b/presto-main/src/main/java/com/facebook/presto/metadata/RemoteNodeState.java index 222295b29070f..6bb16384c8cd4 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/RemoteNodeState.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/RemoteNodeState.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.metadata; +import com.facebook.airlift.http.client.FullJsonResponseHandler.JsonResponse; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpClient.HttpResponseFuture; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.NodeState; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; -import io.airlift.http.client.FullJsonResponseHandler.JsonResponse; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpClient.HttpResponseFuture; -import io.airlift.http.client.Request; -import io.airlift.log.Logger; import io.airlift.units.Duration; import javax.annotation.Nullable; @@ -32,12 +32,12 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import static com.facebook.airlift.http.client.FullJsonResponseHandler.createFullJsonResponseHandler; +import static com.facebook.airlift.http.client.HttpStatus.OK; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.google.common.net.MediaType.JSON_UTF_8; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.http.client.FullJsonResponseHandler.createFullJsonResponseHandler; -import static io.airlift.http.client.HttpStatus.OK; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.units.Duration.nanosSince; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SessionPropertyManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/SessionPropertyManager.java index a78b4ea560ff8..4dac7960ff6a9 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SessionPropertyManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SessionPropertyManager.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.metadata; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; import com.facebook.presto.Session; import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.spi.ConnectorId; @@ -32,8 +34,6 @@ import com.facebook.presto.sql.tree.ExpressionTreeRewriter; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; import javax.annotation.Nullable; import javax.inject.Inject; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBuilder.java b/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBuilder.java index 922e52152f98f..7445b0971b983 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBuilder.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBuilder.java @@ -16,6 +16,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.LongVariableConstraint; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.function.TypeVariableConstraint; import com.facebook.presto.spi.type.TypeSignature; @@ -23,7 +24,7 @@ import java.util.List; -import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; @@ -31,7 +32,7 @@ public final class SignatureBuilder { - private String name; + private QualifiedFunctionName name; private FunctionKind kind; private List typeVariableConstraints = emptyList(); private List longVariableConstraints = emptyList(); @@ -47,6 +48,12 @@ public static SignatureBuilder builder() } public SignatureBuilder name(String name) + { + this.name = QualifiedFunctionName.of(DEFAULT_NAMESPACE, requireNonNull(name, "name is null")); + return this; + } + + public SignatureBuilder name(QualifiedFunctionName name) { this.name = requireNonNull(name, "name is null"); return this; @@ -60,7 +67,7 @@ public SignatureBuilder kind(FunctionKind kind) public SignatureBuilder operatorType(OperatorType operatorType) { - this.name = mangleOperatorName(requireNonNull(operatorType, "operatorType is null")); + this.name = operatorType.getFunctionName(); this.kind = SCALAR; return this; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SpecializedFunctionKey.java b/presto-main/src/main/java/com/facebook/presto/metadata/SpecializedFunctionKey.java index 663b5e2d7452d..17c506b4be101 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SpecializedFunctionKey.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SpecializedFunctionKey.java @@ -19,18 +19,18 @@ public class SpecializedFunctionKey { - private final SqlFunction function; + private final BuiltInFunction function; private final BoundVariables boundVariables; private final int arity; - public SpecializedFunctionKey(SqlFunction function, BoundVariables boundVariables, int arity) + public SpecializedFunctionKey(BuiltInFunction function, BoundVariables boundVariables, int arity) { this.function = requireNonNull(function, "function is null"); this.boundVariables = requireNonNull(boundVariables, "boundVariables is null"); this.arity = arity; } - public SqlFunction getFunction() + public BuiltInFunction getFunction() { return function; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java index 6bcf472fd6857..17c193344b72d 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java @@ -17,6 +17,7 @@ import com.facebook.presto.operator.aggregation.InternalAggregationFunction; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.LongVariableConstraint; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.function.TypeVariableConstraint; import com.facebook.presto.spi.type.TypeManager; @@ -25,13 +26,14 @@ import java.util.List; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.spi.function.FunctionKind.AGGREGATE; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; public abstract class SqlAggregationFunction - implements SqlFunction + extends BuiltInFunction { private final Signature signature; private final boolean hidden; @@ -91,7 +93,7 @@ private static Signature createSignature( requireNonNull(argumentTypes, "argumentTypes is null"); checkArgument(kind == AGGREGATE, "kind must be an aggregate"); return new Signature( - name, + QualifiedFunctionName.of(DEFAULT_NAMESPACE, name), kind, ImmutableList.copyOf(typeVariableConstraints), ImmutableList.copyOf(longVariableConstraints), diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SqlOperator.java b/presto-main/src/main/java/com/facebook/presto/metadata/SqlOperator.java index 1596597300a09..6c0bc3f948890 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SqlOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SqlOperator.java @@ -22,8 +22,6 @@ import java.util.List; -import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; - public abstract class SqlOperator extends SqlScalarFunction { @@ -33,7 +31,7 @@ protected SqlOperator(OperatorType operatorType, List ty { // TODO This should take Signature! super(new Signature( - mangleOperatorName(operatorType), + operatorType.getFunctionName(), FunctionKind.SCALAR, typeVariableConstraints, longVariableConstraints, diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SqlScalarFunction.java b/presto-main/src/main/java/com/facebook/presto/metadata/SqlScalarFunction.java index 7b729f7f94007..da596d6af1e38 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SqlScalarFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SqlScalarFunction.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.metadata; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.TypeManager; @@ -23,7 +23,7 @@ import static java.util.Objects.requireNonNull; public abstract class SqlScalarFunction - implements SqlFunction + extends BuiltInFunction { private final Signature signature; @@ -39,7 +39,7 @@ public final Signature getSignature() return signature; } - public abstract ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager); + public abstract BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager); public static PolymorphicScalarFunctionBuilder builder(Class clazz, OperatorType operatorType) { diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticCatalogStore.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticCatalogStore.java index 2b0b2f7a9ec6c..78510c0dacf0a 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/StaticCatalogStore.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticCatalogStore.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.metadata; +import com.facebook.airlift.log.Logger; import com.facebook.presto.connector.ConnectorManager; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.io.Files; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticCatalogStoreConfig.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticCatalogStoreConfig.java index ad4513be6ec2b..32d125aa2e214 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/StaticCatalogStoreConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticCatalogStoreConfig.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.metadata; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.LegacyConfig; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import io.airlift.configuration.Config; -import io.airlift.configuration.LegacyConfig; import javax.validation.constraints.NotNull; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceStore.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceStore.java new file mode 100644 index 0000000000000..d5004d5f077fb --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceStore.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.airlift.log.Logger; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; + +import javax.inject.Inject; + +import java.io.File; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import static com.facebook.presto.util.PropertiesUtil.loadProperties; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.io.Files.getNameWithoutExtension; + +public class StaticFunctionNamespaceStore +{ + private static final Logger log = Logger.get(StaticFunctionNamespaceStore.class); + private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); + + private final FunctionManager functionManager; + private final File configDir; + private final AtomicBoolean functionNamespaceLoading = new AtomicBoolean(); + + @Inject + public StaticFunctionNamespaceStore(FunctionManager functionManager, StaticFunctionNamespaceStoreConfig config) + { + this.functionManager = functionManager; + this.configDir = config.getFunctionNamespaceConfigurationDir(); + } + + public void loadFunctionNamespaceManagers() + throws Exception + { + if (!functionNamespaceLoading.compareAndSet(false, true)) { + return; + } + + for (File file : listFiles(configDir)) { + if (file.isFile() && file.getName().endsWith(".properties")) { + loadFunctionNamespaceManager(file); + } + } + } + + private void loadFunctionNamespaceManager(File file) + throws Exception + { + String functionNamespaceManagerId = getNameWithoutExtension(file.getName()); + + log.info("-- Loading function namespace manager from %s --", file); + Map properties = new HashMap<>(loadProperties(file)); + + String functionNamespaceManagerName = properties.remove("function-namespace-manager.name"); + checkState(functionNamespaceManagerName != null, "Function namespace configuration %s does not contain function-namespace-manager.name", file.getAbsoluteFile()); + + String functionNamespaces = properties.remove("serving-namespaces"); + checkState(functionNamespaces != null, "Function namespace configuration %s does not contain serving-namespaces", file.getAbsoluteFile()); + + functionManager.loadFunctionNamespaceManager(functionNamespaceManagerName, functionNamespaceManagerId, SPLITTER.splitToList(functionNamespaces), properties); + log.info("-- Added function namespaces [%s] using function namespace manager [%s] --", functionNamespaces, functionNamespaceManagerId); + } + + private static List listFiles(File dir) + { + if (dir != null && dir.isDirectory()) { + File[] files = dir.listFiles(); + if (files != null) { + return ImmutableList.copyOf(files); + } + } + return ImmutableList.of(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceStoreConfig.java similarity index 50% rename from presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java rename to presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceStoreConfig.java index 20bf0729d1909..3d7292e854f93 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespace.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceStoreConfig.java @@ -13,20 +13,26 @@ */ package com.facebook.presto.metadata; -import com.facebook.presto.spi.function.FunctionHandle; -import com.facebook.presto.spi.function.FunctionMetadata; -import com.facebook.presto.sql.analyzer.TypeSignatureProvider; -import com.facebook.presto.sql.tree.QualifiedName; +import com.facebook.airlift.configuration.Config; -import java.util.List; +import javax.validation.constraints.NotNull; -public interface FunctionNamespace -{ - void addFunctions(List functions); +import java.io.File; - List listFunctions(); +public class StaticFunctionNamespaceStoreConfig +{ + private File functionNamespaceConfigurationDir = new File("etc/function-namespace/"); - FunctionHandle resolveFunction(QualifiedName name, List parameterTypes); + @NotNull + public File getFunctionNamespaceConfigurationDir() + { + return functionNamespaceConfigurationDir; + } - FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle); + @Config("function-namespace.config-dir") + public StaticFunctionNamespaceStoreConfig setFunctionNamespaceConfigurationDir(File dir) + { + this.functionNamespaceConfigurationDir = dir; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/AggregationOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/AggregationOperator.java index 4101df5dc9ab7..6e0e814e19212 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/AggregationOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/AggregationOperator.java @@ -18,9 +18,9 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.google.common.collect.ImmutableList; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/Aggregator.java b/presto-main/src/main/java/com/facebook/presto/operator/Aggregator.java index 2f516744094e1..4e8805954cf08 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/Aggregator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/Aggregator.java @@ -17,8 +17,8 @@ import com.facebook.presto.operator.aggregation.AccumulatorFactory; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.planner.plan.AggregationNode; import static com.google.common.base.Preconditions.checkArgument; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/ArrayOfRowsUnnester.java b/presto-main/src/main/java/com/facebook/presto/operator/ArrayOfRowsUnnester.java deleted file mode 100644 index 06624301f7788..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/operator/ArrayOfRowsUnnester.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.operator; - -import com.facebook.presto.spi.PageBuilder; -import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.spi.block.ColumnarRow; -import com.facebook.presto.spi.type.RowType; -import com.facebook.presto.spi.type.Type; -import com.google.common.collect.ImmutableList; - -import java.util.List; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static java.util.Objects.requireNonNull; - -public class ArrayOfRowsUnnester - implements Unnester -{ - private final List fieldTypes; - private ColumnarRow columnarRow; - private int position; - private int nonNullPosition; - private int positionCount; - - public ArrayOfRowsUnnester(Type elementType) - { - requireNonNull(elementType, "elementType is null"); - checkArgument(elementType instanceof RowType, "elementType is not of RowType"); - this.fieldTypes = ImmutableList.copyOf(elementType.getTypeParameters()); - } - - @Override - public int getChannelCount() - { - return fieldTypes.size(); - } - - @Override - public void appendNext(PageBuilder pageBuilder, int outputChannelOffset) - { - checkState(columnarRow != null, "columnarRow is null"); - - for (int i = 0; i < fieldTypes.size(); i++) { - BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(outputChannelOffset + i); - if (columnarRow.isNull(position)) { - blockBuilder.appendNull(); - } - else { - fieldTypes.get(i).appendTo(columnarRow.getField(i), nonNullPosition, blockBuilder); - } - } - if (!columnarRow.isNull(position)) { - nonNullPosition++; - } - position++; - } - - @Override - public boolean hasNext() - { - return position < positionCount; - } - - @Override - public void setBlock(Block block) - { - this.position = 0; - this.nonNullPosition = 0; - if (block == null) { - this.columnarRow = null; - this.positionCount = 0; - } - else { - this.columnarRow = ColumnarRow.toColumnarRow(block); - this.positionCount = block.getPositionCount(); - } - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/ArrayUnnester.java b/presto-main/src/main/java/com/facebook/presto/operator/ArrayUnnester.java deleted file mode 100644 index 7f2ab29eafdaf..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/operator/ArrayUnnester.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.operator; - -import com.facebook.presto.spi.PageBuilder; -import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.spi.type.Type; - -import javax.annotation.Nullable; - -import static com.google.common.base.Preconditions.checkState; -import static java.util.Objects.requireNonNull; - -public class ArrayUnnester - implements Unnester -{ - private final Type elementType; - private Block arrayBlock; - - private int position; - private int positionCount; - - public ArrayUnnester(Type elementType) - { - this.elementType = requireNonNull(elementType, "elementType is null"); - } - - @Override - public boolean hasNext() - { - return position < positionCount; - } - - @Override - public final int getChannelCount() - { - return 1; - } - - @Override - public final void appendNext(PageBuilder pageBuilder, int outputChannelOffset) - { - checkState(arrayBlock != null, "arrayBlock is null"); - BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(outputChannelOffset); - elementType.appendTo(arrayBlock, position, blockBuilder); - position++; - } - - @Override - public void setBlock(@Nullable Block arrayBlock) - { - this.arrayBlock = arrayBlock; - this.position = 0; - this.positionCount = arrayBlock == null ? 0 : arrayBlock.getPositionCount(); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/AssignUniqueIdOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/AssignUniqueIdOperator.java index 734cf2e3901bc..4bed37db8b6ab 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/AssignUniqueIdOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/AssignUniqueIdOperator.java @@ -88,7 +88,7 @@ public AssignUniqueIdOperator(OperatorContext operatorContext, AtomicLong rowIdP this.rowIdPool = requireNonNull(rowIdPool, "rowIdPool is null"); TaskId fullTaskId = operatorContext.getDriverContext().getTaskId(); - uniqueValueMask = (((long) fullTaskId.getStageId().getId()) << 54) | (((long) fullTaskId.getId()) << 40); + uniqueValueMask = (((long) fullTaskId.getStageExecutionId().getStageId().getId()) << 54) | (((long) fullTaskId.getId()) << 40); requestValues(); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/DeleteOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/DeleteOperator.java index 29112f4ba2489..47d925b184ce1 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/DeleteOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/DeleteOperator.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.execution.TaskId; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.UpdatablePageSource; @@ -21,18 +22,17 @@ import com.facebook.presto.spi.block.RunLengthEncodedBlock; import com.facebook.presto.spi.plan.PlanNodeId; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; import java.util.Collection; import java.util.Optional; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.MoreFutures.toListenableFuture; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.MoreFutures.toListenableFuture; import static io.airlift.slice.Slices.wrappedBuffer; import static java.util.Objects.requireNonNull; @@ -190,8 +190,7 @@ public Page getOutput() Slice tableCommitContext = wrappedBuffer(tableCommitContextCodec.toJsonBytes( new TableCommitContext( operatorContext.getDriverContext().getLifespan(), - taskId.getStageId().getId(), - taskId.getId(), + taskId, false, true))); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/Driver.java b/presto-main/src/main/java/com/facebook/presto/operator/Driver.java index 8ced23c1cf84c..a6171d6dd55a0 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/Driver.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/Driver.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.log.Logger; import com.facebook.presto.execution.ScheduledSplit; import com.facebook.presto.execution.TaskSource; import com.facebook.presto.metadata.Split; @@ -27,7 +28,6 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import io.airlift.log.Logger; import io.airlift.units.Duration; import javax.annotation.concurrent.GuardedBy; @@ -44,13 +44,13 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.operator.Operator.NOT_BLOCKED; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Throwables.throwIfUnchecked; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static java.lang.Boolean.TRUE; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/DriverContext.java b/presto-main/src/main/java/com/facebook/presto/operator/DriverContext.java index ac768929847c7..aaf3526b37b31 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/DriverContext.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/DriverContext.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.Session; import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.TaskId; @@ -24,7 +25,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.stats.CounterStat; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/ExchangeClient.java b/presto-main/src/main/java/com/facebook/presto/operator/ExchangeClient.java index cee5eb8a2cd1b..516cb9f437794 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/ExchangeClient.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/ExchangeClient.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.http.client.HttpClient; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.buffer.PageCodecMarker; import com.facebook.presto.execution.buffer.SerializedPage; @@ -23,7 +24,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import io.airlift.http.client.HttpClient; import io.airlift.units.DataSize; import io.airlift.units.Duration; @@ -46,10 +46,15 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import static com.facebook.presto.spi.block.PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Throwables.throwIfUnchecked; import static com.google.common.collect.Sets.newConcurrentHashSet; import static io.airlift.slice.Slices.EMPTY_SLICE; +import static io.airlift.units.DataSize.Unit.BYTE; +import static java.lang.Math.max; +import static java.lang.Math.min; import static java.util.Objects.requireNonNull; /** @@ -103,7 +108,7 @@ public class ExchangeClient @GuardedBy("this") private long successfulRequests; @GuardedBy("this") - private long averageBytesPerRequest; + private final ExponentialMovingAverage responseSizeExponentialMovingAverage; private final AtomicBoolean closed = new AtomicBoolean(); private final AtomicReference failure = new AtomicReference<>(); @@ -119,11 +124,13 @@ public ExchangeClient( int concurrentRequestMultiplier, Duration maxErrorDuration, boolean acknowledgePages, + double responseSizeExponentialMovingAverageDecayingAlpha, HttpClient httpClient, ScheduledExecutorService scheduler, LocalMemoryContext systemMemoryContext, Executor pageBufferClientCallbackExecutor) { + checkArgument(responseSizeExponentialMovingAverageDecayingAlpha >= 0.0 && responseSizeExponentialMovingAverageDecayingAlpha <= 1.0, "responseSizeExponentialMovingAverageDecayingAlpha must be between 0 and 1: %s", responseSizeExponentialMovingAverageDecayingAlpha); this.bufferCapacity = bufferCapacity.toBytes(); this.maxResponseSize = maxResponseSize; this.concurrentRequestMultiplier = concurrentRequestMultiplier; @@ -134,6 +141,7 @@ public ExchangeClient( this.systemMemoryContext = systemMemoryContext; this.maxBufferRetainedSizeInBytes = Long.MIN_VALUE; this.pageBufferClientCallbackExecutor = requireNonNull(pageBufferClientCallbackExecutor, "pageBufferClientCallbackExecutor is null"); + this.responseSizeExponentialMovingAverage = new ExponentialMovingAverage(responseSizeExponentialMovingAverageDecayingAlpha, DEFAULT_MAX_PAGE_SIZE_IN_BYTES); } public ExchangeClientStatus getStatus() @@ -151,7 +159,7 @@ public ExchangeClientStatus getStatus() if (bufferedPages > 0 && pageBuffer.peekLast() == NO_MORE_PAGES) { bufferedPages--; } - return new ExchangeClientStatus(bufferRetainedSizeInBytes, maxBufferRetainedSizeInBytes, averageBytesPerRequest, successfulRequests, bufferedPages, noMoreLocations, pageBufferClientStatus); + return new ExchangeClientStatus(bufferRetainedSizeInBytes, maxBufferRetainedSizeInBytes, responseSizeExponentialMovingAverage.get(), successfulRequests, bufferedPages, noMoreLocations, pageBufferClientStatus); } } @@ -179,7 +187,6 @@ public synchronized void addLocation(URI location, TaskId remoteSourceTaskId) HttpPageBufferClient client = new HttpPageBufferClient( httpClient, - maxResponseSize, maxErrorDuration, acknowledgePages, location, @@ -345,9 +352,9 @@ public synchronized void scheduleRequestIfNecessary() if (neededBytes <= 0) { return; } - - int clientCount = (int) ((1.0 * neededBytes / averageBytesPerRequest) * concurrentRequestMultiplier); - clientCount = Math.max(clientCount, 1); + long averageResponseSize = max(1, responseSizeExponentialMovingAverage.get()); + int clientCount = (int) ((1.0 * neededBytes / averageResponseSize) * concurrentRequestMultiplier); + clientCount = max(clientCount, 1); int pendingClients = allClients.size() - queuedClients.size() - completedClients.size(); clientCount -= pendingClients; @@ -363,7 +370,8 @@ public synchronized void scheduleRequestIfNecessary() continue; } - client.scheduleRequest(); + DataSize max = new DataSize(min(averageResponseSize * 2, maxResponseSize.toBytes()), BYTE); + client.scheduleRequest(max); i++; } } @@ -396,15 +404,14 @@ private synchronized boolean addPages(List pages) .sum(); bufferRetainedSizeInBytes += pagesRetainedSizeInBytes; - maxBufferRetainedSizeInBytes = Math.max(maxBufferRetainedSizeInBytes, bufferRetainedSizeInBytes); + maxBufferRetainedSizeInBytes = max(maxBufferRetainedSizeInBytes, bufferRetainedSizeInBytes); systemMemoryContext.setBytes(bufferRetainedSizeInBytes); successfulRequests++; long responseSize = pages.stream() .mapToLong(SerializedPage::getSizeInBytes) .sum(); - // AVG_n = AVG_(n-1) * (n-1)/n + VALUE_n / n - averageBytesPerRequest = (long) (1.0 * averageBytesPerRequest * (successfulRequests - 1) / successfulRequests + responseSize / successfulRequests); + responseSizeExponentialMovingAverage.update(responseSize); return true; } @@ -506,4 +513,27 @@ private static void closeQuietly(HttpPageBufferClient client) // ignored } } + + private static class ExponentialMovingAverage + { + private final double alpha; + private double oldValue; + + public ExponentialMovingAverage(double alpha, long initialValue) + { + this.alpha = alpha; + this.oldValue = initialValue; + } + + public void update(long value) + { + double newValue = oldValue + (alpha * (value - oldValue)); + oldValue = newValue; + } + + public long get() + { + return (long) oldValue; + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/ExchangeClientConfig.java b/presto-main/src/main/java/com/facebook/presto/operator/ExchangeClientConfig.java index eb942658381fd..a4d54426d5fbf 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/ExchangeClientConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/ExchangeClientConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.operator; -import io.airlift.configuration.Config; -import io.airlift.http.client.HttpClientConfig; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.http.client.HttpClientConfig; import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import io.airlift.units.Duration; @@ -36,6 +36,7 @@ public class ExchangeClientConfig private int clientThreads = 25; private int pageBufferClientMaxCallbackThreads = 25; private boolean acknowledgePages = true; + private double responseSizeExponentialMovingAverageDecayingAlpha = 0.1; @NotNull public DataSize getMaxBufferSize() @@ -141,4 +142,16 @@ public ExchangeClientConfig setAcknowledgePages(boolean acknowledgePages) this.acknowledgePages = acknowledgePages; return this; } + + @Config("exchange.response-size-exponential-moving-average-decaying-alpha") + public ExchangeClientConfig setResponseSizeExponentialMovingAverageDecayingAlpha(double responseSizeExponentialMovingAverageDecayingAlpha) + { + this.responseSizeExponentialMovingAverageDecayingAlpha = responseSizeExponentialMovingAverageDecayingAlpha; + return this; + } + + public double getResponseSizeExponentialMovingAverageDecayingAlpha() + { + return responseSizeExponentialMovingAverageDecayingAlpha; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/ExchangeClientFactory.java b/presto-main/src/main/java/com/facebook/presto/operator/ExchangeClientFactory.java index 3c7d8cfb4e484..d3950865d0280 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/ExchangeClientFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/ExchangeClientFactory.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; +import com.facebook.airlift.http.client.HttpClient; import com.facebook.presto.memory.context.LocalMemoryContext; -import io.airlift.concurrent.ThreadPoolExecutorMBean; -import io.airlift.http.client.HttpClient; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.weakref.jmx.Managed; @@ -28,8 +28,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.BYTE; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newFixedThreadPool; @@ -43,6 +43,7 @@ public class ExchangeClientFactory private final HttpClient httpClient; private final DataSize maxResponseSize; private final boolean acknowledgePages; + private final double responseSizeExponentialMovingAverageDecayingAlpha; private final ScheduledExecutorService scheduler; private final ThreadPoolExecutorMBean executorMBean; private final ExecutorService pageBufferClientCallbackExecutor; @@ -60,6 +61,7 @@ public ExchangeClientFactory( config.getMaxErrorDuration(), config.isAcknowledgePages(), config.getPageBufferClientMaxCallbackThreads(), + config.getResponseSizeExponentialMovingAverageDecayingAlpha(), httpClient, scheduler); } @@ -71,6 +73,7 @@ public ExchangeClientFactory( Duration maxErrorDuration, boolean acknowledgePages, int pageBufferClientMaxCallbackThreads, + double responseSizeExponentialMovingAverageDecayingAlpha, HttpClient httpClient, ScheduledExecutorService scheduler) { @@ -91,9 +94,12 @@ public ExchangeClientFactory( this.pageBufferClientCallbackExecutor = newFixedThreadPool(pageBufferClientMaxCallbackThreads, daemonThreadsNamed("page-buffer-client-callback-%s")); this.executorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) pageBufferClientCallbackExecutor); + this.responseSizeExponentialMovingAverageDecayingAlpha = responseSizeExponentialMovingAverageDecayingAlpha; + checkArgument(maxBufferedBytes.toBytes() > 0, "maxBufferSize must be at least 1 byte: %s", maxBufferedBytes); checkArgument(maxResponseSize.toBytes() > 0, "maxResponseSize must be at least 1 byte: %s", maxResponseSize); checkArgument(concurrentRequestMultiplier > 0, "concurrentRequestMultiplier must be at least 1: %s", concurrentRequestMultiplier); + checkArgument(responseSizeExponentialMovingAverageDecayingAlpha >= 0.0 && responseSizeExponentialMovingAverageDecayingAlpha <= 1.0, "responseSizeExponentialMovingAverageDecayingAlpha must be between 0 and 1: %s", responseSizeExponentialMovingAverageDecayingAlpha); } @PreDestroy @@ -118,6 +124,7 @@ public ExchangeClient get(LocalMemoryContext systemMemoryContext) concurrentRequestMultiplier, maxErrorDuration, acknowledgePages, + responseSizeExponentialMovingAverageDecayingAlpha, httpClient, scheduler, systemMemoryContext, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/ExchangeOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/ExchangeOperator.java index c4ebc9a46d78b..c10bee2458fd2 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/ExchangeOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/ExchangeOperator.java @@ -179,7 +179,7 @@ public Page getOutput() return null; } - operatorContext.recordRawInput(page.getSizeInBytes()); + operatorContext.recordRawInput(page.getSizeInBytes(), page.getPositionCount()); Page deserializedPage = serde.deserialize(page); operatorContext.recordProcessedInput(deserializedPage.getSizeInBytes(), page.getPositionCount()); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/ExplainAnalyzeOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/ExplainAnalyzeOperator.java index e864e150cba49..c3f31146a3b77 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/ExplainAnalyzeOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/ExplainAnalyzeOperator.java @@ -169,7 +169,7 @@ private boolean hasFinalStageInfo(StageInfo stageInfo) private boolean isFinalStageInfo(StageInfo stageInfo) { - List subStages = getSubStagesOf(operatorContext.getDriverContext().getTaskId().getStageId(), stageInfo); + List subStages = getSubStagesOf(operatorContext.getDriverContext().getTaskId().getStageExecutionId().getStageId(), stageInfo); return subStages.stream().allMatch(StageInfo::isFinalStageInfo); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/GroupByIdBlock.java b/presto-main/src/main/java/com/facebook/presto/operator/GroupByIdBlock.java index 7af273ddfa600..3a38062ff9d0a 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/GroupByIdBlock.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/GroupByIdBlock.java @@ -116,9 +116,9 @@ public Slice getSlice(int position, int offset, int length) } @Override - public T getObject(int position, Class clazz) + public Block getBlock(int position) { - return block.getObject(position, clazz); + return block.getBlock(position); } @Override @@ -278,7 +278,7 @@ public int getSliceLengthUnchecked(int internalPosition) @Override public Block getBlockUnchecked(int internalPosition) { - return block.getObject(internalPosition, Block.class); + return block.getBlock(internalPosition); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/HashAggregationOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/HashAggregationOperator.java index 2a636c12e534c..5d40b8ca3daca 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/HashAggregationOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/HashAggregationOperator.java @@ -21,12 +21,12 @@ import com.facebook.presto.operator.scalar.CombineHashFunction; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageBuilder; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.BigintType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spiller.SpillerFactory; import com.facebook.presto.sql.gen.JoinCompiler; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/HashBuilderOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/HashBuilderOperator.java index e1a9d2d75214b..f202d3787808c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/HashBuilderOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/HashBuilderOperator.java @@ -38,12 +38,12 @@ import java.util.OptionalLong; import java.util.Queue; +import static com.facebook.airlift.concurrent.MoreFutures.checkSuccess; +import static com.facebook.airlift.concurrent.MoreFutures.getDone; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static com.google.common.util.concurrent.Futures.immediateFuture; -import static io.airlift.concurrent.MoreFutures.checkSuccess; -import static io.airlift.concurrent.MoreFutures.getDone; import static java.lang.String.format; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/HashGenerator.java b/presto-main/src/main/java/com/facebook/presto/operator/HashGenerator.java index f9e7ee73b40ec..9a247f2faa821 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/HashGenerator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/HashGenerator.java @@ -15,8 +15,6 @@ import com.facebook.presto.spi.Page; -import static com.google.common.base.Preconditions.checkState; - public interface HashGenerator { long hashPosition(int position, Page page); @@ -25,12 +23,8 @@ default int getPartition(int partitionCount, int position, Page page) { long rawHash = hashPosition(position, page); - // clear the sign bit - rawHash &= 0x7fff_ffff_ffff_ffffL; - - int partition = (int) (rawHash % partitionCount); - - checkState(partition >= 0 && partition < partitionCount); - return partition; + // This function reduces the 64 bit rawHash to [0, partitionCount) uniformly. It first reduces the rawHash to 32 bit + // integer x then normalize it to x / 2^32 * partitionCount to reduce the range of x from [0, 2^32) to [0, partitionCount) + return (int) ((Integer.toUnsignedLong(Long.hashCode(rawHash)) * partitionCount) >> 32); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/HashSemiJoinOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/HashSemiJoinOperator.java index b3d47fb6bc6c8..1c3090c1ba55f 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/HashSemiJoinOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/HashSemiJoinOperator.java @@ -23,10 +23,10 @@ import java.util.List; +import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; import static java.util.Objects.requireNonNull; public class HashSemiJoinOperator diff --git a/presto-main/src/main/java/com/facebook/presto/operator/HttpPageBufferClient.java b/presto-main/src/main/java/com/facebook/presto/operator/HttpPageBufferClient.java index 9a5254b5a0cd8..a5446adf15629 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/HttpPageBufferClient.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/HttpPageBufferClient.java @@ -13,6 +13,15 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpClient.HttpResponseFuture; +import com.facebook.airlift.http.client.HttpStatus; +import com.facebook.airlift.http.client.HttpUriBuilder; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.ResponseHandler; +import com.facebook.airlift.http.client.ResponseTooLargeException; +import com.facebook.airlift.log.Logger; import com.facebook.presto.execution.buffer.SerializedPage; import com.facebook.presto.server.remotetask.Backoff; import com.facebook.presto.spi.PrestoException; @@ -21,15 +30,6 @@ import com.google.common.net.MediaType; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpClient.HttpResponseFuture; -import io.airlift.http.client.HttpStatus; -import io.airlift.http.client.HttpUriBuilder; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.ResponseHandler; -import io.airlift.http.client.ResponseTooLargeException; -import io.airlift.log.Logger; import io.airlift.slice.InputStreamSliceInput; import io.airlift.slice.SliceInput; import io.airlift.units.DataSize; @@ -54,6 +54,12 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import static com.facebook.airlift.http.client.HttpStatus.familyForStatusCode; +import static com.facebook.airlift.http.client.Request.Builder.prepareDelete; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.airlift.http.client.ResponseHandlerUtils.propagate; +import static com.facebook.airlift.http.client.StatusResponseHandler.StatusResponse; +import static com.facebook.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; import static com.facebook.presto.PrestoMediaTypes.PRESTO_PAGES_TYPE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_BUFFER_COMPLETE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_SIZE; @@ -72,12 +78,6 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.net.HttpHeaders.CONTENT_TYPE; -import static io.airlift.http.client.HttpStatus.familyForStatusCode; -import static io.airlift.http.client.Request.Builder.prepareDelete; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.http.client.ResponseHandlerUtils.propagate; -import static io.airlift.http.client.StatusResponseHandler.StatusResponse; -import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; @@ -110,7 +110,6 @@ public interface ClientCallback } private final HttpClient httpClient; - private final DataSize maxResponseSize; private final boolean acknowledgePages; private final URI location; private final ClientCallback clientCallback; @@ -146,7 +145,6 @@ public interface ClientCallback public HttpPageBufferClient( HttpClient httpClient, - DataSize maxResponseSize, Duration maxErrorDuration, boolean acknowledgePages, URI location, @@ -154,12 +152,11 @@ public HttpPageBufferClient( ScheduledExecutorService scheduler, Executor pageBufferClientCallbackExecutor) { - this(httpClient, maxResponseSize, maxErrorDuration, acknowledgePages, location, clientCallback, scheduler, Ticker.systemTicker(), pageBufferClientCallbackExecutor); + this(httpClient, maxErrorDuration, acknowledgePages, location, clientCallback, scheduler, Ticker.systemTicker(), pageBufferClientCallbackExecutor); } public HttpPageBufferClient( HttpClient httpClient, - DataSize maxResponseSize, Duration maxErrorDuration, boolean acknowledgePages, URI location, @@ -169,7 +166,6 @@ public HttpPageBufferClient( Executor pageBufferClientCallbackExecutor) { this.httpClient = requireNonNull(httpClient, "httpClient is null"); - this.maxResponseSize = requireNonNull(maxResponseSize, "maxResponseSize is null"); this.acknowledgePages = acknowledgePages; this.location = requireNonNull(location, "location is null"); this.clientCallback = requireNonNull(clientCallback, "clientCallback is null"); @@ -252,7 +248,7 @@ public void close() } } - public synchronized void scheduleRequest() + public synchronized void scheduleRequest(DataSize maxResponseSize) { if (closed || (future != null) || scheduled) { return; @@ -265,7 +261,7 @@ public synchronized void scheduleRequest() long delayNanos = backoff.getBackoffDelayNanos(); scheduler.schedule(() -> { try { - initiateRequest(); + initiateRequest(maxResponseSize); } catch (Throwable t) { // should not happen, but be safe and fail the operator @@ -277,7 +273,7 @@ public synchronized void scheduleRequest() requestsScheduled.incrementAndGet(); } - private synchronized void initiateRequest() + private synchronized void initiateRequest(DataSize maxResponseSize) { scheduled = false; if (closed || (future != null)) { @@ -288,13 +284,13 @@ private synchronized void initiateRequest() sendDelete(); } else { - sendGetResults(); + sendGetResults(maxResponseSize); } lastUpdate = DateTime.now(); } - private synchronized void sendGetResults() + private synchronized void sendGetResults(DataSize maxResponseSize) { URI uri = HttpUriBuilder.uriBuilderFrom(location).appendPath(String.valueOf(token)).build(); HttpResponseFuture resultFuture = httpClient.executeAsync( diff --git a/presto-main/src/main/java/com/facebook/presto/operator/LookupJoinOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/LookupJoinOperator.java index d514cf2e3fe19..58107c01b6084 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/LookupJoinOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/LookupJoinOperator.java @@ -39,13 +39,13 @@ import java.util.function.IntPredicate; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.MoreFutures.addSuccessCallback; +import static com.facebook.airlift.concurrent.MoreFutures.checkSuccess; +import static com.facebook.airlift.concurrent.MoreFutures.getDone; import static com.facebook.presto.operator.LookupJoinOperators.JoinType.FULL_OUTER; import static com.facebook.presto.operator.LookupJoinOperators.JoinType.PROBE_OUTER; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; -import static io.airlift.concurrent.MoreFutures.addSuccessCallback; -import static io.airlift.concurrent.MoreFutures.checkSuccess; -import static io.airlift.concurrent.MoreFutures.getDone; import static java.lang.String.format; import static java.util.Collections.emptyIterator; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/LookupOuterOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/LookupOuterOperator.java index 48cf34c87e80b..de34a14770256 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/LookupOuterOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/LookupOuterOperator.java @@ -25,8 +25,8 @@ import java.util.List; import java.util.Set; +import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; import static java.util.Objects.requireNonNull; public class LookupOuterOperator diff --git a/presto-main/src/main/java/com/facebook/presto/operator/MapUnnester.java b/presto-main/src/main/java/com/facebook/presto/operator/MapUnnester.java deleted file mode 100644 index bbd517871611b..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/operator/MapUnnester.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.operator; - -import com.facebook.presto.spi.PageBuilder; -import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.spi.type.Type; - -import javax.annotation.Nullable; - -import static com.google.common.base.Preconditions.checkState; -import static java.util.Objects.requireNonNull; - -public class MapUnnester - implements Unnester -{ - private final Type keyType; - private final Type valueType; - private Block block; - - private int position; - private int positionCount; - - public MapUnnester(Type keyType, Type valueType) - { - this.keyType = requireNonNull(keyType, "keyType is null"); - this.valueType = requireNonNull(valueType, "valueType is null"); - } - - @Override - public boolean hasNext() - { - return position < positionCount; - } - - @Override - public final int getChannelCount() - { - return 2; - } - - @Override - public final void appendNext(PageBuilder pageBuilder, int outputChannelOffset) - { - checkState(block != null, "block is null"); - BlockBuilder keyBlockBuilder = pageBuilder.getBlockBuilder(outputChannelOffset); - BlockBuilder valueBlockBuilder = pageBuilder.getBlockBuilder(outputChannelOffset + 1); - keyType.appendTo(block, position++, keyBlockBuilder); - valueType.appendTo(block, position++, valueBlockBuilder); - } - - @Override - public void setBlock(@Nullable Block mapBlock) - { - this.block = mapBlock; - this.position = 0; - this.positionCount = mapBlock == null ? 0 : mapBlock.getPositionCount(); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/MergeOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/MergeOperator.java index 08fb0c08e6c66..d145b29b7405c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/MergeOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/MergeOperator.java @@ -164,7 +164,7 @@ public Supplier> addSplit(Split split) exchangeClient.noMoreLocations(); pageProducers.add(exchangeClient.pages() .map(serializedPage -> { - operatorContext.recordRawInput(serializedPage.getSizeInBytes()); + operatorContext.recordRawInput(serializedPage.getSizeInBytes(), serializedPage.getPositionCount()); return pagesSerde.deserialize(serializedPage); })); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/MoreByteArrays.java b/presto-main/src/main/java/com/facebook/presto/operator/MoreByteArrays.java new file mode 100644 index 0000000000000..9d8c3730de152 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/MoreByteArrays.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.operator; + +import com.facebook.presto.spi.api.Experimental; +import sun.misc.Unsafe; + +import java.lang.reflect.Field; + +import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_INT_INDEX_SCALE; + +@Experimental +public class MoreByteArrays +{ + private static final Unsafe unsafe; + + static { + try { + // fetch theUnsafe object + Field field = Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); + unsafe = (Unsafe) field.get(null); + if (unsafe == null) { + throw new RuntimeException("Unsafe access not available"); + } + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static byte getByte(byte[] bytes, int index) + { + checkValidRange(index, SIZE_OF_BYTE, bytes.length); + return unsafe.getByte(bytes, (long) index + ARRAY_BYTE_BASE_OFFSET); + } + + public static int fill(byte[] bytes, int index, int length, byte value) + { + requireNonNull(bytes, "bytes is null"); + checkValidRange(index, length, bytes.length); + + unsafe.setMemory(bytes, index + ARRAY_BYTE_BASE_OFFSET, length, value); + return index + length; + } + + public static int setBytes(byte[] bytes, int index, byte[] values, int offset, int length) + { + requireNonNull(bytes, "bytes is null"); + requireNonNull(values, "values is null"); + + checkValidRange(index, length, bytes.length); + checkValidRange(offset, length, values.length); + + // The performance of one copy and two copies (one big chunk at 8 bytes boundary + rest) are about the same. + unsafe.copyMemory(values, (long) offset + ARRAY_BYTE_BASE_OFFSET, bytes, (long) index + ARRAY_BYTE_BASE_OFFSET, length); + return index + length; + } + + public static int setInts(byte[] bytes, int index, int[] values, int offset, int length) + { + requireNonNull(bytes, "bytes is null"); + requireNonNull(values, "values is null"); + + checkValidRange(index, length * ARRAY_INT_INDEX_SCALE, bytes.length); + checkValidRange(offset, length, values.length); + + for (int i = offset; i < offset + length; i++) { + unsafe.putInt(bytes, (long) index + ARRAY_BYTE_BASE_OFFSET, values[i]); + index += ARRAY_INT_INDEX_SCALE; + } + return index; + } + + private static void checkValidRange(int start, int length, int size) + { + if (start < 0 || length < 0 || start + length > size) { + throw new IndexOutOfBoundsException(format("Invalid start %s and length %s with array size %s", start, length, size)); + } + } + + private MoreByteArrays() + {} +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/NestedLoopJoinOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/NestedLoopJoinOperator.java index ef99941b602ca..ab174733a40c7 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/NestedLoopJoinOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/NestedLoopJoinOperator.java @@ -27,9 +27,9 @@ import java.util.NoSuchElementException; import java.util.Optional; +import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; import static java.lang.Math.multiplyExact; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/OperatorContext.java b/presto-main/src/main/java/com/facebook/presto/operator/OperatorContext.java index 847931218d4a6..7354978268355 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/OperatorContext.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/OperatorContext.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.Session; import com.facebook.presto.memory.QueryContextVisitor; import com.facebook.presto.memory.context.AggregatedMemoryContext; @@ -25,7 +26,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import io.airlift.stats.CounterStat; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; @@ -61,6 +61,7 @@ public class OperatorContext private final Executor executor; private final CounterStat rawInputDataSize = new CounterStat(); + private final CounterStat rawInputPositions = new CounterStat(); private final OperationTiming addInputTiming = new OperationTiming(); private final CounterStat inputDataSize = new CounterStat(); @@ -153,21 +154,23 @@ void recordAddInput(OperationTimer operationTimer, Page page) } /** - * Record the amount of physical bytes that were read by an operator. + * Record the number of rows and amount of physical bytes that were read by an operator. * This metric is valid only for source operators. */ - public void recordRawInput(long sizeInBytes) + public void recordRawInput(long sizeInBytes, long positionCount) { rawInputDataSize.update(sizeInBytes); + rawInputPositions.update(positionCount); } /** - * Record the amount of physical bytes that were read by an operator and + * Record the number of rows and amount of physical bytes that were read by an operator and * the time it took to read the data. This metric is valid only for source operators. */ - public void recordRawInputWithTiming(long sizeInBytes, long readNanos) + public void recordRawInputWithTiming(long sizeInBytes, long positionCount, long readNanos) { rawInputDataSize.update(sizeInBytes); + rawInputPositions.update(positionCount); addInputTiming.record(readNanos, 0); } @@ -438,7 +441,8 @@ public OperatorStats getOperatorStats() long inputPositionsCount = inputPositions.getTotalCount(); return new OperatorStats( - driverContext.getTaskId().getStageId().getId(), + driverContext.getTaskId().getStageExecutionId().getStageId().getId(), + driverContext.getTaskId().getStageExecutionId().getId(), driverContext.getPipelineContext().getPipelineId(), operatorId, planNodeId, @@ -450,6 +454,7 @@ public OperatorStats getOperatorStats() succinctNanos(addInputTiming.getWallNanos()), succinctNanos(addInputTiming.getCpuNanos()), succinctBytes(rawInputDataSize.getTotalCount()), + rawInputPositions.getTotalCount(), succinctBytes(inputDataSize.getTotalCount()), inputPositionsCount, (double) inputPositionsCount * inputPositionsCount, @@ -604,7 +609,12 @@ public ListenableFuture setBytes(long bytes) @Override public boolean trySetBytes(long bytes) { - return delegate.trySetBytes(bytes); + if (delegate.trySetBytes(bytes)) { + allocationListener.run(); + return true; + } + + return false; } @Override @@ -658,6 +668,7 @@ public void close() throw new UnsupportedOperationException("Called close on unclosable aggregated memory context"); } delegate.close(); + allocationListener.run(); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/OperatorInfo.java b/presto-main/src/main/java/com/facebook/presto/operator/OperatorInfo.java index 11050696be4e7..d377ea874876a 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/OperatorInfo.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/OperatorInfo.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.operator; -import com.facebook.presto.operator.PartitionedOutputOperator.PartitionedOutputInfo; import com.facebook.presto.operator.TableWriterOperator.TableWriterInfo; import com.facebook.presto.operator.exchange.LocalExchangeBufferInfo; +import com.facebook.presto.operator.repartition.PartitionedOutputInfo; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -32,7 +32,8 @@ @JsonSubTypes.Type(value = PartitionedOutputInfo.class, name = "partitionedOutput"), @JsonSubTypes.Type(value = JoinOperatorInfo.class, name = "joinOperatorInfo"), @JsonSubTypes.Type(value = WindowInfo.class, name = "windowInfo"), - @JsonSubTypes.Type(value = TableWriterInfo.class, name = "tableWriter")}) + @JsonSubTypes.Type(value = TableWriterInfo.class, name = "tableWriter"), + @JsonSubTypes.Type(value = TableWriterMergeInfo.class, name = "tableWriterMerge")}) public interface OperatorInfo { /** diff --git a/presto-main/src/main/java/com/facebook/presto/operator/OperatorStats.java b/presto-main/src/main/java/com/facebook/presto/operator/OperatorStats.java index 7f78b9ccd0558..e8655b7bb73ac 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/OperatorStats.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/OperatorStats.java @@ -37,6 +37,7 @@ public class OperatorStats { private final int stageId; + private final int stageExecutionId; private final int pipelineId; private final int operatorId; private final PlanNodeId planNodeId; @@ -48,6 +49,7 @@ public class OperatorStats private final Duration addInputWall; private final Duration addInputCpu; private final DataSize rawInputDataSize; + private final long rawInputPositions; private final DataSize inputDataSize; private final long inputPositions; private final double sumSquaredInputPositions; @@ -82,6 +84,7 @@ public class OperatorStats @JsonCreator public OperatorStats( @JsonProperty("stageId") int stageId, + @JsonProperty("stageExecutionId") int stageExecutionId, @JsonProperty("pipelineId") int pipelineId, @JsonProperty("operatorId") int operatorId, @JsonProperty("planNodeId") PlanNodeId planNodeId, @@ -93,6 +96,7 @@ public OperatorStats( @JsonProperty("addInputWall") Duration addInputWall, @JsonProperty("addInputCpu") Duration addInputCpu, @JsonProperty("rawInputDataSize") DataSize rawInputDataSize, + @JsonProperty("rawInputPositions") long rawInputPositions, @JsonProperty("inputDataSize") DataSize inputDataSize, @JsonProperty("inputPositions") long inputPositions, @JsonProperty("sumSquaredInputPositions") double sumSquaredInputPositions, @@ -125,6 +129,7 @@ public OperatorStats( @JsonProperty("info") OperatorInfo info) { this.stageId = stageId; + this.stageExecutionId = stageExecutionId; this.pipelineId = pipelineId; checkArgument(operatorId >= 0, "operatorId is negative"); @@ -138,6 +143,7 @@ public OperatorStats( this.addInputWall = requireNonNull(addInputWall, "addInputWall is null"); this.addInputCpu = requireNonNull(addInputCpu, "addInputCpu is null"); this.rawInputDataSize = requireNonNull(rawInputDataSize, "rawInputDataSize is null"); + this.rawInputPositions = requireNonNull(rawInputPositions, "rawInputPositions is null"); this.inputDataSize = requireNonNull(inputDataSize, "inputDataSize is null"); checkArgument(inputPositions >= 0, "inputPositions is negative"); this.inputPositions = inputPositions; @@ -179,6 +185,12 @@ public int getStageId() return stageId; } + @JsonProperty + public int getStageExecutionId() + { + return stageExecutionId; + } + @JsonProperty public int getPipelineId() { @@ -233,6 +245,12 @@ public DataSize getRawInputDataSize() return rawInputDataSize; } + @JsonProperty + public long getRawInputPositions() + { + return rawInputPositions; + } + @JsonProperty public DataSize getInputDataSize() { @@ -379,6 +397,7 @@ public OperatorStats add(Iterable operators) long addInputWall = this.addInputWall.roundTo(NANOSECONDS); long addInputCpu = this.addInputCpu.roundTo(NANOSECONDS); long rawInputDataSize = this.rawInputDataSize.toBytes(); + long rawInputPositions = this.rawInputPositions; long inputDataSize = this.inputDataSize.toBytes(); long inputPositions = this.inputPositions; double sumSquaredInputPositions = this.sumSquaredInputPositions; @@ -418,6 +437,7 @@ public OperatorStats add(Iterable operators) addInputWall += operator.getAddInputWall().roundTo(NANOSECONDS); addInputCpu += operator.getAddInputCpu().roundTo(NANOSECONDS); rawInputDataSize += operator.getRawInputDataSize().toBytes(); + rawInputPositions += operator.getRawInputPositions(); inputDataSize += operator.getInputDataSize().toBytes(); inputPositions += operator.getInputPositions(); sumSquaredInputPositions += operator.getSumSquaredInputPositions(); @@ -458,6 +478,7 @@ public OperatorStats add(Iterable operators) return new OperatorStats( stageId, + stageExecutionId, pipelineId, operatorId, planNodeId, @@ -469,6 +490,7 @@ public OperatorStats add(Iterable operators) succinctNanos(addInputWall), succinctNanos(addInputCpu), succinctBytes(rawInputDataSize), + rawInputPositions, succinctBytes(inputDataSize), inputPositions, sumSquaredInputPositions, @@ -521,6 +543,7 @@ public OperatorStats summarize() { return new OperatorStats( stageId, + stageExecutionId, pipelineId, operatorId, planNodeId, @@ -530,6 +553,7 @@ public OperatorStats summarize() addInputWall, addInputCpu, rawInputDataSize, + rawInputPositions, inputDataSize, inputPositions, sumSquaredInputPositions, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/PageSourceOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/PageSourceOperator.java index 1e05e41908bc9..de4379b498aaa 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/PageSourceOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/PageSourceOperator.java @@ -22,7 +22,7 @@ import java.io.UncheckedIOException; import java.util.concurrent.CompletableFuture; -import static io.airlift.concurrent.MoreFutures.toListenableFuture; +import static com.facebook.airlift.concurrent.MoreFutures.toListenableFuture; import static java.util.Objects.requireNonNull; public class PageSourceOperator @@ -92,7 +92,7 @@ public Page getOutput() // update operator stats long endCompletedBytes = pageSource.getCompletedBytes(); long endReadTimeNanos = pageSource.getReadTimeNanos(); - operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); + operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, page.getPositionCount(), endReadTimeNanos - readTimeNanos); operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount()); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/PagesIndex.java b/presto-main/src/main/java/com/facebook/presto/operator/PagesIndex.java index 828a906dbc0d1..e8e84a6a1d671 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/PagesIndex.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/PagesIndex.java @@ -13,8 +13,10 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.geospatial.Rectangle; +import com.facebook.presto.memory.context.LocalMemoryContext; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.MetadataManager; @@ -32,7 +34,6 @@ import com.facebook.presto.sql.gen.OrderingCompiler; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import io.airlift.units.DataSize; import it.unimi.dsi.fastutil.Swapper; @@ -464,11 +465,12 @@ public PagesSpatialIndexSupplier createPagesSpatialIndex( SpatialPredicate spatialRelationshipTest, Optional filterFunctionFactory, List outputChannels, - Map partitions) + Map partitions, + LocalMemoryContext localUserMemoryContext) { // TODO probably shouldn't copy to reduce memory and for memory accounting's sake List> channels = ImmutableList.copyOf(this.channels); - return new PagesSpatialIndexSupplier(session, valueAddresses, types, outputChannels, channels, geometryChannel, radiusChannel, partitionChannel, spatialRelationshipTest, filterFunctionFactory, partitions); + return new PagesSpatialIndexSupplier(session, valueAddresses, types, outputChannels, channels, geometryChannel, radiusChannel, partitionChannel, spatialRelationshipTest, filterFunctionFactory, partitions, localUserMemoryContext); } public LookupSourceSupplier createLookupSourceSupplier( diff --git a/presto-main/src/main/java/com/facebook/presto/operator/PagesRTreeIndex.java b/presto-main/src/main/java/com/facebook/presto/operator/PagesRTreeIndex.java index 000aa85e97550..a765c7a9c9af4 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/PagesRTreeIndex.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/PagesRTreeIndex.java @@ -16,7 +16,10 @@ import com.esri.core.geometry.ogc.OGCGeometry; import com.esri.core.geometry.ogc.OGCPoint; import com.facebook.presto.Session; +import com.facebook.presto.geospatial.GeometryUtils; import com.facebook.presto.geospatial.Rectangle; +import com.facebook.presto.geospatial.rtree.Flatbush; +import com.facebook.presto.geospatial.rtree.HasExtent; import com.facebook.presto.operator.SpatialIndexBuilderOperator.SpatialPredicate; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageBuilder; @@ -26,8 +29,6 @@ import io.airlift.slice.Slice; import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; -import org.locationtech.jts.geom.Envelope; -import org.locationtech.jts.index.strtree.STRtree; import org.openjdk.jol.info.ClassLayout; import java.util.List; @@ -35,7 +36,8 @@ import java.util.Optional; import java.util.OptionalDouble; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserialize; +import static com.facebook.presto.geospatial.GeometryUtils.getExtent; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserialize; import static com.facebook.presto.operator.JoinUtils.channelsToPages; import static com.facebook.presto.operator.SyntheticAddress.decodePosition; import static com.facebook.presto.operator.SyntheticAddress.decodeSliceIndex; @@ -54,25 +56,33 @@ public class PagesRTreeIndex private final List types; private final List outputChannels; private final List> channels; - private final STRtree rtree; + private final Flatbush rtree; private final int radiusChannel; private final SpatialPredicate spatialRelationshipTest; private final JoinFilterFunction filterFunction; private final Map partitions; public static final class GeometryWithPosition + implements HasExtent { private static final int INSTANCE_SIZE = ClassLayout.parseClass(GeometryWithPosition.class).instanceSize(); private final OGCGeometry ogcGeometry; private final int partition; private final int position; + private final Rectangle extent; public GeometryWithPosition(OGCGeometry ogcGeometry, int partition, int position) + { + this(ogcGeometry, partition, position, 0.0f); + } + + public GeometryWithPosition(OGCGeometry ogcGeometry, int partition, int position, double radius) { this.ogcGeometry = requireNonNull(ogcGeometry, "ogcGeometry is null"); this.partition = partition; this.position = position; + this.extent = GeometryUtils.getExtent(ogcGeometry, radius); } public OGCGeometry getGeometry() @@ -90,9 +100,16 @@ public int getPosition() return position; } - public long getEstimatedMemorySizeInBytes() + @Override + public Rectangle getExtent() + { + return extent; + } + + @Override + public long getEstimatedSizeInBytes() { - return INSTANCE_SIZE + ogcGeometry.estimateMemorySize(); + return INSTANCE_SIZE + ogcGeometry.estimateMemorySize() + extent.getEstimatedSizeInBytes(); } } @@ -102,7 +119,7 @@ public PagesRTreeIndex( List types, List outputChannels, List> channels, - STRtree rtree, + Flatbush rtree, Optional radiusChannel, SpatialPredicate spatialRelationshipTest, Optional filterFunctionFactory, @@ -119,14 +136,6 @@ public PagesRTreeIndex( this.partitions = requireNonNull(partitions, "partitions is null"); } - private static Envelope getEnvelope(OGCGeometry ogcGeometry) - { - com.esri.core.geometry.Envelope env = new com.esri.core.geometry.Envelope(); - ogcGeometry.getEsriGeometry().queryEnvelope(env); - - return new Envelope(env.getXMin(), env.getXMax(), env.getYMin(), env.getYMax()); - } - /** * Returns an array of addresses from {@link PagesIndex#valueAddresses} corresponding * to rows with matching geometries. @@ -155,11 +164,10 @@ public int[] findJoinPositions(int probePosition, Page probe, int probeGeometryC IntArrayList matchingPositions = new IntArrayList(); - Envelope envelope = getEnvelope(probeGeometry); - rtree.query(envelope, item -> { - GeometryWithPosition geometryWithPosition = (GeometryWithPosition) item; + Rectangle queryRectangle = getExtent(probeGeometry); + rtree.findIntersections(queryRectangle, geometryWithPosition -> { OGCGeometry buildGeometry = geometryWithPosition.getGeometry(); - if (partitions.isEmpty() || (probePartition == geometryWithPosition.getPartition() && (probeIsPoint || (buildGeometry instanceof OGCPoint) || testReferencePoint(envelope, buildGeometry, probePartition)))) { + if (partitions.isEmpty() || (probePartition == geometryWithPosition.getPartition() && (probeIsPoint || (buildGeometry instanceof OGCPoint) || testReferencePoint(queryRectangle, buildGeometry, probePartition)))) { if (radiusChannel == -1) { if (spatialRelationshipTest.apply(buildGeometry, probeGeometry, OptionalDouble.empty())) { matchingPositions.add(geometryWithPosition.getPosition()); @@ -176,18 +184,18 @@ public int[] findJoinPositions(int probePosition, Page probe, int probeGeometryC return matchingPositions.toIntArray(null); } - private boolean testReferencePoint(Envelope probeEnvelope, OGCGeometry buildGeometry, int partition) + private boolean testReferencePoint(Rectangle probeEnvelope, OGCGeometry buildGeometry, int partition) { - Envelope buildEnvelope = getEnvelope(buildGeometry); - Envelope intersection = buildEnvelope.intersection(probeEnvelope); - if (intersection.isNull()) { + Rectangle buildEnvelope = getExtent(buildGeometry); + Rectangle intersection = buildEnvelope.intersection(probeEnvelope); + if (intersection == null) { return false; } Rectangle extent = partitions.get(partition); - double x = intersection.getMinX(); - double y = intersection.getMinY(); + double x = intersection.getXMin(); + double y = intersection.getYMin(); return x >= extent.getXMin() && x < extent.getXMax() && y >= extent.getYMin() && y < extent.getYMax(); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/PagesSpatialIndexSupplier.java b/presto-main/src/main/java/com/facebook/presto/operator/PagesSpatialIndexSupplier.java index e9fd87b57c56a..381727006acad 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/PagesSpatialIndexSupplier.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/PagesSpatialIndexSupplier.java @@ -20,6 +20,8 @@ import com.esri.core.geometry.ogc.OGCGeometry; import com.facebook.presto.Session; import com.facebook.presto.geospatial.Rectangle; +import com.facebook.presto.geospatial.rtree.Flatbush; +import com.facebook.presto.memory.context.LocalMemoryContext; import com.facebook.presto.operator.PagesRTreeIndex.GeometryWithPosition; import com.facebook.presto.operator.SpatialIndexBuilderOperator.SpatialPredicate; import com.facebook.presto.spi.block.Block; @@ -28,10 +30,7 @@ import io.airlift.slice.Slice; import io.airlift.units.DataSize; import it.unimi.dsi.fastutil.longs.LongArrayList; -import org.locationtech.jts.geom.Envelope; -import org.locationtech.jts.index.strtree.AbstractNode; -import org.locationtech.jts.index.strtree.ItemBoundable; -import org.locationtech.jts.index.strtree.STRtree; +import it.unimi.dsi.fastutil.objects.ObjectArrayList; import org.openjdk.jol.info.ClassLayout; import java.util.List; @@ -39,7 +38,7 @@ import java.util.Optional; import java.util.function.Supplier; -import static com.facebook.presto.geospatial.serde.GeometrySerde.deserialize; +import static com.facebook.presto.geospatial.serde.EsriGeometrySerde.deserialize; import static com.facebook.presto.operator.PagesSpatialIndex.EMPTY_INDEX; import static com.facebook.presto.operator.SyntheticAddress.decodePosition; import static com.facebook.presto.operator.SyntheticAddress.decodeSliceIndex; @@ -48,14 +47,13 @@ import static com.google.common.base.Verify.verify; import static io.airlift.units.DataSize.Unit.BYTE; import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; public class PagesSpatialIndexSupplier implements Supplier { private static final int INSTANCE_SIZE = ClassLayout.parseClass(PagesSpatialIndexSupplier.class).instanceSize(); - private static final int ENVELOPE_INSTANCE_SIZE = ClassLayout.parseClass(Envelope.class).instanceSize(); - private static final int STRTREE_INSTANCE_SIZE = ClassLayout.parseClass(STRtree.class).instanceSize(); - private static final int ABSTRACT_NODE_INSTANCE_SIZE = ClassLayout.parseClass(AbstractNode.class).instanceSize(); + private static final int MEMORY_USAGE_UPDATE_INCREMENT_BYTES = 100 * 1024 * 1024; // 100 MB private final Session session; private final LongArrayList addresses; @@ -65,7 +63,7 @@ public class PagesSpatialIndexSupplier private final Optional radiusChannel; private final SpatialPredicate spatialRelationshipTest; private final Optional filterFunctionFactory; - private final STRtree rtree; + private final Flatbush rtree; private final Map partitions; private final long memorySizeInBytes; @@ -80,8 +78,10 @@ public PagesSpatialIndexSupplier( Optional partitionChannel, SpatialPredicate spatialRelationshipTest, Optional filterFunctionFactory, - Map partitions) + Map partitions, + LocalMemoryContext localUserMemoryContext) { + requireNonNull(localUserMemoryContext, "localUserMemoryContext is null"); this.session = session; this.addresses = addresses; this.types = types; @@ -91,17 +91,20 @@ public PagesSpatialIndexSupplier( this.filterFunctionFactory = filterFunctionFactory; this.partitions = partitions; - this.rtree = buildRTree(addresses, channels, geometryChannel, radiusChannel, partitionChannel); + this.rtree = buildRTree(addresses, channels, geometryChannel, radiusChannel, partitionChannel, localUserMemoryContext); this.radiusChannel = radiusChannel; - this.memorySizeInBytes = INSTANCE_SIZE + - (rtree.isEmpty() ? 0 : STRTREE_INSTANCE_SIZE + computeMemorySizeInBytes(rtree.getRoot())); + this.memorySizeInBytes = INSTANCE_SIZE + rtree.getEstimatedSizeInBytes(); } - private static STRtree buildRTree(LongArrayList addresses, List> channels, int geometryChannel, Optional radiusChannel, Optional partitionChannel) + private static Flatbush buildRTree(LongArrayList addresses, List> channels, int geometryChannel, Optional radiusChannel, Optional partitionChannel, LocalMemoryContext localUserMemoryContext) { - STRtree rtree = new STRtree(); Operator relateOperator = OperatorFactoryLocal.getInstance().getOperator(Operator.Type.Relate); + ObjectArrayList geometries = new ObjectArrayList<>(); + + long recordedSizeInBytes = localUserMemoryContext.getBytes(); + long addedSizeInBytes = 0; + for (int position = 0; position < addresses.size(); position++) { long pageAddress = addresses.getLong(position); int blockIndex = decodeSliceIndex(pageAddress); @@ -136,32 +139,19 @@ private static STRtree buildRTree(LongArrayList addresses, List> cha partition = toIntExact(INTEGER.getLong(partitionBlock, blockPosition)); } - rtree.insert(getEnvelope(ogcGeometry, radius), new GeometryWithPosition(ogcGeometry, partition, position)); - } - - rtree.build(); - return rtree; - } - - private static Envelope getEnvelope(OGCGeometry ogcGeometry, double radius) - { - com.esri.core.geometry.Envelope envelope = new com.esri.core.geometry.Envelope(); - ogcGeometry.getEsriGeometry().queryEnvelope(envelope); + GeometryWithPosition geometryWithPosition = new GeometryWithPosition(ogcGeometry, partition, position, radius); + geometries.add(geometryWithPosition); - return new Envelope(envelope.getXMin() - radius, envelope.getXMax() + radius, envelope.getYMin() - radius, envelope.getYMax() + radius); - } + addedSizeInBytes += geometryWithPosition.getEstimatedSizeInBytes(); - private long computeMemorySizeInBytes(AbstractNode root) - { - if (root.getLevel() == 0) { - return ABSTRACT_NODE_INSTANCE_SIZE + ENVELOPE_INSTANCE_SIZE + root.getChildBoundables().stream().mapToLong(child -> computeMemorySizeInBytes((ItemBoundable) child)).sum(); + if (addedSizeInBytes >= MEMORY_USAGE_UPDATE_INCREMENT_BYTES) { + localUserMemoryContext.setBytes(recordedSizeInBytes + addedSizeInBytes); + recordedSizeInBytes += addedSizeInBytes; + addedSizeInBytes = 0; + } } - return ABSTRACT_NODE_INSTANCE_SIZE + ENVELOPE_INSTANCE_SIZE + root.getChildBoundables().stream().mapToLong(child -> computeMemorySizeInBytes((AbstractNode) child)).sum(); - } - private long computeMemorySizeInBytes(ItemBoundable item) - { - return ENVELOPE_INSTANCE_SIZE + ((GeometryWithPosition) item.getItem()).getEstimatedMemorySizeInBytes(); + return new Flatbush<>(geometries.toArray(new GeometryWithPosition[] {})); } private static void accelerateGeometry(OGCGeometry ogcGeometry, Operator relateOperator) diff --git a/presto-main/src/main/java/com/facebook/presto/operator/PipelineContext.java b/presto-main/src/main/java/com/facebook/presto/operator/PipelineContext.java index 056cd9cd319c9..330bac839e0a8 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/PipelineContext.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/PipelineContext.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.Distribution; import com.facebook.presto.Session; import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.TaskId; @@ -25,8 +27,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.stats.CounterStat; -import io.airlift.stats.Distribution; import org.joda.time.DateTime; import javax.annotation.concurrent.ThreadSafe; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/PipelineStats.java b/presto-main/src/main/java/com/facebook/presto/operator/PipelineStats.java index 2d54eaf9fe09f..6d87f796fce2d 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/PipelineStats.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/PipelineStats.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.stats.Distribution.DistributionSnapshot; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.stats.Distribution.DistributionSnapshot; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/ScanFilterAndProjectOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/ScanFilterAndProjectOperator.java index cc1e062e981e7..e1c405bcdc47f 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/ScanFilterAndProjectOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/ScanFilterAndProjectOperator.java @@ -25,6 +25,7 @@ import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.RecordCursor; import com.facebook.presto.spi.RecordPageSource; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.UpdatablePageSource; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.LazyBlock; @@ -47,9 +48,9 @@ import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.MoreFutures.toListenableFuture; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.concurrent.MoreFutures.toListenableFuture; import static java.util.Objects.requireNonNull; public class ScanFilterAndProjectOperator @@ -58,6 +59,7 @@ public class ScanFilterAndProjectOperator private final OperatorContext operatorContext; private final PlanNodeId planNodeId; private final PageSourceProvider pageSourceProvider; + private final TableHandle table; private final List columns; private final PageBuilder pageBuilder; private final CursorProcessor cursorProcessor; @@ -76,6 +78,7 @@ public class ScanFilterAndProjectOperator private boolean finishing; private long completedBytes; + private long completedPositions; private long readTimeNanos; protected ScanFilterAndProjectOperator( @@ -84,6 +87,7 @@ protected ScanFilterAndProjectOperator( PageSourceProvider pageSourceProvider, CursorProcessor cursorProcessor, PageProcessor pageProcessor, + TableHandle table, Iterable columns, Iterable types, MergingPageOutput mergingOutput) @@ -93,6 +97,7 @@ protected ScanFilterAndProjectOperator( this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.planNodeId = requireNonNull(sourceId, "sourceId is null"); this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); + this.table = requireNonNull(table, "table is null"); this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); this.pageSourceMemoryContext = operatorContext.newLocalSystemMemoryContext(ScanFilterAndProjectOperator.class.getSimpleName()); this.pageProcessorMemoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(ScanFilterAndProjectOperator.class.getSimpleName()); @@ -218,7 +223,7 @@ public Page getOutput() } if (!finishing && pageSource == null && cursor == null) { - ConnectorPageSource source = pageSourceProvider.createPageSource(operatorContext.getSession(), split, columns); + ConnectorPageSource source = pageSourceProvider.createPageSource(operatorContext.getSession(), split, table, columns); if (source instanceof RecordPageSource) { cursor = ((RecordPageSource) source).getCursor(); } @@ -244,7 +249,7 @@ private Page processColumnSource() long bytesProcessed = cursor.getCompletedBytes() - completedBytes; long elapsedNanos = cursor.getReadTimeNanos() - readTimeNanos; - operatorContext.recordRawInputWithTiming(bytesProcessed, elapsedNanos); + operatorContext.recordRawInputWithTiming(bytesProcessed, output.getProcessedRows(), elapsedNanos); // TODO: derive better values for cursors operatorContext.recordProcessedInput(bytesProcessed, output.getProcessedRows()); completedBytes = cursor.getCompletedBytes(); @@ -279,9 +284,11 @@ private Page processPageSource() // update operator stats long endCompletedBytes = pageSource.getCompletedBytes(); + long endCompletedPositions = pageSource.getCompletedPositions(); long endReadTimeNanos = pageSource.getReadTimeNanos(); - operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); + operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endCompletedPositions - completedPositions, endReadTimeNanos - readTimeNanos); completedBytes = endCompletedBytes; + completedPositions = endCompletedPositions; readTimeNanos = endReadTimeNanos; Iterator> output = pageProcessor.process(operatorContext.getSession().toConnectorSession(), yieldSignal, pageProcessorMemoryContext, page); @@ -330,6 +337,7 @@ public static class ScanFilterAndProjectOperatorFactory private final Supplier pageProcessor; private final PlanNodeId sourceId; private final PageSourceProvider pageSourceProvider; + private final TableHandle table; private final List columns; private final List types; private final DataSize minOutputPageSize; @@ -343,6 +351,7 @@ public ScanFilterAndProjectOperatorFactory( PageSourceProvider pageSourceProvider, Supplier cursorProcessor, Supplier pageProcessor, + TableHandle table, Iterable columns, List types, DataSize minOutputPageSize, @@ -354,6 +363,7 @@ public ScanFilterAndProjectOperatorFactory( this.pageProcessor = requireNonNull(pageProcessor, "pageProcessor is null"); this.sourceId = requireNonNull(sourceId, "sourceId is null"); this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); + this.table = requireNonNull(table, "table is null"); this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); this.types = requireNonNull(types, "types is null"); this.minOutputPageSize = requireNonNull(minOutputPageSize, "minOutputPageSize is null"); @@ -377,6 +387,7 @@ public SourceOperator createOperator(DriverContext driverContext) pageSourceProvider, cursorProcessor.get(), pageProcessor.get(), + table, columns, types, new MergingPageOutput(types, minOutputPageSize.toBytes(), minOutputPageRowCount)); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/SimpleArrayAllocator.java b/presto-main/src/main/java/com/facebook/presto/operator/SimpleArrayAllocator.java index fbe9a85618211..ee6270fea43cb 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/SimpleArrayAllocator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/SimpleArrayAllocator.java @@ -36,7 +36,7 @@ * constant time access to the appropriately sized array. */ @NotThreadSafe -class SimpleArrayAllocator +public class SimpleArrayAllocator implements ArrayAllocator { private static final int DEFAULT_MAX_OUTSTANDING = 1000; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/SimplePagesHashStrategy.java b/presto-main/src/main/java/com/facebook/presto/operator/SimplePagesHashStrategy.java index 27bf24af7cb1f..aa4a770438cfa 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/SimplePagesHashStrategy.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/SimplePagesHashStrategy.java @@ -75,7 +75,7 @@ public SimplePagesHashStrategy( ImmutableList.Builder distinctFromMethodHandlesBuilder = ImmutableList.builder(); for (Type type : types) { distinctFromMethodHandlesBuilder.add( - functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(IS_DISTINCT_FROM, fromTypes(type, type))).getMethodHandle()); + functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(IS_DISTINCT_FROM, fromTypes(type, type))).getMethodHandle()); } distinctFromMethodHandles = distinctFromMethodHandlesBuilder.build(); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/SpatialIndexBuilderOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/SpatialIndexBuilderOperator.java index fd0d43f0dd59a..afa09927ffefa 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/SpatialIndexBuilderOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/SpatialIndexBuilderOperator.java @@ -230,7 +230,8 @@ public void finish() } finishing = true; - PagesSpatialIndexSupplier spatialIndex = index.createPagesSpatialIndex(operatorContext.getSession(), indexChannel, radiusChannel, partitionChannel, spatialRelationshipTest, filterFunctionFactory, outputChannels, partitions); + localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes()); + PagesSpatialIndexSupplier spatialIndex = index.createPagesSpatialIndex(operatorContext.getSession(), indexChannel, radiusChannel, partitionChannel, spatialRelationshipTest, filterFunctionFactory, outputChannels, partitions, localUserMemoryContext); localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes() + spatialIndex.getEstimatedSize().toBytes()); indexNotNeeded = pagesSpatialIndexFactory.lendPagesSpatialIndex(spatialIndex); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/SpatialJoinOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/SpatialJoinOperator.java index 89651ece81d24..f0f4232c05d2a 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/SpatialJoinOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/SpatialJoinOperator.java @@ -28,12 +28,12 @@ import java.util.List; import java.util.Optional; +import static com.facebook.airlift.concurrent.MoreFutures.getDone; import static com.facebook.presto.sql.planner.plan.SpatialJoinNode.Type.INNER; import static com.facebook.presto.sql.planner.plan.SpatialJoinNode.Type.LEFT; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; -import static io.airlift.concurrent.MoreFutures.getDone; import static io.airlift.slice.SizeOf.sizeOf; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/SpilledLookupSourceHandle.java b/presto-main/src/main/java/com/facebook/presto/operator/SpilledLookupSourceHandle.java index 2cb14295a0ebb..393e9bdcec37f 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/SpilledLookupSourceHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/SpilledLookupSourceHandle.java @@ -23,8 +23,8 @@ import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.MoreFutures.whenAnyComplete; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.concurrent.MoreFutures.whenAnyComplete; import static java.util.Objects.requireNonNull; @ThreadSafe diff --git a/presto-main/src/main/java/com/facebook/presto/operator/StreamingAggregationOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/StreamingAggregationOperator.java index dfe96d8c57e08..184ecc222720d 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/StreamingAggregationOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/StreamingAggregationOperator.java @@ -18,10 +18,10 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.JoinCompiler; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TableCommitContext.java b/presto-main/src/main/java/com/facebook/presto/operator/TableCommitContext.java index 6b85648d13e16..b6a67181b5721 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/TableCommitContext.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/TableCommitContext.java @@ -14,30 +14,29 @@ package com.facebook.presto.operator; import com.facebook.presto.execution.Lifespan; +import com.facebook.presto.execution.TaskId; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; public class TableCommitContext { private final Lifespan lifespan; - private final int stageId; - private final int taskId; + private final TaskId taskId; private final boolean lifespanCommitRequired; private final boolean lastPage; @JsonCreator public TableCommitContext( @JsonProperty("lifespan") Lifespan lifespan, - @JsonProperty("stageId") int stageId, - @JsonProperty("taskId") int taskId, + @JsonProperty("taskId") TaskId taskId, @JsonProperty("lifespanCommitRequired") boolean lifespanCommitRequired, @JsonProperty("lastPage") boolean lastPage) { this.lifespan = requireNonNull(lifespan, "lifespan is null"); - this.stageId = stageId; - this.taskId = taskId; + this.taskId = requireNonNull(taskId, "taskId is null"); this.lifespanCommitRequired = lifespanCommitRequired; this.lastPage = lastPage; } @@ -49,13 +48,7 @@ public Lifespan getLifespan() } @JsonProperty - public int getStageId() - { - return stageId; - } - - @JsonProperty - public int getTaskId() + public TaskId getTaskId() { return taskId; } @@ -71,4 +64,15 @@ public boolean isLastPage() { return lastPage; } + + @Override + public String toString() + { + return toStringHelper(this) + .add("lifespan", lifespan) + .add("taskId", taskId) + .add("lifespanCommitRequired", lifespanCommitRequired) + .add("lastPage", lastPage) + .toString(); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TableFinishInfo.java b/presto-main/src/main/java/com/facebook/presto/operator/TableFinishInfo.java index ea927160cb45b..ec78f8686ff1e 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/TableFinishInfo.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/TableFinishInfo.java @@ -13,18 +13,16 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.connector.ConnectorOutputMetadata; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonRawValue; -import com.fasterxml.jackson.databind.JsonNode; -import io.airlift.json.JsonCodec; import io.airlift.units.DataSize; import io.airlift.units.Duration; import java.util.Optional; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; @@ -34,29 +32,28 @@ public class TableFinishInfo { private static final int JSON_LENGTH_LIMIT = toIntExact(new DataSize(10, MEGABYTE).toBytes()); private static final JsonCodec INFO_CODEC = jsonCodec(Object.class); - private static final JsonCodec JSON_NODE_CODEC = jsonCodec(JsonNode.class); - private final String connectorOutputMetadata; + private final String serializedConnectorOutputMetadata; private final boolean jsonLengthLimitExceeded; private final Duration statisticsWallTime; private final Duration statisticsCpuTime; public TableFinishInfo(Optional metadata, Duration statisticsWallTime, Duration statisticsCpuTime) { - String connectorOutputMetadata = null; + String serializedConnectorOutputMetadata = null; boolean jsonLengthLimitExceeded = false; if (metadata.isPresent()) { Optional serializedMetadata = INFO_CODEC.toJsonWithLengthLimit(metadata.get().getInfo(), JSON_LENGTH_LIMIT); if (!serializedMetadata.isPresent()) { - connectorOutputMetadata = null; + serializedConnectorOutputMetadata = null; jsonLengthLimitExceeded = true; } else { - connectorOutputMetadata = serializedMetadata.get(); + serializedConnectorOutputMetadata = serializedMetadata.get(); jsonLengthLimitExceeded = false; } } - this.connectorOutputMetadata = connectorOutputMetadata; + this.serializedConnectorOutputMetadata = serializedConnectorOutputMetadata; this.jsonLengthLimitExceeded = jsonLengthLimitExceeded; this.statisticsWallTime = requireNonNull(statisticsWallTime, "statisticsWallTime is null"); this.statisticsCpuTime = requireNonNull(statisticsCpuTime, "statisticsCpuTime is null"); @@ -64,22 +61,21 @@ public TableFinishInfo(Optional metadata, Duration stat @JsonCreator public TableFinishInfo( - @JsonProperty("connectorOutputMetadata") JsonNode connectorOutputMetadata, + @JsonProperty("serializedConnectorOutputMetadata") String serializedConnectorOutputMetadata, @JsonProperty("jsonLengthLimitExceeded") boolean jsonLengthLimitExceeded, @JsonProperty("statisticsWallTime") Duration statisticsWallTime, @JsonProperty("statisticsCpuTime") Duration statisticsCpuTime) { - this.connectorOutputMetadata = JSON_NODE_CODEC.toJson(connectorOutputMetadata); + this.serializedConnectorOutputMetadata = serializedConnectorOutputMetadata; this.jsonLengthLimitExceeded = jsonLengthLimitExceeded; this.statisticsWallTime = requireNonNull(statisticsWallTime, "statisticsWallTime is null"); this.statisticsCpuTime = requireNonNull(statisticsCpuTime, "statisticsCpuTime is null"); } @JsonProperty - @JsonRawValue - public String getConnectorOutputMetadata() + public String getSerializedConnectorOutputMetadata() { - return connectorOutputMetadata; + return serializedConnectorOutputMetadata; } @JsonProperty diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TableFinishOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/TableFinishOperator.java index 1c907662062d5..e0cc8148d160d 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/TableFinishOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/TableFinishOperator.java @@ -13,11 +13,14 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.Session; import com.facebook.presto.execution.Lifespan; +import com.facebook.presto.execution.TaskId; import com.facebook.presto.operator.OperationTimer.OperationTiming; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageBuilder; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.connector.ConnectorOutputMetadata; import com.facebook.presto.spi.plan.PlanNodeId; @@ -27,28 +30,34 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.ExecutionException; import java.util.stream.Stream; import static com.facebook.presto.SystemSessionProperties.isStatisticsCpuTimerEnabled; -import static com.facebook.presto.operator.TableWriterOperator.CONTEXT_CHANNEL; -import static com.facebook.presto.operator.TableWriterOperator.FRAGMENT_CHANNEL; -import static com.facebook.presto.operator.TableWriterOperator.ROW_COUNT_CHANNEL; +import static com.facebook.presto.operator.TableWriterUtils.FRAGMENT_CHANNEL; +import static com.facebook.presto.operator.TableWriterUtils.ROW_COUNT_CHANNEL; +import static com.facebook.presto.operator.TableWriterUtils.extractStatisticsRows; +import static com.facebook.presto.operator.TableWriterUtils.getTableCommitContext; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Throwables.propagateIfPossible; import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.util.concurrent.Futures.whenAllSucceed; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static io.airlift.units.Duration.succinctNanos; +import static java.lang.Thread.currentThread; import static java.util.Objects.requireNonNull; public class TableFinishOperator @@ -202,7 +211,7 @@ public void addInput(Page page) requireNonNull(page, "page is null"); checkState(state == State.RUNNING, "Operator is %s", state); - TableCommitContext tableCommitContext = getTableCommitContext(page); + TableCommitContext tableCommitContext = getTableCommitContext(page, tableCommitContextCodec); lifespanAndStageStateTracker.update(page, tableCommitContext); lifespanAndStageStateTracker.getStatisticsPagesToProcess(page, tableCommitContext).forEach(statisticsPage -> { OperationTimer timer = new OperationTimer(statisticsCpuTimerEnabled); @@ -211,79 +220,6 @@ public void addInput(Page page) }); } - private TableCommitContext getTableCommitContext(Page page) - { - checkState(page.getPositionCount() > 0, "TableFinishOperator receives empty page"); - Block operatorExecutionContextBlock = page.getBlock(CONTEXT_CHANNEL); - return tableCommitContextCodec.fromJson(operatorExecutionContextBlock.getSlice(0, 0, operatorExecutionContextBlock.getSliceLength(0)).getBytes()); - } - - private static Optional extractStatisticsRows(Page page) - { - int statisticsPositionCount = 0; - for (int position = 0; position < page.getPositionCount(); position++) { - if (isStatisticsPosition(page, position)) { - statisticsPositionCount++; - } - } - - if (statisticsPositionCount == 0) { - return Optional.empty(); - } - - if (statisticsPositionCount == page.getPositionCount()) { - return Optional.of(page); - } - - int selectedPositionsIndex = 0; - int[] selectedPositions = new int[statisticsPositionCount]; - for (int position = 0; position < page.getPositionCount(); position++) { - if (isStatisticsPosition(page, position)) { - selectedPositions[selectedPositionsIndex] = position; - selectedPositionsIndex++; - } - } - - Block[] blocks = new Block[page.getChannelCount()]; - for (int channel = 0; channel < page.getChannelCount(); channel++) { - blocks[channel] = page.getBlock(channel).getPositions(selectedPositions, 0, statisticsPositionCount); - } - return Optional.of(new Page(statisticsPositionCount, blocks)); - } - - /** - * Both the statistics and the row_count + fragments are transferred over the same communication - * link between the TableWriterOperator and the TableFinishOperator. Thus the multiplexing is needed. - *

- * The transferred page layout looks like: - *

- * [[row_count_channel], [fragment_channel], [statistic_channel_1] ... [statistic_channel_N]] - *

- * [row_count_channel] - contains number of rows processed by a TableWriterOperator instance - * [fragment_channel] - contains arbitrary binary data provided by the ConnectorPageSink#finish for - * the further post processing on the coordinator - *

- * [statistic_channel_1] ... [statistic_channel_N] - contain pre-aggregated statistics computed by the - * statistics aggregation operator within the - * TableWriterOperator - *

- * Since the final aggregation operator in the TableFinishOperator doesn't know what to do with the - * first two channels, those must be pruned. For the convenience we never set both, the - * [row_count_channel] + [fragment_channel] and the [statistic_channel_1] ... [statistic_channel_N]. - *

- * If this is a row that holds statistics - the [row_count_channel] + [fragment_channel] will be NULL. - *

- * It this is a row that holds the row count or the fragment - all the statistics channels will be set - * to NULL. - *

- * Since neither [row_count_channel] or [fragment_channel] cannot hold the NULL value naturally, by - * checking isNull on these two channels we can determine if this is a row that contains statistics. - */ - private static boolean isStatisticsPosition(Page page, int position) - { - return page.getBlock(ROW_COUNT_CHANNEL).isNull(position) && page.getBlock(FRAGMENT_CHANNEL).isNull(position); - } - @Override public Page getOutput() { @@ -312,6 +248,7 @@ public Page getOutput() } state = State.FINISHED; + lifespanAndStageStateTracker.waitForAllLifespanCommitted(); outputMetadata = tableFinisher.finishTable(lifespanAndStageStateTracker.getFinalFragments(), computedStatisticsBuilder.build()); // output page will only be constructed once, @@ -364,7 +301,7 @@ public interface TableFinisher public interface LifespanCommitter { - void commitLifespan(Collection fragments); + ListenableFuture commitLifespan(Collection fragments); } // A lifespan in a stage defines the unit for commit and recovery in recoverable grouped execution @@ -374,37 +311,69 @@ private static class LifespanAndStageStateTracker // For recoverable execution, it is possible to receive pages of the same lifespan-stage from different tasks. We track all of them and commit the one // which finishes sending pages first. - private final Map> uncommittedRecoverableLifespanAndStageStates = new HashMap<>(); + private final Map> uncommittedRecoverableLifespanAndStageStates = new HashMap<>(); private final Map committedRecoverableLifespanAndStages = new HashMap<>(); private final LifespanCommitter lifespanCommitter; + private final List> commitFutures = new ArrayList<>(); LifespanAndStageStateTracker(LifespanCommitter lifespanCommitter) { this.lifespanCommitter = requireNonNull(lifespanCommitter, "lifespanCommitter is null"); } + public void waitForAllLifespanCommitted() + { + ListenableFuture future = whenAllSucceed(commitFutures).call(() -> null, directExecutor()); + try { + future.get(); + } + catch (InterruptedException e) { + future.cancel(true); + currentThread().interrupt(); + throw new RuntimeException(e); + } + catch (ExecutionException e) { + future.cancel(true); + propagateIfPossible(e.getCause(), PrestoException.class); + throw new RuntimeException(e.getCause()); + } + } + public void update(Page page, TableCommitContext tableCommitContext) { LifespanAndStage lifespanAndStage = LifespanAndStage.fromTableCommitContext(tableCommitContext); - if (committedRecoverableLifespanAndStages.containsKey(lifespanAndStage)) { + + // Case 1: lifespan commit is not required, this can be one of the following cases: + // - The source fragment is ungrouped execution (lifespan is TASK_WIDE). + // - The source fragment is grouped execution but not recoverable. + if (!tableCommitContext.isLifespanCommitRequired()) { + unrecoverableLifespanAndStageStates.computeIfAbsent(lifespanAndStage, ignored -> new LifespanAndStageState(tableCommitContext.getTaskId())).update(page); return; } - if (!tableCommitContext.isLifespanCommitRequired()) { - unrecoverableLifespanAndStageStates.computeIfAbsent(lifespanAndStage, ignored -> new LifespanAndStageState()).update(page); + // Case 2: lifespan commit is required + checkState(lifespanAndStage.lifespan != Lifespan.taskWide(), "Recoverable lifespan cannot be TASK_WIDE"); + + // Case 2a: current (stage, lifespan) combination is already committed + if (committedRecoverableLifespanAndStages.containsKey(lifespanAndStage)) { + checkState( + !committedRecoverableLifespanAndStages.get(lifespanAndStage).getTaskId().equals(tableCommitContext.getTaskId()), + "Received page from same task of committed lifespan and stage combination"); + return; } - Map lifespanStageStatesPerTask = uncommittedRecoverableLifespanAndStageStates.computeIfAbsent(lifespanAndStage, ignored -> new HashMap<>()); - lifespanStageStatesPerTask.computeIfAbsent(tableCommitContext.getTaskId(), ignored -> new LifespanAndStageState()).update(page); + // Case 2b: current (stage, lifespan) combination is not yet committed + Map lifespanStageStatesPerTask = uncommittedRecoverableLifespanAndStageStates.computeIfAbsent(lifespanAndStage, ignored -> new HashMap<>()); + lifespanStageStatesPerTask.computeIfAbsent(tableCommitContext.getTaskId(), ignored -> new LifespanAndStageState(tableCommitContext.getTaskId())).update(page); if (tableCommitContext.isLastPage()) { checkState(!committedRecoverableLifespanAndStages.containsKey(lifespanAndStage), "LifespanAndStage already finished"); LifespanAndStageState lifespanAndStageState = lifespanStageStatesPerTask.get(tableCommitContext.getTaskId()); committedRecoverableLifespanAndStages.put(lifespanAndStage, lifespanAndStageState); uncommittedRecoverableLifespanAndStageStates.remove(lifespanAndStage); - lifespanCommitter.commitLifespan(lifespanAndStageState.getFragments()); + commitFutures.add(lifespanCommitter.commitLifespan(lifespanAndStageState.getFragments())); } } @@ -451,7 +420,7 @@ private LifespanAndStage(Lifespan lifespan, int stageId) public static LifespanAndStage fromTableCommitContext(TableCommitContext operatorExecutionContext) { - return new LifespanAndStage(operatorExecutionContext.getLifespan(), operatorExecutionContext.getStageId()); + return new LifespanAndStage(operatorExecutionContext.getLifespan(), operatorExecutionContext.getTaskId().getStageExecutionId().getStageId().getId()); } public Lifespan getLifespan() @@ -500,6 +469,13 @@ private static class LifespanAndStageState private ImmutableList.Builder fragmentBuilder = ImmutableList.builder(); private ImmutableList.Builder statisticsPages = ImmutableList.builder(); + private TaskId taskId; + + public LifespanAndStageState(TaskId taskId) + { + this.taskId = requireNonNull(taskId, "taskId is null"); + } + public void update(Page page) { Block rowCountBlock = page.getBlock(ROW_COUNT_CHANNEL); @@ -529,6 +505,11 @@ public List getStatisticsPages() { return statisticsPages.build(); } + + public TaskId getTaskId() + { + return taskId; + } } } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TableScanOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/TableScanOperator.java index fcb597625cba2..ba8775dc6994f 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/TableScanOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/TableScanOperator.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.UpdatablePageSource; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.split.EmptySplit; @@ -35,8 +36,8 @@ import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.MoreFutures.toListenableFuture; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.concurrent.MoreFutures.toListenableFuture; import static java.util.Objects.requireNonNull; public class TableScanOperator @@ -48,6 +49,7 @@ public static class TableScanOperatorFactory private final int operatorId; private final PlanNodeId sourceId; private final PageSourceProvider pageSourceProvider; + private final TableHandle table; private final List columns; private boolean closed; @@ -55,11 +57,13 @@ public TableScanOperatorFactory( int operatorId, PlanNodeId sourceId, PageSourceProvider pageSourceProvider, + TableHandle table, Iterable columns) { this.operatorId = operatorId; this.sourceId = requireNonNull(sourceId, "sourceId is null"); this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); + this.table = requireNonNull(table, "table is null"); this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); } @@ -78,6 +82,7 @@ public SourceOperator createOperator(DriverContext driverContext) operatorContext, sourceId, pageSourceProvider, + table, columns); } @@ -91,6 +96,7 @@ public void noMoreOperators() private final OperatorContext operatorContext; private final PlanNodeId planNodeId; private final PageSourceProvider pageSourceProvider; + private final TableHandle table; private final List columns; private final LocalMemoryContext systemMemoryContext; private final SettableFuture blocked = SettableFuture.create(); @@ -101,17 +107,20 @@ public void noMoreOperators() private boolean finished; private long completedBytes; + private long completedPositions; private long readTimeNanos; public TableScanOperator( OperatorContext operatorContext, PlanNodeId planNodeId, PageSourceProvider pageSourceProvider, + TableHandle table, Iterable columns) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); + this.table = requireNonNull(table, "table is null"); this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(TableScanOperator.class.getSimpleName()); } @@ -236,7 +245,7 @@ public Page getOutput() return null; } if (source == null) { - source = pageSourceProvider.createPageSource(operatorContext.getSession(), split, columns); + source = pageSourceProvider.createPageSource(operatorContext.getSession(), split, table, columns); } Page page = source.getNextPage(); @@ -246,10 +255,12 @@ public Page getOutput() // update operator stats long endCompletedBytes = source.getCompletedBytes(); + long endCompletedPositions = source.getCompletedPositions(); long endReadTimeNanos = source.getReadTimeNanos(); - operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); + operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endCompletedPositions - completedPositions, endReadTimeNanos - readTimeNanos); operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount()); completedBytes = endCompletedBytes; + completedPositions = endCompletedPositions; readTimeNanos = endReadTimeNanos; } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TableWriterMergeInfo.java b/presto-main/src/main/java/com/facebook/presto/operator/TableWriterMergeInfo.java new file mode 100644 index 0000000000000..9855446b37f3e --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/TableWriterMergeInfo.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import com.facebook.presto.util.Mergeable; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.airlift.units.Duration; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +public class TableWriterMergeInfo + implements Mergeable, OperatorInfo +{ + private final Duration statisticsWallTime; + private final Duration statisticsCpuTime; + + @JsonCreator + public TableWriterMergeInfo( + @JsonProperty("statisticsWallTime") Duration statisticsWallTime, + @JsonProperty("statisticsCpuTime") Duration statisticsCpuTime) + { + this.statisticsWallTime = requireNonNull(statisticsWallTime, "statisticsWallTime is null"); + this.statisticsCpuTime = requireNonNull(statisticsCpuTime, "statisticsCpuTime is null"); + } + + @JsonProperty + public Duration getStatisticsWallTime() + { + return statisticsWallTime; + } + + @JsonProperty + public Duration getStatisticsCpuTime() + { + return statisticsCpuTime; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("statisticsWallTime", statisticsWallTime) + .add("statisticsCpuTime", statisticsCpuTime) + .toString(); + } + + @Override + public TableWriterMergeInfo mergeWith(TableWriterMergeInfo other) + { + return new TableWriterMergeInfo( + new Duration(this.statisticsWallTime.toMillis() + other.statisticsWallTime.toMillis(), MILLISECONDS), + new Duration(this.statisticsCpuTime.toMillis() + other.statisticsCpuTime.toMillis(), MILLISECONDS)); + } + + @Override + public boolean isFinal() + { + return true; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TableWriterMergeOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/TableWriterMergeOperator.java new file mode 100644 index 0000000000000..23166a47c4f4d --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/TableWriterMergeOperator.java @@ -0,0 +1,356 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.Session; +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.operator.OperationTimer.OperationTiming; +import com.facebook.presto.operator.aggregation.builder.InMemoryHashAggregationBuilder; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.PageBuilder; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListenableFuture; +import io.airlift.slice.Slice; + +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +import static com.facebook.presto.SystemSessionProperties.isStatisticsCpuTimerEnabled; +import static com.facebook.presto.operator.TableWriterUtils.CONTEXT_CHANNEL; +import static com.facebook.presto.operator.TableWriterUtils.FRAGMENT_CHANNEL; +import static com.facebook.presto.operator.TableWriterUtils.ROW_COUNT_CHANNEL; +import static com.facebook.presto.operator.TableWriterUtils.createStatisticsPage; +import static com.facebook.presto.operator.TableWriterUtils.extractStatisticsRows; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Verify.verify; +import static io.airlift.slice.Slices.wrappedBuffer; +import static io.airlift.units.Duration.succinctNanos; +import static java.util.Objects.requireNonNull; + +public class TableWriterMergeOperator + implements Operator +{ + public static class TableWriterMergeOperatorFactory + implements OperatorFactory + { + private final int operatorId; + private final PlanNodeId planNodeId; + private final OperatorFactory statisticsAggregationOperatorFactory; + private final JsonCodec tableCommitContextCodec; + private final Session session; + private final List types; + private boolean closed; + + public TableWriterMergeOperatorFactory( + int operatorId, + PlanNodeId planNodeId, + OperatorFactory statisticsAggregationOperatorFactory, + JsonCodec tableCommitContextCodec, + Session session, + List types) + { + this.operatorId = operatorId; + this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); + this.statisticsAggregationOperatorFactory = requireNonNull(statisticsAggregationOperatorFactory, "statisticsAggregationOperatorFactory is null"); + this.tableCommitContextCodec = requireNonNull(tableCommitContextCodec, "tableCommitContextCodec is null"); + this.session = requireNonNull(session, "session is null"); + this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); + } + + @Override + public Operator createOperator(DriverContext driverContext) + { + checkState(!closed, "Factory is already closed"); + OperatorContext context = driverContext.addOperatorContext(operatorId, planNodeId, TableWriterMergeOperator.class.getSimpleName()); + Operator statisticsAggregationOperator = statisticsAggregationOperatorFactory.createOperator(driverContext); + boolean statisticsCpuTimerEnabled = !(statisticsAggregationOperator instanceof DevNullOperator) && isStatisticsCpuTimerEnabled(session); + return new TableWriterMergeOperator(context, statisticsAggregationOperator, tableCommitContextCodec, statisticsCpuTimerEnabled, types); + } + + @Override + public void noMoreOperators() + { + closed = true; + } + + @Override + public OperatorFactory duplicate() + { + return new TableWriterMergeOperatorFactory(operatorId, planNodeId, statisticsAggregationOperatorFactory, tableCommitContextCodec, session, types); + } + } + + private enum State + { + RUNNING, FINISHING, FINISHED + } + + private final OperatorContext context; + private final Operator statisticsAggregationOperator; + private final JsonCodec tableCommitContextCodec; + private final LocalMemoryContext systemMemoryContext; + private final List types; + + private final OperationTiming statisticsTiming = new OperationTiming(); + private final boolean statisticsCpuTimerEnabled; + + private TableCommitContext lastTableCommitContext; + private long rowCount; + private final Queue fragmentsBlocks = new LinkedList<>(); + + private State state = State.RUNNING; + + public TableWriterMergeOperator( + OperatorContext context, + Operator statisticsAggregationOperator, + JsonCodec tableCommitContextCodec, + boolean statisticsCpuTimerEnabled, + List types) + { + this.context = requireNonNull(context, "context is null"); + this.statisticsAggregationOperator = requireNonNull(statisticsAggregationOperator, "statisticAggregationOperator is null"); + this.systemMemoryContext = context.newLocalSystemMemoryContext(InMemoryHashAggregationBuilder.class.getSimpleName()); + this.tableCommitContextCodec = requireNonNull(tableCommitContextCodec, "tableCommitContextCodec is null"); + this.statisticsCpuTimerEnabled = statisticsCpuTimerEnabled; + this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); + this.context.setInfoSupplier(this::getInfo); + } + + @Override + public OperatorContext getOperatorContext() + { + return context; + } + + @Override + public ListenableFuture isBlocked() + { + return statisticsAggregationOperator.isBlocked(); + } + + @Override + public boolean needsInput() + { + if (state != State.RUNNING) { + return false; + } + return statisticsAggregationOperator.needsInput(); + } + + @Override + public void addInput(Page page) + { + requireNonNull(page, "page is null"); + checkState(state == State.RUNNING, "Operator is %s", state); + + // make sure the lifespan is the same + TableCommitContext tableCommitContext = TableWriterUtils.getTableCommitContext(page, tableCommitContextCodec); + if (lastTableCommitContext != null) { + checkArgument( + isSameTaskAndLifespan(lastTableCommitContext, tableCommitContext), + "incompatible table commit context: %s is not compatible with %s", + lastTableCommitContext, + tableCommitContext); + } + lastTableCommitContext = tableCommitContext; + + // increment rows + rowCount += getRowCount(page); + + // Add fragments to the buffer. + // Fragments will be outputted as soon as possible to avoid using extra memory. + Block fragmentsBlock = page.getBlock(FRAGMENT_CHANNEL); + if (containsNonNullRows(fragmentsBlock)) { + fragmentsBlocks.add(fragmentsBlock); + } + + extractStatisticsRows(page).ifPresent(statisticsPage -> { + OperationTimer timer = new OperationTimer(statisticsCpuTimerEnabled); + statisticsAggregationOperator.addInput(statisticsPage); + timer.end(statisticsTiming); + }); + + systemMemoryContext.setBytes(getRetainedMemoryBytes()); + } + + private static long getRowCount(Page page) + { + long rowCount = 0; + Block rowCountBlock = page.getBlock(ROW_COUNT_CHANNEL); + for (int position = 0; position < page.getPositionCount(); position++) { + if (!rowCountBlock.isNull(position)) { + rowCount += BIGINT.getLong(rowCountBlock, position); + } + } + return rowCount; + } + + private static boolean containsNonNullRows(Block block) + { + // shortcut for RunLengthEncodedBlock + if (block instanceof RunLengthEncodedBlock) { + RunLengthEncodedBlock runLengthEncodedBlock = (RunLengthEncodedBlock) block; + return !runLengthEncodedBlock.getValue().isNull(0); + } + for (int position = 0; position < block.getPositionCount(); position++) { + if (!block.isNull(position)) { + return true; + } + } + return false; + } + + private static boolean isSameTaskAndLifespan(TableCommitContext first, TableCommitContext second) + { + return first.getLifespan().equals(second.getLifespan()) && + first.getTaskId().equals(second.getTaskId()) && + first.isLifespanCommitRequired() == second.isLifespanCommitRequired(); + } + + private long getRetainedMemoryBytes() + { + return fragmentsBlocks.stream().mapToLong(Block::getRetainedSizeInBytes).sum(); + } + + @Override + public Page getOutput() + { + // pass through fragment pages first to avoid use extra memory + if (!fragmentsBlocks.isEmpty()) { + Block fragmentsBlock = fragmentsBlocks.poll(); + systemMemoryContext.setBytes(getRetainedMemoryBytes()); + return createFragmentsPage(fragmentsBlock); + } + + // still working on merging statistic pages + if (!isBlocked().isDone()) { + return null; + } + + if (!statisticsAggregationOperator.isFinished()) { + verify(statisticsAggregationOperator.isBlocked().isDone(), "aggregation operator should not be blocked"); + + OperationTimer timer = new OperationTimer(statisticsCpuTimerEnabled); + Page page = statisticsAggregationOperator.getOutput(); + timer.end(statisticsTiming); + + if (page == null) { + return null; + } + + return createStatisticsPage(types, page, createTableCommitContext(false)); + } + + if (state != State.FINISHING) { + return null; + } + state = State.FINISHED; + + Page finalPage = createFinalPage(); + systemMemoryContext.setBytes(getRetainedMemoryBytes()); + return finalPage; + } + + private Page createFragmentsPage(Block fragmentsBlock) + { + int positionCount = fragmentsBlock.getPositionCount(); + Block[] outputBlocks = new Block[types.size()]; + for (int channel = 0; channel < types.size(); channel++) { + if (channel == FRAGMENT_CHANNEL) { + outputBlocks[channel] = fragmentsBlock; + } + else if (channel == CONTEXT_CHANNEL) { + outputBlocks[channel] = RunLengthEncodedBlock.create(types.get(channel), createTableCommitContext(false), positionCount); + } + else { + outputBlocks[channel] = RunLengthEncodedBlock.create(types.get(channel), null, positionCount); + } + } + return new Page(outputBlocks); + } + + private Page createFinalPage() + { + checkState(lastTableCommitContext.isLastPage(), "unexpected last table commit context: %s", lastTableCommitContext); + PageBuilder pageBuilder = new PageBuilder(1, types); + pageBuilder.declarePosition(); + for (int channel = 0; channel < types.size(); channel++) { + if (channel == ROW_COUNT_CHANNEL) { + types.get(channel).writeLong(pageBuilder.getBlockBuilder(channel), rowCount); + } + else if (channel == CONTEXT_CHANNEL) { + types.get(channel).writeSlice(pageBuilder.getBlockBuilder(channel), createTableCommitContext(true)); + } + else { + pageBuilder.getBlockBuilder(channel).appendNull(); + } + } + return pageBuilder.build(); + } + + private Slice createTableCommitContext(boolean lastPage) + { + checkState(tableCommitContextCodec != null, "tableCommitContextCodec is null"); + return wrappedBuffer(tableCommitContextCodec.toJsonBytes(new TableCommitContext( + lastTableCommitContext.getLifespan(), + lastTableCommitContext.getTaskId(), + lastTableCommitContext.isLifespanCommitRequired(), + lastPage))); + } + + @Override + public void finish() + { + OperationTimer timer = new OperationTimer(statisticsCpuTimerEnabled); + statisticsAggregationOperator.finish(); + timer.end(statisticsTiming); + + if (state == State.RUNNING) { + state = State.FINISHING; + } + } + + @Override + public boolean isFinished() + { + if (state == State.FINISHED) { + verify(statisticsAggregationOperator.isFinished()); + return true; + } + return false; + } + + @Override + public void close() + throws Exception + { + statisticsAggregationOperator.close(); + systemMemoryContext.setBytes(0); + } + + public TableWriterMergeInfo getInfo() + { + return new TableWriterMergeInfo( + succinctNanos(statisticsTiming.getWallNanos()), + succinctNanos(statisticsTiming.getCpuNanos())); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TableWriterOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/TableWriterOperator.java index 6691240dee4f3..8a67c09efefb3 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/TableWriterOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/TableWriterOperator.java @@ -13,8 +13,12 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.Session; import com.facebook.presto.execution.TaskId; +import com.facebook.presto.execution.scheduler.ExecutionWriterTarget; +import com.facebook.presto.execution.scheduler.ExecutionWriterTarget.CreateHandle; +import com.facebook.presto.execution.scheduler.ExecutionWriterTarget.InsertHandle; import com.facebook.presto.memory.context.LocalMemoryContext; import com.facebook.presto.operator.OperationTimer.OperationTiming; import com.facebook.presto.spi.ConnectorPageSink; @@ -26,7 +30,6 @@ import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.Type; import com.facebook.presto.split.PageSinkManager; -import com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget; import com.facebook.presto.util.AutoCloseableCloser; import com.facebook.presto.util.Mergeable; import com.fasterxml.jackson.annotation.JsonCreator; @@ -34,7 +37,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; import io.airlift.units.Duration; @@ -43,17 +45,17 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.MoreFutures.toListenableFuture; import static com.facebook.presto.SystemSessionProperties.isStatisticsCpuTimerEnabled; +import static com.facebook.presto.operator.TableWriterUtils.STATS_START_CHANNEL; +import static com.facebook.presto.operator.TableWriterUtils.createStatisticsPage; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; -import static com.facebook.presto.sql.planner.plan.TableWriterNode.CreateHandle; -import static com.facebook.presto.sql.planner.plan.TableWriterNode.InsertHandle; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.util.concurrent.Futures.allAsList; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.MoreFutures.toListenableFuture; import static io.airlift.slice.Slices.wrappedBuffer; import static io.airlift.units.Duration.succinctNanos; import static java.util.Objects.requireNonNull; @@ -62,18 +64,13 @@ public class TableWriterOperator implements Operator { - public static final int ROW_COUNT_CHANNEL = 0; - public static final int FRAGMENT_CHANNEL = 1; - public static final int CONTEXT_CHANNEL = 2; - public static final int STATS_START_CHANNEL = 3; - public static class TableWriterOperatorFactory implements OperatorFactory { private final int operatorId; private final PlanNodeId planNodeId; private final PageSinkManager pageSinkManager; - private final WriterTarget target; + private final ExecutionWriterTarget target; private final List columnChannels; private final Session session; private final OperatorFactory statisticsAggregationOperatorFactory; @@ -86,7 +83,7 @@ public TableWriterOperatorFactory( int operatorId, PlanNodeId planNodeId, PageSinkManager pageSinkManager, - WriterTarget writerTarget, + ExecutionWriterTarget writerTarget, List columnChannels, Session session, OperatorFactory statisticsAggregationOperatorFactory, @@ -284,7 +281,7 @@ public Page getOutput() if (aggregationOutput == null) { return null; } - return createStatisticsPage(aggregationOutput); + return createStatisticsPage(types, aggregationOutput, createTableCommitContext(false)); } if (state != State.FINISHING) { @@ -307,35 +304,6 @@ public Page getOutput() return new Page(positionCount, outputBlocks); } - // Statistics page layout: - // - // row fragments context stats1 stats2 ... - // null null X X X - // null null X X X - // null null X X X - // null null X X X - // ... - private Page createStatisticsPage(Page aggregationOutput) - { - int positionCount = aggregationOutput.getPositionCount(); - Block[] outputBlocks = new Block[types.size()]; - for (int channel = 0; channel < types.size(); channel++) { - if (channel < STATS_START_CHANNEL) { - // Include table commit context into statistics page to allow TableFinishOperator publish correct statistics for recoverable grouped execution. - if (channel == CONTEXT_CHANNEL) { - outputBlocks[channel] = RunLengthEncodedBlock.create(types.get(channel), getTableCommitContext(false), positionCount); - } - else { - outputBlocks[channel] = RunLengthEncodedBlock.create(types.get(channel), null, positionCount); - } - } - else { - outputBlocks[channel] = aggregationOutput.getBlock(channel - STATS_START_CHANNEL); - } - } - return new Page(positionCount, outputBlocks); - } - // Fragments page layout: // // row fragments context @@ -366,17 +334,16 @@ private Page createFragmentsPage() VARBINARY.writeSlice(fragmentBuilder, fragment); } - return new Page(positionCount, rowsBuilder.build(), fragmentBuilder.build(), RunLengthEncodedBlock.create(VARBINARY, getTableCommitContext(true), positionCount)); + return new Page(positionCount, rowsBuilder.build(), fragmentBuilder.build(), RunLengthEncodedBlock.create(VARBINARY, createTableCommitContext(true), positionCount)); } - private Slice getTableCommitContext(boolean lastPage) + private Slice createTableCommitContext(boolean lastPage) { TaskId taskId = operatorContext.getDriverContext().getPipelineContext().getTaskId(); return wrappedBuffer(tableCommitContextCodec.toJsonBytes( new TableCommitContext( operatorContext.getDriverContext().getLifespan(), - taskId.getStageId().getId(), - taskId.getId(), + taskId, partitionCommitRequired, lastPage))); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TableWriterUtils.java b/presto-main/src/main/java/com/facebook/presto/operator/TableWriterUtils.java new file mode 100644 index 0000000000000..6b5d67047187d --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/TableWriterUtils.java @@ -0,0 +1,140 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.Type; +import io.airlift.slice.Slice; + +import java.util.List; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public final class TableWriterUtils +{ + public static final int ROW_COUNT_CHANNEL = 0; + public static final int FRAGMENT_CHANNEL = 1; + public static final int CONTEXT_CHANNEL = 2; + public static final int STATS_START_CHANNEL = 3; + + private TableWriterUtils() {} + + public static Optional extractStatisticsRows(Page page) + { + int statisticsPositionCount = 0; + for (int position = 0; position < page.getPositionCount(); position++) { + if (isStatisticsPosition(page, position)) { + statisticsPositionCount++; + } + } + + if (statisticsPositionCount == 0) { + return Optional.empty(); + } + + if (statisticsPositionCount == page.getPositionCount()) { + return Optional.of(page); + } + + int selectedPositionsIndex = 0; + int[] selectedPositions = new int[statisticsPositionCount]; + for (int position = 0; position < page.getPositionCount(); position++) { + if (isStatisticsPosition(page, position)) { + selectedPositions[selectedPositionsIndex] = position; + selectedPositionsIndex++; + } + } + + Block[] blocks = new Block[page.getChannelCount()]; + for (int channel = 0; channel < page.getChannelCount(); channel++) { + blocks[channel] = page.getBlock(channel).getPositions(selectedPositions, 0, statisticsPositionCount); + } + return Optional.of(new Page(statisticsPositionCount, blocks)); + } + + /** + * Both the statistics and the row_count + fragments are transferred over the same communication + * link between the TableWriterOperator and the TableFinishOperator. Thus the multiplexing is needed. + *

+ * The transferred page layout looks like: + *

+ * [[row_count_channel], [fragment_channel], [statistic_channel_1] ... [statistic_channel_N]] + *

+ * [row_count_channel] - contains number of rows processed by a TableWriterOperator instance + * [fragment_channel] - contains arbitrary binary data provided by the ConnectorPageSink#finish for + * the further post processing on the coordinator + *

+ * [statistic_channel_1] ... [statistic_channel_N] - contain pre-aggregated statistics computed by the + * statistics aggregation operator within the + * TableWriterOperator + *

+ * Since the final aggregation operator in the TableFinishOperator doesn't know what to do with the + * first two channels, those must be pruned. For the convenience we never set both, the + * [row_count_channel] + [fragment_channel] and the [statistic_channel_1] ... [statistic_channel_N]. + *

+ * If this is a row that holds statistics - the [row_count_channel] + [fragment_channel] will be NULL. + *

+ * It this is a row that holds the row count or the fragment - all the statistics channels will be set + * to NULL. + *

+ * Since neither [row_count_channel] or [fragment_channel] cannot hold the NULL value naturally, by + * checking isNull on these two channels we can determine if this is a row that contains statistics. + */ + private static boolean isStatisticsPosition(Page page, int position) + { + return page.getBlock(ROW_COUNT_CHANNEL).isNull(position) && page.getBlock(FRAGMENT_CHANNEL).isNull(position); + } + + // Statistics page layout: + // + // row fragments context stats1 stats2 ... + // null null X X X + // null null X X X + // null null X X X + // null null X X X + // ... + public static Page createStatisticsPage(List types, Page aggregationOutput, Slice tableCommitContext) + { + int positionCount = aggregationOutput.getPositionCount(); + Block[] outputBlocks = new Block[types.size()]; + for (int channel = 0; channel < types.size(); channel++) { + if (channel < STATS_START_CHANNEL) { + // Include table commit context into statistics page to allow TableFinishOperator publish correct statistics for recoverable grouped execution. + if (channel == CONTEXT_CHANNEL) { + outputBlocks[channel] = RunLengthEncodedBlock.create(types.get(channel), tableCommitContext, positionCount); + } + else { + outputBlocks[channel] = RunLengthEncodedBlock.create(types.get(channel), null, positionCount); + } + } + else { + outputBlocks[channel] = aggregationOutput.getBlock(channel - STATS_START_CHANNEL); + } + } + return new Page(positionCount, outputBlocks); + } + + public static TableCommitContext getTableCommitContext(Page page, JsonCodec tableCommitContextCodec) + { + checkState(page.getPositionCount() > 0, "page is empty"); + Block operatorExecutionContextBlock = page.getBlock(CONTEXT_CHANNEL); + TableCommitContext context = tableCommitContextCodec.fromJson(operatorExecutionContextBlock.getSlice(0, 0, operatorExecutionContextBlock.getSliceLength(0)).getBytes()); + return requireNonNull(context, "context is null"); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TaskContext.java b/presto-main/src/main/java/com/facebook/presto/operator/TaskContext.java index a01036ef0f4a7..980f7c769be7a 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/TaskContext.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/TaskContext.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.GcMonitor; import com.facebook.presto.Session; import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.TaskId; @@ -28,8 +30,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.AtomicDouble; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.stats.CounterStat; -import io.airlift.stats.GcMonitor; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -96,6 +96,7 @@ public class TaskContext private final Object cumulativeMemoryLock = new Object(); private final AtomicDouble cumulativeUserMemory = new AtomicDouble(0.0); + private final AtomicLong peakTotalMemoryInBytes = new AtomicLong(0); @GuardedBy("cumulativeMemoryLock") private long lastUserMemoryReservation; @@ -475,6 +476,9 @@ public TaskStats getTaskStats() Duration fullGcTime = getFullGcTime(); long userMemory = taskMemoryContext.getUserMemory(); + long systemMemory = taskMemoryContext.getSystemMemory(); + + peakTotalMemoryInBytes.accumulateAndGet(userMemory + systemMemory, Math::max); synchronized (cumulativeMemoryLock) { double sinceLastPeriodMillis = (System.nanoTime() - lastTaskStatCallNanos) / 1_000_000.0; @@ -512,7 +516,8 @@ public TaskStats getTaskStats() cumulativeUserMemory.get(), succinctBytes(userMemory), succinctBytes(taskMemoryContext.getRevocableMemory()), - succinctBytes(taskMemoryContext.getSystemMemory()), + succinctBytes(systemMemory), + peakTotalMemoryInBytes.get(), succinctNanos(totalScheduledTime), succinctNanos(totalCpuTime), succinctNanos(totalBlockedTime), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TaskStats.java b/presto-main/src/main/java/com/facebook/presto/operator/TaskStats.java index 88ce4423a5ae0..a66faf8d14980 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/TaskStats.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/TaskStats.java @@ -55,6 +55,7 @@ public class TaskStats private final DataSize userMemoryReservation; private final DataSize revocableMemoryReservation; private final DataSize systemMemoryReservation; + private final long peakTotalMemoryInBytes; private final Duration totalScheduledTime; private final Duration totalCpuTime; @@ -98,6 +99,7 @@ public TaskStats(DateTime createTime, DateTime endTime) new DataSize(0, BYTE), new DataSize(0, BYTE), new DataSize(0, BYTE), + 0, new Duration(0, MILLISECONDS), new Duration(0, MILLISECONDS), new Duration(0, MILLISECONDS), @@ -137,6 +139,7 @@ public TaskStats( @JsonProperty("userMemoryReservation") DataSize userMemoryReservation, @JsonProperty("revocableMemoryReservation") DataSize revocableMemoryReservation, @JsonProperty("systemMemoryReservation") DataSize systemMemoryReservation, + @JsonProperty("peakTotalMemoryInBytes") long peakTotalMemoryInBytes, @JsonProperty("totalScheduledTime") Duration totalScheduledTime, @JsonProperty("totalCpuTime") Duration totalCpuTime, @@ -190,6 +193,7 @@ public TaskStats( this.userMemoryReservation = requireNonNull(userMemoryReservation, "userMemoryReservation is null"); this.revocableMemoryReservation = requireNonNull(revocableMemoryReservation, "revocableMemoryReservation is null"); this.systemMemoryReservation = requireNonNull(systemMemoryReservation, "systemMemoryReservation is null"); + this.peakTotalMemoryInBytes = peakTotalMemoryInBytes; this.totalScheduledTime = requireNonNull(totalScheduledTime, "totalScheduledTime is null"); this.totalCpuTime = requireNonNull(totalCpuTime, "totalCpuTime is null"); @@ -318,6 +322,12 @@ public DataSize getSystemMemoryReservation() return systemMemoryReservation; } + @JsonProperty + public long getPeakTotalMemoryInBytes() + { + return peakTotalMemoryInBytes; + } + @JsonProperty public Duration getTotalScheduledTime() { @@ -441,6 +451,7 @@ public TaskStats summarize() userMemoryReservation, revocableMemoryReservation, systemMemoryReservation, + peakTotalMemoryInBytes, totalScheduledTime, totalCpuTime, totalBlockedTime, @@ -479,6 +490,7 @@ public TaskStats summarizeFinal() userMemoryReservation, revocableMemoryReservation, systemMemoryReservation, + peakTotalMemoryInBytes, totalScheduledTime, totalCpuTime, totalBlockedTime, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/UncheckedByteArrays.java b/presto-main/src/main/java/com/facebook/presto/operator/UncheckedByteArrays.java new file mode 100644 index 0000000000000..ffe4f480060da --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/UncheckedByteArrays.java @@ -0,0 +1,94 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import com.facebook.presto.spi.api.Experimental; +import sun.misc.Unsafe; + +import java.lang.reflect.Field; + +import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_BYTE_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_INT_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_LONG_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_SHORT_INDEX_SCALE; + +// This class is a copy-paste of ByteArrays from airlift with range checks removed. +@Experimental +public class UncheckedByteArrays +{ + private static final Unsafe unsafe; + + static { + try { + // fetch theUnsafe object + Field field = Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); + unsafe = (Unsafe) field.get(null); + if (unsafe == null) { + throw new RuntimeException("Unsafe access not available"); + } + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static byte getByteUnchecked(byte[] bytes, int index) + { + return unsafe.getByte(bytes, (long) index + ARRAY_BYTE_BASE_OFFSET); + } + + public static int setByteUnchecked(byte[] bytes, int index, byte value) + { + unsafe.putByte(bytes, (long) index + ARRAY_BYTE_BASE_OFFSET, value); + return index + ARRAY_BYTE_INDEX_SCALE; + } + + public static short getShortUnchecked(byte[] bytes, int index) + { + return unsafe.getShort(bytes, (long) index + ARRAY_BYTE_BASE_OFFSET); + } + + public static int setShortUnchecked(byte[] bytes, int index, short value) + { + unsafe.putShort(bytes, (long) index + ARRAY_BYTE_BASE_OFFSET, value); + return index + ARRAY_SHORT_INDEX_SCALE; + } + + public static int getIntUnchecked(byte[] bytes, int index) + { + return unsafe.getInt(bytes, (long) index + ARRAY_BYTE_BASE_OFFSET); + } + + public static int setIntUnchecked(byte[] bytes, int index, int value) + { + unsafe.putInt(bytes, (long) index + ARRAY_BYTE_BASE_OFFSET, value); + return index + ARRAY_INT_INDEX_SCALE; + } + + public static long getLongUnchecked(byte[] bytes, int index) + { + return unsafe.getLong(bytes, (long) index + ARRAY_BYTE_BASE_OFFSET); + } + + public static int setLongUnchecked(byte[] bytes, int index, long value) + { + unsafe.putLong(bytes, (long) index + ARRAY_BYTE_BASE_OFFSET, value); + return index + ARRAY_LONG_INDEX_SCALE; + } + + private UncheckedByteArrays() + {} +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/UnnestOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/UnnestOperator.java deleted file mode 100644 index 85a8094a3709a..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/operator/UnnestOperator.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.operator; - -import com.facebook.presto.SystemSessionProperties; -import com.facebook.presto.spi.Page; -import com.facebook.presto.spi.PageBuilder; -import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.plan.PlanNodeId; -import com.facebook.presto.spi.type.ArrayType; -import com.facebook.presto.spi.type.MapType; -import com.facebook.presto.spi.type.RowType; -import com.facebook.presto.spi.type.Type; -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.List; - -import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static java.util.Objects.requireNonNull; - -public class UnnestOperator - implements Operator -{ - public static class UnnestOperatorFactory - implements OperatorFactory - { - private final int operatorId; - private final PlanNodeId planNodeId; - private final List replicateChannels; - private final List replicateTypes; - private final List unnestChannels; - private final List unnestTypes; - private final boolean withOrdinality; - private boolean closed; - - public UnnestOperatorFactory(int operatorId, PlanNodeId planNodeId, List replicateChannels, List replicateTypes, List unnestChannels, List unnestTypes, boolean withOrdinality) - { - this.operatorId = operatorId; - this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); - this.replicateChannels = ImmutableList.copyOf(requireNonNull(replicateChannels, "replicateChannels is null")); - this.replicateTypes = ImmutableList.copyOf(requireNonNull(replicateTypes, "replicateTypes is null")); - checkArgument(replicateChannels.size() == replicateTypes.size(), "replicateChannels and replicateTypes do not match"); - this.unnestChannels = ImmutableList.copyOf(requireNonNull(unnestChannels, "unnestChannels is null")); - this.unnestTypes = ImmutableList.copyOf(requireNonNull(unnestTypes, "unnestTypes is null")); - checkArgument(unnestChannels.size() == unnestTypes.size(), "unnestChannels and unnestTypes do not match"); - this.withOrdinality = withOrdinality; - } - - @Override - public Operator createOperator(DriverContext driverContext) - { - checkState(!closed, "Factory is already closed"); - OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, UnnestOperator.class.getSimpleName()); - return new UnnestOperator(operatorContext, replicateChannels, replicateTypes, unnestChannels, unnestTypes, withOrdinality, SystemSessionProperties.isLegacyUnnest(driverContext.getSession())); - } - - @Override - public void noMoreOperators() - { - closed = true; - } - - @Override - public OperatorFactory duplicate() - { - return new UnnestOperatorFactory(operatorId, planNodeId, replicateChannels, replicateTypes, unnestChannels, unnestTypes, withOrdinality); - } - } - - private final OperatorContext operatorContext; - private final List replicateChannels; - private final List replicateTypes; - private final List unnestChannels; - private final List unnestTypes; - private final boolean withOrdinality; - private final PageBuilder pageBuilder; - private final List unnesters; - private boolean finishing; - private Page currentPage; - private int currentPosition; - private int ordinalityCount; - - public UnnestOperator(OperatorContext operatorContext, List replicateChannels, List replicateTypes, List unnestChannels, List unnestTypes, boolean withOrdinality, boolean isLegacyUnnest) - { - this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); - this.replicateChannels = ImmutableList.copyOf(requireNonNull(replicateChannels, "replicateChannels is null")); - this.replicateTypes = ImmutableList.copyOf(requireNonNull(replicateTypes, "replicateTypes is null")); - this.unnestChannels = ImmutableList.copyOf(requireNonNull(unnestChannels, "unnestChannels is null")); - this.unnestTypes = ImmutableList.copyOf(requireNonNull(unnestTypes, "unnestTypes is null")); - this.withOrdinality = withOrdinality; - checkArgument(replicateChannels.size() == replicateTypes.size(), "replicate channels or types has wrong size"); - checkArgument(unnestChannels.size() == unnestTypes.size(), "unnest channels or types has wrong size"); - ImmutableList.Builder outputTypesBuilder = ImmutableList.builder() - .addAll(replicateTypes) - .addAll(getUnnestedTypes(unnestTypes, isLegacyUnnest)); - if (withOrdinality) { - outputTypesBuilder.add(BIGINT); - } - this.pageBuilder = new PageBuilder(outputTypesBuilder.build()); - this.unnesters = new ArrayList<>(unnestTypes.size()); - for (Type type : unnestTypes) { - if (type instanceof ArrayType) { - Type elementType = ((ArrayType) type).getElementType(); - if (!isLegacyUnnest && elementType instanceof RowType) { - unnesters.add(new ArrayOfRowsUnnester(elementType)); - } - else { - unnesters.add(new ArrayUnnester(elementType)); - } - } - else if (type instanceof MapType) { - MapType mapType = (MapType) type; - unnesters.add(new MapUnnester(mapType.getKeyType(), mapType.getValueType())); - } - else { - throw new IllegalArgumentException("Cannot unnest type: " + type); - } - } - } - - private static List getUnnestedTypes(List types, boolean isLegacyUnnest) - { - ImmutableList.Builder builder = ImmutableList.builder(); - for (Type type : types) { - checkArgument(type instanceof ArrayType || type instanceof MapType, "Can only unnest map and array types"); - if (type instanceof ArrayType && !isLegacyUnnest && ((ArrayType) type).getElementType() instanceof RowType) { - builder.addAll(((ArrayType) type).getElementType().getTypeParameters()); - } - else { - builder.addAll(type.getTypeParameters()); - } - } - return builder.build(); - } - - @Override - public OperatorContext getOperatorContext() - { - return operatorContext; - } - - @Override - public void finish() - { - finishing = true; - } - - @Override - public boolean isFinished() - { - return finishing && pageBuilder.isEmpty() && currentPage == null; - } - - @Override - public boolean needsInput() - { - return !finishing && !pageBuilder.isFull() && currentPage == null; - } - - @Override - public void addInput(Page page) - { - checkState(!finishing, "Operator is already finishing"); - requireNonNull(page, "page is null"); - checkState(currentPage == null, "currentPage is not null"); - checkState(!pageBuilder.isFull(), "Page buffer is full"); - - currentPage = page; - currentPosition = 0; - fillUnnesters(); - } - - private void fillUnnesters() - { - for (int i = 0; i < unnestTypes.size(); i++) { - Type type = unnestTypes.get(i); - int channel = unnestChannels.get(i); - Block block = null; - if (!currentPage.getBlock(channel).isNull(currentPosition)) { - block = (Block) type.getObject(currentPage.getBlock(channel), currentPosition); - } - unnesters.get(i).setBlock(block); - } - ordinalityCount = 0; - } - - private boolean anyUnnesterHasData() - { - for (Unnester unnester : unnesters) { - if (unnester.hasNext()) { - return true; - } - } - return false; - } - - @Override - public Page getOutput() - { - while (!pageBuilder.isFull() && currentPage != null) { - // Advance until we find data to unnest - while (!anyUnnesterHasData()) { - currentPosition++; - if (currentPosition == currentPage.getPositionCount()) { - currentPage = null; - currentPosition = 0; - break; - } - fillUnnesters(); - } - while (!pageBuilder.isFull() && anyUnnesterHasData()) { - // Copy all the channels marked for replication - for (int replicateChannel = 0; replicateChannel < replicateTypes.size(); replicateChannel++) { - Type type = replicateTypes.get(replicateChannel); - int channel = replicateChannels.get(replicateChannel); - type.appendTo(currentPage.getBlock(channel), currentPosition, pageBuilder.getBlockBuilder(replicateChannel)); - } - int offset = replicateTypes.size(); - - pageBuilder.declarePosition(); - for (Unnester unnester : unnesters) { - if (unnester.hasNext()) { - unnester.appendNext(pageBuilder, offset); - } - else { - for (int unnesterChannelIndex = 0; unnesterChannelIndex < unnester.getChannelCount(); unnesterChannelIndex++) { - pageBuilder.getBlockBuilder(offset + unnesterChannelIndex).appendNull(); - } - } - offset += unnester.getChannelCount(); - } - - if (withOrdinality) { - ordinalityCount++; - BIGINT.writeLong(pageBuilder.getBlockBuilder(offset), ordinalityCount); - } - } - } - - if ((!finishing && !pageBuilder.isFull()) || pageBuilder.isEmpty()) { - return null; - } - - Page page = pageBuilder.build(); - pageBuilder.reset(); - return page; - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/WindowFunctionDefinition.java b/presto-main/src/main/java/com/facebook/presto/operator/WindowFunctionDefinition.java index 6da2a00014e21..6a339a3ec78bc 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/WindowFunctionDefinition.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/WindowFunctionDefinition.java @@ -30,10 +30,11 @@ public class WindowFunctionDefinition private final Type type; private final FrameInfo frameInfo; private final List argumentChannels; + private final boolean ignoreNulls; public static WindowFunctionDefinition window(WindowFunctionSupplier functionSupplier, Type type, FrameInfo frameInfo, List inputs) { - return new WindowFunctionDefinition(functionSupplier, type, frameInfo, inputs); + return new WindowFunctionDefinition(functionSupplier, type, frameInfo, false, inputs); } public static WindowFunctionDefinition window(WindowFunctionSupplier functionSupplier, Type type, FrameInfo frameInfo, Integer... inputs) @@ -41,7 +42,17 @@ public static WindowFunctionDefinition window(WindowFunctionSupplier functionSup return window(functionSupplier, type, frameInfo, Arrays.asList(inputs)); } - WindowFunctionDefinition(WindowFunctionSupplier functionSupplier, Type type, FrameInfo frameInfo, List argumentChannels) + public static WindowFunctionDefinition window(WindowFunctionSupplier functionSupplier, Type type, FrameInfo frameInfo, boolean ignoreNulls, List inputs) + { + return new WindowFunctionDefinition(functionSupplier, type, frameInfo, ignoreNulls, inputs); + } + + public static WindowFunctionDefinition window(WindowFunctionSupplier functionSupplier, Type type, FrameInfo frameInfo, boolean ignoreNulls, Integer... inputs) + { + return window(functionSupplier, type, frameInfo, ignoreNulls, Arrays.asList(inputs)); + } + + WindowFunctionDefinition(WindowFunctionSupplier functionSupplier, Type type, FrameInfo frameInfo, boolean ignoreNulls, List argumentChannels) { requireNonNull(functionSupplier, "functionSupplier is null"); requireNonNull(type, "type is null"); @@ -51,6 +62,7 @@ public static WindowFunctionDefinition window(WindowFunctionSupplier functionSup this.functionSupplier = functionSupplier; this.type = type; this.frameInfo = frameInfo; + this.ignoreNulls = ignoreNulls; this.argumentChannels = ImmutableList.copyOf(argumentChannels); } @@ -66,6 +78,6 @@ public Type getType() public WindowFunction createWindowFunction() { - return functionSupplier.createWindowFunction(argumentChannels); + return functionSupplier.createWindowFunction(argumentChannels, ignoreNulls); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxAggregationFunction.java index 18489d2f1f060..4d479af727235 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxAggregationFunction.java @@ -90,7 +90,7 @@ protected AbstractMinMaxAggregationFunction(String name, boolean min) public InternalAggregationFunction specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type type = boundVariables.getTypeVariable("E"); - MethodHandle compareMethodHandle = functionManager.getScalarFunctionImplementation( + MethodHandle compareMethodHandle = functionManager.getBuiltInScalarFunctionImplementation( functionManager.resolveOperator(operatorType, fromTypes(type, type))).getMethodHandle(); return generateAggregation(type, compareMethodHandle); } @@ -141,7 +141,7 @@ else if (type.getJavaType() == boolean.class) { Type intermediateType = stateSerializer.getSerializedType(); AggregationMetadata metadata = new AggregationMetadata( - generateAggregationName(getSignature().getName(), type.getTypeSignature(), inputTypes.stream().map(Type::getTypeSignature).collect(toImmutableList())), + generateAggregationName(getSignature().getNameSuffix(), type.getTypeSignature(), inputTypes.stream().map(Type::getTypeSignature).collect(toImmutableList())), createParameterMetadata(type), inputFunction, combineFunction, @@ -153,7 +153,7 @@ else if (type.getJavaType() == boolean.class) { type); GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader); - return new InternalAggregationFunction(getSignature().getName(), inputTypes, ImmutableList.of(intermediateType), type, true, false, factory); + return new InternalAggregationFunction(getSignature().getNameSuffix(), inputTypes, ImmutableList.of(intermediateType), type, true, false, factory); } private static List createParameterMetadata(Type type) diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxNAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxNAggregationFunction.java index f66200bf86b45..4ece5e7312b62 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxNAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AbstractMinMaxNAggregationFunction.java @@ -95,7 +95,7 @@ protected InternalAggregationFunction generateAggregation(Type type) new ParameterMetadata(BLOCK_INDEX)); AggregationMetadata metadata = new AggregationMetadata( - generateAggregationName(getSignature().getName(), type.getTypeSignature(), inputTypes.stream().map(Type::getTypeSignature).collect(toImmutableList())), + generateAggregationName(getSignature().getNameSuffix(), type.getTypeSignature(), inputTypes.stream().map(Type::getTypeSignature).collect(toImmutableList())), inputParameterMetadata, INPUT_FUNCTION.bindTo(comparator).bindTo(type), COMBINE_FUNCTION, @@ -107,7 +107,7 @@ protected InternalAggregationFunction generateAggregation(Type type) outputType); GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader); - return new InternalAggregationFunction(getSignature().getName(), inputTypes, ImmutableList.of(intermediateType), outputType, true, false, factory); + return new InternalAggregationFunction(getSignature().getNameSuffix(), inputTypes, ImmutableList.of(intermediateType), outputType, true, false, factory); } public static void input(BlockComparator comparator, Type type, MinMaxNState state, Block block, long n, int blockIndex) diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationImplementation.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationImplementation.java index 44ac7588062a5..fc81c17bedaf0 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationImplementation.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationImplementation.java @@ -26,6 +26,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.LongVariableConstraint; import com.facebook.presto.spi.function.OutputFunction; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.function.TypeParameter; @@ -44,6 +45,7 @@ import java.util.Set; import java.util.stream.Stream; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.BLOCK_INDEX; import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.STATE; import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.inputChannelParameterType; @@ -304,7 +306,7 @@ private Parser( private AggregationImplementation get() { Signature signature = new Signature( - header.getName(), + QualifiedFunctionName.of(DEFAULT_NAMESPACE, header.getName()), FunctionKind.AGGREGATE, typeVariableConstraints, longVariableConstraints, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationUtils.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationUtils.java index 5873d07c3b6d3..889cf4fbf7d01 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationUtils.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.operator.aggregation.state.CentralMomentsState; import com.facebook.presto.operator.aggregation.state.CorrelationState; import com.facebook.presto.operator.aggregation.state.CovarianceState; @@ -20,10 +21,12 @@ import com.facebook.presto.operator.aggregation.state.VarianceState; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.type.TypeSignature; import com.google.common.base.CaseFormat; import java.util.List; +import java.util.Optional; import java.util.function.Function; import static com.google.common.base.Preconditions.checkArgument; @@ -35,6 +38,38 @@ private AggregationUtils() { } + public static boolean isDecomposable(AggregationNode aggregationNode, FunctionManager functionManager) + { + boolean hasOrderBy = aggregationNode.getAggregations().values().stream() + .map(AggregationNode.Aggregation::getOrderBy) + .anyMatch(Optional::isPresent); + + boolean hasDistinct = aggregationNode.getAggregations().values().stream() + .anyMatch(AggregationNode.Aggregation::isDistinct); + + boolean decomposableFunctions = aggregationNode.getAggregations().values().stream() + .map(AggregationNode.Aggregation::getFunctionHandle) + .map(functionManager::getAggregateFunctionImplementation) + .allMatch(InternalAggregationFunction::isDecomposable); + + return !hasOrderBy && !hasDistinct && decomposableFunctions; + } + + public static boolean hasSingleNodeExecutionPreference(AggregationNode aggregationNode, FunctionManager functionManager) + { + // There are two kinds of aggregations the have single node execution preference: + // + // 1. aggregations with only empty grouping sets like + // + // SELECT count(*) FROM lineitem; + // + // there is no need for distributed aggregation. Single node FINAL aggregation will suffice, + // since all input have to be aggregated into one line output. + // + // 2. aggregations that must produce default output and are not decomposable, we can not distribute them. + return (aggregationNode.hasEmptyGroupingSet() && !aggregationNode.hasNonEmptyGroupingSet()) || (aggregationNode.hasDefaultOutput() && !isDecomposable(aggregationNode, functionManager)); + } + public static void updateVarianceState(VarianceState state, double value) { state.setCount(state.getCount() + 1); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateCountDistinctAggregation.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateCountDistinctAggregation.java index 6a96bfb43ab09..ad72863473afe 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateCountDistinctAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateCountDistinctAggregation.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.operator.aggregation.state.BooleanDistinctState; import com.facebook.presto.operator.aggregation.state.HyperLogLogState; import com.facebook.presto.spi.block.Block; @@ -29,7 +30,6 @@ import com.facebook.presto.spi.function.TypeParameter; import com.facebook.presto.spi.type.StandardTypes; import io.airlift.slice.Slice; -import io.airlift.stats.cardinality.HyperLogLog; import java.lang.invoke.MethodHandle; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateDoublePercentileAggregations.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateDoublePercentileAggregations.java index 6bb8774061fdf..eb57f0baae48d 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateDoublePercentileAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateDoublePercentileAggregations.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.operator.aggregation.state.DigestAndPercentileState; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AggregationFunction; @@ -22,7 +23,6 @@ import com.facebook.presto.spi.function.OutputFunction; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; -import io.airlift.stats.QuantileDigest; import static com.facebook.presto.operator.aggregation.FloatingPointBitsConverterUtil.doubleToSortableLong; import static com.facebook.presto.operator.aggregation.FloatingPointBitsConverterUtil.sortableLongToDouble; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateDoublePercentileArrayAggregations.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateDoublePercentileArrayAggregations.java index 150952d2fb2f2..19920ff9b9553 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateDoublePercentileArrayAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateDoublePercentileArrayAggregations.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.operator.aggregation.state.DigestAndPercentileArrayState; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; @@ -23,7 +24,6 @@ import com.facebook.presto.spi.function.OutputFunction; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; -import io.airlift.stats.QuantileDigest; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateLongPercentileAggregations.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateLongPercentileAggregations.java index e07382db85d93..de59693367d68 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateLongPercentileAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateLongPercentileAggregations.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.operator.aggregation.state.DigestAndPercentileState; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AggregationFunction; @@ -22,7 +23,6 @@ import com.facebook.presto.spi.function.OutputFunction; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; -import io.airlift.stats.QuantileDigest; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.type.BigintType.BIGINT; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateLongPercentileArrayAggregations.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateLongPercentileArrayAggregations.java index c0954f937915a..51259bb7830a4 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateLongPercentileArrayAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateLongPercentileArrayAggregations.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.operator.aggregation.state.DigestAndPercentileArrayState; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; @@ -24,7 +25,6 @@ import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; import com.google.common.collect.ImmutableList; -import io.airlift.stats.QuantileDigest; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateRealPercentileAggregations.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateRealPercentileAggregations.java index 570d2a9706b3e..252a22c68caef 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateRealPercentileAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateRealPercentileAggregations.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.operator.aggregation.state.DigestAndPercentileState; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AggregationFunction; @@ -22,7 +23,6 @@ import com.facebook.presto.spi.function.OutputFunction; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; -import io.airlift.stats.QuantileDigest; import static com.facebook.presto.operator.aggregation.FloatingPointBitsConverterUtil.floatToSortableInt; import static com.facebook.presto.operator.aggregation.FloatingPointBitsConverterUtil.sortableIntToFloat; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateRealPercentileArrayAggregations.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateRealPercentileArrayAggregations.java index f22b563734fc8..25e7bc47f2d7c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateRealPercentileArrayAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateRealPercentileArrayAggregations.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.operator.aggregation.state.DigestAndPercentileArrayState; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; @@ -23,7 +24,6 @@ import com.facebook.presto.spi.function.OutputFunction; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; -import io.airlift.stats.QuantileDigest; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateSetAggregation.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateSetAggregation.java index 0ee9d8a250841..d86da8275e477 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateSetAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ApproximateSetAggregation.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.operator.aggregation.state.HyperLogLogState; import com.facebook.presto.operator.aggregation.state.StateCompiler; import com.facebook.presto.spi.block.BlockBuilder; @@ -26,7 +27,6 @@ import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; import io.airlift.slice.Slice; -import io.airlift.stats.cardinality.HyperLogLog; @AggregationFunction("approx_set") public final class ApproximateSetAggregation diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/HyperLogLogUtils.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/HyperLogLogUtils.java index 960ea6dd5409d..aad5984e68516 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/HyperLogLogUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/HyperLogLogUtils.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.operator.aggregation.state.HyperLogLogState; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.AggregationState; -import io.airlift.stats.cardinality.HyperLogLog; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.util.Failures.checkCondition; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeHyperLogLogAggregation.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeHyperLogLogAggregation.java index f5da5a6decd5c..1760129f498b3 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeHyperLogLogAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeHyperLogLogAggregation.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.operator.aggregation.state.HyperLogLogState; import com.facebook.presto.operator.aggregation.state.StateCompiler; import com.facebook.presto.spi.block.BlockBuilder; @@ -25,7 +26,6 @@ import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; import io.airlift.slice.Slice; -import io.airlift.stats.cardinality.HyperLogLog; @AggregationFunction("merge") public final class MergeHyperLogLogAggregation diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java index 4f1a7b9223309..6fc810bc15d70 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.bytecode.DynamicClassLoader; import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.FunctionManager; @@ -31,7 +32,6 @@ import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignatureParameter; import com.google.common.collect.ImmutableList; -import io.airlift.stats.QuantileDigest; import java.lang.invoke.MethodHandle; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ParametricAggregation.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ParametricAggregation.java index f96e51468417c..6d44fa1933779 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ParametricAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ParametricAggregation.java @@ -94,7 +94,7 @@ public InternalAggregationFunction specialize(BoundVariables variables, int arit List parametersMetadata = buildParameterMetadata(concreteImplementation.getInputParameterMetadataTypes(), inputTypes); // Generate Aggregation name - String aggregationName = generateAggregationName(getSignature().getName(), outputType.getTypeSignature(), signaturesFromTypes(inputTypes)); + String aggregationName = generateAggregationName(getSignature().getNameSuffix(), outputType.getTypeSignature(), signaturesFromTypes(inputTypes)); // Collect all collected data in Metadata AggregationMetadata metadata = new AggregationMetadata( @@ -110,7 +110,7 @@ public InternalAggregationFunction specialize(BoundVariables variables, int arit outputType); // Create specialized InternalAggregregationFunction for Presto - return new InternalAggregationFunction(getSignature().getName(), + return new InternalAggregationFunction(getSignature().getNameSuffix(), inputTypes, ImmutableList.of(stateSerializer.getSerializedType()), outputType, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java index 085b8f15690d1..e14e5708df592 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.bytecode.DynamicClassLoader; import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.FunctionManager; @@ -28,7 +29,6 @@ import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.spi.type.TypeSignatureParameter; import com.google.common.collect.ImmutableList; -import io.airlift.stats.QuantileDigest; import java.lang.invoke.MethodHandle; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ReduceAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ReduceAggregationFunction.java index 5b6c6fea1eed5..39f951e58327f 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ReduceAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ReduceAggregationFunction.java @@ -131,7 +131,7 @@ else if (stateType.getJavaType() == boolean.class) { } AggregationMetadata metadata = new AggregationMetadata( - generateAggregationName(getSignature().getName(), inputType.getTypeSignature(), ImmutableList.of(inputType.getTypeSignature())), + generateAggregationName(getSignature().getNameSuffix(), inputType.getTypeSignature(), ImmutableList.of(inputType.getTypeSignature())), createInputParameterMetadata(inputType, stateType), inputMethodHandle.asType( inputMethodHandle.type() @@ -144,7 +144,7 @@ else if (stateType.getJavaType() == boolean.class) { GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader); return new InternalAggregationFunction( - getSignature().getName(), + getSignature().getNameSuffix(), ImmutableList.of(inputType), ImmutableList.of(stateType), stateType, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TypedSet.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TypedSet.java index 5135afb062ed5..6843e5cfa3bb4 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TypedSet.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TypedSet.java @@ -106,7 +106,7 @@ public boolean contains(Block block, int position) } } - public void add(Block block, int position) + public boolean add(Block block, int position) { requireNonNull(block, "block must not be null"); checkArgument(position >= 0, "position must be >= 0"); @@ -119,7 +119,24 @@ public void add(Block block, int position) int hashPosition = getHashPositionOfElement(block, position); if (blockPositionByHash.get(hashPosition) == EMPTY_SLOT) { addNewElement(hashPosition, block, position); + return true; } + + return false; + } + + public boolean addNonNull(Block block, int position) + { + requireNonNull(block, "block must not be null"); + checkArgument(position >= 0, "position must be >= 0"); + + int hashPosition = getHashPositionOfElement(block, position); + if (blockPositionByHash.get(hashPosition) == EMPTY_SLOT) { + addNewElement(hashPosition, block, position); + return true; + } + + return false; } public int size() diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/builder/InMemoryHashAggregationBuilder.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/builder/InMemoryHashAggregationBuilder.java index cdd7ad23517d1..2d5b047c918d0 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/builder/InMemoryHashAggregationBuilder.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/builder/InMemoryHashAggregationBuilder.java @@ -29,10 +29,10 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.JoinCompiler; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/builder/MergingHashAggregationBuilder.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/builder/MergingHashAggregationBuilder.java index f54c3ceb576db..7c61a54e6f10d 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/builder/MergingHashAggregationBuilder.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/builder/MergingHashAggregationBuilder.java @@ -20,9 +20,9 @@ import com.facebook.presto.operator.WorkProcessor.TransformationState; import com.facebook.presto.operator.aggregation.AccumulatorFactory; import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.JoinCompiler; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/builder/SpillableHashAggregationBuilder.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/builder/SpillableHashAggregationBuilder.java index 900302cdd610d..b368335940135 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/builder/SpillableHashAggregationBuilder.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/builder/SpillableHashAggregationBuilder.java @@ -21,11 +21,11 @@ import com.facebook.presto.operator.WorkProcessor; import com.facebook.presto.operator.aggregation.AccumulatorFactory; import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spiller.Spiller; import com.facebook.presto.spiller.SpillerFactory; import com.facebook.presto.sql.gen.JoinCompiler; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.google.common.collect.ImmutableList; import com.google.common.io.Closer; import com.google.common.util.concurrent.ListenableFuture; @@ -35,10 +35,10 @@ import java.util.List; import java.util.Optional; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Futures.immediateFuture; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static java.lang.Math.max; public class SpillableHashAggregationBuilder diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyAggregation.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyAggregation.java new file mode 100644 index 0000000000000..3aaa7aa46b9ae --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyAggregation.java @@ -0,0 +1,111 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.function.AggregationFunction; +import com.facebook.presto.spi.function.AggregationState; +import com.facebook.presto.spi.function.CombineFunction; +import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.InputFunction; +import com.facebook.presto.spi.function.OutputFunction; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.type.StandardTypes; +import io.airlift.slice.Slice; + +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static java.util.Locale.ENGLISH; + +@AggregationFunction("differential_entropy") +@Description("Computes differential entropy based on random-variable samples") +public final class DifferentialEntropyAggregation +{ + private DifferentialEntropyAggregation() {} + + @InputFunction + public static void input( + @AggregationState DifferentialEntropyState state, + @SqlType(StandardTypes.BIGINT) long size, + @SqlType(StandardTypes.DOUBLE) double sample, + @SqlType(StandardTypes.DOUBLE) double weight, + @SqlType(StandardTypes.VARCHAR) Slice method, + @SqlType(StandardTypes.DOUBLE) double min, + @SqlType(StandardTypes.DOUBLE) double max) + { + DifferentialEntropyStateStrategy strategy = DifferentialEntropyStateStrategy.getStrategy( + state.getStrategy(), + size, + sample, + weight, + method.toStringUtf8().toLowerCase(ENGLISH), + min, + max); + state.setStrategy(strategy); + strategy.add(sample, weight); + } + + @InputFunction + public static void input( + @AggregationState DifferentialEntropyState state, + @SqlType(StandardTypes.BIGINT) long size, + @SqlType(StandardTypes.DOUBLE) double sample, + @SqlType(StandardTypes.DOUBLE) double weight) + { + DifferentialEntropyStateStrategy strategy = DifferentialEntropyStateStrategy.getStrategy( + state.getStrategy(), + size, + sample, + weight); + state.setStrategy(strategy); + strategy.add(sample, weight); + } + + @InputFunction + public static void input( + @AggregationState DifferentialEntropyState state, + @SqlType(StandardTypes.BIGINT) long size, + @SqlType(StandardTypes.DOUBLE) double sample) + { + DifferentialEntropyStateStrategy strategy = DifferentialEntropyStateStrategy.getStrategy( + state.getStrategy(), + size, + sample); + state.setStrategy(strategy); + strategy.add(sample); + } + + @CombineFunction + public static void combine( + @AggregationState DifferentialEntropyState state, + @AggregationState DifferentialEntropyState otherState) + { + DifferentialEntropyStateStrategy strategy = state.getStrategy(); + DifferentialEntropyStateStrategy otherStrategy = otherState.getStrategy(); + if (strategy == null && otherStrategy != null) { + state.setStrategy(otherStrategy); + return; + } + if (otherStrategy == null) { + return; + } + DifferentialEntropyStateStrategy.combine(strategy, otherStrategy); + } + + @OutputFunction("double") + public static void output(@AggregationState DifferentialEntropyState state, BlockBuilder out) + { + DifferentialEntropyStateStrategy strategy = state.getStrategy(); + DOUBLE.writeDouble(out, strategy == null ? Double.NaN : strategy.calculateEntropy()); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyState.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyState.java new file mode 100644 index 0000000000000..92e7c70eeb343 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyState.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.spi.function.AccumulatorState; +import com.facebook.presto.spi.function.AccumulatorStateMetadata; + +@AccumulatorStateMetadata( + stateSerializerClass = DifferentialEntropyStateSerializer.class, + stateFactoryClass = DifferentialEntropyStateFactory.class) +public interface DifferentialEntropyState + extends AccumulatorState +{ + void setStrategy(DifferentialEntropyStateStrategy strategy); + + DifferentialEntropyStateStrategy getStrategy(); +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyStateFactory.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyStateFactory.java new file mode 100644 index 0000000000000..6ddbbaa4b069e --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyStateFactory.java @@ -0,0 +1,110 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.array.ObjectBigArray; +import com.facebook.presto.operator.aggregation.state.AbstractGroupedAccumulatorState; +import com.facebook.presto.spi.function.AccumulatorStateFactory; + +import static java.util.Objects.requireNonNull; + +public class DifferentialEntropyStateFactory + implements AccumulatorStateFactory +{ + @Override + public DifferentialEntropyState createSingleState() + { + return new SingleState(); + } + + @Override + public Class getSingleStateClass() + { + return SingleState.class; + } + + @Override + public DifferentialEntropyState createGroupedState() + { + return new GroupedState(); + } + + @Override + public Class getGroupedStateClass() + { + return GroupedState.class; + } + + public static class GroupedState + extends AbstractGroupedAccumulatorState + implements DifferentialEntropyState + { + private ObjectBigArray strategies = new ObjectBigArray<>(); + private long size; + + @Override + public void ensureCapacity(long size) + { + strategies.ensureCapacity(size); + } + + @Override + public void setStrategy(DifferentialEntropyStateStrategy strategy) + { + DifferentialEntropyStateStrategy previous = requireNonNull(strategy, "strategy is null"); + if (previous != null) { + size -= previous.getEstimatedSize(); + } + + strategies.set(getGroupId(), strategy); + size += strategy.getEstimatedSize(); + } + + @Override + public DifferentialEntropyStateStrategy getStrategy() + { + return strategies.get(getGroupId()); + } + + @Override + public long getEstimatedSize() + { + return size + strategies.sizeOf(); + } + } + + public static class SingleState + implements DifferentialEntropyState + { + private DifferentialEntropyStateStrategy strategy; + + @Override + public void setStrategy(DifferentialEntropyStateStrategy strategy) + { + this.strategy = requireNonNull(strategy, "strategy is null"); + } + + @Override + public DifferentialEntropyStateStrategy getStrategy() + { + return strategy; + } + + @Override + public long getEstimatedSize() + { + return strategy == null ? 0 : strategy.getEstimatedSize(); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyStateSerializer.java new file mode 100644 index 0000000000000..d07ec5a5b4c77 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyStateSerializer.java @@ -0,0 +1,57 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.function.AccumulatorStateSerializer; +import com.facebook.presto.spi.type.Type; +import io.airlift.slice.SliceInput; +import io.airlift.slice.SliceOutput; +import io.airlift.slice.Slices; + +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; + +public class DifferentialEntropyStateSerializer + implements AccumulatorStateSerializer +{ + @Override + public Type getSerializedType() + { + return VARBINARY; + } + + @Override + public void serialize(DifferentialEntropyState state, BlockBuilder output) + { + DifferentialEntropyStateStrategy strategy = state.getStrategy(); + int requiredBytes = DifferentialEntropyStateStrategy.getRequiredBytesForSerialization(strategy); + SliceOutput sliceOut = Slices.allocate(requiredBytes).getOutput(); + DifferentialEntropyStateStrategy.serialize(strategy, sliceOut); + VARBINARY.writeSlice(output, sliceOut.getUnderlyingSlice()); + } + + @Override + public void deserialize( + Block block, + int index, + DifferentialEntropyState state) + { + SliceInput input = VARBINARY.getSlice(block, index).getInput(); + DifferentialEntropyStateStrategy strategy = DifferentialEntropyStateStrategy.deserialize(input); + if (strategy != null) { + state.setStrategy(strategy); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyStateStrategy.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyStateStrategy.java new file mode 100644 index 0000000000000..957952fd960cb --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/DifferentialEntropyStateStrategy.java @@ -0,0 +1,230 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.spi.PrestoException; +import com.google.common.annotations.VisibleForTesting; +import io.airlift.slice.SizeOf; +import io.airlift.slice.SliceInput; +import io.airlift.slice.SliceOutput; + +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.google.common.base.Verify.verify; +import static java.lang.String.format; + +/** + * Interface for different strategies for calculating entropy: MLE (maximum likelihood + * estimator) using NumericHistogram, jacknife estimates using a fixed histogram, compressed + * counting and Renyi entropy, and so forth. + */ +public interface DifferentialEntropyStateStrategy + extends Cloneable +{ + @VisibleForTesting + String FIXED_HISTOGRAM_MLE_METHOD_NAME = "fixed_histogram_mle"; + @VisibleForTesting + String FIXED_HISTOGRAM_JACKNIFE_METHOD_NAME = "fixed_histogram_jacknife"; + + static DifferentialEntropyStateStrategy getStrategy( + DifferentialEntropyStateStrategy strategy, + long size, + double sample, + double weight, + String method, + double min, + double max) + { + if (strategy == null) { + switch (method) { + case DifferentialEntropyStateStrategy.FIXED_HISTOGRAM_MLE_METHOD_NAME: + strategy = new FixedHistogramMleStateStrategy(size, min, max); + break; + case DifferentialEntropyStateStrategy.FIXED_HISTOGRAM_JACKNIFE_METHOD_NAME: + strategy = new FixedHistogramJacknifeStateStrategy(size, min, max); + break; + default: + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, invalid method: %s", method)); + } + } + else { + switch (method) { + case DifferentialEntropyStateStrategy.FIXED_HISTOGRAM_MLE_METHOD_NAME: + if (!(strategy instanceof FixedHistogramMleStateStrategy)) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy, strategy class is not compatible with entropy method: %s %s", strategy.getClass().getSimpleName(), method)); + } + break; + case DifferentialEntropyStateStrategy.FIXED_HISTOGRAM_JACKNIFE_METHOD_NAME: + if (!(strategy instanceof FixedHistogramJacknifeStateStrategy)) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy, strategy class is not compatible with entropy method: %s %s", strategy.getClass().getSimpleName(), method)); + } + break; + default: + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy, unknown entropy method: %s", method)); + } + } + strategy.validateParameters(size, sample, weight, min, max); + return strategy; + } + + static DifferentialEntropyStateStrategy getStrategy( + DifferentialEntropyStateStrategy strategy, + long size, + double sample, + double weight) + { + if (strategy == null) { + strategy = new WeightedReservoirSampleStateStrategy(size); + } + else { + verify(strategy instanceof WeightedReservoirSampleStateStrategy, + format("In differential entropy, expected WeightedReservoirSampleStateStrategy, got: %s", strategy.getClass().getSimpleName())); + } + strategy.validateParameters(size, sample, weight); + return strategy; + } + + static DifferentialEntropyStateStrategy getStrategy( + DifferentialEntropyStateStrategy strategy, + long size, + double sample) + { + if (strategy == null) { + strategy = new UnweightedReservoirSampleStateStrategy(size); + } + else { + verify(strategy instanceof UnweightedReservoirSampleStateStrategy, + format("In differential entropy, expected UnweightedReservoirSampleStateStrategy, got: %s", strategy.getClass().getSimpleName())); + } + return strategy; + } + + default void add(double sample) + { + verify(false, format("Unweighted unsupported for type: %s", getClass().getSimpleName())); + } + + default void add(double sample, double weight) + { + verify(false, format("Weighted unsupported for type: %s", getClass().getSimpleName())); + } + + double calculateEntropy(); + + long getEstimatedSize(); + + static int getRequiredBytesForSerialization(DifferentialEntropyStateStrategy strategy) + { + return SizeOf.SIZE_OF_INT + // magic hash + SizeOf.SIZE_OF_INT + // method + (strategy == null ? 0 : strategy.getRequiredBytesForSpecificSerialization()); + } + + int getRequiredBytesForSpecificSerialization(); + + void serialize(SliceOutput out); + + void mergeWith(DifferentialEntropyStateStrategy other); + + DifferentialEntropyStateStrategy clone(); + + default void validateParameters(long size, double sample, double weight, double min, double max) + { + throw new UnsupportedOperationException( + format("In differential_entropy UDF, unsupported arguments for type: %s", getClass().getSimpleName())); + } + + default void validateParameters(long size, double sample, double weight) + { + throw new UnsupportedOperationException( + format("In differential_entropy UDF, unsupported arguments for type: %s", getClass().getSimpleName())); + } + + default void validateParameters(long size, double sample) + { + throw new UnsupportedOperationException( + format("In differential_entropy UDF, unsupported arguments for type: %s", getClass().getSimpleName())); + } + + static void serialize(DifferentialEntropyStateStrategy strategy, SliceOutput sliceOut) + { + sliceOut.appendInt(DifferentialEntropyStateStrategy.class.getSimpleName().hashCode()); + if (strategy == null) { + sliceOut.appendInt(0); + return; + } + + if (strategy instanceof UnweightedReservoirSampleStateStrategy) { + sliceOut.appendInt(1); + } + else if (strategy instanceof WeightedReservoirSampleStateStrategy) { + sliceOut.appendInt(2); + } + else if (strategy instanceof FixedHistogramMleStateStrategy) { + sliceOut.appendInt(3); + } + else if (strategy instanceof FixedHistogramJacknifeStateStrategy) { + sliceOut.appendInt(4); + } + else { + verify(false, format("Strategy cannot be serialized: %s", strategy.getClass().getSimpleName())); + } + + strategy.serialize(sliceOut); + } + + static DifferentialEntropyStateStrategy deserialize(SliceInput input) + { + verify( + input.readInt() == DifferentialEntropyStateStrategy.class.getSimpleName().hashCode(), + "magic failed"); + int method = input.readInt(); + switch (method) { + case 0: + return null; + case 1: + return UnweightedReservoirSampleStateStrategy.deserialize(input); + case 2: + return WeightedReservoirSampleStateStrategy.deserialize(input); + case 3: + return FixedHistogramMleStateStrategy.deserialize(input); + case 4: + return FixedHistogramJacknifeStateStrategy.deserialize(input); + default: + verify(false, format("In differential_entropy UDF, Unknown method code when deserializing: %s", method)); + return null; + } + } + + static void combine( + DifferentialEntropyStateStrategy strategy, + DifferentialEntropyStateStrategy otherStrategy) + { + verify(strategy.getClass() == otherStrategy.getClass(), + format("In combine, %s != %s", strategy.getClass().getSimpleName(), otherStrategy.getClass().getSimpleName())); + + strategy.mergeWith(otherStrategy); + } + + DifferentialEntropyStateStrategy cloneEmpty(); + + double getTotalPopulationWeight(); +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/EntropyCalculations.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/EntropyCalculations.java new file mode 100644 index 0000000000000..479a6b99d7ee6 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/EntropyCalculations.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import java.util.Arrays; + +import static com.google.common.base.Verify.verify; +import static java.lang.Math.toIntExact; + +public class EntropyCalculations +{ + private EntropyCalculations() {} + + /** + * @implNote Based on Alizadeh Noughabi, Hadi & Arghami, N. (2010). "A New Estimator of Entropy". + */ + public static double calculateFromSamplesUsingVasicek(double[] samples) + { + if (samples.length == 0) { + return Double.NaN; + } + + Arrays.sort(samples); + int n = samples.length; + int m = toIntExact(Math.max(Math.round(Math.sqrt(n)), 2)); + double entropy = 0; + for (int i = 0; i < n; i++) { + double sIPlusM = i + m < n ? samples[i + m] : samples[n - 1]; + double sIMinusM = i - m > 0 ? samples[i - m] : samples[0]; + double aI = i + m < n && i - m > 0 ? 2 : 1; + entropy += Math.log(n / (aI * m) * (sIPlusM - sIMinusM)); + } + return entropy / n / Math.log(2); + } + + static double calculateEntropyFromHistogramAggregates(double width, double sumWeight, double sumWeightLogWeight) + { + verify(sumWeight > 0.0); + return Math.max((Math.log(width * sumWeight) - sumWeightLogWeight / sumWeight) / Math.log(2.0), 0.0); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/FixedHistogramJacknifeStateStrategy.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/FixedHistogramJacknifeStateStrategy.java new file mode 100644 index 0000000000000..3d9316b60149f --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/FixedHistogramJacknifeStateStrategy.java @@ -0,0 +1,181 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.operator.aggregation.fixedhistogram.FixedDoubleBreakdownHistogram; +import io.airlift.slice.SliceInput; +import io.airlift.slice.SliceOutput; + +import java.util.Map; + +import static com.facebook.presto.operator.aggregation.differentialentropy.EntropyCalculations.calculateEntropyFromHistogramAggregates; +import static com.facebook.presto.operator.aggregation.differentialentropy.FixedHistogramStateStrategyUtils.getXLogX; +import static com.google.common.collect.Streams.stream; +import static java.lang.Math.toIntExact; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.summingDouble; + +/** + * Calculates sample entropy using jacknife estimates on a fixed histogram. + * See http://cs.brown.edu/~pvaliant/unseen_nips.pdf. + */ +public class FixedHistogramJacknifeStateStrategy + implements DifferentialEntropyStateStrategy +{ + private final FixedDoubleBreakdownHistogram histogram; + + public FixedHistogramJacknifeStateStrategy(long bucketCount, double min, double max) + { + FixedHistogramStateStrategyUtils.validateParameters( + bucketCount, + min, + max); + + histogram = new FixedDoubleBreakdownHistogram(toIntExact(bucketCount), min, max); + } + + private FixedHistogramJacknifeStateStrategy(FixedDoubleBreakdownHistogram histogram) + { + this.histogram = histogram; + } + + private FixedHistogramJacknifeStateStrategy(FixedHistogramJacknifeStateStrategy other) + { + histogram = other.histogram.clone(); + } + + @Override + public void validateParameters(long bucketCount, double sample, double weight, double min, double max) + { + FixedHistogramStateStrategyUtils.validateParameters( + histogram.getBucketCount(), + histogram.getMin(), + histogram.getMax(), + bucketCount, + sample, + weight, + min, + max); + } + + @Override + public void mergeWith(DifferentialEntropyStateStrategy other) + { + histogram.mergeWith(((FixedHistogramJacknifeStateStrategy) other).histogram); + } + + @Override + public void add(double value, double weight) + { + histogram.add(value, weight); + } + + @Override + public double getTotalPopulationWeight() + { + return stream(histogram.iterator()) + .mapToDouble(FixedDoubleBreakdownHistogram.Bucket::getWeight) + .sum(); + } + + @Override + public double calculateEntropy() + { + Map bucketWeights = stream(histogram.iterator()).collect( + groupingBy( + FixedDoubleBreakdownHistogram.Bucket::getLeft, + summingDouble(e -> e.getCount() * e.getWeight()))); + double sumWeight = bucketWeights.values().stream().mapToDouble(Double::doubleValue).sum(); + if (sumWeight == 0.0) { + return Double.NaN; + } + long n = stream(histogram.iterator()) + .mapToLong(FixedDoubleBreakdownHistogram.Bucket::getCount) + .sum(); + double sumWeightLogWeight = + bucketWeights.values().stream().mapToDouble(w -> w == 0.0 ? 0.0 : w * Math.log(w)).sum(); + + double entropy = n * calculateEntropyFromHistogramAggregates(histogram.getWidth(), sumWeight, sumWeightLogWeight); + for (FixedDoubleBreakdownHistogram.Bucket bucketWeight : histogram) { + double weight = bucketWeights.get(bucketWeight.getLeft()); + if (weight > 0.0) { + entropy -= getHoldOutEntropy( + n, + bucketWeight.getRight() - bucketWeight.getLeft(), + sumWeight, + sumWeightLogWeight, + weight, + bucketWeight.getWeight(), + bucketWeight.getCount()); + } + } + return entropy; + } + + private static double getHoldOutEntropy( + long n, + double width, + double sumW, + double sumWeightLogWeight, + double bucketWeight, + double entryWeight, + long entryMultiplicity) + { + double holdoutBucketWeight = Math.max(bucketWeight - entryWeight, 0); + double holdoutSumWeight = + sumW - bucketWeight + holdoutBucketWeight; + double holdoutSumWeightLogWeight = + sumWeightLogWeight - getXLogX(bucketWeight) + getXLogX(holdoutBucketWeight); + double holdoutEntropy = entryMultiplicity * (n - 1) * + calculateEntropyFromHistogramAggregates(width, holdoutSumWeight, holdoutSumWeightLogWeight) / + n; + return holdoutEntropy; + } + + @Override + public long getEstimatedSize() + { + return histogram.estimatedInMemorySize(); + } + + @Override + public int getRequiredBytesForSpecificSerialization() + { + return histogram.getRequiredBytesForSerialization(); + } + + public static FixedHistogramJacknifeStateStrategy deserialize(SliceInput input) + { + FixedDoubleBreakdownHistogram histogram = FixedDoubleBreakdownHistogram.deserialize(input); + return new FixedHistogramJacknifeStateStrategy(histogram); + } + + @Override + public void serialize(SliceOutput out) + { + histogram.serialize(out); + } + + @Override + public DifferentialEntropyStateStrategy clone() + { + return new FixedHistogramJacknifeStateStrategy(this); + } + + @Override + public DifferentialEntropyStateStrategy cloneEmpty() + { + return new FixedHistogramJacknifeStateStrategy(histogram.getBucketCount(), histogram.getMin(), histogram.getMax()); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/FixedHistogramMleStateStrategy.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/FixedHistogramMleStateStrategy.java new file mode 100644 index 0000000000000..6edd2bc7865cc --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/FixedHistogramMleStateStrategy.java @@ -0,0 +1,146 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.operator.aggregation.fixedhistogram.FixedDoubleHistogram; +import io.airlift.slice.SliceInput; +import io.airlift.slice.SliceOutput; + +import static com.facebook.presto.operator.aggregation.differentialentropy.FixedHistogramStateStrategyUtils.getXLogX; +import static com.google.common.collect.Streams.stream; +import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; + +/** + * Calculates sample entropy using MLE (maximum likelihood estimates) on a NumericHistogram. + */ +public class FixedHistogramMleStateStrategy + implements DifferentialEntropyStateStrategy +{ + private final FixedDoubleHistogram histogram; + + public FixedHistogramMleStateStrategy(long bucketCount, double min, double max) + { + FixedHistogramStateStrategyUtils.validateParameters( + bucketCount, + min, + max); + + histogram = new FixedDoubleHistogram(toIntExact(bucketCount), min, max); + } + + private FixedHistogramMleStateStrategy(FixedHistogramMleStateStrategy other) + { + histogram = other.histogram.clone(); + } + + private FixedHistogramMleStateStrategy(FixedDoubleHistogram histogram) + { + this.histogram = requireNonNull(histogram, "histogram is null"); + } + + @Override + public void validateParameters( + long bucketCount, + double sample, + double weight, + double min, + double max) + { + FixedHistogramStateStrategyUtils.validateParameters( + histogram.getBucketCount(), + histogram.getMin(), + histogram.getMax(), + bucketCount, + sample, + weight, + min, + max); + } + + @Override + public void add(double sample, double weight) + { + histogram.add(sample, weight); + } + + @Override + public double getTotalPopulationWeight() + { + return stream(histogram.iterator()) + .mapToDouble(FixedDoubleHistogram.Bucket::getWeight) + .sum(); + } + + @Override + public double calculateEntropy() + { + double sum = 0; + for (FixedDoubleHistogram.Bucket bucket : histogram) { + sum += bucket.getWeight(); + } + if (sum == 0.0) { + return Double.NaN; + } + + double rawEntropy = 0; + for (FixedDoubleHistogram.Bucket bucket : histogram) { + rawEntropy -= getXLogX(bucket.getWeight() / sum); + } + return (rawEntropy + Math.log(histogram.getWidth())) / Math.log(2); + } + + @Override + public long getEstimatedSize() + { + return histogram.estimatedInMemorySize(); + } + + @Override + public int getRequiredBytesForSpecificSerialization() + { + return histogram.getRequiredBytesForSerialization(); + } + + @Override + public void mergeWith(DifferentialEntropyStateStrategy other) + { + histogram.mergeWith(((FixedHistogramMleStateStrategy) other).histogram); + } + + public static FixedHistogramMleStateStrategy deserialize(SliceInput input) + { + FixedDoubleHistogram histogram = FixedDoubleHistogram.deserialize(input); + + return new FixedHistogramMleStateStrategy(histogram); + } + + @Override + public void serialize(SliceOutput out) + { + histogram.serialize(out); + } + + @Override + public DifferentialEntropyStateStrategy clone() + { + return new FixedHistogramMleStateStrategy(this); + } + + @Override + public DifferentialEntropyStateStrategy cloneEmpty() + { + return new FixedHistogramMleStateStrategy(histogram.getBucketCount(), histogram.getMin(), histogram.getMax()); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/FixedHistogramStateStrategyUtils.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/FixedHistogramStateStrategyUtils.java new file mode 100644 index 0000000000000..3d55b977bf582 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/FixedHistogramStateStrategyUtils.java @@ -0,0 +1,96 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.spi.PrestoException; + +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static java.lang.String.format; + +/** + * Utility class for different strategies for calculating entropy based on fixed histograms. + */ +public class FixedHistogramStateStrategyUtils +{ + private FixedHistogramStateStrategyUtils() {} + + public static void validateParameters( + long bucketCount, + double min, + double max) + { + if (bucketCount < 0) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, bucket count must be non-negative: %s", bucketCount)); + } + + if (min >= max) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, format( + "In differential_entropy UDF, min must be larger than max: min=%s, max=%s", min, max)); + } + } + + public static void validateParameters( + long histogramBucketCount, + double histogramMin, + double histogramMax, + long bucketCount, + double sample, + double weight, + double min, + double max) + { + validateParameters(histogramBucketCount, histogramMin, histogramMax); + validateParameters(bucketCount, min, max); + + if (weight < 0.0) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, weight must be non-negative: %s", weight)); + } + + if (histogramBucketCount != bucketCount) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, inconsistent bucket count: prev=%s, current=%s", histogramBucketCount, bucketCount)); + } + if (histogramMin != min) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, inconsistent min: prev=%s, current=%s", histogramMin, min)); + } + if (histogramMax != max) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, inconsistent max: prev=%s, current=%s", histogramMax, max)); + } + if (sample < min) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, sample must be at least min: sample=%s, min=%s", sample, min)); + } + if (sample > max) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, sample must be at most max: sample=%s, max=%s", sample, max)); + } + } + + public static double getXLogX(double x) + { + return x <= 0.0 ? 0.0 : x * Math.log(x); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/UnweightedReservoirSampleStateStrategy.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/UnweightedReservoirSampleStateStrategy.java new file mode 100644 index 0000000000000..b1ab327501783 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/UnweightedReservoirSampleStateStrategy.java @@ -0,0 +1,141 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.operator.aggregation.reservoirsample.UnweightedDoubleReservoirSample; +import com.facebook.presto.spi.PrestoException; +import io.airlift.slice.SliceInput; +import io.airlift.slice.SliceOutput; + +import static com.facebook.presto.operator.aggregation.differentialentropy.EntropyCalculations.calculateFromSamplesUsingVasicek; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static java.lang.Math.toIntExact; +import static java.lang.String.format; + +public class UnweightedReservoirSampleStateStrategy + implements DifferentialEntropyStateStrategy +{ + private final UnweightedDoubleReservoirSample reservoir; + + public UnweightedReservoirSampleStateStrategy(long maxSamples) + { + if (maxSamples <= 0) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, max samples must be positive: %s", maxSamples)); + } + if (maxSamples >= UnweightedDoubleReservoirSample.MAX_SAMPLES_LIMIT) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, max samples must be capped: max_samples=%s, cap=%s", maxSamples, UnweightedDoubleReservoirSample.MAX_SAMPLES_LIMIT)); + } + + reservoir = new UnweightedDoubleReservoirSample(toIntExact(maxSamples)); + } + + private UnweightedReservoirSampleStateStrategy(UnweightedReservoirSampleStateStrategy other) + { + reservoir = other.reservoir.clone(); + } + + private UnweightedReservoirSampleStateStrategy(UnweightedDoubleReservoirSample reservoir) + { + this.reservoir = reservoir; + } + + @Override + public void validateParameters(long maxSamples, double sample, double weight) + { + if (weight != 1.0) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, weight must be 1.0: %s", weight)); + } + + if (maxSamples != reservoir.getMaxSamples()) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, inconsistent maxSamples: %s, %s", maxSamples, reservoir.getMaxSamples())); + } + } + + @Override + public void validateParameters(long maxSamples, double sample) + { + if (maxSamples != reservoir.getMaxSamples()) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, inconsistent maxSamples: %s, %s", maxSamples, reservoir.getMaxSamples())); + } + } + + @Override + public void mergeWith(DifferentialEntropyStateStrategy other) + { + reservoir.mergeWith(((UnweightedReservoirSampleStateStrategy) other).reservoir); + } + + @Override + public void add(double value) + { + reservoir.add(value); + } + + @Override + public double getTotalPopulationWeight() + { + return reservoir.getTotalPopulationCount(); + } + + @Override + public double calculateEntropy() + { + return calculateFromSamplesUsingVasicek(reservoir.getSamples()); + } + + @Override + public long getEstimatedSize() + { + return reservoir.estimatedInMemorySize(); + } + + @Override + public int getRequiredBytesForSpecificSerialization() + { + return reservoir.getRequiredBytesForSerialization(); + } + + public static UnweightedReservoirSampleStateStrategy deserialize(SliceInput input) + { + return new UnweightedReservoirSampleStateStrategy(UnweightedDoubleReservoirSample.deserialize(input)); + } + + @Override + public void serialize(SliceOutput out) + { + reservoir.serialize(out); + } + + @Override + public DifferentialEntropyStateStrategy clone() + { + return new UnweightedReservoirSampleStateStrategy(this); + } + + @Override + public DifferentialEntropyStateStrategy cloneEmpty() + { + return new UnweightedReservoirSampleStateStrategy(reservoir.getMaxSamples()); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/WeightedReservoirSampleStateStrategy.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/WeightedReservoirSampleStateStrategy.java new file mode 100644 index 0000000000000..e65b81ef845df --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/differentialentropy/WeightedReservoirSampleStateStrategy.java @@ -0,0 +1,134 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.operator.aggregation.reservoirsample.WeightedDoubleReservoirSample; +import com.facebook.presto.spi.PrestoException; +import io.airlift.slice.SliceInput; +import io.airlift.slice.SliceOutput; + +import static com.facebook.presto.operator.aggregation.differentialentropy.EntropyCalculations.calculateFromSamplesUsingVasicek; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.google.common.base.Verify.verify; +import static java.lang.String.format; + +public class WeightedReservoirSampleStateStrategy + implements DifferentialEntropyStateStrategy +{ + private final WeightedDoubleReservoirSample reservoir; + + public WeightedReservoirSampleStateStrategy(long maxSamples) + { + if (maxSamples <= 0) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, max samples must be positive: %s", maxSamples)); + } + if (maxSamples >= WeightedDoubleReservoirSample.MAX_SAMPLES_LIMIT) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, max samples must be capped: max_samples=%s, cap=%s", maxSamples, WeightedDoubleReservoirSample.MAX_SAMPLES_LIMIT)); + } + + reservoir = new WeightedDoubleReservoirSample((int) maxSamples); + } + + private WeightedReservoirSampleStateStrategy(WeightedReservoirSampleStateStrategy other) + { + reservoir = other.reservoir.clone(); + } + + private WeightedReservoirSampleStateStrategy(WeightedDoubleReservoirSample reservoir) + { + this.reservoir = reservoir; + } + + @Override + public void validateParameters( + long maxSamples, + double sample, + double weight) + { + if (weight < 0.0) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("In differential_entropy UDF, weight must be non-negative: %s", weight)); + } + + if (maxSamples != reservoir.getMaxSamples()) { + throw new PrestoException( + INVALID_FUNCTION_ARGUMENT, + format("In differential_entropy UDF, inconsistent maxSamples: %s, %s", maxSamples, reservoir.getMaxSamples())); + } + } + + @Override + public void mergeWith(DifferentialEntropyStateStrategy other) + { + verify(other instanceof WeightedReservoirSampleStateStrategy, + format("class should be an instance of WeightedReservoirSampleStateStrategy: %s", other.getClass().getSimpleName())); + reservoir.mergeWith(((WeightedReservoirSampleStateStrategy) other).reservoir); + } + + @Override + public void add(double value, double weight) + { + reservoir.add(value, weight); + } + + @Override + public double getTotalPopulationWeight() + { + return reservoir.getTotalPopulationWeight(); + } + + @Override + public double calculateEntropy() + { + return calculateFromSamplesUsingVasicek(reservoir.getSamples()); + } + + @Override + public long getEstimatedSize() + { + return reservoir.estimatedInMemorySize(); + } + + @Override + public int getRequiredBytesForSpecificSerialization() + { + return reservoir.getRequiredBytesForSerialization(); + } + + public static WeightedReservoirSampleStateStrategy deserialize(SliceInput input) + { + return new WeightedReservoirSampleStateStrategy(WeightedDoubleReservoirSample.deserialize(input)); + } + + @Override + public void serialize(SliceOutput out) + { + reservoir.serialize(out); + } + + @Override + public DifferentialEntropyStateStrategy clone() + { + return new WeightedReservoirSampleStateStrategy(this); + } + + @Override + public DifferentialEntropyStateStrategy cloneEmpty() + { + return new WeightedReservoirSampleStateStrategy(reservoir.getMaxSamples()); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/fixedhistogram/FixedDoubleBreakdownHistogram.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/fixedhistogram/FixedDoubleBreakdownHistogram.java index 5224aadb19fef..e1960929badce 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/fixedhistogram/FixedDoubleBreakdownHistogram.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/fixedhistogram/FixedDoubleBreakdownHistogram.java @@ -19,6 +19,7 @@ import io.airlift.slice.Slices; import org.openjdk.jol.info.ClassLayout; +import java.util.Arrays; import java.util.Iterator; import java.util.NoSuchElementException; import java.util.stream.IntStream; @@ -26,8 +27,10 @@ import static com.facebook.presto.operator.aggregation.fixedhistogram.FixedHistogramUtils.getIndexForValue; import static com.facebook.presto.operator.aggregation.fixedhistogram.FixedHistogramUtils.getLeftValueForIndex; import static com.facebook.presto.operator.aggregation.fixedhistogram.FixedHistogramUtils.getRightValueForIndex; -import static com.facebook.presto.operator.aggregation.fixedhistogram.FixedHistogramUtils.verifyParameters; +import static com.facebook.presto.operator.aggregation.fixedhistogram.FixedHistogramUtils.validateParameters; import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; /** * Fixed-bucket histogram of weights breakdowns. For each bucket, it stores the number of times @@ -38,18 +41,19 @@ * and space complexities can be very large. */ public class FixedDoubleBreakdownHistogram + implements Iterable { private static final int INSTANCE_SIZE = ClassLayout.parseClass(FixedDoubleBreakdownHistogram.class).instanceSize(); - private int bucketCount; - private double min; - private double max; + private final int bucketCount; + private final double min; + private final double max; private int[] indices; private double[] weights; private long[] counts; - public static class BucketWeight + public static class Bucket { private final double left; private final double right; @@ -76,7 +80,7 @@ public long getCount() return count; } - public BucketWeight(double left, double right, double weight, long count) + public Bucket(double left, double right, double weight, long count) { this.left = left; this.right = right; @@ -87,50 +91,40 @@ public BucketWeight(double left, double right, double weight, long count) public FixedDoubleBreakdownHistogram(int bucketCount, double min, double max) { + validateParameters(bucketCount, min, max); this.bucketCount = bucketCount; this.min = min; this.max = max; - verifyParameters(bucketCount, min, max); this.indices = new int[0]; this.weights = new double[0]; this.counts = new long[0]; } - public FixedDoubleBreakdownHistogram(SliceInput input) - { - this.bucketCount = input.readInt(); - this.min = input.readDouble(); - this.max = input.readDouble(); - verifyParameters(bucketCount, min, max); - int size = input.readInt(); - this.indices = new int[size]; - this.weights = new double[size]; - this.counts = new long[size]; - input.readBytes( - Slices.wrappedIntArray(indices), - size * SizeOf.SIZE_OF_INT); - input.readBytes( - Slices.wrappedDoubleArray(weights), - size * SizeOf.SIZE_OF_DOUBLE); - input.readBytes( - Slices.wrappedLongArray(counts), - size * SizeOf.SIZE_OF_LONG); - } - - protected FixedDoubleBreakdownHistogram(FixedDoubleBreakdownHistogram other) + private FixedDoubleBreakdownHistogram(FixedDoubleBreakdownHistogram other) { this.bucketCount = other.bucketCount; this.min = other.min; this.max = other.max; - verifyParameters(bucketCount, min, max); - this.indices = new int[other.indices.length]; - this.weights = new double[other.weights.length]; - this.counts = new long[other.counts.length]; - for (int i = 0; i < other.indices.length; ++i) { - this.indices[i] = other.indices[i]; - this.weights[i] = other.weights[i]; - this.counts[i] = other.counts[i]; - } + this.indices = Arrays.copyOf(other.indices, other.indices.length); + this.weights = Arrays.copyOf(other.weights, other.weights.length); + this.counts = Arrays.copyOf(other.counts, other.counts.length); + } + + private FixedDoubleBreakdownHistogram( + int bucketCount, + double min, + double max, + int[] indices, + double[] weights, + long[] counts) + { + validateParameters(bucketCount, min, max); + this.bucketCount = bucketCount; + this.min = min; + this.max = max; + this.indices = requireNonNull(indices, "indices is null"); + this.weights = requireNonNull(weights, "weights is null"); + this.counts = requireNonNull(counts, "counts is null"); } public int getBucketCount() @@ -155,15 +149,36 @@ public double getWidth() public long estimatedInMemorySize() { - return INSTANCE_SIZE + indices.length * (SizeOf.SIZE_OF_INT + SizeOf.SIZE_OF_DOUBLE + SizeOf.SIZE_OF_LONG); + return INSTANCE_SIZE + + SizeOf.sizeOf(indices) + + SizeOf.sizeOf(weights) + + SizeOf.sizeOf(counts); } public int getRequiredBytesForSerialization() { return SizeOf.SIZE_OF_INT + // bucketCount - // 2 * SizeOf.SIZE_OF_DOUBLE + // min, max + 2 * SizeOf.SIZE_OF_DOUBLE + // min, max SizeOf.SIZE_OF_INT + // size - indices.length * (SizeOf.SIZE_OF_INT + SizeOf.SIZE_OF_DOUBLE + SizeOf.SIZE_OF_LONG); // arrays + indices.length * + (SizeOf.SIZE_OF_INT + SizeOf.SIZE_OF_DOUBLE + SizeOf.SIZE_OF_LONG); // indices, weights, counts + } + + public static FixedDoubleBreakdownHistogram deserialize(SliceInput input) + { + int bucketCount = input.readInt(); + double min = input.readDouble(); + double max = input.readDouble(); + validateParameters(bucketCount, min, max); + + int size = input.readInt(); + int[] indices = new int[size]; + double[] weights = new double[size]; + long[] counts = new long[size]; + input.readBytes(Slices.wrappedIntArray(indices), size * SizeOf.SIZE_OF_INT); + input.readBytes(Slices.wrappedDoubleArray(weights), size * SizeOf.SIZE_OF_DOUBLE); + input.readBytes(Slices.wrappedLongArray(counts), size * SizeOf.SIZE_OF_LONG); + return new FixedDoubleBreakdownHistogram(bucketCount, min, max, indices, weights, counts); } public void serialize(SliceOutput out) @@ -223,20 +238,24 @@ private void add(int logicalBucketIndex, double weight, long count) public void mergeWith(FixedDoubleBreakdownHistogram other) { - checkArgument(bucketCount == other.bucketCount, "bucket counts must be equal."); - checkArgument(min == other.min, "minimums must be equal."); - checkArgument(max == other.max, "maximums must be equal."); - - for (int i = 0; i < other.indices.length; ++i) { - add(other.indices[i], - other.weights[i], - other.counts[i]); + checkArgument( + bucketCount == other.bucketCount, + format("bucket count must be equal to other bucket count: %s %s", bucketCount, other.bucketCount)); + checkArgument( + min == other.min, + format("minimum must be equal to other minimum: %s %s", min, other.min)); + checkArgument( + max == other.max, + format("Maximum must be equal to other maximum: %s %s", max, other.max)); + + for (int i = 0; i < other.indices.length; i++) { + add(other.indices[i], other.weights[i], other.counts[i]); } } - public Iterator iterator() + public Iterator iterator() { - return new Iterator() { + return new Iterator() { private int currentIndex; @Override @@ -246,13 +265,13 @@ public boolean hasNext() } @Override - public BucketWeight next() + public Bucket next() { if (!hasNext()) { throw new NoSuchElementException(); } - final BucketWeight bucket = new BucketWeight( + final Bucket bucket = new Bucket( getLeftValueForIndex(bucketCount, min, max, indices[currentIndex]), getRightValueForIndex(bucketCount, min, max, indices[currentIndex]), weights[currentIndex], diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/fixedhistogram/FixedDoubleHistogram.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/fixedhistogram/FixedDoubleHistogram.java index a04a44cc631ad..504a07ac55973 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/fixedhistogram/FixedDoubleHistogram.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/fixedhistogram/FixedDoubleHistogram.java @@ -25,10 +25,15 @@ import static com.facebook.presto.operator.aggregation.fixedhistogram.FixedHistogramUtils.getIndexForValue; import static com.facebook.presto.operator.aggregation.fixedhistogram.FixedHistogramUtils.getLeftValueForIndex; import static com.facebook.presto.operator.aggregation.fixedhistogram.FixedHistogramUtils.getRightValueForIndex; -import static com.facebook.presto.operator.aggregation.fixedhistogram.FixedHistogramUtils.verifyParameters; +import static com.facebook.presto.operator.aggregation.fixedhistogram.FixedHistogramUtils.validateParameters; import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; +/** + * Fixed-bucket histogram of weights. For each bucket, it stores the total weights it accrued. + */ public class FixedDoubleHistogram + implements Iterable { public static class Bucket { @@ -63,29 +68,26 @@ public Bucket(double left, double right, double weight) private final double[] weights; - private int bucketCount; - private double min; - private double max; + private final int bucketCount; + private final double min; + private final double max; public FixedDoubleHistogram(int bucketCount, double min, double max) { + validateParameters(bucketCount, min, max); this.bucketCount = bucketCount; this.min = min; this.max = max; - verifyParameters(bucketCount, min, max); this.weights = new double[bucketCount]; } - public FixedDoubleHistogram(SliceInput input) + private FixedDoubleHistogram(int bucketCount, double min, double max, double[] weights) { - this.bucketCount = input.readInt(); - this.min = input.readDouble(); - this.max = input.readDouble(); - verifyParameters(bucketCount, min, max); - this.weights = new double[bucketCount]; - input.readBytes( - Slices.wrappedDoubleArray(weights), - bucketCount * SizeOf.SIZE_OF_DOUBLE); + validateParameters(bucketCount, min, max); + this.bucketCount = bucketCount; + this.min = min; + this.max = max; + this.weights = requireNonNull(weights, "weights is null"); } public int getBucketCount() @@ -108,7 +110,7 @@ public double getWidth() return (max - min) / bucketCount; } - protected FixedDoubleHistogram(FixedDoubleHistogram other) + private FixedDoubleHistogram(FixedDoubleHistogram other) { bucketCount = other.bucketCount; min = other.min; @@ -126,6 +128,17 @@ public int getRequiredBytesForSerialization() SizeOf.SIZE_OF_DOUBLE * bucketCount; // weights } + public static FixedDoubleHistogram deserialize(SliceInput input) + { + int bucketCount = input.readInt(); + double min = input.readDouble(); + double max = input.readDouble(); + validateParameters(bucketCount, min, max); + double[] weights = new double[bucketCount]; + input.readBytes(Slices.wrappedDoubleArray(weights), bucketCount * SizeOf.SIZE_OF_DOUBLE); + return new FixedDoubleHistogram(bucketCount, min, max, weights); + } + public void serialize(SliceOutput out) { out.appendInt(bucketCount) @@ -161,7 +174,7 @@ public void mergeWith(FixedDoubleHistogram other) checkArgument( bucketCount == other.bucketCount, "bucketCount %s must be equal to other bucketCount %s", bucketCount, other.bucketCount); - for (int i = 0; i < bucketCount; ++i) { + for (int i = 0; i < bucketCount; i++) { weights[i] += other.weights[i]; } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/fixedhistogram/FixedHistogramUtils.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/fixedhistogram/FixedHistogramUtils.java index 952132dd973d7..2c7610fa7cb72 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/fixedhistogram/FixedHistogramUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/fixedhistogram/FixedHistogramUtils.java @@ -19,17 +19,17 @@ class FixedHistogramUtils { private FixedHistogramUtils() {} - public static void verifyParameters(int bucketCount, double min, double max) + public static void validateParameters(int bucketCount, double min, double max) { - checkArgument(bucketCount >= 2, "bucketCount %s must be at least 2", bucketCount); - checkArgument(min < max, "min %s must be smaller than max %s ", min, max); + checkArgument(bucketCount >= 2, "bucketCount must be at least 2: %s", bucketCount); + checkArgument(min < max, "min must be smaller than max: %s %s", min, max); } public static int getIndexForValue(int bucketCount, double min, double max, double value) { checkArgument( value >= min && value < max, - "value must be within range [%s, %s]", min, max); + "value must be within range: %s [%s, %s]", value, min, max); return Math.min( (int) (bucketCount * (value - min) / (max - min)), bucketCount - 1); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/AbstractMinMaxBy.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/AbstractMinMaxBy.java index 588bf39e48717..f5df7c17990df 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/AbstractMinMaxBy.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/minmaxby/AbstractMinMaxBy.java @@ -109,7 +109,7 @@ private InternalAggregationFunction generateAggregation(Type valueType, Type key // Generate states and serializers: // For value that is a Block or Slice, we store them as Block/position combination - // to avoid generating long-living objects through getSlice or getObject. + // to avoid generating long-living objects through getSlice or getBlock. // This can also help reducing cross-region reference in G1GC engine. // TODO: keys can have the same problem. But usually they are primitive types (given the nature of comparison). AccumulatorStateFactory stateFactory; @@ -136,7 +136,7 @@ private InternalAggregationFunction generateAggregation(Type valueType, Type key CallSiteBinder binder = new CallSiteBinder(); OperatorType operator = min ? LESS_THAN : GREATER_THAN; - MethodHandle compareMethod = functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(operator, fromTypes(keyType, keyType))).getMethodHandle(); + MethodHandle compareMethod = functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(operator, fromTypes(keyType, keyType))).getMethodHandle(); ClassDefinition definition = new ClassDefinition( a(PUBLIC, FINAL), @@ -151,7 +151,7 @@ private InternalAggregationFunction generateAggregation(Type valueType, Type key MethodHandle combineMethod = methodHandle(generatedClass, "combine", stateClazz, stateClazz); MethodHandle outputMethod = methodHandle(generatedClass, "output", stateClazz, BlockBuilder.class); AggregationMetadata metadata = new AggregationMetadata( - generateAggregationName(getSignature().getName(), valueType.getTypeSignature(), inputTypes.stream().map(Type::getTypeSignature).collect(toImmutableList())), + generateAggregationName(getSignature().getNameSuffix(), valueType.getTypeSignature(), inputTypes.stream().map(Type::getTypeSignature).collect(toImmutableList())), createInputParameterMetadata(valueType, keyType), inputMethod, combineMethod, @@ -162,7 +162,7 @@ private InternalAggregationFunction generateAggregation(Type valueType, Type key stateFactory)), valueType); GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader); - return new InternalAggregationFunction(getSignature().getName(), inputTypes, ImmutableList.of(intermediateType), valueType, true, false, factory); + return new InternalAggregationFunction(getSignature().getNameSuffix(), inputTypes, ImmutableList.of(intermediateType), valueType, true, false, factory); } private static List createInputParameterMetadata(Type value, Type key) diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/reservoirsample/UnweightedDoubleReservoirSample.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/reservoirsample/UnweightedDoubleReservoirSample.java new file mode 100644 index 0000000000000..57d527db14a27 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/reservoirsample/UnweightedDoubleReservoirSample.java @@ -0,0 +1,173 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.reservoirsample; + +import io.airlift.slice.SizeOf; +import io.airlift.slice.SliceInput; +import io.airlift.slice.SliceOutput; +import io.airlift.slice.Slices; +import org.openjdk.jol.info.ClassLayout; + +import java.util.Arrays; +import java.util.concurrent.ThreadLocalRandom; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class UnweightedDoubleReservoirSample + implements Cloneable +{ + public static final int MAX_SAMPLES_LIMIT = 1_000_000; + private static final int INSTANCE_SIZE = ClassLayout.parseClass(UnweightedDoubleReservoirSample.class).instanceSize(); + + private int seenCount; + private double[] samples; + + public UnweightedDoubleReservoirSample(int maxSamples) + { + checkArgument( + maxSamples > 0, + format("Maximum number of samples must be positive: %s", maxSamples)); + checkArgument( + maxSamples <= MAX_SAMPLES_LIMIT, + format("Maximum number of samples must not exceed maximum: %s %s", maxSamples, MAX_SAMPLES_LIMIT)); + + this.samples = new double[maxSamples]; + } + + private UnweightedDoubleReservoirSample(UnweightedDoubleReservoirSample other) + { + this.seenCount = other.seenCount; + this.samples = Arrays.copyOf(requireNonNull(other.samples, "samples is null"), other.samples.length); + } + + private UnweightedDoubleReservoirSample(int seenCount, double[] samples) + { + this.seenCount = seenCount; + this.samples = samples; + } + + public int getMaxSamples() + { + return samples.length; + } + + public void add(double sample) + { + seenCount++; + if (seenCount <= samples.length) { + samples[seenCount - 1] = sample; + return; + } + int index = ThreadLocalRandom.current().nextInt(0, seenCount); + if (index < samples.length) { + samples[index] = sample; + } + } + + public void mergeWith(UnweightedDoubleReservoirSample other) + { + checkArgument( + samples.length == other.samples.length, + format("Maximum number of samples %s must be equal to that of other %s", samples.length, other.samples.length)); + if (other.seenCount < other.samples.length) { + for (int i = 0; i < other.seenCount; i++) { + add(other.samples[i]); + } + return; + } + if (seenCount < samples.length) { + UnweightedDoubleReservoirSample target = ((UnweightedDoubleReservoirSample) other.clone()); + for (int i = 0; i < seenCount; i++) { + target.add(samples[i]); + } + seenCount = target.seenCount; + samples = target.samples; + return; + } + + shuffleArray(samples); + shuffleArray(other.samples); + int nextIndex = 0; + int otherNextIndex = 0; + double[] merged = new double[samples.length]; + for (int i = 0; i < samples.length; i++) { + if (ThreadLocalRandom.current().nextLong(0, seenCount + other.seenCount) < seenCount) { + merged[i] = samples[nextIndex++]; + } + else { + merged[i] = other.samples[otherNextIndex++]; + } + } + seenCount += other.seenCount; + samples = merged; + } + + public int getTotalPopulationCount() + { + return seenCount; + } + + @Override + public UnweightedDoubleReservoirSample clone() + { + return new UnweightedDoubleReservoirSample(this); + } + + public double[] getSamples() + { + return Arrays.copyOf(samples, Math.min(seenCount, samples.length)); + } + + private static void shuffleArray(double[] samples) + { + for (int i = samples.length - 1; i > 0; i--) { + int index = ThreadLocalRandom.current().nextInt(0, i + 1); + double sample = samples[index]; + samples[index] = samples[i]; + samples[i] = sample; + } + } + + public static UnweightedDoubleReservoirSample deserialize(SliceInput input) + { + int seenCount = input.readInt(); + int maxSamples = input.readInt(); + double[] samples = new double[maxSamples]; + input.readBytes(Slices.wrappedDoubleArray(samples), Math.min(seenCount, samples.length) * SizeOf.SIZE_OF_DOUBLE); + return new UnweightedDoubleReservoirSample(seenCount, samples); + } + + public void serialize(SliceOutput output) + { + output.appendInt(seenCount); + output.appendInt(samples.length); + for (int i = 0; i < Math.min(seenCount, samples.length); i++) { + output.appendDouble(samples[i]); + } + } + + public int getRequiredBytesForSerialization() + { + return SizeOf.SIZE_OF_INT + // seenCount + SizeOf.SIZE_OF_INT + SizeOf.SIZE_OF_DOUBLE * Math.min(seenCount, samples.length); // samples + } + + public long estimatedInMemorySize() + { + return INSTANCE_SIZE + + SizeOf.sizeOf(samples); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/reservoirsample/WeightedDoubleReservoirSample.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/reservoirsample/WeightedDoubleReservoirSample.java new file mode 100644 index 0000000000000..102d1bef05155 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/reservoirsample/WeightedDoubleReservoirSample.java @@ -0,0 +1,219 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.reservoirsample; + +import io.airlift.slice.SizeOf; +import io.airlift.slice.SliceInput; +import io.airlift.slice.SliceOutput; +import io.airlift.slice.Slices; +import org.openjdk.jol.info.ClassLayout; + +import java.util.Arrays; +import java.util.concurrent.ThreadLocalRandom; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class WeightedDoubleReservoirSample + implements Cloneable +{ + public static final int MAX_SAMPLES_LIMIT = 1_000_000; + private static final int INSTANCE_SIZE = ClassLayout.parseClass(WeightedDoubleReservoirSample.class).instanceSize(); + + private int count; + private double[] samples; + private double[] weights; + private double totalPopulationWeight; + + public WeightedDoubleReservoirSample(int maxSamples) + { + checkArgument(maxSamples > 0, format("Maximum number of samples must be positive: %s", maxSamples)); + checkArgument( + maxSamples <= MAX_SAMPLES_LIMIT, + format("Maximum number of samples must not exceed limit: %s %s", maxSamples, MAX_SAMPLES_LIMIT)); + + this.samples = new double[maxSamples]; + this.weights = new double[maxSamples]; + } + + private WeightedDoubleReservoirSample(WeightedDoubleReservoirSample other) + { + this.count = other.count; + this.samples = Arrays.copyOf(other.samples, other.samples.length); + this.weights = Arrays.copyOf(other.weights, other.weights.length); + this.totalPopulationWeight = other.totalPopulationWeight; + } + + private WeightedDoubleReservoirSample(int count, double[] samples, double[] weights, double totalPopulationWeight) + { + this.count = count; + this.samples = requireNonNull(samples, "samples is null"); + this.weights = requireNonNull(weights, "weights is null"); + this.totalPopulationWeight = totalPopulationWeight; + } + + public long getMaxSamples() + { + return samples.length; + } + + public void add(double sample, double weight) + { + checkArgument(weight >= 0, format("Weight %s cannot be negative", weight)); + totalPopulationWeight += weight; + double adjustedWeight = Math.pow( + ThreadLocalRandom.current().nextDouble(), + 1.0 / weight); + addWithAdjustedWeight(sample, adjustedWeight); + } + + private void addWithAdjustedWeight(double sample, double adjustedWeight) + { + if (count < samples.length) { + samples[count] = sample; + weights[count] = adjustedWeight; + count++; + bubbleUp(); + return; + } + + if (adjustedWeight <= weights[0]) { + return; + } + + samples[0] = sample; + weights[0] = adjustedWeight; + bubbleDown(); + } + + public void mergeWith(WeightedDoubleReservoirSample other) + { + totalPopulationWeight += other.totalPopulationWeight; + for (int i = 0; i < other.count; i++) { + addWithAdjustedWeight(other.samples[i], other.weights[i]); + } + } + + @Override + public WeightedDoubleReservoirSample clone() + { + return new WeightedDoubleReservoirSample(this); + } + + public double[] getSamples() + { + return Arrays.copyOf(samples, count); + } + + private void swap(int i, int j) + { + double tmpElement = samples[i]; + double tmpWeight = weights[i]; + samples[i] = samples[j]; + weights[i] = weights[j]; + samples[j] = tmpElement; + weights[j] = tmpWeight; + } + + private void bubbleDown() + { + int index = 0; + while (leftChild(index) < count) { + int smallestChildIndex = leftChild(index); + + if (rightChild(index) < count && weights[leftChild(index)] > weights[rightChild(index)]) { + smallestChildIndex = rightChild(index); + } + + if (weights[index] > weights[smallestChildIndex]) { + swap(index, smallestChildIndex); + } + else { + break; + } + + index = smallestChildIndex; + } + } + + private void bubbleUp() + { + int index = count - 1; + while (index > 0 && weights[index] < weights[parent(index)]) { + swap(index, parent(index)); + index = parent(index); + } + } + + private static int parent(int pos) + { + return pos / 2; + } + + private static int leftChild(int pos) + { + return 2 * pos; + } + + private static int rightChild(int pos) + { + return 2 * pos + 1; + } + + public static WeightedDoubleReservoirSample deserialize(SliceInput input) + { + int count = input.readInt(); + int maxSamples = input.readInt(); + checkArgument(count <= maxSamples, "count must not be larger than number of samples"); + double[] samples = new double[maxSamples]; + input.readBytes(Slices.wrappedDoubleArray(samples), count * SizeOf.SIZE_OF_DOUBLE); + double[] weights = new double[maxSamples]; + input.readBytes(Slices.wrappedDoubleArray(weights), count * SizeOf.SIZE_OF_DOUBLE); + double totalPopulationWeight = input.readDouble(); + return new WeightedDoubleReservoirSample(count, samples, weights, totalPopulationWeight); + } + + public void serialize(SliceOutput output) + { + output.appendInt(count); + output.appendInt(samples.length); + for (int i = 0; i < count; i++) { + output.appendDouble(samples[i]); + } + for (int i = 0; i < count; i++) { + output.appendDouble(weights[i]); + } + output.appendDouble(totalPopulationWeight); + } + + public int getRequiredBytesForSerialization() + { + return SizeOf.SIZE_OF_INT + // count + SizeOf.SIZE_OF_INT + 2 * SizeOf.SIZE_OF_DOUBLE * Math.min(count, samples.length) + // samples, weights + SizeOf.SIZE_OF_DOUBLE; // totalPopulationWeight; + } + + public long estimatedInMemorySize() + { + return INSTANCE_SIZE + + SizeOf.sizeOf(samples) + + SizeOf.sizeOf(weights); + } + + public double getTotalPopulationWeight() + { + return totalPopulationWeight; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileArrayState.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileArrayState.java index ecead3a6908df..f688fa783e7fb 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileArrayState.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileArrayState.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.operator.aggregation.state; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.spi.function.AccumulatorState; import com.facebook.presto.spi.function.AccumulatorStateMetadata; -import io.airlift.stats.QuantileDigest; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileArrayStateFactory.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileArrayStateFactory.java index 0e6b14ad98195..1e83ac703bd68 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileArrayStateFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileArrayStateFactory.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.operator.aggregation.state; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.array.ObjectBigArray; import com.facebook.presto.spi.function.AccumulatorStateFactory; import io.airlift.slice.SizeOf; -import io.airlift.stats.QuantileDigest; import org.openjdk.jol.info.ClassLayout; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileArrayStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileArrayStateSerializer.java index 53848fdca6497..e83ee39687c9f 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileArrayStateSerializer.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileArrayStateSerializer.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation.state; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AccumulatorStateSerializer; @@ -22,7 +23,6 @@ import io.airlift.slice.SliceInput; import io.airlift.slice.SliceOutput; import io.airlift.slice.Slices; -import io.airlift.stats.QuantileDigest; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileState.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileState.java index 896aebfb4c9f1..89c779c257a50 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileState.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileState.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.operator.aggregation.state; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.spi.function.AccumulatorState; import com.facebook.presto.spi.function.AccumulatorStateMetadata; -import io.airlift.stats.QuantileDigest; @AccumulatorStateMetadata(stateSerializerClass = DigestAndPercentileStateSerializer.class, stateFactoryClass = DigestAndPercentileStateFactory.class) public interface DigestAndPercentileState diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileStateFactory.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileStateFactory.java index e602d7a9f9dd9..137e3e4e0b258 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileStateFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileStateFactory.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.operator.aggregation.state; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.array.DoubleBigArray; import com.facebook.presto.array.ObjectBigArray; import com.facebook.presto.spi.function.AccumulatorStateFactory; -import io.airlift.stats.QuantileDigest; import org.openjdk.jol.info.ClassLayout; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileStateSerializer.java index b9291fad7ac20..90f91b56c2c17 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileStateSerializer.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/DigestAndPercentileStateSerializer.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation.state; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AccumulatorStateSerializer; @@ -21,7 +22,6 @@ import io.airlift.slice.SliceInput; import io.airlift.slice.SliceOutput; import io.airlift.slice.Slices; -import io.airlift.stats.QuantileDigest; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static io.airlift.slice.SizeOf.SIZE_OF_DOUBLE; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogState.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogState.java index e5464d6307d27..6f9e8363d7dd6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogState.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogState.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.operator.aggregation.state; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.spi.function.AccumulatorState; import com.facebook.presto.spi.function.AccumulatorStateMetadata; -import io.airlift.stats.cardinality.HyperLogLog; @AccumulatorStateMetadata(stateSerializerClass = HyperLogLogStateSerializer.class, stateFactoryClass = HyperLogLogStateFactory.class) public interface HyperLogLogState diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogStateFactory.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogStateFactory.java index ffd79f27c8247..0138c7331a7a7 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogStateFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogStateFactory.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.operator.aggregation.state; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.array.ObjectBigArray; import com.facebook.presto.spi.function.AccumulatorStateFactory; -import io.airlift.stats.cardinality.HyperLogLog; import org.openjdk.jol.info.ClassLayout; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogStateSerializer.java index 4b900b6074466..4c94c90f22ee1 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogStateSerializer.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/HyperLogLogStateSerializer.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.operator.aggregation.state; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AccumulatorStateSerializer; import com.facebook.presto.spi.type.Type; -import io.airlift.stats.cardinality.HyperLogLog; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/PrecisionRecallStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/PrecisionRecallStateSerializer.java index ce36085f59a85..9c049e07ae291 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/PrecisionRecallStateSerializer.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/PrecisionRecallStateSerializer.java @@ -67,8 +67,8 @@ public void deserialize( hasHistograms == 0 || hasHistograms == 1, "hasHistogram %s should be boolean-convertible", hasHistograms); if (hasHistograms == 1) { - state.setTrueWeights(new FixedDoubleHistogram(input)); - state.setFalseWeights(new FixedDoubleHistogram(input)); + state.setTrueWeights(FixedDoubleHistogram.deserialize(input)); + state.setFalseWeights(FixedDoubleHistogram.deserialize(input)); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestState.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestState.java index da3488bb9d8a2..9b4a080f0e4b8 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestState.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestState.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.operator.aggregation.state; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.spi.function.AccumulatorState; -import io.airlift.stats.QuantileDigest; public interface QuantileDigestState extends AccumulatorState diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateFactory.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateFactory.java index e9e00efe31b8d..688a4b681e3c4 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateFactory.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.operator.aggregation.state; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.array.ObjectBigArray; import com.facebook.presto.spi.function.AccumulatorStateFactory; -import io.airlift.stats.QuantileDigest; import org.openjdk.jol.info.ClassLayout; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java index aa1d33ea14dd1..05c2231e0672f 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.operator.aggregation.state; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AccumulatorStateSerializer; import com.facebook.presto.spi.type.Type; -import io.airlift.stats.QuantileDigest; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StateCompiler.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StateCompiler.java index df659cfa17f59..708027f9a1f5c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StateCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StateCompiler.java @@ -68,7 +68,6 @@ import static com.facebook.presto.bytecode.ParameterizedType.type; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.add; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantBoolean; -import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantClass; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantInt; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantNull; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantNumber; @@ -244,7 +243,7 @@ private static void generateDeserialize(ClassDefinition definition, CallSite } else if (fields.size() > 1) { Variable row = scope.declareVariable(Block.class, "row"); - deserializerBody.append(row.set(block.invoke("getObject", Object.class, index, constantClass(Block.class)).cast(Block.class))); + deserializerBody.append(row.set(block.invoke("getBlock", Block.class, index))); int position = 0; for (StateField field : fields) { Method setter = getSetter(clazz, field); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/annotations/ScalarImplementationDependency.java b/presto-main/src/main/java/com/facebook/presto/operator/annotations/ScalarImplementationDependency.java index c07b6e92377d3..8ec12e632ef4c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/annotations/ScalarImplementationDependency.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/annotations/ScalarImplementationDependency.java @@ -41,7 +41,7 @@ public MethodHandle resolve(BoundVariables boundVariables, TypeManager typeManag return functionManager.getFunctionInvokerProvider().createFunctionInvoker(functionHandle, invocationConvention).methodHandle(); } else { - return functionManager.getScalarFunctionImplementation(functionHandle).getMethodHandle(); + return functionManager.getBuiltInScalarFunctionImplementation(functionHandle).getMethodHandle(); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/exchange/LocalExchange.java b/presto-main/src/main/java/com/facebook/presto/operator/exchange/LocalExchange.java index db83f95a4c2bd..9c8f09725a4d6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/exchange/LocalExchange.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/exchange/LocalExchange.java @@ -13,10 +13,21 @@ */ package com.facebook.presto.operator.exchange; +import com.facebook.presto.Session; import com.facebook.presto.execution.Lifespan; +import com.facebook.presto.operator.BucketPartitionFunction; +import com.facebook.presto.operator.HashGenerator; +import com.facebook.presto.operator.InterpretedHashGenerator; +import com.facebook.presto.operator.PartitionFunction; import com.facebook.presto.operator.PipelineExecutionStrategy; +import com.facebook.presto.operator.PrecomputedHashGenerator; +import com.facebook.presto.spi.BucketFunction; +import com.facebook.presto.spi.connector.ConnectorBucketNodeMap; +import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.planner.PartitioningHandle; +import com.facebook.presto.sql.planner.PartitioningProviderManager; +import com.facebook.presto.sql.planner.SystemPartitioningHandle; import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; @@ -34,6 +45,7 @@ import java.util.Set; import java.util.function.Consumer; import java.util.function.Supplier; +import java.util.stream.IntStream; import java.util.stream.Stream; import static com.facebook.presto.operator.PipelineExecutionStrategy.UNGROUPED_EXECUTION; @@ -76,11 +88,13 @@ public class LocalExchange private int nextSourceIndex; public LocalExchange( + PartitioningProviderManager partitioningProviderManager, + Session session, int sinkFactoryCount, int bufferCount, PartitioningHandle partitioning, - List types, List partitionChannels, + List partitioningChannelTypes, Optional partitionHashChannel, DataSize maxBufferedBytes) { @@ -110,9 +124,6 @@ else if (partitioning.equals(FIXED_BROADCAST_DISTRIBUTION)) { else if (partitioning.equals(FIXED_ARBITRARY_DISTRIBUTION)) { exchangerSupplier = () -> new RandomExchanger(buffers, memoryManager); } - else if (partitioning.equals(FIXED_HASH_DISTRIBUTION)) { - exchangerSupplier = () -> new PartitioningExchanger(buffers, memoryManager, types, partitionChannels, partitionHashChannel); - } else if (partitioning.equals(FIXED_PASSTHROUGH_DISTRIBUTION)) { Iterator sourceIterator = this.sources.iterator(); exchangerSupplier = () -> { @@ -120,11 +131,69 @@ else if (partitioning.equals(FIXED_PASSTHROUGH_DISTRIBUTION)) { return new PassthroughExchanger(sourceIterator.next(), maxBufferedBytes.toBytes() / bufferCount, memoryManager::updateMemoryUsage); }; } + else if (partitioning.equals(FIXED_HASH_DISTRIBUTION) || partitioning.getConnectorId().isPresent()) { + // partitioned exchange + exchangerSupplier = () -> new PartitioningExchanger( + buffers, + memoryManager, + createPartitionFunction( + partitioningProviderManager, + session, + partitioning, + bufferCount, + partitioningChannelTypes, + partitionHashChannel.isPresent()), + partitionChannels, + partitionHashChannel); + } else { throw new IllegalArgumentException("Unsupported local exchange partitioning " + partitioning); } } + private static PartitionFunction createPartitionFunction( + PartitioningProviderManager partitioningProviderManager, + Session session, + PartitioningHandle partitioning, + int partitionCount, + List partitioningChannelTypes, + boolean isHashPrecomputed) + { + if (partitioning.getConnectorHandle() instanceof SystemPartitioningHandle) { + HashGenerator hashGenerator; + if (isHashPrecomputed) { + hashGenerator = new PrecomputedHashGenerator(0); + } + else { + hashGenerator = new InterpretedHashGenerator(partitioningChannelTypes, IntStream.range(0, partitioningChannelTypes.size()).toArray()); + } + return new LocalPartitionGenerator(hashGenerator, partitionCount); + } + + ConnectorNodePartitioningProvider partitioningProvider = partitioningProviderManager.getPartitioningProvider(partitioning.getConnectorId().get()); + ConnectorBucketNodeMap connectorBucketNodeMap = partitioningProvider.getBucketNodeMap( + partitioning.getTransactionHandle().orElse(null), + session.toConnectorSession(), + partitioning.getConnectorHandle()); + checkArgument(connectorBucketNodeMap != null, "No partition map %s", partitioning); + + int bucketCount = connectorBucketNodeMap.getBucketCount(); + int[] bucketToPartition = new int[bucketCount]; + for (int bucket = 0; bucket < bucketCount; bucket++) { + bucketToPartition[bucket] = bucket % partitionCount; + } + + BucketFunction bucketFunction = partitioningProvider.getBucketFunction( + partitioning.getTransactionHandle().orElse(null), + session.toConnectorSession(), + partitioning.getConnectorHandle(), + partitioningChannelTypes, + bucketCount); + + checkArgument(bucketFunction != null, "No bucket function for partitioning: %s", partitioning); + return new BucketPartitionFunction(bucketFunction, bucketToPartition); + } + public int getBufferCount() { return sources.size(); @@ -255,9 +324,11 @@ private static void checkNotHoldsLock(Object lock) @ThreadSafe public static class LocalExchangeFactory { + private final PartitioningProviderManager partitioningProviderManager; + private final Session session; private final PartitioningHandle partitioning; - private final List types; private final List partitionChannels; + private final List partitioningChannelTypes; private final Optional partitionHashChannel; private final PipelineExecutionStrategy exchangeSourcePipelineExecutionStrategy; private final DataSize maxBufferedBytes; @@ -276,6 +347,8 @@ public static class LocalExchangeFactory private final List closedSinkFactories = new ArrayList<>(); public LocalExchangeFactory( + PartitioningProviderManager partitioningProviderManager, + Session session, PartitioningHandle partitioning, int defaultConcurrency, List types, @@ -284,14 +357,18 @@ public LocalExchangeFactory( PipelineExecutionStrategy exchangeSourcePipelineExecutionStrategy, DataSize maxBufferedBytes) { + this.partitioningProviderManager = requireNonNull(partitioningProviderManager, "partitioningProviderManager is null"); + this.session = requireNonNull(session, "session is null"); this.partitioning = requireNonNull(partitioning, "partitioning is null"); - this.types = requireNonNull(types, "types is null"); - this.partitionChannels = requireNonNull(partitionChannels, "partitioningChannels is null"); + this.bufferCount = computeBufferCount(partitioning, defaultConcurrency, partitionChannels); + this.partitionChannels = ImmutableList.copyOf(requireNonNull(partitionChannels, "partitionChannels is null")); + requireNonNull(types, "types is null"); + this.partitioningChannelTypes = partitionChannels.stream() + .map(types::get) + .collect(toImmutableList()); this.partitionHashChannel = requireNonNull(partitionHashChannel, "partitionHashChannel is null"); this.exchangeSourcePipelineExecutionStrategy = requireNonNull(exchangeSourcePipelineExecutionStrategy, "exchangeSourcePipelineExecutionStrategy is null"); this.maxBufferedBytes = requireNonNull(maxBufferedBytes, "maxBufferedBytes is null"); - - this.bufferCount = computeBufferCount(partitioning, defaultConcurrency, partitionChannels); } public synchronized LocalExchangeSinkFactoryId newSinkFactoryId() @@ -322,8 +399,16 @@ public synchronized LocalExchange getLocalExchange(Lifespan lifespan) } return localExchangeMap.computeIfAbsent(lifespan, ignored -> { checkState(noMoreSinkFactories); - LocalExchange localExchange = - new LocalExchange(numSinkFactories, bufferCount, partitioning, types, partitionChannels, partitionHashChannel, maxBufferedBytes); + LocalExchange localExchange = new LocalExchange( + partitioningProviderManager, + session, + numSinkFactories, + bufferCount, + partitioning, + partitionChannels, + partitioningChannelTypes, + partitionHashChannel, + maxBufferedBytes); for (LocalExchangeSinkFactoryId closedSinkFactoryId : closedSinkFactories) { localExchange.getSinkFactory(closedSinkFactoryId).close(); } @@ -355,14 +440,14 @@ else if (partitioning.equals(FIXED_ARBITRARY_DISTRIBUTION)) { bufferCount = defaultConcurrency; checkArgument(partitionChannels.isEmpty(), "Arbitrary exchange must not have partition channels"); } - else if (partitioning.equals(FIXED_HASH_DISTRIBUTION)) { - bufferCount = defaultConcurrency; - checkArgument(!partitionChannels.isEmpty(), "Partitioned exchange must have partition channels"); - } else if (partitioning.equals(FIXED_PASSTHROUGH_DISTRIBUTION)) { bufferCount = defaultConcurrency; checkArgument(partitionChannels.isEmpty(), "Passthrough exchange must not have partition channels"); } + else if (partitioning.equals(FIXED_HASH_DISTRIBUTION) || partitioning.getConnectorId().isPresent()) { + // partitioned exchange + bufferCount = defaultConcurrency; + } else { throw new IllegalArgumentException("Unsupported local exchange partitioning " + partitioning); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/exchange/PartitioningExchanger.java b/presto-main/src/main/java/com/facebook/presto/operator/exchange/PartitioningExchanger.java index bada02f85bb02..f9c8ad2de80b9 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/exchange/PartitioningExchanger.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/exchange/PartitioningExchanger.java @@ -13,14 +13,10 @@ */ package com.facebook.presto.operator.exchange; -import com.facebook.presto.operator.HashGenerator; -import com.facebook.presto.operator.InterpretedHashGenerator; -import com.facebook.presto.operator.PrecomputedHashGenerator; +import com.facebook.presto.operator.PartitionFunction; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableList; -import com.google.common.primitives.Ints; import com.google.common.util.concurrent.ListenableFuture; import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.ints.IntList; @@ -29,7 +25,6 @@ import java.util.Optional; import java.util.function.Consumer; -import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; class PartitioningExchanger @@ -37,30 +32,23 @@ class PartitioningExchanger { private final List> buffers; private final LocalExchangeMemoryManager memoryManager; - private final LocalPartitionGenerator partitionGenerator; + private final PartitionFunction partitionFunction; + private final List partitioningChannels; + private final Optional hashChannel; private final IntArrayList[] partitionAssignments; public PartitioningExchanger( List> partitions, LocalExchangeMemoryManager memoryManager, - List types, - List partitionChannels, + PartitionFunction partitionFunction, + List partitioningChannels, Optional hashChannel) { this.buffers = ImmutableList.copyOf(requireNonNull(partitions, "partitions is null")); this.memoryManager = requireNonNull(memoryManager, "memoryManager is null"); - - HashGenerator hashGenerator; - if (hashChannel.isPresent()) { - hashGenerator = new PrecomputedHashGenerator(hashChannel.get()); - } - else { - List partitionChannelTypes = partitionChannels.stream() - .map(types::get) - .collect(toImmutableList()); - hashGenerator = new InterpretedHashGenerator(partitionChannelTypes, Ints.toArray(partitionChannels)); - } - partitionGenerator = new LocalPartitionGenerator(hashGenerator, buffers.size()); + this.partitionFunction = requireNonNull(partitionFunction, "partitionFunction is null"); + this.partitioningChannels = ImmutableList.copyOf(requireNonNull(partitioningChannels, "partitioningChannels is null")); + this.hashChannel = requireNonNull(hashChannel, "hashChannel is null"); partitionAssignments = new IntArrayList[partitions.size()]; for (int i = 0; i < partitionAssignments.length; i++) { @@ -77,8 +65,9 @@ public synchronized void accept(Page page) } // assign each row to a partition - for (int position = 0; position < page.getPositionCount(); position++) { - int partition = partitionGenerator.getPartition(page, position); + Page partitioningChannelsPage = extractPartitioningChannels(page); + for (int position = 0; position < partitioningChannelsPage.getPositionCount(); position++) { + int partition = partitionFunction.getPartition(partitioningChannelsPage, position); partitionAssignments[partition].add(position); } @@ -98,6 +87,21 @@ public synchronized void accept(Page page) } } + private Page extractPartitioningChannels(Page inputPage) + { + // hash value is pre-computed, only needs to extract that channel + if (hashChannel.isPresent()) { + return new Page(inputPage.getBlock(hashChannel.get())); + } + + // extract partitioning channels + Block[] blocks = new Block[partitioningChannels.size()]; + for (int i = 0; i < partitioningChannels.size(); i++) { + blocks[i] = inputPage.getBlock(partitioningChannels.get(i)); + } + return new Page(inputPage.getPositionCount(), blocks); + } + @Override public ListenableFuture waitForWriting() { diff --git a/presto-main/src/main/java/com/facebook/presto/operator/index/DynamicTupleFilterFactory.java b/presto-main/src/main/java/com/facebook/presto/operator/index/DynamicTupleFilterFactory.java index 926a377689f7b..09c839e88f7d2 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/index/DynamicTupleFilterFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/index/DynamicTupleFilterFactory.java @@ -18,6 +18,7 @@ import com.facebook.presto.operator.project.PageProjection; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.PageFunctionCompiler; @@ -57,6 +58,7 @@ public DynamicTupleFilterFactory( int[] tupleFilterChannels, int[] outputFilterChannels, List outputTypes, + SqlFunctionProperties sqlFunctionProperties, PageFunctionCompiler pageFunctionCompiler) { requireNonNull(planNodeId, "planNodeId is null"); @@ -79,7 +81,7 @@ public DynamicTupleFilterFactory( this.outputTypes = ImmutableList.copyOf(outputTypes); this.outputProjections = IntStream.range(0, outputTypes.size()) - .mapToObj(field -> pageFunctionCompiler.compileProjection(Expressions.field(field, outputTypes.get(field)), Optional.empty())) + .mapToObj(field -> pageFunctionCompiler.compileProjection(sqlFunctionProperties, Expressions.field(field, outputTypes.get(field)), Optional.empty())) .collect(toImmutableList()); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/index/FieldSetFilteringRecordSet.java b/presto-main/src/main/java/com/facebook/presto/operator/index/FieldSetFilteringRecordSet.java index 9c0f314c42712..b2c065453f236 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/index/FieldSetFilteringRecordSet.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/index/FieldSetFilteringRecordSet.java @@ -54,7 +54,7 @@ public FieldSetFilteringRecordSet(FunctionManager functionManager, RecordSet del for (int field : fieldSet) { fieldSetBuilder.add(new Field( field, - functionManager.getScalarFunctionImplementation( + functionManager.getBuiltInScalarFunctionImplementation( functionManager.resolveOperator(OperatorType.EQUAL, fromTypes(columnTypes.get(field), columnTypes.get(field)))).getMethodHandle())); } fieldSetsBuilder.add(fieldSetBuilder.build()); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/index/IndexJoinLookupStats.java b/presto-main/src/main/java/com/facebook/presto/operator/index/IndexJoinLookupStats.java index f6bef1ce8d1ed..449dbbd2b6577 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/index/IndexJoinLookupStats.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/index/IndexJoinLookupStats.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.operator.index; -import io.airlift.stats.CounterStat; +import com.facebook.airlift.stats.CounterStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/index/IndexLoader.java b/presto-main/src/main/java/com/facebook/presto/operator/index/IndexLoader.java index d7e7a13755273..c848257f7780b 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/index/IndexLoader.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/index/IndexLoader.java @@ -25,6 +25,7 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageBuilder; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.Type; @@ -34,6 +35,7 @@ import com.google.common.collect.Iterables; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.DataSize; +import io.airlift.units.Duration; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.NotThreadSafe; @@ -44,15 +46,21 @@ import java.util.OptionalInt; import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.INDEX_LOADER_TIMEOUT; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Predicates.equalTo; import static com.google.common.base.Predicates.not; import static com.google.common.collect.Iterables.filter; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; @ThreadSafe public class IndexLoader @@ -73,6 +81,7 @@ public class IndexLoader private final List keyTypes; private final PagesIndex.Factory pagesIndexFactory; private final JoinCompiler joinCompiler; + private final Duration indexLoaderTimeout; @GuardedBy("this") private IndexSnapshotLoader indexSnapshotLoader; // Lazily initialized @@ -93,7 +102,8 @@ public IndexLoader( DataSize maxIndexMemorySize, IndexJoinLookupStats stats, PagesIndex.Factory pagesIndexFactory, - JoinCompiler joinCompiler) + JoinCompiler joinCompiler, + Duration indexLoaderTimeout) { requireNonNull(lookupSourceInputChannels, "lookupSourceInputChannels is null"); checkArgument(!lookupSourceInputChannels.isEmpty(), "lookupSourceInputChannels must not be empty"); @@ -107,6 +117,7 @@ public IndexLoader( requireNonNull(stats, "stats is null"); requireNonNull(pagesIndexFactory, "pagesIndexFactory is null"); requireNonNull(joinCompiler, "joinCompiler is null"); + requireNonNull(indexLoaderTimeout, "indexLoaderTimeout is null"); this.lookupSourceInputChannels = ImmutableSet.copyOf(lookupSourceInputChannels); this.keyOutputChannels = ImmutableList.copyOf(keyOutputChannels); @@ -118,6 +129,7 @@ public IndexLoader( this.stats = stats; this.pagesIndexFactory = pagesIndexFactory; this.joinCompiler = joinCompiler; + this.indexLoaderTimeout = indexLoaderTimeout; ImmutableList.Builder keyTypeBuilder = ImmutableList.builder(); for (int keyOutputChannel : keyOutputChannels) { @@ -263,7 +275,8 @@ private synchronized void initializeStateIfNecessary() expectedPositions, maxIndexMemorySize, pagesIndexFactory, - joinCompiler); + joinCompiler, + indexLoaderTimeout); } } @@ -280,6 +293,7 @@ private static class IndexSnapshotLoader private final JoinCompiler joinCompiler; private final IndexSnapshotBuilder indexSnapshotBuilder; + private final Duration timeout; private IndexSnapshotLoader(IndexBuildDriverFactoryProvider indexBuildDriverFactoryProvider, PipelineContext pipelineContext, @@ -291,7 +305,8 @@ private IndexSnapshotLoader(IndexBuildDriverFactoryProvider indexBuildDriverFact int expectedPositions, DataSize maxIndexMemorySize, PagesIndex.Factory pagesIndexFactory, - JoinCompiler joinCompiler) + JoinCompiler joinCompiler, + Duration timeout) { this.pipelineContext = pipelineContext; this.indexSnapshotReference = indexSnapshotReference; @@ -299,6 +314,7 @@ private IndexSnapshotLoader(IndexBuildDriverFactoryProvider indexBuildDriverFact this.outputTypes = indexBuildDriverFactoryProvider.getOutputTypes(); this.indexTypes = indexTypes; this.joinCompiler = joinCompiler; + this.timeout = timeout; this.indexSnapshotBuilder = new IndexSnapshotBuilder( outputTypes, @@ -334,7 +350,19 @@ public boolean load(List requests) driver.updateSource(new TaskSource(sourcePlanNodeId, ImmutableSet.of(split), true)); while (!driver.isFinished()) { ListenableFuture process = driver.process(); - checkState(process.isDone(), "Driver should never block"); + try { + process.get(timeout.toMillis(), MILLISECONDS); + } + catch (TimeoutException e) { + throw new PrestoException(INDEX_LOADER_TIMEOUT, format("Exceeded the time limit of %s loading indexes for index join", timeout)); + } + catch (ExecutionException e) { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Error loading index for join", e); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Error loading index for join", e); + } } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/index/IndexLookupSourceFactory.java b/presto-main/src/main/java/com/facebook/presto/operator/index/IndexLookupSourceFactory.java index 12ffdc49cc2f0..706aaafa6ad2d 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/index/IndexLookupSourceFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/index/IndexLookupSourceFactory.java @@ -28,6 +28,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import io.airlift.units.DataSize; +import io.airlift.units.Duration; import java.util.List; import java.util.Map; @@ -60,17 +61,18 @@ public IndexLookupSourceFactory( IndexJoinLookupStats stats, boolean shareIndexLoading, PagesIndex.Factory pagesIndexFactory, - JoinCompiler joinCompiler) + JoinCompiler joinCompiler, + Duration indexLoaderTimeout) { this.outputTypes = ImmutableList.copyOf(requireNonNull(outputTypes, "outputTypes is null")); this.layout = ImmutableMap.copyOf(requireNonNull(layout, "layout is null")); if (shareIndexLoading) { - IndexLoader shared = new IndexLoader(lookupSourceInputChannels, keyOutputChannels, keyOutputHashChannel, outputTypes, indexBuildDriverFactoryProvider, 10_000, maxIndexMemorySize, stats, pagesIndexFactory, joinCompiler); + IndexLoader shared = new IndexLoader(lookupSourceInputChannels, keyOutputChannels, keyOutputHashChannel, outputTypes, indexBuildDriverFactoryProvider, 10_000, maxIndexMemorySize, stats, pagesIndexFactory, joinCompiler, indexLoaderTimeout); this.indexLoaderSupplier = () -> shared; } else { - this.indexLoaderSupplier = () -> new IndexLoader(lookupSourceInputChannels, keyOutputChannels, keyOutputHashChannel, outputTypes, indexBuildDriverFactoryProvider, 10_000, maxIndexMemorySize, stats, pagesIndexFactory, joinCompiler); + this.indexLoaderSupplier = () -> new IndexLoader(lookupSourceInputChannels, keyOutputChannels, keyOutputHashChannel, outputTypes, indexBuildDriverFactoryProvider, 10_000, maxIndexMemorySize, stats, pagesIndexFactory, joinCompiler, indexLoaderTimeout); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/index/UpdateRequest.java b/presto-main/src/main/java/com/facebook/presto/operator/index/UpdateRequest.java index 74cd79353d9ab..295db64f086ae 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/index/UpdateRequest.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/index/UpdateRequest.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.operator.index; +import com.facebook.airlift.concurrent.MoreFutures; import com.facebook.presto.spi.Page; import com.google.common.util.concurrent.SettableFuture; -import io.airlift.concurrent.MoreFutures; import javax.annotation.concurrent.ThreadSafe; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/AbstractBlockEncodingBuffer.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/AbstractBlockEncodingBuffer.java new file mode 100644 index 0000000000000..1827597f3dacf --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/AbstractBlockEncodingBuffer.java @@ -0,0 +1,465 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.ByteArrayBlock; +import com.facebook.presto.spi.block.ColumnarArray; +import com.facebook.presto.spi.block.ColumnarMap; +import com.facebook.presto.spi.block.ColumnarRow; +import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.block.Int128ArrayBlock; +import com.facebook.presto.spi.block.IntArrayBlock; +import com.facebook.presto.spi.block.LongArrayBlock; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.block.ShortArrayBlock; +import com.facebook.presto.spi.block.VariableWidthBlock; +import com.google.common.annotations.VisibleForTesting; +import io.airlift.slice.SliceOutput; + +import javax.annotation.Nullable; + +import java.util.Arrays; + +import static com.facebook.presto.array.Arrays.ExpansionFactor.LARGE; +import static com.facebook.presto.array.Arrays.ExpansionFactor.SMALL; +import static com.facebook.presto.array.Arrays.ExpansionOption.INITIALIZE; +import static com.facebook.presto.array.Arrays.ExpansionOption.PRESERVE; +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.operator.MoreByteArrays.fill; +import static com.facebook.presto.operator.UncheckedByteArrays.setByteUnchecked; +import static com.google.common.base.Verify.verify; +import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.lang.String.format; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; + +public abstract class AbstractBlockEncodingBuffer + implements BlockEncodingBuffer +{ + // The block after peeling off the Dictionary or RLE wrappings. + protected Block decodedBlock; + + // The number of positions (rows) to be copied from the incoming block. + protected int positionCount; + + // The number of positions (rows) to be copied from the incoming block within current batch. + protected int batchSize; + + // The offset into positions/mappedPositions array that the current batch starts copying. + // I.e. in each batch we copy the values of rows from positions[positionsOffset] to positions[positionsOffset + batchSize] + protected int positionsOffset; + + // The number of positions (rows) buffered so far + protected int bufferedPositionCount; + + // Whether the positions array has already been mapped to mappedPositions + protected boolean positionsMapped; + + // The positions (row numbers) of the incoming Block to be copied to this buffer. + // All top level AbstractBlockEncodingBuffer for the same partition share the same positions array. + private int[] positions; + + // The mapped positions of the incoming Dictionary or RLE Block. For other blocks, mappedPosition + // is either null or meaningless. + // + // For example, a VARCHAR column with 5 rows - ['b', 'a', 'b', 'a', 'c'] is represented as + // a DictionaryBlock with dictionary ['a', 'b', 'c'] and ids [1, 0, 1, 0, 2]. The mappedPositions + // array for positions [0, 1, 4] for this VARCHAR column will be [1, 0, 2]. + @Nullable + private int[] mappedPositions; + + @Nullable + private byte[] nullsBuffer; + + // The address offset in the nullsBuffer if new values are to be added. + private int nullsBufferIndex; + + // For each batch we put the nulls values up to a multiple of 8 into nullsBuffer. The rest is kept in remainingNulls. + private final boolean[] remainingNulls = new boolean[Byte.SIZE]; + + // Number of null values not put into nullsBuffer from last batch + private int remainingNullsCount; + + // Boolean indicating whether there are any null values in the nullsBuffer. It is possible that all values are non-null. + private boolean hasEncodedNulls; + + public static BlockEncodingBuffer createBlockEncodingBuffers(DecodedBlockNode decodedBlockNode) + { + requireNonNull(decodedBlockNode, "decodedBlockNode is null"); + + // decodedBlock could be a block or ColumnarArray/Map/Row + Object decodedBlock = decodedBlockNode.getDecodedBlock(); + + // Skip the Dictionary/Rle block node. The mapping info is not needed when creating buffers. + // This is because the AbstractBlockEncodingBuffer is only created once, while position mapping for Dictionar/Rle blocks + // need to be done for every incoming block. + if (decodedBlock instanceof DictionaryBlock) { + decodedBlockNode = decodedBlockNode.getChildren().get(0); + decodedBlock = decodedBlockNode.getDecodedBlock(); + } + else if (decodedBlock instanceof RunLengthEncodedBlock) { + decodedBlockNode = decodedBlockNode.getChildren().get(0); + decodedBlock = decodedBlockNode.getDecodedBlock(); + } + + verify(!(decodedBlock instanceof DictionaryBlock), "Nested RLEs and dictionaries are not supported"); + verify(!(decodedBlock instanceof RunLengthEncodedBlock), "Nested RLEs and dictionaries are not supported"); + + if (decodedBlock instanceof LongArrayBlock) { + return new LongArrayBlockEncodingBuffer(); + } + + if (decodedBlock instanceof Int128ArrayBlock) { + return new Int128ArrayBlockEncodingBuffer(); + } + + if (decodedBlock instanceof IntArrayBlock) { + return new IntArrayBlockEncodingBuffer(); + } + + if (decodedBlock instanceof ShortArrayBlock) { + return new ShortArrayBlockEncodingBuffer(); + } + + if (decodedBlock instanceof ByteArrayBlock) { + return new ByteArrayBlockEncodingBuffer(); + } + + if (decodedBlock instanceof VariableWidthBlock) { + return new VariableWidthBlockEncodingBuffer(); + } + + if (decodedBlock instanceof ColumnarArray) { + return new ArrayBlockEncodingBuffer(decodedBlockNode); + } + + if (decodedBlock instanceof ColumnarMap) { + return new MapBlockEncodingBuffer(decodedBlockNode); + } + + if (decodedBlock instanceof ColumnarRow) { + return new RowBlockEncodingBuffer(decodedBlockNode); + } + + throw new IllegalArgumentException("Unsupported encoding: " + decodedBlock.getClass().getSimpleName()); + } + + @Override + public void setupDecodedBlocksAndPositions(DecodedBlockNode decodedBlockNode, int[] positions, int positionCount) + { + requireNonNull(decodedBlockNode, "decodedBlockNode is null"); + requireNonNull(positions, "positions is null"); + + this.positions = positions; + this.positionCount = positionCount; + this.positionsOffset = 0; + this.positionsMapped = false; + + setupDecodedBlockAndMapPositions(decodedBlockNode); + } + + @Override + public void setNextBatch(int positionsOffset, int batchSize) + { + this.positionsOffset = positionsOffset; + this.batchSize = batchSize; + } + + protected void setupDecodedBlockAndMapPositions(DecodedBlockNode decodedBlockNode) + { + requireNonNull(decodedBlockNode, "decodedBlockNode is null"); + decodedBlock = (Block) mapPositionsToNestedBlock(decodedBlockNode).getDecodedBlock(); + } + + /** + * Map the positions for DictionaryBlock to its nested dictionaryBlock, and positions for RunLengthEncodedBlock + * to its nested value block. For example, positions [0, 2, 5] on DictionaryBlock with dictionary block ['a', 'b', 'c'] + * and ids [1, 1, 2, 0, 2, 1] will be mapped to [1, 2, 1], and the copied data will be ['b', 'c', 'b']. + */ + protected DecodedBlockNode mapPositionsToNestedBlock(DecodedBlockNode decodedBlockNode) + { + Object decodedObject = decodedBlockNode.getDecodedBlock(); + + if (decodedObject instanceof DictionaryBlock) { + DictionaryBlock dictionaryBlock = (DictionaryBlock) decodedObject; + mappedPositions = ensureCapacity(mappedPositions, positionCount); + + for (int i = 0; i < positionCount; i++) { + mappedPositions[i] = dictionaryBlock.getId(positions[i]); + } + positionsMapped = true; + return decodedBlockNode.getChildren().get(0); + } + + if (decodedObject instanceof RunLengthEncodedBlock) { + mappedPositions = ensureCapacity(mappedPositions, positionCount, SMALL, INITIALIZE); + positionsMapped = true; + return decodedBlockNode.getChildren().get(0); + } + + positionsMapped = false; + return decodedBlockNode; + } + + protected void appendPositionRange(int offset, int length) + { + positions = ensureCapacity(positions, positionCount + length, LARGE, PRESERVE); + + for (int i = 0; i < length; i++) { + positions[positionCount++] = offset + i; + } + } + + /** + * Called by accumulateSerializedRowSizes(int[] serializedRowSizes) for composite types to add row sizes for nested blocks. + *

+ * For example, a AbstractBlockEncodingBuffer for an array(int) column with 3 rows - [{1, 2}, {3}, {4, 5, 6}] - will call accumulateSerializedRowSizes + * with positionOffsets = [0, 2, 3, 6] and serializedRowSizes = [5, 5, 5] (5 = 4 + 1 where 4 bytes for offset and 1 for null flag). + * After the method returns, serializedRowSizes will be [15, 10, 20]. + * + * @param positionOffsets The offsets of the top level rows + * @param positionCount Number of top level rows + */ + protected abstract void accumulateSerializedRowSizes(int[] positionOffsets, int positionCount, int[] serializedRowSizes); + + protected void resetPositions() + { + positionsOffset = 0; + positionCount = 0; + positionsMapped = false; + } + + protected int[] getPositions() + { + if (positionsMapped) { + verify(mappedPositions != null); + return mappedPositions; + } + + return positions; + } + + protected void appendNulls() + { + if (decodedBlock.mayHaveNull()) { + // Write to nullsBuffer if there is a possibility to have nulls. It is possible that the + // decodedBlock contains nulls, but rows that go into this partition don't. Write to nullsBuffer anyway. + nullsBuffer = ensureCapacity(nullsBuffer, (bufferedPositionCount + batchSize) / Byte.SIZE + 1, LARGE, PRESERVE); + + int bufferedNullsCount = nullsBufferIndex * Byte.SIZE + remainingNullsCount; + if (bufferedPositionCount > bufferedNullsCount) { + // There are no nulls in positions (bufferedNullsCount, bufferedPositionCount] + encodeNonNullsAsBits(bufferedPositionCount - bufferedNullsCount); + } + + // Append this batch + encodeNullsAsBits(decodedBlock); + } + else if (containsNull()) { + // There were nulls in previously buffered rows, but for this batch there can't be any nulls. + // Any how we need to append 0's for this batch. + nullsBuffer = ensureCapacity(nullsBuffer, (bufferedPositionCount + batchSize) / Byte.SIZE + 1, LARGE, PRESERVE); + encodeNonNullsAsBits(batchSize); + } + } + + protected void serializeNullsTo(SliceOutput output) + { + encodeRemainingNullsAsBits(); + + if (hasEncodedNulls) { + output.writeBoolean(true); + output.appendBytes(nullsBuffer, 0, nullsBufferIndex); + } + else { + output.writeBoolean(false); + } + } + + protected static void writeLengthPrefixedString(SliceOutput output, String value) + { + byte[] bytes = value.getBytes(UTF_8); + output.writeInt(bytes.length); + output.writeBytes(bytes); + } + + protected void resetNullsBuffer() + { + nullsBufferIndex = 0; + remainingNullsCount = 0; + hasEncodedNulls = false; + } + + protected long getPositionsRetainedSizeInBytes() + { + return sizeOf(positions) + sizeOf(mappedPositions); + } + + protected long getNullsBufferRetainedSizeInBytes() + { + return sizeOf(nullsBuffer) + sizeOf(remainingNulls); + } + + protected long getNullsBufferSerializedSizeInBytes() + { + long size = SIZE_OF_BYTE; // nulls uses 1 byte for mayHaveNull + if (containsNull()) { + size += nullsBufferIndex + // nulls buffer + (remainingNullsCount > 0 ? SIZE_OF_BYTE : 0); // The remaining nulls not serialized yet. We have to add it here because at the time of calling this function, the remaining nulls was not put into the nullsBuffer yet. + } + return size; + } + + private void encodeNullsAsBits(Block block) + { + int[] positions = getPositions(); + + if (remainingNullsCount + batchSize < Byte.SIZE) { + // just put all of this batch to remainingNulls + for (int i = 0; i < batchSize; i++) { + remainingNulls[remainingNullsCount++] = block.isNull(positions[positionsOffset + i]); + } + + return; + } + + // Process the remaining nulls from last batch + int offset = positionsOffset; + + if (remainingNullsCount > 0) { + byte value = 0; + + for (int i = 0; i < remainingNullsCount; i++) { + value |= remainingNulls[i] ? 0b1000_0000 >>> i : 0; + } + + // process a few more nulls to make up one byte + for (int i = remainingNullsCount; i < Byte.SIZE; i++) { + value |= block.isNull(positions[offset++]) ? 0b1000_0000 >>> i : 0; + } + + hasEncodedNulls |= (value != 0); + nullsBufferIndex = setByteUnchecked(nullsBuffer, nullsBufferIndex, value); + + remainingNullsCount = 0; + } + + // Process the next Byte.SIZE * n positions. We now have processed (offset - positionsOffset) positions + int remainingPositions = batchSize - (offset - positionsOffset); + int positionsToEncode = remainingPositions & ~0b111; + for (int i = 0; i < positionsToEncode; i += Byte.SIZE) { + byte value = 0; + value |= block.isNull(positions[offset]) ? 0b1000_0000 : 0; + value |= block.isNull(positions[offset + 1]) ? 0b0100_0000 : 0; + value |= block.isNull(positions[offset + 2]) ? 0b0010_0000 : 0; + value |= block.isNull(positions[offset + 3]) ? 0b0001_0000 : 0; + value |= block.isNull(positions[offset + 4]) ? 0b0000_1000 : 0; + value |= block.isNull(positions[offset + 5]) ? 0b0000_0100 : 0; + value |= block.isNull(positions[offset + 6]) ? 0b0000_0010 : 0; + value |= block.isNull(positions[offset + 7]) ? 0b0000_0001 : 0; + + hasEncodedNulls |= (value != 0); + nullsBufferIndex = setByteUnchecked(nullsBuffer, nullsBufferIndex, value); + offset += Byte.SIZE; + } + + // Process the remaining positions + remainingNullsCount = remainingPositions & 0b111; + for (int i = 0; i < remainingNullsCount; i++) { + remainingNulls[i] = block.isNull(positions[offset++]); + } + } + + private void encodeNonNullsAsBits(int count) + { + if (remainingNullsCount + count < Byte.SIZE) { + // just put all of this batch to remainingNulls + for (int i = 0; i < count; i++) { + remainingNulls[remainingNullsCount++] = false; + } + return; + } + + int remainingPositions = count - encodeRemainingNullsAsBits(); + + nullsBufferIndex = fill(nullsBuffer, nullsBufferIndex, remainingPositions >>> 3, (byte) 0); + + remainingNullsCount = remainingPositions & 0b111; + Arrays.fill(remainingNulls, false); + } + + private int encodeRemainingNullsAsBits() + { + if (remainingNullsCount == 0) { + return 0; + } + + byte value = 0; + for (int i = 0; i < remainingNullsCount; i++) { + value |= remainingNulls[i] ? 0b1000_0000 >>> i : 0; + } + + hasEncodedNulls |= (value != 0); + nullsBufferIndex = setByteUnchecked(nullsBuffer, nullsBufferIndex, value); + + int padding = Byte.SIZE - remainingNullsCount; + remainingNullsCount = 0; + return padding; + } + + private boolean containsNull() + { + if (hasEncodedNulls) { + return true; + } + + for (int i = 0; i < remainingNullsCount; i++) { + if (remainingNulls[i]) { + return true; + } + } + return false; + } + + @VisibleForTesting + void checkValidPositions() + { + verify(decodedBlock != null, "decodedBlock should have been set up"); + + int positionCount = decodedBlock.getPositionCount(); + + int[] positions = getPositions(); + for (int i = 0; i < this.positionCount; i++) { + if (positions[i] < 0 || positions[i] >= positionCount) { + throw new IndexOutOfBoundsException(format("Invalid position %d for decodedBlock with %d positions", positions[i], positionCount)); + } + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/ArrayBlockEncodingBuffer.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/ArrayBlockEncodingBuffer.java new file mode 100644 index 0000000000000..3c9fb196277c2 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/ArrayBlockEncodingBuffer.java @@ -0,0 +1,242 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.facebook.presto.spi.block.ColumnarArray; +import com.google.common.annotations.VisibleForTesting; +import io.airlift.slice.SliceOutput; +import org.openjdk.jol.info.ClassLayout; + +import static com.facebook.presto.array.Arrays.ExpansionFactor.LARGE; +import static com.facebook.presto.array.Arrays.ExpansionOption.PRESERVE; +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.operator.UncheckedByteArrays.setIntUnchecked; +import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.util.Objects.requireNonNull; +import static sun.misc.Unsafe.ARRAY_INT_INDEX_SCALE; + +public class ArrayBlockEncodingBuffer + extends AbstractBlockEncodingBuffer +{ + @VisibleForTesting + static final int POSITION_SIZE = SIZE_OF_INT + SIZE_OF_BYTE; + + private static final String NAME = "ARRAY"; + private static final int INSTANCE_SIZE = ClassLayout.parseClass(ArrayBlockEncodingBuffer.class).instanceSize(); + + // The buffer for the offsets for all incoming blocks so far + private byte[] offsetsBuffer; + + // The address that the next offset value will be written to. + private int offsetsBufferIndex; + + // This array holds the condensed offsets for each position for the incoming block. + private int[] offsets; + + // The last offset in the offsets buffer + private int lastOffset; + + // This array holds the offsets into its nested values block for each row in the ArrayBlock. + private int[] offsetsCopy; + + // The AbstractBlockEncodingBuffer for the nested values Block of the ArrayBlock + private final BlockEncodingBuffer valuesBuffers; + + public ArrayBlockEncodingBuffer(DecodedBlockNode decodedBlockNode) + { + valuesBuffers = createBlockEncodingBuffers(decodedBlockNode.getChildren().get(0)); + } + + @Override + public void accumulateSerializedRowSizes(int[] serializedRowSizes) + { + for (int i = 0; i < positionCount; i++) { + serializedRowSizes[i] += POSITION_SIZE; + } + + offsetsCopy = ensureCapacity(offsetsCopy, positionCount + 1); + System.arraycopy(offsets, 0, offsetsCopy, 0, positionCount + 1); + + ((AbstractBlockEncodingBuffer) valuesBuffers).accumulateSerializedRowSizes(offsetsCopy, positionCount, serializedRowSizes); + } + + @Override + public void setNextBatch(int positionsOffset, int batchSize) + { + this.positionsOffset = positionsOffset; + this.batchSize = batchSize; + + // If all positions for the ArrayBlock to be copied are null, the number of positions to copy for its + // nested values block could be 0. In such case we don't need to proceed. + if (this.positionCount == 0) { + return; + } + + int offset = offsets[positionsOffset]; + + valuesBuffers.setNextBatch(offset, offsets[positionsOffset + batchSize] - offset); + } + + @Override + public void appendDataInBatch() + { + if (batchSize == 0) { + return; + } + + appendNulls(); + appendOffsets(); + valuesBuffers.appendDataInBatch(); + + bufferedPositionCount += batchSize; + } + + @Override + public void serializeTo(SliceOutput output) + { + writeLengthPrefixedString(output, NAME); + + valuesBuffers.serializeTo(output); + + output.writeInt(bufferedPositionCount); + + // offsets + output.writeInt(0); // the base position + if (offsetsBufferIndex > 0) { + output.appendBytes(offsetsBuffer, 0, offsetsBufferIndex); + } + + serializeNullsTo(output); + } + + @Override + public void resetBuffers() + { + bufferedPositionCount = 0; + offsetsBufferIndex = 0; + lastOffset = 0; + resetNullsBuffer(); + + valuesBuffers.resetBuffers(); + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + + getPositionsRetainedSizeInBytes() + + sizeOf(offsetsBuffer) + + sizeOf(offsets) + + sizeOf(offsetsCopy) + + getNullsBufferRetainedSizeInBytes(); + } + + @Override + public long getSerializedSizeInBytes() + { + return NAME.length() + SIZE_OF_INT + // encoding name + valuesBuffers.getSerializedSizeInBytes() + // nested block + SIZE_OF_INT + // positionCount + SIZE_OF_INT + // offset 0. The offsetsBuffer doesn't contain the offset 0 so we need to add it here. + offsetsBufferIndex + // offsets buffer. + getNullsBufferSerializedSizeInBytes(); // nulls + } + + @Override + protected void setupDecodedBlockAndMapPositions(DecodedBlockNode decodedBlockNode) + { + requireNonNull(decodedBlockNode, "decodedBlockNode is null"); + + decodedBlockNode = mapPositionsToNestedBlock(decodedBlockNode); + ColumnarArray columnarArray = (ColumnarArray) decodedBlockNode.getDecodedBlock(); + decodedBlock = columnarArray.getNullCheckBlock(); + + populateNestedPositions(columnarArray); + + ((AbstractBlockEncodingBuffer) valuesBuffers).setupDecodedBlockAndMapPositions(decodedBlockNode.getChildren().get(0)); + } + + @Override + protected void accumulateSerializedRowSizes(int[] positionOffsets, int positionCount, int[] serializedRowSizes) + { + if (this.positionCount == 0) { + return; + } + + int lastOffset = positionOffsets[0]; + for (int i = 0; i < positionCount; i++) { + int offset = positionOffsets[i + 1]; + serializedRowSizes[i] += POSITION_SIZE * (offset - lastOffset); + lastOffset = offset; + positionOffsets[i + 1] = offsets[offset]; + } + + ((AbstractBlockEncodingBuffer) valuesBuffers).accumulateSerializedRowSizes(positionOffsets, positionCount, serializedRowSizes); + } + + private void populateNestedPositions(ColumnarArray columnarArray) + { + // Reset nested level positions before checking positionCount. Failing to do so may result in valuesBuffers having stale values when positionCount is 0. + ((AbstractBlockEncodingBuffer) valuesBuffers).resetPositions(); + + if (positionCount == 0) { + return; + } + + offsets = ensureCapacity(offsets, positionCount + 1); + + int[] positions = getPositions(); + + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + int beginOffset = columnarArray.getOffset(position); + int endOffset = columnarArray.getOffset(position + 1); + int length = endOffset - beginOffset; + + offsets[i + 1] = offsets[i] + length; + + if (length > 0) { + // beginOffset is the absolute position in the nested block. We need to subtract the base offset from it to get the logical position. + ((AbstractBlockEncodingBuffer) valuesBuffers).appendPositionRange(beginOffset, length); + } + } + } + + private void appendOffsets() + { + offsetsBuffer = ensureCapacity(offsetsBuffer, offsetsBufferIndex + batchSize * ARRAY_INT_INDEX_SCALE, LARGE, PRESERVE); + + int baseOffset = lastOffset - offsets[positionsOffset]; + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + offsetsBufferIndex = setIntUnchecked(offsetsBuffer, offsetsBufferIndex, offsets[i + 1] + baseOffset); + } + lastOffset = offsets[positionsOffset + batchSize] + baseOffset; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/BlockEncodingBuffer.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/BlockEncodingBuffer.java new file mode 100644 index 0000000000000..10f54f3fc4b14 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/BlockEncodingBuffer.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.operator.repartition; + +import io.airlift.slice.SliceOutput; + +public interface BlockEncodingBuffer +{ + /** + * Pass in the decoded block and positions in this block to copy. Called when a new page is being processed. + */ + void setupDecodedBlocksAndPositions(DecodedBlockNode decodedBlockNode, int[] positions, int positionCount); + + /** + * Adds serialized row sizes to serializedRowSizes array. Called for top level columns. + */ + void accumulateSerializedRowSizes(int[] serializedRowSizes); + + /** + * Set the position offset and batch size on the positions array in the BlockEncodingBuffer. Called before the next batch is being copied + */ + void setNextBatch(int positionsOffset, int batchSize); + + /** + * Append the rows at the end of the buffers in BlockEncodingBuffer + */ + void appendDataInBatch(); + + /** + * Copy the data from the buffers in the BlockEncodingBuffer to the SliceOutput object. Called when the buffers are full and flushing happens. + */ + void serializeTo(SliceOutput output); + + /** + * Reset the BlockEncodingBuffer to its initial state. The next incoming page will be appended from the beginning. + */ + void resetBuffers(); + + long getRetainedSizeInBytes(); + + long getSerializedSizeInBytes(); +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/ByteArrayBlockEncodingBuffer.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/ByteArrayBlockEncodingBuffer.java new file mode 100644 index 0000000000000..2f1f75eb07bf3 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/ByteArrayBlockEncodingBuffer.java @@ -0,0 +1,142 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.google.common.annotations.VisibleForTesting; +import io.airlift.slice.SliceOutput; +import org.openjdk.jol.info.ClassLayout; + +import static com.facebook.presto.array.Arrays.ExpansionFactor.LARGE; +import static com.facebook.presto.array.Arrays.ExpansionOption.PRESERVE; +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.operator.UncheckedByteArrays.setByteUnchecked; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; +import static io.airlift.slice.SizeOf.sizeOf; + +public class ByteArrayBlockEncodingBuffer + extends AbstractBlockEncodingBuffer +{ + @VisibleForTesting + static final int POSITION_SIZE = Byte.BYTES + Byte.BYTES; + + private static final String NAME = "BYTE_ARRAY"; + private static final int INSTANCE_SIZE = ClassLayout.parseClass(ByteArrayBlockEncodingBuffer.class).instanceSize(); + + private byte[] valuesBuffer; + private int valuesBufferIndex; + + @Override + public void accumulateSerializedRowSizes(int[] serializedRowSizes) + { + throw new UnsupportedOperationException("accumulateSerializedRowSizes is not supported for fixed width types"); + } + + @Override + protected void accumulateSerializedRowSizes(int[] positionOffsets, int positionCount, int[] serializedRowSizes) + { + for (int i = 0; i < positionCount; i++) { + serializedRowSizes[i] += (positionOffsets[i + 1] - positionOffsets[i]) * POSITION_SIZE; + } + } + + @Override + public void appendDataInBatch() + { + if (batchSize == 0) { + return; + } + + appendValuesToBuffer(); + appendNulls(); + bufferedPositionCount += batchSize; + } + + @Override + public void serializeTo(SliceOutput output) + { + writeLengthPrefixedString(output, NAME); + + output.writeInt(bufferedPositionCount); + + serializeNullsTo(output); + + if (valuesBufferIndex > 0) { + output.appendBytes(valuesBuffer, 0, valuesBufferIndex); + } + } + + @Override + public void resetBuffers() + { + bufferedPositionCount = 0; + valuesBufferIndex = 0; + resetNullsBuffer(); + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + + getPositionsRetainedSizeInBytes() + + sizeOf(valuesBuffer) + + getNullsBufferRetainedSizeInBytes(); + } + + @Override + public long getSerializedSizeInBytes() + { + return NAME.length() + SIZE_OF_INT + // NAME + SIZE_OF_INT + // positionCount + valuesBufferIndex + // values buffer + getNullsBufferSerializedSizeInBytes(); // nulls buffer + } + + private void appendValuesToBuffer() + { + valuesBuffer = ensureCapacity(valuesBuffer, valuesBufferIndex + batchSize, LARGE, PRESERVE); + + int[] positions = getPositions(); + if (decodedBlock.mayHaveNull()) { + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + int position = positions[i]; + byte value = decodedBlock.getByte(position); + int newIndex = setByteUnchecked(valuesBuffer, valuesBufferIndex, value); + + if (!decodedBlock.isNull(position)) { + valuesBufferIndex = newIndex; + } + } + } + else { + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + byte value = decodedBlock.getByte(positions[i]); + valuesBufferIndex = setByteUnchecked(valuesBuffer, valuesBufferIndex, value); + } + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/DecodedBlockNode.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/DecodedBlockNode.java new file mode 100644 index 0000000000000..df18ae84db8b5 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/DecodedBlockNode.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.ColumnarArray; +import com.facebook.presto.spi.block.ColumnarMap; +import com.facebook.presto.spi.block.ColumnarRow; +import org.openjdk.jol.info.ClassLayout; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * A tree structure to represent a decoded block. Dictionary/Rle blocks will be kept as a + * separate node because we need the position mappings. For example, a block of map where the key + * is dictionary encoded will result in the following tree structure: + * + * ColumnarMap + * / \ + * DictionaryBlock LongArrayBlock + * | + * IntArrayBlock + */ +class DecodedBlockNode +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(DecodedBlockNode.class).instanceSize(); + + // The decodedBlock could be primitive block, Dictionary/Rle block, or ColumnarArray/Map/Row object + private final Object decodedBlock; + private final List children; + + public DecodedBlockNode(Object decodedBlock, List children) + { + this.decodedBlock = requireNonNull(decodedBlock, "decodedBlock is null"); + this.children = requireNonNull(children, "children is null"); + } + + public Object getDecodedBlock() + { + return decodedBlock; + } + + public List getChildren() + { + return children; + } + + public long getRetainedSizeInBytes() + { + long size = INSTANCE_SIZE; + if (decodedBlock instanceof Block) { + size += ((Block) decodedBlock).getRetainedSizeInBytes(); + } + else if (decodedBlock instanceof ColumnarArray) { + size += ((ColumnarArray) decodedBlock).getRetainedSizeInBytes(); + } + else if (decodedBlock instanceof ColumnarMap) { + size += ((ColumnarMap) decodedBlock).getRetainedSizeInBytes(); + } + else if (decodedBlock instanceof ColumnarRow) { + size += ((ColumnarRow) decodedBlock).getRetainedSizeInBytes(); + } + + for (DecodedBlockNode child : children) { + size += child.getRetainedSizeInBytes(); + } + return size; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/Int128ArrayBlockEncodingBuffer.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/Int128ArrayBlockEncodingBuffer.java new file mode 100644 index 0000000000000..b4217e9d31528 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/Int128ArrayBlockEncodingBuffer.java @@ -0,0 +1,153 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.google.common.annotations.VisibleForTesting; +import io.airlift.slice.SliceOutput; +import org.openjdk.jol.info.ClassLayout; + +import static com.facebook.presto.array.Arrays.ExpansionFactor.LARGE; +import static com.facebook.presto.array.Arrays.ExpansionOption.PRESERVE; +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.operator.UncheckedByteArrays.setLongUnchecked; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; +import static io.airlift.slice.SizeOf.sizeOf; +import static sun.misc.Unsafe.ARRAY_LONG_INDEX_SCALE; + +public class Int128ArrayBlockEncodingBuffer + extends AbstractBlockEncodingBuffer +{ + @VisibleForTesting + static final int POSITION_SIZE = Long.BYTES * 2 + Byte.BYTES; + + private static final String NAME = "INT128_ARRAY"; + private static final int INSTANCE_SIZE = ClassLayout.parseClass(Int128ArrayBlockEncodingBuffer.class).instanceSize(); + + private byte[] valuesBuffer; + private int valuesBufferIndex; + + @Override + public void accumulateSerializedRowSizes(int[] serializedRowSizes) + { + throw new UnsupportedOperationException("accumulateSerializedRowSizes is not supported for fixed width types"); + } + + @Override + public void appendDataInBatch() + { + if (batchSize == 0) { + return; + } + + appendValuesToBuffer(); + appendNulls(); + bufferedPositionCount += batchSize; + } + + @Override + public void serializeTo(SliceOutput output) + { + writeLengthPrefixedString(output, NAME); + + output.writeInt(bufferedPositionCount); + + serializeNullsTo(output); + + if (valuesBufferIndex > 0) { + output.appendBytes(valuesBuffer, 0, valuesBufferIndex); + } + } + + @Override + public void resetBuffers() + { + bufferedPositionCount = 0; + valuesBufferIndex = 0; + resetNullsBuffer(); + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + + getPositionsRetainedSizeInBytes() + + sizeOf(valuesBuffer) + + getNullsBufferRetainedSizeInBytes(); + } + + @Override + public long getSerializedSizeInBytes() + { + return NAME.length() + SIZE_OF_INT + // NAME + SIZE_OF_INT + // positionCount + valuesBufferIndex + // values buffer + getNullsBufferSerializedSizeInBytes(); // nulls buffer + } + + @Override + protected void accumulateSerializedRowSizes(int[] positionOffsets, int positionCount, int[] serializedRowSizes) + { + for (int i = 0; i < positionCount; i++) { + serializedRowSizes[i] += (positionOffsets[i + 1] - positionOffsets[i]) * POSITION_SIZE; + } + } + + private void appendValuesToBuffer() + { + valuesBuffer = ensureCapacity(valuesBuffer, valuesBufferIndex + batchSize * ARRAY_LONG_INDEX_SCALE * 2, LARGE, PRESERVE); + + int[] positions = getPositions(); + + if (decodedBlock.mayHaveNull()) { + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + int position = positions[i]; + + long value = decodedBlock.getLong(position, 0); + valuesBufferIndex = setLongUnchecked(valuesBuffer, valuesBufferIndex, value); + + value = decodedBlock.getLong(position, 8); + valuesBufferIndex = setLongUnchecked(valuesBuffer, valuesBufferIndex, value); + + if (decodedBlock.isNull(position)) { + valuesBufferIndex -= ARRAY_LONG_INDEX_SCALE * 2; + } + } + } + else { + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + int position = positions[i]; + + long value = decodedBlock.getLong(position, 0); + valuesBufferIndex = setLongUnchecked(valuesBuffer, valuesBufferIndex, value); + + value = decodedBlock.getLong(position, 8); + valuesBufferIndex = setLongUnchecked(valuesBuffer, valuesBufferIndex, value); + } + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/IntArrayBlockEncodingBuffer.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/IntArrayBlockEncodingBuffer.java new file mode 100644 index 0000000000000..49bc871974186 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/IntArrayBlockEncodingBuffer.java @@ -0,0 +1,144 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.google.common.annotations.VisibleForTesting; +import io.airlift.slice.SliceOutput; +import org.openjdk.jol.info.ClassLayout; + +import static com.facebook.presto.array.Arrays.ExpansionFactor.LARGE; +import static com.facebook.presto.array.Arrays.ExpansionOption.PRESERVE; +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.operator.UncheckedByteArrays.setIntUnchecked; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; +import static io.airlift.slice.SizeOf.sizeOf; +import static sun.misc.Unsafe.ARRAY_INT_INDEX_SCALE; + +public class IntArrayBlockEncodingBuffer + extends AbstractBlockEncodingBuffer +{ + @VisibleForTesting + static final int POSITION_SIZE = Integer.BYTES + Byte.BYTES; + + private static final String NAME = "INT_ARRAY"; + private static final int INSTANCE_SIZE = ClassLayout.parseClass(IntArrayBlockEncodingBuffer.class).instanceSize(); + + private byte[] valuesBuffer; + private int valuesBufferIndex; + + @Override + public void accumulateSerializedRowSizes(int[] serializedRowSizes) + { + throw new UnsupportedOperationException("accumulateSerializedRowSizes is not supported for fixed width types"); + } + + @Override + public void appendDataInBatch() + { + if (batchSize == 0) { + return; + } + + appendValuesToBuffer(); + appendNulls(); + bufferedPositionCount += batchSize; + } + + @Override + public void serializeTo(SliceOutput output) + { + writeLengthPrefixedString(output, NAME); + + output.writeInt(bufferedPositionCount); + + serializeNullsTo(output); + + if (valuesBufferIndex > 0) { + output.appendBytes(valuesBuffer, 0, valuesBufferIndex); + } + } + + @Override + public void resetBuffers() + { + bufferedPositionCount = 0; + valuesBufferIndex = 0; + resetNullsBuffer(); + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + + getPositionsRetainedSizeInBytes() + + sizeOf(valuesBuffer) + + getNullsBufferRetainedSizeInBytes(); + } + + @Override + public long getSerializedSizeInBytes() + { + return NAME.length() + SIZE_OF_INT + // NAME + SIZE_OF_INT + // positionCount + getNullsBufferSerializedSizeInBytes() + // nulls buffer + valuesBufferIndex; // values buffer + } + + @Override + protected void accumulateSerializedRowSizes(int[] positionOffsets, int positionCount, int[] serializedRowSizes) + { + for (int i = 0; i < positionCount; i++) { + serializedRowSizes[i] += (positionOffsets[i + 1] - positionOffsets[i]) * POSITION_SIZE; + } + } + + private void appendValuesToBuffer() + { + valuesBuffer = ensureCapacity(valuesBuffer, valuesBufferIndex + batchSize * ARRAY_INT_INDEX_SCALE, LARGE, PRESERVE); + + int[] positions = getPositions(); + + if (decodedBlock.mayHaveNull()) { + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + int position = positions[i]; + int value = decodedBlock.getInt(position); + int newIndex = setIntUnchecked(valuesBuffer, valuesBufferIndex, value); + + if (!decodedBlock.isNull(position)) { + valuesBufferIndex = newIndex; + } + } + } + else { + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + int value = decodedBlock.getInt(positions[i]); + valuesBufferIndex = setIntUnchecked(valuesBuffer, valuesBufferIndex, value); + } + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/LongArrayBlockEncodingBuffer.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/LongArrayBlockEncodingBuffer.java new file mode 100644 index 0000000000000..12be805cb8672 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/LongArrayBlockEncodingBuffer.java @@ -0,0 +1,128 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.google.common.annotations.VisibleForTesting; +import io.airlift.slice.SliceOutput; +import org.openjdk.jol.info.ClassLayout; + +import static com.facebook.presto.array.Arrays.ExpansionFactor.LARGE; +import static com.facebook.presto.array.Arrays.ExpansionOption.PRESERVE; +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.operator.UncheckedByteArrays.setLongUnchecked; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; +import static io.airlift.slice.SizeOf.sizeOf; +import static sun.misc.Unsafe.ARRAY_LONG_INDEX_SCALE; + +public class LongArrayBlockEncodingBuffer + extends AbstractBlockEncodingBuffer +{ + @VisibleForTesting + static final int POSITION_SIZE = Long.BYTES + Byte.BYTES; + + private static final String NAME = "LONG_ARRAY"; + private static final int INSTANCE_SIZE = ClassLayout.parseClass(LongArrayBlockEncodingBuffer.class).instanceSize(); + + private byte[] valuesBuffer; + private int valuesBufferIndex; + + @Override + public void accumulateSerializedRowSizes(int[] serializedRowSizes) + { + throw new UnsupportedOperationException("accumulateSerializedRowSizes is not supported for fixed width types"); + } + + @Override + public void appendDataInBatch() + { + if (batchSize == 0) { + return; + } + + appendValuesToBuffer(); + appendNulls(); + bufferedPositionCount += batchSize; + } + + @Override + public void serializeTo(SliceOutput output) + { + writeLengthPrefixedString(output, NAME); + + output.writeInt(bufferedPositionCount); + + serializeNullsTo(output); + + if (valuesBufferIndex > 0) { + output.appendBytes(valuesBuffer, 0, valuesBufferIndex); + } + } + + @Override + public void resetBuffers() + { + bufferedPositionCount = 0; + valuesBufferIndex = 0; + resetNullsBuffer(); + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + + getPositionsRetainedSizeInBytes() + + sizeOf(valuesBuffer) + + getNullsBufferRetainedSizeInBytes(); + } + + @Override + public long getSerializedSizeInBytes() + { + return NAME.length() + SIZE_OF_INT + // NAME + SIZE_OF_INT + // positionCount + getNullsBufferSerializedSizeInBytes() + // nulls buffer + valuesBufferIndex; // values buffer + } + + @Override + protected void accumulateSerializedRowSizes(int[] positionOffsets, int positionCount, int[] serializedRowSizes) + { + for (int i = 0; i < positionCount; i++) { + serializedRowSizes[i] += (positionOffsets[i + 1] - positionOffsets[i]) * POSITION_SIZE; + } + } + + private void appendValuesToBuffer() + { + valuesBuffer = ensureCapacity(valuesBuffer, valuesBufferIndex + batchSize * ARRAY_LONG_INDEX_SCALE, LARGE, PRESERVE); + + int[] positions = getPositions(); + if (decodedBlock.mayHaveNull()) { + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + int position = positions[i]; + int newIndex = setLongUnchecked(valuesBuffer, valuesBufferIndex, decodedBlock.getLong(position)); + + // Make sure the branch statement contains only one instruction, so that JVM can compile to a conditional move (cmov) + if (!decodedBlock.isNull(position)) { + valuesBufferIndex = newIndex; + } + } + } + else { + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + valuesBufferIndex = setLongUnchecked(valuesBuffer, valuesBufferIndex, decodedBlock.getLong(positions[i])); + } + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/MapBlockEncodingBuffer.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/MapBlockEncodingBuffer.java new file mode 100644 index 0000000000000..d826cba1dac26 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/MapBlockEncodingBuffer.java @@ -0,0 +1,335 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.facebook.presto.spi.block.ColumnarMap; +import com.facebook.presto.spi.type.TypeSerde; +import com.google.common.annotations.VisibleForTesting; +import io.airlift.slice.SliceOutput; +import org.openjdk.jol.info.ClassLayout; + +import java.util.Optional; + +import static com.facebook.presto.array.Arrays.ExpansionFactor.LARGE; +import static com.facebook.presto.array.Arrays.ExpansionOption.PRESERVE; +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.operator.MoreByteArrays.setInts; +import static com.facebook.presto.operator.UncheckedByteArrays.setIntUnchecked; +import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.util.Objects.requireNonNull; +import static sun.misc.Unsafe.ARRAY_INT_INDEX_SCALE; + +public class MapBlockEncodingBuffer + extends AbstractBlockEncodingBuffer +{ + @VisibleForTesting + static final int POSITION_SIZE = SIZE_OF_INT + SIZE_OF_BYTE; + + private static final String NAME = "MAP"; + private static final int INSTANCE_SIZE = ClassLayout.parseClass(MapBlockEncodingBuffer.class).instanceSize(); + + private static final int HASH_MULTIPLIER = 2; + + // The buffer for the hashtables for all incoming blocks so far + private byte[] hashTablesBuffer; + + // The address that the next hashtable entry will be written to. + private int hashTableBufferIndex; + + // If any of the incoming blocks do not contain the hashtable, noHashTables is set to true. + private boolean noHashTables; + + // The buffer for the offsets for all incoming blocks so far + private byte[] offsetsBuffer; + + // The address that the next offset value will be written to. + private int offsetsBufferIndex; + + // This array holds the condensed offsets for each position for the incoming block. + private int[] offsets; + + // The last offset in the offsets buffer + private int lastOffset; + + // This array holds the offsets into its nested key and value blocks for each row in the MapBlock. + private int[] offsetsCopy; + + // The current incoming MapBlock is converted into ColumnarMap + private ColumnarMap columnarMap; + + // The AbstractBlockEncodingBuffer for the nested key and value Block of the MapBlock + private final BlockEncodingBuffer keyBuffers; + private final BlockEncodingBuffer valueBuffers; + + public MapBlockEncodingBuffer(DecodedBlockNode decodedBlockNode) + { + keyBuffers = createBlockEncodingBuffers(decodedBlockNode.getChildren().get(0)); + valueBuffers = createBlockEncodingBuffers(decodedBlockNode.getChildren().get(1)); + } + + @Override + public void accumulateSerializedRowSizes(int[] serializedRowSizes) + { + for (int i = 0; i < positionCount; i++) { + serializedRowSizes[i] += POSITION_SIZE; + } + + offsetsCopy = ensureCapacity(offsetsCopy, positionCount + 1); + + System.arraycopy(offsets, 0, offsetsCopy, 0, positionCount + 1); + ((AbstractBlockEncodingBuffer) keyBuffers).accumulateSerializedRowSizes(offsetsCopy, positionCount, serializedRowSizes); + + System.arraycopy(offsets, 0, offsetsCopy, 0, positionCount + 1); + ((AbstractBlockEncodingBuffer) valueBuffers).accumulateSerializedRowSizes(offsetsCopy, positionCount, serializedRowSizes); + } + + @Override + public void setNextBatch(int positionsOffset, int batchSize) + { + this.positionsOffset = positionsOffset; + this.batchSize = batchSize; + + if (this.positionCount == 0) { + return; + } + + int beginOffset = offsets[positionsOffset]; + int endOffset = offsets[positionsOffset + batchSize]; + + keyBuffers.setNextBatch(beginOffset, endOffset - beginOffset); + valueBuffers.setNextBatch(beginOffset, endOffset - beginOffset); + } + + @Override + public void appendDataInBatch() + { + if (batchSize == 0) { + return; + } + + appendNulls(); + appendOffsets(); + appendHashTables(); + + keyBuffers.appendDataInBatch(); + valueBuffers.appendDataInBatch(); + + bufferedPositionCount += batchSize; + } + + @Override + public void serializeTo(SliceOutput output) + { + writeLengthPrefixedString(output, NAME); + + TypeSerde.writeType(output, columnarMap.getKeyType()); + + keyBuffers.serializeTo(output); + valueBuffers.serializeTo(output); + + // Hash tables + if (hashTableBufferIndex == 0) { + output.appendInt(-1); + } + else { + output.appendInt(lastOffset * HASH_MULTIPLIER); // Hash table length + output.appendBytes(hashTablesBuffer, 0, hashTableBufferIndex); + } + + output.writeInt(bufferedPositionCount); + + // offsets + output.appendInt(0); + if (offsetsBufferIndex > 0) { + output.appendBytes(offsetsBuffer, 0, offsetsBufferIndex); + } + + serializeNullsTo(output); + } + + @Override + public void resetBuffers() + { + bufferedPositionCount = 0; + offsetsBufferIndex = 0; + lastOffset = 0; + hashTableBufferIndex = 0; + noHashTables = false; + resetNullsBuffer(); + + keyBuffers.resetBuffers(); + valueBuffers.resetBuffers(); + } + + @Override + public long getRetainedSizeInBytes() + { + // columnarMap is counted as part of DecodedBlockNode in OptimizedPartitionedOutputOperator and won't be counted here. + // This is because the same columnarMap would be hold in all partitions/AbstractBlockEncodingBuffer and thus counting it here would be counting it multiple times. + return INSTANCE_SIZE + + getPositionsRetainedSizeInBytes() + + sizeOf(offsets) + + sizeOf(offsetsBuffer) + + sizeOf(offsetsCopy) + + getNullsBufferRetainedSizeInBytes() + + keyBuffers.getRetainedSizeInBytes() + + valueBuffers.getRetainedSizeInBytes() + + sizeOf(hashTablesBuffer); + } + + @Override + public long getSerializedSizeInBytes() + { + return NAME.length() + SIZE_OF_INT + // length prefixed encoding name + columnarMap.getKeyType().getTypeSignature().toString().length() + SIZE_OF_INT + // length prefixed type string + keyBuffers.getSerializedSizeInBytes() + // nested key block + valueBuffers.getSerializedSizeInBytes() + // nested value block + SIZE_OF_INT + // hash tables size + hashTableBufferIndex + // hash tables + SIZE_OF_INT + // positionCount + offsetsBufferIndex + SIZE_OF_INT + // offsets buffer. + getNullsBufferSerializedSizeInBytes(); // nulls + } + + @Override + protected void setupDecodedBlockAndMapPositions(DecodedBlockNode decodedBlockNode) + { + requireNonNull(decodedBlockNode, "decodedBlockNode is null"); + + decodedBlockNode = mapPositionsToNestedBlock(decodedBlockNode); + columnarMap = (ColumnarMap) decodedBlockNode.getDecodedBlock(); + decodedBlock = columnarMap.getNullCheckBlock(); + + populateNestedPositions(); + + ((AbstractBlockEncodingBuffer) keyBuffers).setupDecodedBlockAndMapPositions(decodedBlockNode.getChildren().get(0)); + ((AbstractBlockEncodingBuffer) valueBuffers).setupDecodedBlockAndMapPositions(decodedBlockNode.getChildren().get(1)); + } + + @Override + protected void accumulateSerializedRowSizes(int[] positionOffsets, int positionCount, int[] serializedRowSizes) + { + // If all positions for the MapBlock to be copied are null, the number of positions to copy for its + // nested key and value blocks could be 0. In such case we don't need to proceed. + if (this.positionCount == 0) { + return; + } + + int lastOffset = positionOffsets[0]; + for (int i = 0; i < positionCount; i++) { + int offset = positionOffsets[i + 1]; + serializedRowSizes[i] += POSITION_SIZE * (offset - lastOffset); + lastOffset = offset; + positionOffsets[i + 1] = offsets[offset]; + } + + // positionOffsets might be modified by the next level. Save it for the valueBuffers first. + offsetsCopy = ensureCapacity(offsetsCopy, positionCount + 1); + System.arraycopy(positionOffsets, 0, offsetsCopy, 0, positionCount + 1); + + ((AbstractBlockEncodingBuffer) keyBuffers).accumulateSerializedRowSizes(positionOffsets, positionCount, serializedRowSizes); + ((AbstractBlockEncodingBuffer) valueBuffers).accumulateSerializedRowSizes(offsetsCopy, positionCount, serializedRowSizes); + } + + private void populateNestedPositions() + { + // Reset nested level positions before checking positionCount. Failing to do so may result in elementsBuffers having stale values when positionCount is 0. + ((AbstractBlockEncodingBuffer) keyBuffers).resetPositions(); + ((AbstractBlockEncodingBuffer) valueBuffers).resetPositions(); + + if (positionCount == 0) { + return; + } + + offsets = ensureCapacity(offsets, positionCount + 1); + + int[] positions = getPositions(); + + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + int beginOffset = columnarMap.getOffset(position); + int endOffset = columnarMap.getOffset(position + 1); + int currentRowSize = endOffset - beginOffset; + + offsets[i + 1] = offsets[i] + currentRowSize; + + if (currentRowSize > 0) { + ((AbstractBlockEncodingBuffer) keyBuffers).appendPositionRange(beginOffset, currentRowSize); + ((AbstractBlockEncodingBuffer) valueBuffers).appendPositionRange(beginOffset, currentRowSize); + } + } + } + + private void appendOffsets() + { + offsetsBuffer = ensureCapacity(offsetsBuffer, offsetsBufferIndex + batchSize * ARRAY_INT_INDEX_SCALE, LARGE, PRESERVE); + + int baseOffset = lastOffset - offsets[positionsOffset]; + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + offsetsBufferIndex = setIntUnchecked(offsetsBuffer, offsetsBufferIndex, offsets[i + 1] + baseOffset); + } + lastOffset = offsets[positionsOffset + batchSize] + baseOffset; + } + + private void appendHashTables() + { + // MergingPageOutput may build hash tables for some of the small blocks. But if there're some blocks + // without hash tables, it means hash tables are not needed so far. In this case we don't send the hash tables. + if (noHashTables) { + return; + } + + Optional hashTables = columnarMap.getHashTables(); + if (!hashTables.isPresent()) { + noHashTables = true; + hashTableBufferIndex = 0; + return; + } + + int hashTablesSize = (offsets[positionsOffset + batchSize] - offsets[positionsOffset]) * HASH_MULTIPLIER; + hashTablesBuffer = ensureCapacity(hashTablesBuffer, hashTableBufferIndex + hashTablesSize * ARRAY_INT_INDEX_SCALE, LARGE, PRESERVE); + + int[] positions = getPositions(); + + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + int position = positions[i]; + + int beginOffset = columnarMap.getAbsoluteOffset(position); + int endOffset = columnarMap.getAbsoluteOffset(position + 1); + + hashTableBufferIndex = setInts( + hashTablesBuffer, + hashTableBufferIndex, + hashTables.get(), + beginOffset * HASH_MULTIPLIER, + (endOffset - beginOffset) * HASH_MULTIPLIER); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/OptimizedPartitionedOutputOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/OptimizedPartitionedOutputOperator.java new file mode 100644 index 0000000000000..7b226f37d27f0 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/OptimizedPartitionedOutputOperator.java @@ -0,0 +1,734 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.facebook.presto.execution.Lifespan; +import com.facebook.presto.execution.buffer.OutputBuffer; +import com.facebook.presto.execution.buffer.PagesSerde; +import com.facebook.presto.execution.buffer.PagesSerdeFactory; +import com.facebook.presto.execution.buffer.SerializedPage; +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.operator.DriverContext; +import com.facebook.presto.operator.Operator; +import com.facebook.presto.operator.OperatorContext; +import com.facebook.presto.operator.OperatorFactory; +import com.facebook.presto.operator.OutputFactory; +import com.facebook.presto.operator.PartitionFunction; +import com.facebook.presto.operator.SimpleArrayAllocator; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.ArrayAllocator; +import com.facebook.presto.spi.block.ArrayBlock; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockFlattener; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.block.ColumnarArray; +import com.facebook.presto.spi.block.ColumnarMap; +import com.facebook.presto.spi.block.ColumnarRow; +import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.block.MapBlock; +import com.facebook.presto.spi.block.RowBlock; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.type.FixedWidthType; +import com.facebook.presto.spi.type.Type; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Closer; +import com.google.common.util.concurrent.ListenableFuture; +import io.airlift.slice.DynamicSliceOutput; +import io.airlift.slice.SliceOutput; +import io.airlift.units.DataSize; +import org.openjdk.jol.info.ClassLayout; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; + +import static com.facebook.presto.array.Arrays.ExpansionFactor.SMALL; +import static com.facebook.presto.array.Arrays.ExpansionOption.INITIALIZE; +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.operator.repartition.AbstractBlockEncodingBuffer.createBlockEncodingBuffers; +import static com.facebook.presto.spi.block.PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES; +import static com.google.common.base.Verify.verify; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.lang.Math.toIntExact; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class OptimizedPartitionedOutputOperator + implements Operator +{ + private final OperatorContext operatorContext; + private final Function pagePreprocessor; + private final PagePartitioner pagePartitioner; + private final LocalMemoryContext systemMemoryContext; + private boolean finished; + + public OptimizedPartitionedOutputOperator( + OperatorContext operatorContext, + List sourceTypes, + Function pagePreprocessor, + PartitionFunction partitionFunction, + List partitionChannels, + List> partitionConstants, + boolean replicatesAnyRow, + OptionalInt nullChannel, + OutputBuffer outputBuffer, + PagesSerdeFactory serdeFactory, + DataSize maxMemory) + { + this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); + this.pagePreprocessor = requireNonNull(pagePreprocessor, "pagePreprocessor is null"); + this.pagePartitioner = new PagePartitioner( + partitionFunction, + partitionChannels, + partitionConstants, + replicatesAnyRow, + nullChannel, + outputBuffer, + serdeFactory, + sourceTypes, + maxMemory, + operatorContext.getDriverContext().getLifespan()); + + operatorContext.setInfoSupplier(this::getInfo); + this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(PartitionedOutputOperator.class.getSimpleName()); + this.systemMemoryContext.setBytes(pagePartitioner.getRetainedSizeInBytes()); + } + + @Override + public OperatorContext getOperatorContext() + { + return operatorContext; + } + + public PartitionedOutputInfo getInfo() + { + return pagePartitioner.getInfo(); + } + + @Override + public void finish() + { + finished = true; + pagePartitioner.flush(); + } + + @Override + public boolean isFinished() + { + return finished && isBlocked().isDone(); + } + + @Override + public ListenableFuture isBlocked() + { + ListenableFuture blocked = pagePartitioner.isFull(); + return blocked.isDone() ? NOT_BLOCKED : blocked; + } + + @Override + public boolean needsInput() + { + return !finished && isBlocked().isDone(); + } + + @Override + public void addInput(Page page) + { + requireNonNull(page, "page is null"); + + if (page.getPositionCount() == 0) { + return; + } + + page = pagePreprocessor.apply(page); + pagePartitioner.partitionPage(page); + + // TODO: PartitionedOutputOperator reports incorrect output data size #11770 + operatorContext.recordOutput(page.getSizeInBytes(), page.getPositionCount()); + + systemMemoryContext.setBytes(pagePartitioner.getRetainedSizeInBytes()); + } + + @Override + public Page getOutput() + { + return null; + } + + /** + * Flatten the block and convert the nested-typed block into ColumnarArray/Map/Row. + * For performance considerations we decode the block only once for each block instead of for each batch. + * + * @return A tree structure that contains the decoded block + */ + @VisibleForTesting + static DecodedBlockNode decodeBlock(BlockFlattener flattener, Closer blockLeaseCloser, Block block) + { + BlockLease lease = flattener.flatten(block); + blockLeaseCloser.register(lease::close); + Block decodedBlock = lease.get(); + + if (decodedBlock instanceof ArrayBlock) { + ColumnarArray columnarArray = ColumnarArray.toColumnarArray(decodedBlock); + return new DecodedBlockNode(columnarArray, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, columnarArray.getElementsBlock()))); + } + + if (decodedBlock instanceof MapBlock) { + ColumnarMap columnarMap = ColumnarMap.toColumnarMap(decodedBlock); + return new DecodedBlockNode(columnarMap, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, columnarMap.getKeysBlock()), decodeBlock(flattener, blockLeaseCloser, columnarMap.getValuesBlock()))); + } + + if (decodedBlock instanceof RowBlock) { + ColumnarRow columnarRow = ColumnarRow.toColumnarRow(decodedBlock); + ImmutableList.Builder children = ImmutableList.builder(); + for (int i = 0; i < columnarRow.getFieldCount(); i++) { + children.add(decodeBlock(flattener, blockLeaseCloser, columnarRow.getField(i))); + } + return new DecodedBlockNode(columnarRow, children.build()); + } + + if (decodedBlock instanceof DictionaryBlock) { + return new DecodedBlockNode(decodedBlock, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, ((DictionaryBlock) decodedBlock).getDictionary()))); + } + + if (decodedBlock instanceof RunLengthEncodedBlock) { + return new DecodedBlockNode(decodedBlock, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, ((RunLengthEncodedBlock) decodedBlock).getValue()))); + } + + return new DecodedBlockNode(decodedBlock, ImmutableList.of()); + } + + public static class OptimizedPartitionedOutputFactory + implements OutputFactory + { + private final PartitionFunction partitionFunction; + private final List partitionChannels; + private final List> partitionConstants; + private final OutputBuffer outputBuffer; + private final boolean replicatesAnyRow; + private final OptionalInt nullChannel; + private final DataSize maxMemory; + + public OptimizedPartitionedOutputFactory( + PartitionFunction partitionFunction, + List partitionChannels, + List> partitionConstants, + boolean replicatesAnyRow, + OptionalInt nullChannel, + OutputBuffer outputBuffer, + DataSize maxMemory) + { + this.partitionFunction = requireNonNull(partitionFunction, "partitionFunction is null"); + this.partitionChannels = requireNonNull(partitionChannels, "partitionChannels is null"); + this.partitionConstants = requireNonNull(partitionConstants, "partitionConstants is null"); + this.replicatesAnyRow = replicatesAnyRow; + this.nullChannel = requireNonNull(nullChannel, "nullChannel is null"); + this.outputBuffer = requireNonNull(outputBuffer, "outputBuffer is null"); + this.maxMemory = requireNonNull(maxMemory, "maxMemory is null"); + } + + @Override + public OperatorFactory createOutputOperator( + int operatorId, + PlanNodeId planNodeId, + List types, + Function pagePreprocessor, + PagesSerdeFactory serdeFactory) + { + return new OptimizedPartitionedOutputOperatorFactory( + operatorId, + planNodeId, + types, + pagePreprocessor, + partitionFunction, + partitionChannels, + partitionConstants, + replicatesAnyRow, + nullChannel, + outputBuffer, + serdeFactory, + maxMemory); + } + } + + public static class OptimizedPartitionedOutputOperatorFactory + implements OperatorFactory + { + private final int operatorId; + private final PlanNodeId planNodeId; + private final List sourceTypes; + private final Function pagePreprocessor; + private final PartitionFunction partitionFunction; + private final List partitionChannels; + private final List> partitionConstants; + private final boolean replicatesAnyRow; + private final OptionalInt nullChannel; + private final OutputBuffer outputBuffer; + private final PagesSerdeFactory serdeFactory; + private final DataSize maxMemory; + + public OptimizedPartitionedOutputOperatorFactory( + int operatorId, + PlanNodeId planNodeId, + List sourceTypes, + Function pagePreprocessor, + PartitionFunction partitionFunction, + List partitionChannels, + List> partitionConstants, + boolean replicatesAnyRow, + OptionalInt nullChannel, + OutputBuffer outputBuffer, + PagesSerdeFactory serdeFactory, + DataSize maxMemory) + { + this.operatorId = operatorId; + this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); + this.sourceTypes = requireNonNull(sourceTypes, "sourceTypes is null"); + this.pagePreprocessor = requireNonNull(pagePreprocessor, "pagePreprocessor is null"); + this.partitionFunction = requireNonNull(partitionFunction, "partitionFunction is null"); + this.partitionChannels = requireNonNull(partitionChannels, "partitionChannels is null"); + this.partitionConstants = requireNonNull(partitionConstants, "partitionConstants is null"); + this.replicatesAnyRow = replicatesAnyRow; + this.nullChannel = requireNonNull(nullChannel, "nullChannel is null"); + this.outputBuffer = requireNonNull(outputBuffer, "outputBuffer is null"); + this.serdeFactory = requireNonNull(serdeFactory, "serdeFactory is null"); + this.maxMemory = requireNonNull(maxMemory, "maxMemory is null"); + } + + @Override + public Operator createOperator(DriverContext driverContext) + { + OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, PartitionedOutputOperator.class.getSimpleName()); + return new OptimizedPartitionedOutputOperator( + operatorContext, + sourceTypes, + pagePreprocessor, + partitionFunction, + partitionChannels, + partitionConstants, + replicatesAnyRow, + nullChannel, + outputBuffer, + serdeFactory, + maxMemory); + } + + @Override + public void noMoreOperators() + { + } + + @Override + public OperatorFactory duplicate() + { + return new OptimizedPartitionedOutputOperatorFactory( + operatorId, + planNodeId, + sourceTypes, + pagePreprocessor, + partitionFunction, + partitionChannels, + partitionConstants, + replicatesAnyRow, + nullChannel, + outputBuffer, + serdeFactory, + maxMemory); + } + } + + private static class PagePartitioner + { + private final OutputBuffer outputBuffer; + private final PartitionFunction partitionFunction; + private final List partitionChannels; + private final List> partitionConstants; + private final PagesSerde serde; + private final boolean replicatesAnyRow; + private final OptionalInt nullChannel; // when present, send the position to every partition if this channel is null. + private final AtomicLong rowsAdded = new AtomicLong(); + private final AtomicLong pagesAdded = new AtomicLong(); + + // There could be queries that shuffles data with up to 1000 columns so we need to set the maxOutstandingArrays a high number. + private final ArrayAllocator arrayAllocator = new SimpleArrayAllocator(5000); + private final BlockFlattener flattener = new BlockFlattener(arrayAllocator); + private final Closer blockLeaseCloser = Closer.create(); + + private final PartitionBuffer[] partitionBuffers; + private final List sourceTypes; + private final List variableWidthChannels; + private final int fixedWidthRowSize; + private final DecodedBlockNode[] decodedBlocks; + + private boolean hasAnyRowBeenReplicated; + + public PagePartitioner( + PartitionFunction partitionFunction, + List partitionChannels, + List> partitionConstants, + boolean replicatesAnyRow, + OptionalInt nullChannel, + OutputBuffer outputBuffer, + PagesSerdeFactory serdeFactory, + List sourceTypes, + DataSize maxMemory, + Lifespan lifespan) + { + this.partitionFunction = requireNonNull(partitionFunction, "pagePartitioner is null"); + this.partitionChannels = requireNonNull(partitionChannels, "partitionChannels is null"); + this.partitionConstants = requireNonNull(partitionConstants, "partitionConstants is null").stream() + .map(constant -> constant.map(ConstantExpression::getValueBlock)) + .collect(toImmutableList()); + this.replicatesAnyRow = replicatesAnyRow; + this.nullChannel = requireNonNull(nullChannel, "nullChannel is null"); + this.outputBuffer = requireNonNull(outputBuffer, "outputBuffer is null"); + this.serde = requireNonNull(serdeFactory, "serdeFactory is null").createPagesSerde(); + + int partitionCount = partitionFunction.getPartitionCount(); + + int pageSize = max(1, min(DEFAULT_MAX_PAGE_SIZE_IN_BYTES, toIntExact(maxMemory.toBytes()) / partitionCount)); + + partitionBuffers = new PartitionBuffer[partitionCount]; + for (int i = 0; i < partitionCount; i++) { + partitionBuffers[i] = new PartitionBuffer(i, sourceTypes.size(), pageSize, pagesAdded, rowsAdded, serde, lifespan); + } + + this.sourceTypes = sourceTypes; + decodedBlocks = new DecodedBlockNode[sourceTypes.size()]; + + ImmutableList.Builder variableWidthChannels = ImmutableList.builder(); + int fixedWidthRowSize = 0; + for (int i = 0; i < sourceTypes.size(); i++) { + int bytesPerPosition = getFixedWidthTypeSize(sourceTypes.get(i)); + fixedWidthRowSize += bytesPerPosition; + + if (bytesPerPosition == 0) { + variableWidthChannels.add(i); + } + } + this.variableWidthChannels = variableWidthChannels.build(); + this.fixedWidthRowSize = fixedWidthRowSize; + } + + public ListenableFuture isFull() + { + return outputBuffer.isFull(); + } + + public PartitionedOutputInfo getInfo() + { + return new PartitionedOutputInfo(rowsAdded.get(), pagesAdded.get(), outputBuffer.getPeakMemoryUsage()); + } + + public void partitionPage(Page page) + { + // Populate positions to copy for each destination partition. + int positionCount = page.getPositionCount(); + + for (int i = 0; i < partitionBuffers.length; i++) { + partitionBuffers[i].resetPositions(positionCount); + } + + Block nullBlock = nullChannel.isPresent() ? page.getBlock(nullChannel.getAsInt()) : null; + Page partitionFunctionArgs = getPartitionFunctionArguments(page); + + for (int position = 0; position < positionCount; position++) { + boolean shouldReplicate = (replicatesAnyRow && !hasAnyRowBeenReplicated) || + nullBlock != null && nullBlock.isNull(position); + + if (shouldReplicate) { + for (int i = 0; i < partitionBuffers.length; i++) { + partitionBuffers[i].addPosition(position); + } + hasAnyRowBeenReplicated = true; + } + else { + int partition = partitionFunction.getPartition(partitionFunctionArgs, position); + partitionBuffers[partition].addPosition(position); + } + } + + // Decode the page just once. The decoded blocks will be fed to each PartitionBuffer object to set up AbstractBlockEncodingBuffer. + for (int i = 0; i < decodedBlocks.length; i++) { + decodedBlocks[i] = decodeBlock(flattener, blockLeaseCloser, page.getBlock(i)); + } + + // Copy the data to their destination partitions and flush when the buffer is full. + for (int i = 0; i < partitionBuffers.length; i++) { + partitionBuffers[i].appendData(decodedBlocks, fixedWidthRowSize, variableWidthChannels, outputBuffer); + } + + // Return all borrowed arrays + try { + blockLeaseCloser.close(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public void flush() + { + for (int i = 0; i < partitionBuffers.length; i++) { + partitionBuffers[i].flush(outputBuffer); + } + } + + public long getRetainedSizeInBytes() + { + // When called by the operator constructor, the arrayAllocator was empty at the moment. + // When called in addInput(), the arrays have been returned to the arrayAllocator already, + // but they're still owned by the decodedBlock which will be counted as part of the decodedBlock. + // In both cases, the arrayAllocator doesn't need to be counted. + long size = 0; + + for (int i = 0; i < partitionBuffers.length; i++) { + size += partitionBuffers[i].getRetainedSizeInBytes(); + } + + for (int i = 0; i < decodedBlocks.length; i++) { + size += decodedBlocks[i] == null ? 0 : decodedBlocks[i].getRetainedSizeInBytes(); + } + + return size; + } + + private Page getPartitionFunctionArguments(Page page) + { + Block[] blocks = new Block[partitionChannels.size()]; + for (int i = 0; i < blocks.length; i++) { + Optional partitionConstant = partitionConstants.get(i); + if (partitionConstant.isPresent()) { + blocks[i] = new RunLengthEncodedBlock(partitionConstant.get(), page.getPositionCount()); + } + else { + blocks[i] = page.getBlock(partitionChannels.get(i)); + } + } + return new Page(page.getPositionCount(), blocks); + } + + private static int getFixedWidthTypeSize(Type type) + { + int bytesPerPosition = 0; + if (type instanceof FixedWidthType) { + bytesPerPosition = ((FixedWidthType) type).getFixedSize() + 1; + } + + return bytesPerPosition; + } + } + + private static class PartitionBuffer + { + private static final int INSTANCE_SIZE = ClassLayout.parseClass(PartitionBuffer.class).instanceSize(); + + private final int partition; + private final AtomicLong rowsAdded; + private final AtomicLong pagesAdded; + private final PagesSerde serde; + private final Lifespan lifespan; + private final int capacity; + private final int channelCount; + + private int[] positions; // the default positions array for top level BlockEncodingBuffer + private int[] serializedRowSizes; // The sizes of the rows in bytes if they were serialized + private int positionCount; // number of positions to be copied for this partition + private BlockEncodingBuffer[] blockEncodingBuffers; + + private int bufferedRowCount; + private boolean bufferFull; + + PartitionBuffer(int partition, int channelCount, int capacity, AtomicLong pagesAdded, AtomicLong rowsAdded, PagesSerde serde, Lifespan lifespan) + { + this.partition = partition; + this.channelCount = channelCount; + this.capacity = capacity; + this.pagesAdded = requireNonNull(pagesAdded, "pagesAdded is null"); + this.rowsAdded = requireNonNull(rowsAdded, "rowsAdded is null"); + this.serde = requireNonNull(serde, "serde is null"); + this.lifespan = requireNonNull(lifespan, "lifespan is null"); + } + + private void resetPositions(int positionCount) + { + positions = ensureCapacity(positions, positionCount); + this.positionCount = 0; + } + + private void addPosition(int position) + { + positions[positionCount++] = position; + } + + private void appendData(DecodedBlockNode[] decodedBlocks, int fixedWidthRowSize, List variableWidthChannels, OutputBuffer outputBuffer) + { + if (decodedBlocks.length != channelCount) { + throw new IllegalArgumentException(format("Unexpected number of decoded blocks %d. It should be %d.", decodedBlocks.length, channelCount)); + } + + if (positionCount == 0) { + return; + } + + if (channelCount == 0) { + bufferedRowCount += positionCount; + return; + } + + initializeBlockEncodingBuffers(decodedBlocks); + + for (int i = 0; i < channelCount; i++) { + blockEncodingBuffers[i].setupDecodedBlocksAndPositions(decodedBlocks[i], positions, positionCount); + } + + populateSerializedRowSizes(fixedWidthRowSize, variableWidthChannels); + + // Due to the limitation of buffer size, we append the data batch by batch + int offset = 0; + do { + int batchSize = calculateNextBatchSize(fixedWidthRowSize, variableWidthChannels, offset); + + for (int i = 0; i < channelCount; i++) { + blockEncodingBuffers[i].setNextBatch(offset, batchSize); + blockEncodingBuffers[i].appendDataInBatch(); + } + + bufferedRowCount += batchSize; + offset += batchSize; + + if (bufferFull) { + flush(outputBuffer); + bufferFull = false; + } + } + while (offset < positionCount); + } + + private void initializeBlockEncodingBuffers(DecodedBlockNode[] decodedBlocks) + { + // Create buffers has to be done after seeing the first page. + if (blockEncodingBuffers == null) { + blockEncodingBuffers = new BlockEncodingBuffer[channelCount]; + for (int i = 0; i < channelCount; i++) { + blockEncodingBuffers[i] = createBlockEncodingBuffers(decodedBlocks[i]); + } + } + } + + /** + * Calculate the row sizes in bytes and write them to serializedRowSizes. + */ + private void populateSerializedRowSizes(int fixedWidthRowSize, List variableWidthChannels) + { + if (variableWidthChannels.isEmpty()) { + return; + } + + serializedRowSizes = ensureCapacity(serializedRowSizes, positionCount, SMALL, INITIALIZE); + + for (int i : variableWidthChannels) { + blockEncodingBuffers[i].accumulateSerializedRowSizes(serializedRowSizes); + } + + for (int i = 0; i < positionCount; i++) { + serializedRowSizes[i] += fixedWidthRowSize; + } + } + + private int calculateNextBatchSize(int fixedWidthRowSize, List variableWidthChannels, int startPosition) + { + int bytesRemaining = capacity - getSerializedBuffersSizeInBytes(); + + if (variableWidthChannels.isEmpty()) { + int maxPositionsFit = max(bytesRemaining / fixedWidthRowSize, 1); + if (maxPositionsFit <= positionCount - startPosition) { + bufferFull = true; + return maxPositionsFit; + } + return positionCount - startPosition; + } + + verify(serializedRowSizes != null); + for (int i = startPosition; i < positionCount; i++) { + bytesRemaining -= serializedRowSizes[i]; + + if (bytesRemaining <= 0) { + bufferFull = true; + return max(i - startPosition, 1); + } + } + + return positionCount - startPosition; + } + + private void flush(OutputBuffer outputBuffer) + { + if (bufferedRowCount == 0) { + return; + } + + SliceOutput output = new DynamicSliceOutput(toIntExact(getSerializedBuffersSizeInBytes())); + output.writeInt(channelCount); + + for (int i = 0; i < channelCount; i++) { + blockEncodingBuffers[i].serializeTo(output); + blockEncodingBuffers[i].resetBuffers(); + } + + SerializedPage serializedPage = serde.serialize(output.slice(), bufferedRowCount); + outputBuffer.enqueue(lifespan, partition, ImmutableList.of(serializedPage)); + pagesAdded.incrementAndGet(); + rowsAdded.addAndGet(bufferedRowCount); + + bufferedRowCount = 0; + } + + private long getRetainedSizeInBytes() + { + long size = INSTANCE_SIZE + sizeOf(positions) + sizeOf(serializedRowSizes); + + // Some destination partitions might get 0 rows. In that case the BlockEncodingBuffer won't be created. + if (blockEncodingBuffers != null) { + for (int i = 0; i < channelCount; i++) { + size += blockEncodingBuffers[i].getRetainedSizeInBytes(); + } + } + + return size; + } + + private int getSerializedBuffersSizeInBytes() + { + int size = 0; + + for (int i = 0; i < channelCount; i++) { + size += blockEncodingBuffers[i].getSerializedSizeInBytes(); + } + + return SIZE_OF_INT + size; // channelCount takes one int + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/PartitionedOutputInfo.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/PartitionedOutputInfo.java new file mode 100644 index 0000000000000..1d2914d08557b --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/PartitionedOutputInfo.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.facebook.presto.operator.OperatorInfo; +import com.facebook.presto.util.Mergeable; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import static com.google.common.base.MoreObjects.toStringHelper; + +public class PartitionedOutputInfo + implements Mergeable, OperatorInfo +{ + private final long rowsAdded; + private final long pagesAdded; + private final long outputBufferPeakMemoryUsage; + + @JsonCreator + public PartitionedOutputInfo( + @JsonProperty("rowsAdded") long rowsAdded, + @JsonProperty("pagesAdded") long pagesAdded, + @JsonProperty("outputBufferPeakMemoryUsage") long outputBufferPeakMemoryUsage) + { + this.rowsAdded = rowsAdded; + this.pagesAdded = pagesAdded; + this.outputBufferPeakMemoryUsage = outputBufferPeakMemoryUsage; + } + + @JsonProperty + public long getRowsAdded() + { + return rowsAdded; + } + + @JsonProperty + public long getPagesAdded() + { + return pagesAdded; + } + + @JsonProperty + public long getOutputBufferPeakMemoryUsage() + { + return outputBufferPeakMemoryUsage; + } + + @Override + public PartitionedOutputInfo mergeWith(PartitionedOutputInfo other) + { + return new PartitionedOutputInfo( + rowsAdded + other.rowsAdded, + pagesAdded + other.pagesAdded, + Math.max(outputBufferPeakMemoryUsage, other.outputBufferPeakMemoryUsage)); + } + + @Override + public boolean isFinal() + { + return true; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("rowsAdded", rowsAdded) + .add("pagesAdded", pagesAdded) + .add("outputBufferPeakMemoryUsage", outputBufferPeakMemoryUsage) + .toString(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/PartitionedOutputOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/PartitionedOutputOperator.java similarity index 87% rename from presto-main/src/main/java/com/facebook/presto/operator/PartitionedOutputOperator.java rename to presto-main/src/main/java/com/facebook/presto/operator/repartition/PartitionedOutputOperator.java index 80c6d42a99dd3..74e53f0bd9f80 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/PartitionedOutputOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/PartitionedOutputOperator.java @@ -11,14 +11,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.operator; +package com.facebook.presto.operator.repartition; -import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.buffer.OutputBuffer; import com.facebook.presto.execution.buffer.PagesSerde; import com.facebook.presto.execution.buffer.PagesSerdeFactory; import com.facebook.presto.execution.buffer.SerializedPage; import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.operator.DriverContext; +import com.facebook.presto.operator.Operator; +import com.facebook.presto.operator.OperatorContext; +import com.facebook.presto.operator.OperatorFactory; +import com.facebook.presto.operator.OutputFactory; +import com.facebook.presto.operator.PartitionFunction; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.block.Block; @@ -26,9 +31,6 @@ import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.util.Mergeable; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.DataSize; @@ -40,7 +42,6 @@ import static com.facebook.presto.execution.buffer.PageSplitterUtil.splitPage; import static com.facebook.presto.spi.block.PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES; -import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.Math.max; import static java.lang.Math.min; @@ -220,7 +221,7 @@ public PartitionedOutputOperator( serdeFactory, sourceTypes, maxMemory, - operatorContext.getDriverContext().getLifespan()); + operatorContext); operatorContext.setInfoSupplier(this::getInfo); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(PartitionedOutputOperator.class.getSimpleName()); @@ -277,8 +278,6 @@ public void addInput(Page page) page = pagePreprocessor.apply(page); partitionFunction.partitionPage(page); - operatorContext.recordOutput(page.getSizeInBytes(), page.getPositionCount()); - // We use getSizeInBytes() here instead of getRetainedSizeInBytes() for an approximation of // the amount of memory used by the pageBuilders, because calculating the retained // size can be expensive especially for complex types. @@ -302,13 +301,13 @@ private static class PagePartitioner private final List partitionChannels; private final List> partitionConstants; private final PagesSerde serde; - private final Lifespan lifespan; private final PageBuilder[] pageBuilders; private final boolean replicatesAnyRow; private final OptionalInt nullChannel; // when present, send the position to every partition if this channel is null. private final AtomicLong rowsAdded = new AtomicLong(); private final AtomicLong pagesAdded = new AtomicLong(); private boolean hasAnyRowBeenReplicated; + private final OperatorContext operatorContext; public PagePartitioner( PartitionFunction partitionFunction, @@ -320,7 +319,7 @@ public PagePartitioner( PagesSerdeFactory serdeFactory, List sourceTypes, DataSize maxMemory, - Lifespan lifespan) + OperatorContext operatorContext) { this.partitionFunction = requireNonNull(partitionFunction, "partitionFunction is null"); this.partitionChannels = requireNonNull(partitionChannels, "partitionChannels is null"); @@ -332,7 +331,7 @@ public PagePartitioner( this.outputBuffer = requireNonNull(outputBuffer, "outputBuffer is null"); this.sourceTypes = requireNonNull(sourceTypes, "sourceTypes is null"); this.serde = requireNonNull(serdeFactory, "serdeFactory is null").createPagesSerde(); - this.lifespan = requireNonNull(lifespan, "lifespan is null"); + this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); int partitionCount = partitionFunction.getPartitionCount(); int pageSize = min(DEFAULT_MAX_PAGE_SIZE_IN_BYTES, ((int) maxMemory.toBytes()) / partitionCount); @@ -353,7 +352,7 @@ public long getSizeInBytes() { // We use a foreach loop instead of streams // as it has much better performance. - long sizeInBytes = 0; + long sizeInBytes = serde.getSizeInBytes(); for (PageBuilder pageBuilder : pageBuilders) { sizeInBytes += pageBuilder.getSizeInBytes(); } @@ -365,7 +364,7 @@ public long getSizeInBytes() */ public long getRetainedSizeInBytes() { - long sizeInBytes = 0; + long sizeInBytes = serde.getRetainedSizeInBytes(); for (PageBuilder pageBuilder : pageBuilders) { sizeInBytes += pageBuilder.getRetainedSizeInBytes(); } @@ -433,77 +432,17 @@ public void flush(boolean force) Page pagePartition = partitionPageBuilder.build(); partitionPageBuilder.reset(); + operatorContext.recordOutput(pagePartition.getSizeInBytes(), pagePartition.getPositionCount()); + List serializedPages = splitPage(pagePartition, DEFAULT_MAX_PAGE_SIZE_IN_BYTES).stream() .map(serde::serialize) .collect(toImmutableList()); - outputBuffer.enqueue(lifespan, partition, serializedPages); + outputBuffer.enqueue(operatorContext.getDriverContext().getLifespan(), partition, serializedPages); pagesAdded.incrementAndGet(); rowsAdded.addAndGet(pagePartition.getPositionCount()); } } } } - - public static class PartitionedOutputInfo - implements Mergeable, OperatorInfo - { - private final long rowsAdded; - private final long pagesAdded; - private final long outputBufferPeakMemoryUsage; - - @JsonCreator - public PartitionedOutputInfo( - @JsonProperty("rowsAdded") long rowsAdded, - @JsonProperty("pagesAdded") long pagesAdded, - @JsonProperty("outputBufferPeakMemoryUsage") long outputBufferPeakMemoryUsage) - { - this.rowsAdded = rowsAdded; - this.pagesAdded = pagesAdded; - this.outputBufferPeakMemoryUsage = outputBufferPeakMemoryUsage; - } - - @JsonProperty - public long getRowsAdded() - { - return rowsAdded; - } - - @JsonProperty - public long getPagesAdded() - { - return pagesAdded; - } - - @JsonProperty - public long getOutputBufferPeakMemoryUsage() - { - return outputBufferPeakMemoryUsage; - } - - @Override - public PartitionedOutputInfo mergeWith(PartitionedOutputInfo other) - { - return new PartitionedOutputInfo( - rowsAdded + other.rowsAdded, - pagesAdded + other.pagesAdded, - Math.max(outputBufferPeakMemoryUsage, other.outputBufferPeakMemoryUsage)); - } - - @Override - public boolean isFinal() - { - return true; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("rowsAdded", rowsAdded) - .add("pagesAdded", pagesAdded) - .add("outputBufferPeakMemoryUsage", outputBufferPeakMemoryUsage) - .toString(); - } - } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/RowBlockEncodingBuffer.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/RowBlockEncodingBuffer.java new file mode 100644 index 0000000000000..da535ac0678cc --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/RowBlockEncodingBuffer.java @@ -0,0 +1,284 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.facebook.presto.spi.block.ColumnarRow; +import com.google.common.annotations.VisibleForTesting; +import io.airlift.slice.SliceOutput; +import org.openjdk.jol.info.ClassLayout; + +import java.util.List; + +import static com.facebook.presto.array.Arrays.ExpansionFactor.LARGE; +import static com.facebook.presto.array.Arrays.ExpansionOption.PRESERVE; +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.operator.UncheckedByteArrays.setIntUnchecked; +import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.util.Objects.requireNonNull; +import static sun.misc.Unsafe.ARRAY_INT_INDEX_SCALE; + +public class RowBlockEncodingBuffer + extends AbstractBlockEncodingBuffer +{ + @VisibleForTesting + static final int POSITION_SIZE = SIZE_OF_INT + SIZE_OF_BYTE; + + private static final String NAME = "ROW"; + private static final int INSTANCE_SIZE = ClassLayout.parseClass(RowBlockEncodingBuffer.class).instanceSize(); + + // The buffer for the offsets for all incoming blocks so far + private byte[] offsetsBuffer; + + // The address next offset value will be written to. + private int offsetsBufferIndex; + + // This array holds the condensed offsets for each position for the incoming block. + private int[] offsets; + + // The last offset in the offsets buffer + private int lastOffset; + + // This array holds the offsets into its nested field blocks for each row in the RowBlock. + private int[] offsetsCopy; + + // The AbstractBlockEncodingBuffer for the nested field blocks of the RowBlock + private final BlockEncodingBuffer[] fieldBuffers; + + public RowBlockEncodingBuffer(DecodedBlockNode decodedBlockNode) + { + List childrenNodes = decodedBlockNode.getChildren(); + fieldBuffers = new AbstractBlockEncodingBuffer[childrenNodes.size()]; + for (int i = 0; i < childrenNodes.size(); i++) { + fieldBuffers[i] = createBlockEncodingBuffers(decodedBlockNode.getChildren().get(i)); + } + } + + @Override + public void accumulateSerializedRowSizes(int[] serializedRowSizes) + { + for (int i = 0; i < positionCount; i++) { + serializedRowSizes[i] += POSITION_SIZE; + } + + offsetsCopy = ensureCapacity(offsetsCopy, positionCount + 1); + + for (int i = 0; i < fieldBuffers.length; i++) { + System.arraycopy(offsets, 0, offsetsCopy, 0, positionCount + 1); + ((AbstractBlockEncodingBuffer) fieldBuffers[i]).accumulateSerializedRowSizes(offsetsCopy, positionCount, serializedRowSizes); + } + } + + @Override + public void setNextBatch(int positionsOffset, int batchSize) + { + this.positionsOffset = positionsOffset; + this.batchSize = batchSize; + + // The nested level positionCount could be 0. + if (this.positionCount == 0) { + return; + } + + int offset = offsets[positionsOffset]; + + for (int i = 0; i < fieldBuffers.length; i++) { + fieldBuffers[i].setNextBatch(offset, offsets[positionsOffset + batchSize] - offset); + } + } + + @Override + public void appendDataInBatch() + { + if (batchSize == 0) { + return; + } + + appendNulls(); + appendOffsets(); + + for (int i = 0; i < fieldBuffers.length; i++) { + fieldBuffers[i].appendDataInBatch(); + } + + bufferedPositionCount += batchSize; + } + + @Override + public void serializeTo(SliceOutput output) + { + writeLengthPrefixedString(output, NAME); + + output.writeInt(fieldBuffers.length); + for (int i = 0; i < fieldBuffers.length; i++) { + fieldBuffers[i].serializeTo(output); + } + + // positionCount + output.writeInt(bufferedPositionCount); + + // offsets + output.writeInt(0); // the base position + if (offsetsBufferIndex > 0) { + output.appendBytes(offsetsBuffer, 0, offsetsBufferIndex); + } + + serializeNullsTo(output); + } + + @Override + public void resetBuffers() + { + bufferedPositionCount = 0; + offsetsBufferIndex = 0; + lastOffset = 0; + resetNullsBuffer(); + + for (int i = 0; i < fieldBuffers.length; i++) { + fieldBuffers[i].resetBuffers(); + } + } + + @Override + public long getRetainedSizeInBytes() + { + int size = 0; + for (int i = 0; i < fieldBuffers.length; i++) { + size += fieldBuffers[i].getRetainedSizeInBytes(); + } + + return INSTANCE_SIZE + + getPositionsRetainedSizeInBytes() + + sizeOf(offsetsBuffer) + + sizeOf(offsets) + + sizeOf(offsetsCopy) + + getNullsBufferRetainedSizeInBytes() + + size; + } + + @Override + public long getSerializedSizeInBytes() + { + int size = 0; + for (int i = 0; i < fieldBuffers.length; i++) { + size += fieldBuffers[i].getSerializedSizeInBytes(); + } + + return NAME.length() + SIZE_OF_INT + // encoding name + SIZE_OF_INT + // field count + size + // field blocks + SIZE_OF_INT + // positionCount + SIZE_OF_INT + // offset 0. The offsetsBuffer doesn't contain the offset 0 so we need to add it here. + offsetsBufferIndex + // offsets buffer. + getNullsBufferSerializedSizeInBytes(); // nulls + } + + @Override + protected void setupDecodedBlockAndMapPositions(DecodedBlockNode decodedBlockNode) + { + requireNonNull(decodedBlockNode, "decodedBlockNode is null"); + + decodedBlockNode = mapPositionsToNestedBlock(decodedBlockNode); + ColumnarRow columnarRow = (ColumnarRow) decodedBlockNode.getDecodedBlock(); + decodedBlock = columnarRow.getNullCheckBlock(); + + populateNestedPositions(columnarRow); + + for (int i = 0; i < fieldBuffers.length; i++) { + ((AbstractBlockEncodingBuffer) fieldBuffers[i]).setupDecodedBlockAndMapPositions(decodedBlockNode.getChildren().get(i)); + } + } + + @Override + protected void accumulateSerializedRowSizes(int[] positionOffsets, int positionCount, int[] serializedRowSizes) + { + // If all positions for the RowBlock to be copied are null, the number of positions to copy for its + // nested field blocks could be 0. In such case we don't need to proceed. + if (this.positionCount == 0) { + return; + } + + int lastOffset = positionOffsets[0]; + for (int i = 0; i < positionCount; i++) { + int offset = positionOffsets[i + 1]; + serializedRowSizes[i] += POSITION_SIZE * (offset - lastOffset); + lastOffset = offset; + positionOffsets[i + 1] = this.offsets[offset]; + } + + offsetsCopy = ensureCapacity(offsetsCopy, positionCount + 1); + + for (int i = 0; i < fieldBuffers.length; i++) { + System.arraycopy(positionOffsets, 0, offsetsCopy, 0, positionCount + 1); + ((AbstractBlockEncodingBuffer) fieldBuffers[i]).accumulateSerializedRowSizes(offsetsCopy, positionCount, serializedRowSizes); + } + } + + private void populateNestedPositions(ColumnarRow columnarRow) + { + // Reset nested level positions before checking positionCount. Failing to do so may result in elementsBuffers having stale values when positionCount is 0. + for (int i = 0; i < fieldBuffers.length; i++) { + ((AbstractBlockEncodingBuffer) fieldBuffers[i]).resetPositions(); + } + + if (positionCount == 0) { + return; + } + + offsets = ensureCapacity(offsets, positionCount + 1); + + int[] positions = getPositions(); + + int columnarRowBaseOffset = columnarRow.getOffset(0); + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + int beginOffset = columnarRow.getOffset(position); + int endOffset = columnarRow.getOffset(position + 1); // if the row is null, endOffsetInBlock == beginOffsetInBlock + int currentRowSize = endOffset - beginOffset; + offsets[i + 1] = offsets[i] + currentRowSize; + + if (currentRowSize > 0) { + for (int j = 0; j < fieldBuffers.length; j++) { + ((AbstractBlockEncodingBuffer) fieldBuffers[j]).appendPositionRange(beginOffset - columnarRowBaseOffset, currentRowSize); + } + } + } + } + + private void appendOffsets() + { + offsetsBuffer = ensureCapacity(offsetsBuffer, offsetsBufferIndex + batchSize * ARRAY_INT_INDEX_SCALE, LARGE, PRESERVE); + + int baseOffset = lastOffset - offsets[positionsOffset]; + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + offsetsBufferIndex = setIntUnchecked(offsetsBuffer, offsetsBufferIndex, offsets[i + 1] + baseOffset); + } + lastOffset = offsets[positionsOffset + batchSize] + baseOffset; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/ShortArrayBlockEncodingBuffer.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/ShortArrayBlockEncodingBuffer.java new file mode 100644 index 0000000000000..67cefbf495b44 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/ShortArrayBlockEncodingBuffer.java @@ -0,0 +1,144 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.google.common.annotations.VisibleForTesting; +import io.airlift.slice.SliceOutput; +import org.openjdk.jol.info.ClassLayout; + +import static com.facebook.presto.array.Arrays.ExpansionFactor.LARGE; +import static com.facebook.presto.array.Arrays.ExpansionOption.PRESERVE; +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.operator.UncheckedByteArrays.setShortUnchecked; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; +import static io.airlift.slice.SizeOf.sizeOf; +import static sun.misc.Unsafe.ARRAY_SHORT_INDEX_SCALE; + +public class ShortArrayBlockEncodingBuffer + extends AbstractBlockEncodingBuffer +{ + @VisibleForTesting + static final int POSITION_SIZE = Short.BYTES + Byte.BYTES; + + private static final String NAME = "SHORT_ARRAY"; + private static final int INSTANCE_SIZE = ClassLayout.parseClass(ShortArrayBlockEncodingBuffer.class).instanceSize(); + + private byte[] valuesBuffer; + private int valuesBufferIndex; + + @Override + public void accumulateSerializedRowSizes(int[] serializedRowSizes) + { + throw new UnsupportedOperationException("accumulateSerializedRowSizes is not supported for fixed width types"); + } + + @Override + public void appendDataInBatch() + { + if (batchSize == 0) { + return; + } + + appendValuesToBuffer(); + appendNulls(); + bufferedPositionCount += batchSize; + } + + @Override + public void serializeTo(SliceOutput output) + { + writeLengthPrefixedString(output, NAME); + + output.writeInt(bufferedPositionCount); + + serializeNullsTo(output); + + if (valuesBufferIndex > 0) { + output.appendBytes(valuesBuffer, 0, valuesBufferIndex); + } + } + + @Override + public void resetBuffers() + { + bufferedPositionCount = 0; + valuesBufferIndex = 0; + resetNullsBuffer(); + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + + getPositionsRetainedSizeInBytes() + + sizeOf(valuesBuffer) + + getNullsBufferRetainedSizeInBytes(); + } + + @Override + public long getSerializedSizeInBytes() + { + return NAME.length() + SIZE_OF_INT + // NAME + SIZE_OF_INT + // positionCount + getNullsBufferSerializedSizeInBytes() + // nulls + valuesBufferIndex; // values buffer + } + + @Override + protected void accumulateSerializedRowSizes(int[] positionOffsets, int positionCount, int[] serializedRowSizes) + { + for (int i = 0; i < positionCount; i++) { + serializedRowSizes[i] += (positionOffsets[i + 1] - positionOffsets[i]) * POSITION_SIZE; + } + } + + private void appendValuesToBuffer() + { + valuesBuffer = ensureCapacity(valuesBuffer, valuesBufferIndex + batchSize * ARRAY_SHORT_INDEX_SCALE, LARGE, PRESERVE); + + int[] positions = getPositions(); + if (decodedBlock.mayHaveNull()) { + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + int position = positions[i]; + + short value = decodedBlock.getShort(position); + int newIndex = setShortUnchecked(valuesBuffer, valuesBufferIndex, value); + + if (!decodedBlock.isNull(position)) { + valuesBufferIndex = newIndex; + } + } + } + else { + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + short value = decodedBlock.getShort(positions[i]); + valuesBufferIndex = setShortUnchecked(valuesBuffer, valuesBufferIndex, value); + } + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/repartition/VariableWidthBlockEncodingBuffer.java b/presto-main/src/main/java/com/facebook/presto/operator/repartition/VariableWidthBlockEncodingBuffer.java new file mode 100644 index 0000000000000..f4dd6900e4f5b --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/repartition/VariableWidthBlockEncodingBuffer.java @@ -0,0 +1,197 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.facebook.presto.spi.block.AbstractVariableWidthBlock; +import com.google.common.annotations.VisibleForTesting; +import io.airlift.slice.Slice; +import io.airlift.slice.SliceOutput; +import org.openjdk.jol.info.ClassLayout; + +import static com.facebook.presto.array.Arrays.ExpansionFactor.LARGE; +import static com.facebook.presto.array.Arrays.ExpansionOption.PRESERVE; +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.operator.MoreByteArrays.setBytes; +import static com.facebook.presto.operator.UncheckedByteArrays.setIntUnchecked; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; +import static io.airlift.slice.SizeOf.sizeOf; +import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_INT_INDEX_SCALE; + +public class VariableWidthBlockEncodingBuffer + extends AbstractBlockEncodingBuffer +{ + @VisibleForTesting + static final int POSITION_SIZE = Integer.BYTES + Byte.BYTES; + + private static final String NAME = "VARIABLE_WIDTH"; + private static final int INSTANCE_SIZE = ClassLayout.parseClass(VariableWidthBlockEncodingBuffer.class).instanceSize(); + + // The buffer for the slice for all incoming blocks so far + private byte[] sliceBuffer; + + // The address that the next slice will be written to. + private int sliceBufferIndex; + + // The buffer for the offsets for all incoming blocks so far + private byte[] offsetsBuffer; + + // The address that the next offset value will be written to. + private int offsetsBufferIndex; + + // The last offset in the offsets buffer + private int lastOffset; + + @Override + public void accumulateSerializedRowSizes(int[] serializedRowSizes) + { + int[] positions = getPositions(); + for (int i = 0; i < positionCount; i++) { + serializedRowSizes[i] += POSITION_SIZE + decodedBlock.getSliceLength(positions[i]); + } + } + + @Override + public void appendDataInBatch() + { + if (batchSize == 0) { + return; + } + + appendOffsetsAndSlices(); + appendNulls(); + bufferedPositionCount += batchSize; + } + + @Override + public void serializeTo(SliceOutput output) + { + writeLengthPrefixedString(output, NAME); + + output.writeInt(bufferedPositionCount); + + // offsets + // note that VariableWidthBlock doesn't write the initial offset 0 + if (offsetsBufferIndex > 0) { + output.appendBytes(offsetsBuffer, 0, offsetsBufferIndex); + } + + // nulls + serializeNullsTo(output); + + // slice + output.writeInt(sliceBufferIndex); // totalLength + if (sliceBufferIndex > 0) { + output.appendBytes(sliceBuffer, 0, sliceBufferIndex); + } + } + + @Override + public void resetBuffers() + { + bufferedPositionCount = 0; + sliceBufferIndex = 0; + offsetsBufferIndex = 0; + lastOffset = 0; + resetNullsBuffer(); + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + + getPositionsRetainedSizeInBytes() + + sizeOf(offsetsBuffer) + + getNullsBufferRetainedSizeInBytes() + + sizeOf(sliceBuffer); + } + + @Override + public long getSerializedSizeInBytes() + { + return NAME.length() + SIZE_OF_INT + // NAME + SIZE_OF_INT + // positionCount + offsetsBufferIndex + // offsets buffer. + SIZE_OF_INT + // sliceBuffer size. + sliceBufferIndex + // sliceBuffer + getNullsBufferSerializedSizeInBytes(); // nulls + } + + @Override + protected void accumulateSerializedRowSizes(int[] positionOffsets, int positionCount, int[] serializedRowSizes) + { + // The nested level positionCount could be 0. + if (this.positionCount == 0) { + return; + } + + int[] positions = getPositions(); + + for (int i = 0; i < positionCount; i++) { + for (int j = positionOffsets[i]; j < positionOffsets[i + 1]; j++) { + serializedRowSizes[i] += POSITION_SIZE + decodedBlock.getSliceLength(positions[j]); + } + } + } + + // This implementation uses variableWidthBlock.getRawSlice() and variableWidthBlock.getPositionOffset() to achieve high performance + private void appendOffsetsAndSlices() + { + offsetsBuffer = ensureCapacity(offsetsBuffer, offsetsBufferIndex + batchSize * ARRAY_INT_INDEX_SCALE, LARGE, PRESERVE); + + AbstractVariableWidthBlock variableWidthBlock = (AbstractVariableWidthBlock) decodedBlock; + int[] positions = getPositions(); + + // We need to use getRawSlice() to get the raw slice whose address is not advanced by getSlice(). It's incorrect to call getSlice() + // because the returned slice's address may be advanced if it's based on a slice view. + Slice rawSlice = variableWidthBlock.getRawSlice(0); + byte[] sliceBase = (byte[]) rawSlice.getBase(); + + // The slice's address starts from ARRAY_BYTE_BASE_OFFSET but when we read the bytes later in setBytes() the ARRAY_BYTE_BASE_OFFSET was added + // inside, so we need to subtract it here. If sliceAddress < ARRAY_BYTE_BASE_OFFSET it's an empty slice and the sliceBuffer writing will be + // guarded by the length check in the for loop, so the subtraction doesn't matter. + int sliceAddress = (int) rawSlice.getAddress() - ARRAY_BYTE_BASE_OFFSET; + + for (int i = positionsOffset; i < positionsOffset + batchSize; i++) { + int position = positions[i]; + int beginOffset = variableWidthBlock.getPositionOffset(position); + int endOffset = variableWidthBlock.getPositionOffset(position + 1); + int length = endOffset - beginOffset; + + lastOffset += length; + offsetsBufferIndex = setIntUnchecked(offsetsBuffer, offsetsBufferIndex, lastOffset); + + if (length > 0) { + sliceBuffer = ensureCapacity(sliceBuffer, sliceBufferIndex + length, LARGE, PRESERVE); + + // The slice address may be greater than 0. Since we are reading from the raw slice, we need to read from beginOffset + sliceAddress. + sliceBufferIndex = setBytes(sliceBuffer, sliceBufferIndex, sliceBase, beginOffset + sliceAddress, length); + } + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/AbstractGreatestLeast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/AbstractGreatestLeast.java index 6308ab26fe889..0ac39aafe9740 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/AbstractGreatestLeast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/AbstractGreatestLeast.java @@ -28,6 +28,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; @@ -46,8 +47,8 @@ import static com.facebook.presto.bytecode.Access.a; import static com.facebook.presto.bytecode.Parameter.arg; import static com.facebook.presto.bytecode.ParameterizedType.type; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.function.Signature.orderableTypeParameter; @@ -72,7 +73,7 @@ public abstract class AbstractGreatestLeast private final OperatorType operatorType; - protected AbstractGreatestLeast(String name, OperatorType operatorType) + protected AbstractGreatestLeast(QualifiedFunctionName name, OperatorType operatorType) { super(new Signature( name, @@ -98,12 +99,12 @@ public boolean isDeterministic() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type type = boundVariables.getTypeVariable("E"); checkArgument(type.isOrderable(), "Type must be orderable"); - MethodHandle compareMethod = functionManager.getScalarFunctionImplementation( + MethodHandle compareMethod = functionManager.getBuiltInScalarFunctionImplementation( functionManager.resolveOperator(operatorType, fromTypes(type, type))).getMethodHandle(); List> javaTypes = IntStream.range(0, arity) @@ -111,9 +112,9 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in .collect(toImmutableList()); Class clazz = generate(javaTypes, type, compareMethod); - MethodHandle methodHandle = methodHandle(clazz, getSignature().getName(), javaTypes.toArray(new Class[javaTypes.size()])); + MethodHandle methodHandle = methodHandle(clazz, getSignature().getNameSuffix(), javaTypes.toArray(new Class[javaTypes.size()])); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, nCopies(javaTypes.size(), valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); @@ -129,14 +130,14 @@ public static void checkNotNaN(String name, double value) private Class generate(List> javaTypes, Type type, MethodHandle compareMethod) { - checkCondition(javaTypes.size() <= 127, NOT_SUPPORTED, "Too many arguments for function call %s()", getSignature().getName()); + checkCondition(javaTypes.size() <= 127, NOT_SUPPORTED, "Too many arguments for function call %s()", getSignature().getNameSuffix()); String javaTypeName = javaTypes.stream() .map(Class::getSimpleName) .collect(joining()); ClassDefinition definition = new ClassDefinition( a(PUBLIC, FINAL), - makeClassName(javaTypeName + "$" + getSignature().getName()), + makeClassName(javaTypeName + "$" + getSignature().getNameSuffix()), type(Object.class)); definition.declareDefaultConstructor(a(PRIVATE)); @@ -147,7 +148,7 @@ private Class generate(List> javaTypes, Type type, MethodHandle comp MethodDefinition method = definition.declareMethod( a(PUBLIC, STATIC), - getSignature().getName(), + getSignature().getNameSuffix(), type(javaTypes.get(0)), parameters); @@ -159,7 +160,7 @@ private Class generate(List> javaTypes, Type type, MethodHandle comp if (type.getTypeSignature().getBase().equals(StandardTypes.DOUBLE)) { for (Parameter parameter : parameters) { body.append(parameter); - body.append(invoke(binder.bind(CHECK_NOT_NAN.bindTo(getSignature().getName())), "checkNotNaN")); + body.append(invoke(binder.bind(CHECK_NOT_NAN.bindTo(getSignature().getNameSuffix())), "checkNotNaN")); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ApplyFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ApplyFunction.java index ebbd574662791..c681f9c68f83e 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ApplyFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ApplyFunction.java @@ -17,6 +17,7 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -25,9 +26,10 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; @@ -46,7 +48,7 @@ public final class ApplyFunction private ApplyFunction() { super(new Signature( - "apply", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "apply"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T"), typeVariable("U")), ImmutableList.of(), @@ -80,11 +82,11 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type argumentType = boundVariables.getTypeVariable("T"); Type returnType = boundVariables.getTypeVariable("U"); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( true, ImmutableList.of( valueTypeArgumentProperty(USE_BOXED_TYPE), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayAllMatchFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayAllMatchFunction.java new file mode 100644 index 0000000000000..4f5be07409038 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayAllMatchFunction.java @@ -0,0 +1,183 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.SqlNullable; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.function.TypeParameter; +import com.facebook.presto.spi.function.TypeParameterSpecialization; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; +import io.airlift.slice.Slice; + +import static java.lang.Boolean.FALSE; + +@Description("Returns true if all elements of the array match the given predicate") +@ScalarFunction(value = "all_match") +public final class ArrayAllMatchFunction +{ + private ArrayAllMatchFunction() {} + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = Block.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean allMatchBlock( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") BlockToBooleanFunction function) + { + boolean hasNullResult = false; + for (int i = 0; i < arrayBlock.getPositionCount(); i++) { + Block element = null; + if (!arrayBlock.isNull(i)) { + element = (Block) elementType.getObject(arrayBlock, i); + } + Boolean match = function.apply(element); + if (FALSE.equals(match)) { + return false; + } + if (match == null) { + hasNullResult = true; + } + } + if (hasNullResult) { + return null; + } + return true; + } + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = Slice.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean allMatchSlice( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") SliceToBooleanFunction function) + { + boolean hasNullResult = false; + int positionCount = arrayBlock.getPositionCount(); + for (int i = 0; i < positionCount; i++) { + Slice element = null; + if (!arrayBlock.isNull(i)) { + element = elementType.getSlice(arrayBlock, i); + } + Boolean match = function.apply(element); + if (FALSE.equals(match)) { + return false; + } + if (match == null) { + hasNullResult = true; + } + } + if (hasNullResult) { + return null; + } + return true; + } + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = long.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean allMatchLong( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") LongToBooleanFunction function) + { + boolean hasNullResult = false; + int positionCount = arrayBlock.getPositionCount(); + for (int i = 0; i < positionCount; i++) { + Long element = null; + if (!arrayBlock.isNull(i)) { + element = elementType.getLong(arrayBlock, i); + } + Boolean match = function.apply(element); + if (FALSE.equals(match)) { + return false; + } + if (match == null) { + hasNullResult = true; + } + } + if (hasNullResult) { + return null; + } + return true; + } + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = double.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean allMatchDouble( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") DoubleToBooleanFunction function) + { + boolean hasNullResult = false; + int positionCount = arrayBlock.getPositionCount(); + for (int i = 0; i < positionCount; i++) { + Double element = null; + if (!arrayBlock.isNull(i)) { + element = elementType.getDouble(arrayBlock, i); + } + Boolean match = function.apply(element); + if (FALSE.equals(match)) { + return false; + } + if (match == null) { + hasNullResult = true; + } + } + if (hasNullResult) { + return null; + } + return true; + } + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = boolean.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean allMatchBoolean( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") BooleanToBooleanFunction function) + { + boolean hasNullResult = false; + int positionCount = arrayBlock.getPositionCount(); + for (int i = 0; i < positionCount; i++) { + Boolean element = null; + if (!arrayBlock.isNull(i)) { + element = elementType.getBoolean(arrayBlock, i); + } + Boolean match = function.apply(element); + if (FALSE.equals(match)) { + return false; + } + if (match == null) { + hasNullResult = true; + } + } + if (hasNullResult) { + return null; + } + return true; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayAnyMatchFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayAnyMatchFunction.java new file mode 100644 index 0000000000000..43590dbee9154 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayAnyMatchFunction.java @@ -0,0 +1,183 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.SqlNullable; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.function.TypeParameter; +import com.facebook.presto.spi.function.TypeParameterSpecialization; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; +import io.airlift.slice.Slice; + +import static java.lang.Boolean.TRUE; + +@Description("Returns true if the array contains one or more elements that match the given predicate") +@ScalarFunction(value = "any_match") +public final class ArrayAnyMatchFunction +{ + private ArrayAnyMatchFunction() {} + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = Block.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean anyMatchBlock( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") BlockToBooleanFunction function) + { + boolean hasNullResult = false; + for (int i = 0; i < arrayBlock.getPositionCount(); i++) { + Block element = null; + if (!arrayBlock.isNull(i)) { + element = (Block) elementType.getObject(arrayBlock, i); + } + Boolean match = function.apply(element); + if (TRUE.equals(match)) { + return true; + } + if (match == null) { + hasNullResult = true; + } + } + if (hasNullResult) { + return null; + } + return false; + } + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = Slice.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean anyMatchSlice( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") SliceToBooleanFunction function) + { + boolean hasNullResult = false; + int positionCount = arrayBlock.getPositionCount(); + for (int i = 0; i < positionCount; i++) { + Slice element = null; + if (!arrayBlock.isNull(i)) { + element = elementType.getSlice(arrayBlock, i); + } + Boolean match = function.apply(element); + if (TRUE.equals(match)) { + return true; + } + if (match == null) { + hasNullResult = true; + } + } + if (hasNullResult) { + return null; + } + return false; + } + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = long.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean anyMatchLong( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") LongToBooleanFunction function) + { + boolean hasNullResult = false; + int positionCount = arrayBlock.getPositionCount(); + for (int i = 0; i < positionCount; i++) { + Long element = null; + if (!arrayBlock.isNull(i)) { + element = elementType.getLong(arrayBlock, i); + } + Boolean match = function.apply(element); + if (TRUE.equals(match)) { + return true; + } + if (match == null) { + hasNullResult = true; + } + } + if (hasNullResult) { + return null; + } + return false; + } + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = double.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean anyMatchDouble( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") DoubleToBooleanFunction function) + { + boolean hasNullResult = false; + int positionCount = arrayBlock.getPositionCount(); + for (int i = 0; i < positionCount; i++) { + Double element = null; + if (!arrayBlock.isNull(i)) { + element = elementType.getDouble(arrayBlock, i); + } + Boolean match = function.apply(element); + if (TRUE.equals(match)) { + return true; + } + if (match == null) { + hasNullResult = true; + } + } + if (hasNullResult) { + return null; + } + return false; + } + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = boolean.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean anyMatchBoolean( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") BooleanToBooleanFunction function) + { + boolean hasNullResult = false; + int positionCount = arrayBlock.getPositionCount(); + for (int i = 0; i < positionCount; i++) { + Boolean element = null; + if (!arrayBlock.isNull(i)) { + element = elementType.getBoolean(arrayBlock, i); + } + Boolean match = function.apply(element); + if (TRUE.equals(match)) { + return true; + } + if (match == null) { + hasNullResult = true; + } + } + if (hasNullResult) { + return null; + } + return false; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayCombinationsFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayCombinationsFunction.java new file mode 100644 index 0000000000000..7b205e2328c96 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayCombinationsFunction.java @@ -0,0 +1,137 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.ArrayBlock; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.block.PageBuilderStatus; +import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.function.TypeParameter; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; +import com.google.common.annotations.VisibleForTesting; + +import java.util.Optional; + +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.util.Failures.checkCondition; +import static com.google.common.base.Verify.verify; +import static java.lang.Math.multiplyExact; +import static java.lang.StrictMath.toIntExact; +import static java.lang.String.format; +import static java.lang.System.arraycopy; +import static java.util.Arrays.setAll; + +@Description("Returns n-element combinations from array") +@ScalarFunction("combinations") +public final class ArrayCombinationsFunction +{ + private ArrayCombinationsFunction() {} + + private static final int MAX_COMBINATION_LENGTH = 5; + private static final int MAX_RESULT_ELEMENTS = 100_000; + + @TypeParameter("T") + @SqlType("array(array(T))") + public static Block combinations( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block array, + @SqlType(StandardTypes.INTEGER) long n) + { + int arrayLength = array.getPositionCount(); + int combinationLength = toIntExact(n); + checkCondition(combinationLength >= 0, INVALID_FUNCTION_ARGUMENT, "combination size must not be negative: %s", combinationLength); + checkCondition(combinationLength <= MAX_COMBINATION_LENGTH, INVALID_FUNCTION_ARGUMENT, "combination size must not exceed %s: %s", MAX_COMBINATION_LENGTH, combinationLength); + + ArrayType arrayType = new ArrayType(elementType); + if (combinationLength > arrayLength) { + return arrayType.createBlockBuilder(new PageBuilderStatus().createBlockBuilderStatus(), 0).build(); + } + + int combinationCount = combinationCount(arrayLength, combinationLength); + checkCondition(combinationCount * (long) combinationLength <= MAX_RESULT_ELEMENTS, INVALID_FUNCTION_ARGUMENT, "combinations exceed max size"); + + int[] ids = new int[combinationCount * combinationLength]; + int idsPosition = 0; + + int[] combination = firstCombination(combinationLength); + do { + arraycopy(combination, 0, ids, idsPosition, combinationLength); + idsPosition += combinationLength; + } + while (nextCombination(combination, arrayLength)); + verify(idsPosition == ids.length, "idsPosition != ids.length, %s and %s respectively", idsPosition, ids.length); + + int[] offsets = new int[combinationCount + 1]; + setAll(offsets, i -> i * combinationLength); + + return ArrayBlock.fromElementBlock(combinationCount, Optional.empty(), offsets, new DictionaryBlock(array, ids)); + } + + @VisibleForTesting + static int combinationCount(int arrayLength, int combinationLength) + { + try { + /* + * Then combinationCount(n, k) = combinationCount(n-1, k-1) * n/k (https://en.wikipedia.org/wiki/Combination#Number_of_k-combinations) + * The formula is recursive. Here, instead of starting with k=combinationCount, n=arrayLength and recursing, + * we start with k=0 n=(arrayLength-combinationLength) and proceed "bottom up". + */ + int combinations = 1; + for (int i = 1; i <= combinationLength; i++) { + combinations = multiplyExact(combinations, arrayLength - combinationLength + i) / i; + } + return combinations; + } + catch (ArithmeticException e) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Number of combinations too large for array of size %s and combination length %s", arrayLength, combinationLength)); + } + } + + private static int[] firstCombination(int combinationLength) + { + int[] combination = new int[combinationLength]; + setAll(combination, i -> i); + return combination; + } + + private static boolean nextCombination(int[] combination, int arrayLength) + { + for (int i = 0; i < combination.length - 1; i++) { + if (combination[i] + 1 < combination[i + 1]) { + combination[i]++; + resetCombination(combination, i); + return true; + } + } + if (combination.length > 0 && combination[combination.length - 1] + 1 < arrayLength) { + combination[combination.length - 1]++; + resetCombination(combination, combination.length - 1); + return true; + } + return false; + } + + private static void resetCombination(int[] combination, int to) + { + for (int i = 0; i < to; i++) { + combination[i] = i; + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConcatFunction.java index d2fefc2f19df9..9179b02329a7e 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConcatFunction.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -31,8 +32,9 @@ import java.lang.invoke.MethodHandle; import java.util.Optional; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -53,7 +55,8 @@ public final class ArrayConcatFunction private ArrayConcatFunction() { - super(new Signature(FUNCTION_NAME, + super(new Signature( + QualifiedFunctionName.of(DEFAULT_NAMESPACE, FUNCTION_NAME), FunctionKind.SCALAR, ImmutableList.of(typeVariable("E")), ImmutableList.of(), @@ -81,7 +84,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { if (arity < 2) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "There must be two or more arguments to " + FUNCTION_NAME); @@ -96,7 +99,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in METHOD_HANDLE.bindTo(elementType), USER_STATE_FACTORY.bindTo(elementType)); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, nCopies(arity, valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandleAndConstructor.getMethodHandle(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConstructor.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConstructor.java index f2046e7500524..99c449b84b2cb 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConstructor.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConstructor.java @@ -29,6 +29,7 @@ import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.BlockBuilderStatus; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -52,8 +53,9 @@ import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantInt; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantNull; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.equal; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -73,7 +75,7 @@ public final class ArrayConstructor public ArrayConstructor() { - super(new Signature("array_constructor", + super(new Signature(QualifiedFunctionName.of(DEFAULT_NAMESPACE, "array_constructor"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("E")), ImmutableList.of(), @@ -108,7 +110,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Map types = boundVariables.getTypeVariables(); checkArgument(types.size() == 1, "Can only construct arrays from exactly matching types"); @@ -132,7 +134,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in catch (ReflectiveOperationException e) { throw new RuntimeException(e); } - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, nCopies(stackTypes.size(), valueTypeArgumentProperty(USE_BOXED_TYPE)), methodHandle); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayDistinctFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayDistinctFunction.java index 18aa15f8e83c2..df626ca5c1821 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayDistinctFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayDistinctFunction.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.scalar; import com.facebook.presto.operator.aggregation.TypedSet; -import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.Description; @@ -23,7 +22,6 @@ import com.facebook.presto.spi.function.TypeParameter; import com.facebook.presto.spi.type.Type; import com.facebook.presto.type.TypeUtils; -import com.google.common.collect.ImmutableList; import it.unimi.dsi.fastutil.longs.LongOpenHashSet; import it.unimi.dsi.fastutil.longs.LongSet; @@ -33,23 +31,18 @@ @Description("Remove duplicate values from the given array") public final class ArrayDistinctFunction { - private final PageBuilder pageBuilder; - - @TypeParameter("E") - public ArrayDistinctFunction(@TypeParameter("E") Type elementType) - { - pageBuilder = new PageBuilder(ImmutableList.of(elementType)); - } + private ArrayDistinctFunction() {} @TypeParameter("E") @SqlType("array(E)") - public Block distinct(@TypeParameter("E") Type type, @SqlType("array(E)") Block array) + public static Block distinct(@TypeParameter("E") Type type, @SqlType("array(E)") Block array) { - if (array.getPositionCount() < 2) { + int arrayLength = array.getPositionCount(); + if (arrayLength < 2) { return array; } - if (array.getPositionCount() == 2) { + if (arrayLength == 2) { if (TypeUtils.positionEqualsPosition(type, array, 0, array, 1)) { return array.getSingleValueBlock(0); } @@ -58,62 +51,139 @@ public Block distinct(@TypeParameter("E") Type type, @SqlType("array(E)") Block } } - TypedSet typedSet = new TypedSet(type, array.getPositionCount(), "array_distinct"); - int distinctCount = 0; + TypedSet typedSet = new TypedSet(type, arrayLength, "array_distinct"); + BlockBuilder distinctElementBlockBuilder; - if (pageBuilder.isFull()) { - pageBuilder.reset(); - } + if (array.mayHaveNull()) { + int firstDuplicatePosition = 0; + // Keep adding the element to the set as long as there are no dupes. + while (firstDuplicatePosition < arrayLength && typedSet.add(array, firstDuplicatePosition)) { + firstDuplicatePosition++; + } - BlockBuilder distinctElementBlockBuilder = pageBuilder.getBlockBuilder(0); - for (int i = 0; i < array.getPositionCount(); i++) { - if (!typedSet.contains(array, i)) { - typedSet.add(array, i); - distinctCount++; - type.appendTo(array, i, distinctElementBlockBuilder); + if (firstDuplicatePosition == arrayLength) { + // All elements are distinct, so just return the original. + return array; + } + + int position = 0; + distinctElementBlockBuilder = type.createBlockBuilder(null, arrayLength); + while (position < firstDuplicatePosition) { + type.appendTo(array, position, distinctElementBlockBuilder); + position++; + } + while (position < arrayLength) { + if (typedSet.add(array, position)) { + type.appendTo(array, position, distinctElementBlockBuilder); + } + position++; } } + else { + int firstDuplicatePosition = 0; + while (firstDuplicatePosition < arrayLength && typedSet.addNonNull(array, firstDuplicatePosition)) { + firstDuplicatePosition++; + } + + if (firstDuplicatePosition == arrayLength) { + // All elements are distinct, so just return the original. + return array; + } - pageBuilder.declarePositions(distinctCount); + int position = 0; + distinctElementBlockBuilder = type.createBlockBuilder(null, arrayLength); + while (position < firstDuplicatePosition) { + type.appendTo(array, position, distinctElementBlockBuilder); + position++; + } + while (position < arrayLength) { + if (typedSet.addNonNull(array, position)) { + type.appendTo(array, position, distinctElementBlockBuilder); + } + position++; + } + } - return distinctElementBlockBuilder.getRegion(distinctElementBlockBuilder.getPositionCount() - distinctCount, distinctCount); + return distinctElementBlockBuilder.build(); } @SqlType("array(bigint)") - public Block bigintDistinct(@SqlType("array(bigint)") Block array) + public static Block bigintDistinct(@SqlType("array(bigint)") Block array) { - if (array.getPositionCount() == 0) { + final int arrayLength = array.getPositionCount(); + if (arrayLength < 2) { return array; } - boolean containsNull = false; - LongSet set = new LongOpenHashSet(array.getPositionCount()); - int distinctCount = 0; + BlockBuilder distinctElementBlockBuilder; + LongSet set = new LongOpenHashSet(arrayLength); + + if (array.mayHaveNull()) { + int position = 0; + boolean containsNull = false; + + // Keep adding the element to the set as long as there are no dupes. + while (position < arrayLength) { + if (array.isNull(position)) { + if (!containsNull) { + containsNull = true; + } + else { + // Second null. + break; + } + } + else if (!set.add(BIGINT.getLong(array, position))) { + // Dupe found. + break; + } + position++; + } - if (pageBuilder.isFull()) { - pageBuilder.reset(); - } + if (position == arrayLength) { + // All elements are distinct, so just return the original. + return array; + } - BlockBuilder distinctElementBlockBuilder = pageBuilder.getBlockBuilder(0); - for (int i = 0; i < array.getPositionCount(); i++) { - if (array.isNull(i)) { - if (!containsNull) { - containsNull = true; - distinctElementBlockBuilder.appendNull(); - distinctCount++; + distinctElementBlockBuilder = BIGINT.createBlockBuilder(null, arrayLength); + for (int i = 0; i < position; i++) { + BIGINT.appendTo(array, i, distinctElementBlockBuilder); + } + + for (position++; position < arrayLength; position++) { + if (array.isNull(position)) { + if (!containsNull) { + BIGINT.appendTo(array, position, distinctElementBlockBuilder); + } + } + else if (set.add(BIGINT.getLong(array, position))) { + BIGINT.appendTo(array, position, distinctElementBlockBuilder); } - continue; } - long value = BIGINT.getLong(array, i); - if (!set.contains(value)) { - set.add(value); - distinctCount++; + } + else { + int position = 0; + // Keep adding the element to the set as long as there are no dupes. + while (position < arrayLength && set.add(BIGINT.getLong(array, position))) { + position++; + } + if (position == arrayLength) { + // All elements are distinct, so just return the original. + return array; + } + + distinctElementBlockBuilder = BIGINT.createBlockBuilder(null, arrayLength); + for (int i = 0; i < position; i++) { BIGINT.appendTo(array, i, distinctElementBlockBuilder); } - } - pageBuilder.declarePositions(distinctCount); + for (position++; position < arrayLength; position++) { + if (set.add(BIGINT.getLong(array, position))) { + BIGINT.appendTo(array, position, distinctElementBlockBuilder); + } + } + } - return distinctElementBlockBuilder.getRegion(distinctElementBlockBuilder.getPositionCount() - distinctCount, distinctCount); + return distinctElementBlockBuilder.build(); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayExceptFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayExceptFunction.java index 0454d1e1b9c94..05c2580824495 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayExceptFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayExceptFunction.java @@ -47,8 +47,7 @@ public static Block except( typedSet.add(rightArray, i); } for (int i = 0; i < leftPositionCount; i++) { - if (!typedSet.contains(leftArray, i)) { - typedSet.add(leftArray, i); + if (typedSet.add(leftArray, i)) { type.appendTo(leftArray, i, distinctElementBlockBuilder); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFilterFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFilterFunction.java index e2507d9cd0e99..dae21f4c3b066 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFilterFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFilterFunction.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.function.TypeParameter; import com.facebook.presto.spi.function.TypeParameterSpecialization; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.gen.lambda.LambdaFunctionInterface; import io.airlift.slice.Slice; import static java.lang.Boolean.TRUE; @@ -38,21 +37,59 @@ private ArrayFilterFunction() {} public static Block filterLong( @TypeParameter("T") Type elementType, @SqlType("array(T)") Block arrayBlock, - @SqlType("function(T, boolean)") FilterLongLambda function) + @SqlType("function(T, boolean)") LongToBooleanFunction function) { int positionCount = arrayBlock.getPositionCount(); - BlockBuilder resultBuilder = elementType.createBlockBuilder(null, positionCount); - for (int position = 0; position < positionCount; position++) { - Long input = null; - if (!arrayBlock.isNull(position)) { - input = elementType.getLong(arrayBlock, position); + int position = 0; + BlockBuilder resultBuilder; + + if (arrayBlock.mayHaveNull()) { + while (position < positionCount && + TRUE.equals(function.apply(arrayBlock.isNull(position) ? + null : + elementType.getLong(arrayBlock, position)))) { + position++; + } + + if (position == positionCount) { + // Nothing fitered out. So just return the original. + return arrayBlock; + } + + resultBuilder = elementType.createBlockBuilder(null, positionCount); + for (int i = 0; i < position; i++) { + elementType.appendTo(arrayBlock, i, resultBuilder); } - Boolean keep = function.apply(input); - if (TRUE.equals(keep)) { - elementType.appendTo(arrayBlock, position, resultBuilder); + for (position++; position < positionCount; position++) { + if (TRUE.equals(function.apply(arrayBlock.isNull(position) ? + null : + elementType.getLong(arrayBlock, position)))) { + elementType.appendTo(arrayBlock, position, resultBuilder); + } } } + else { + while (position < positionCount && TRUE.equals(function.apply(elementType.getLong(arrayBlock, position)))) { + position++; + } + + if (position == positionCount) { + // Nothing filtered out. So just return the original. + return arrayBlock; + } + + resultBuilder = elementType.createBlockBuilder(null, positionCount); + for (int i = 0; i < position; i++) { + elementType.appendTo(arrayBlock, i, resultBuilder); + } + for (position++; position < positionCount; position++) { + if (TRUE.equals(function.apply(elementType.getLong(arrayBlock, position)))) { + elementType.appendTo(arrayBlock, position, resultBuilder); + } + } + } + return resultBuilder.build(); } @@ -62,21 +99,58 @@ public static Block filterLong( public static Block filterDouble( @TypeParameter("T") Type elementType, @SqlType("array(T)") Block arrayBlock, - @SqlType("function(T, boolean)") FilterDoubleLambda function) + @SqlType("function(T, boolean)") DoubleToBooleanFunction function) { int positionCount = arrayBlock.getPositionCount(); - BlockBuilder resultBuilder = elementType.createBlockBuilder(null, positionCount); - for (int position = 0; position < positionCount; position++) { - Double input = null; - if (!arrayBlock.isNull(position)) { - input = elementType.getDouble(arrayBlock, position); + int position = 0; + BlockBuilder resultBuilder; + + if (arrayBlock.mayHaveNull()) { + while (position < positionCount && + TRUE.equals(function.apply(arrayBlock.isNull(position) ? + null : + elementType.getDouble(arrayBlock, position)))) { + position++; + } + + if (position == positionCount) { + // Nothing fitered out. So just return the original. + return arrayBlock; + } + + resultBuilder = elementType.createBlockBuilder(null, positionCount); + for (int i = 0; i < position; i++) { + elementType.appendTo(arrayBlock, i, resultBuilder); + } + for (position++; position < positionCount; position++) { + if (TRUE.equals(function.apply(arrayBlock.isNull(position) ? + null : + elementType.getDouble(arrayBlock, position)))) { + elementType.appendTo(arrayBlock, position, resultBuilder); + } + } + } + else { + while (position < positionCount && TRUE.equals(function.apply(elementType.getDouble(arrayBlock, position)))) { + position++; } - Boolean keep = function.apply(input); - if (TRUE.equals(keep)) { - elementType.appendTo(arrayBlock, position, resultBuilder); + if (position == positionCount) { + // Nothing filtered out. So just return the original. + return arrayBlock; + } + + resultBuilder = elementType.createBlockBuilder(null, positionCount); + for (int i = 0; i < position; i++) { + elementType.appendTo(arrayBlock, i, resultBuilder); + } + for (position++; position < positionCount; position++) { + if (TRUE.equals(function.apply(elementType.getDouble(arrayBlock, position)))) { + elementType.appendTo(arrayBlock, position, resultBuilder); + } } } + return resultBuilder.build(); } @@ -86,21 +160,58 @@ public static Block filterDouble( public static Block filterBoolean( @TypeParameter("T") Type elementType, @SqlType("array(T)") Block arrayBlock, - @SqlType("function(T, boolean)") FilterBooleanLambda function) + @SqlType("function(T, boolean)") BooleanToBooleanFunction function) { int positionCount = arrayBlock.getPositionCount(); - BlockBuilder resultBuilder = elementType.createBlockBuilder(null, positionCount); - for (int position = 0; position < positionCount; position++) { - Boolean input = null; - if (!arrayBlock.isNull(position)) { - input = elementType.getBoolean(arrayBlock, position); + int position = 0; + BlockBuilder resultBuilder; + + if (arrayBlock.mayHaveNull()) { + while (position < positionCount && + TRUE.equals(function.apply(arrayBlock.isNull(position) ? + null : + elementType.getBoolean(arrayBlock, position)))) { + position++; } - Boolean keep = function.apply(input); - if (TRUE.equals(keep)) { - elementType.appendTo(arrayBlock, position, resultBuilder); + if (position == positionCount) { + // Nothing fitered out. So just return the original. + return arrayBlock; + } + + resultBuilder = elementType.createBlockBuilder(null, positionCount); + for (int i = 0; i < position; i++) { + elementType.appendTo(arrayBlock, i, resultBuilder); + } + for (position++; position < positionCount; position++) { + if (TRUE.equals(function.apply(arrayBlock.isNull(position) ? + null : + elementType.getBoolean(arrayBlock, position)))) { + elementType.appendTo(arrayBlock, position, resultBuilder); + } } } + else { + while (position < positionCount && TRUE.equals(function.apply(elementType.getBoolean(arrayBlock, position)))) { + position++; + } + + if (position == positionCount) { + // Nothing filtered out. So just return the original. + return arrayBlock; + } + + resultBuilder = elementType.createBlockBuilder(null, positionCount); + for (int i = 0; i < position; i++) { + elementType.appendTo(arrayBlock, i, resultBuilder); + } + for (position++; position < positionCount; position++) { + if (TRUE.equals(function.apply(elementType.getBoolean(arrayBlock, position)))) { + elementType.appendTo(arrayBlock, position, resultBuilder); + } + } + } + return resultBuilder.build(); } @@ -110,21 +221,58 @@ public static Block filterBoolean( public static Block filterSlice( @TypeParameter("T") Type elementType, @SqlType("array(T)") Block arrayBlock, - @SqlType("function(T, boolean)") FilterSliceLambda function) + @SqlType("function(T, boolean)") SliceToBooleanFunction function) { int positionCount = arrayBlock.getPositionCount(); - BlockBuilder resultBuilder = elementType.createBlockBuilder(null, positionCount); - for (int position = 0; position < positionCount; position++) { - Slice input = null; - if (!arrayBlock.isNull(position)) { - input = elementType.getSlice(arrayBlock, position); + int position = 0; + BlockBuilder resultBuilder; + + if (arrayBlock.mayHaveNull()) { + while (position < positionCount && + TRUE.equals(function.apply(arrayBlock.isNull(position) ? + null : + elementType.getSlice(arrayBlock, position)))) { + position++; } - Boolean keep = function.apply(input); - if (TRUE.equals(keep)) { - elementType.appendTo(arrayBlock, position, resultBuilder); + if (position == positionCount) { + // Nothing fitered out. So just return the original. + return arrayBlock; + } + + resultBuilder = elementType.createBlockBuilder(null, positionCount); + for (int i = 0; i < position; i++) { + elementType.appendTo(arrayBlock, i, resultBuilder); + } + for (position++; position < positionCount; position++) { + if (TRUE.equals(function.apply(arrayBlock.isNull(position) ? + null : + elementType.getSlice(arrayBlock, position)))) { + elementType.appendTo(arrayBlock, position, resultBuilder); + } + } + } + else { + while (position < positionCount && TRUE.equals(function.apply(elementType.getSlice(arrayBlock, position)))) { + position++; + } + + if (position == positionCount) { + // Nothing filtered out. So just return the original. + return arrayBlock; + } + + resultBuilder = elementType.createBlockBuilder(null, positionCount); + for (int i = 0; i < position; i++) { + elementType.appendTo(arrayBlock, i, resultBuilder); + } + for (position++; position < positionCount; position++) { + if (TRUE.equals(function.apply(elementType.getSlice(arrayBlock, position)))) { + elementType.appendTo(arrayBlock, position, resultBuilder); + } } } + return resultBuilder.build(); } @@ -134,63 +282,58 @@ public static Block filterSlice( public static Block filterBlock( @TypeParameter("T") Type elementType, @SqlType("array(T)") Block arrayBlock, - @SqlType("function(T, boolean)") FilterBlockLambda function) + @SqlType("function(T, boolean)") BlockToBooleanFunction function) { int positionCount = arrayBlock.getPositionCount(); - BlockBuilder resultBuilder = elementType.createBlockBuilder(null, positionCount); - for (int position = 0; position < positionCount; position++) { - Block input = null; - if (!arrayBlock.isNull(position)) { - input = (Block) elementType.getObject(arrayBlock, position); - } + int position = 0; + BlockBuilder resultBuilder; - Boolean keep = function.apply(input); - if (TRUE.equals(keep)) { - elementType.appendTo(arrayBlock, position, resultBuilder); + if (arrayBlock.mayHaveNull()) { + while (position < positionCount && + TRUE.equals(function.apply(arrayBlock.isNull(position) ? + null : + (Block) elementType.getObject(arrayBlock, position)))) { + position++; } - } - return resultBuilder.build(); - } - @FunctionalInterface - public interface FilterLongLambda - extends LambdaFunctionInterface - { - Boolean apply(Long x); - } - - @FunctionalInterface - public interface FilterDoubleLambda - extends LambdaFunctionInterface - { - Boolean apply(Double x); - } + if (position == positionCount) { + // Nothing fitered out. So just return the original. + return arrayBlock; + } - @FunctionalInterface - public interface FilterBooleanLambda - extends LambdaFunctionInterface - { - Boolean apply(Boolean x); - } + resultBuilder = elementType.createBlockBuilder(null, positionCount); + for (int i = 0; i < position; i++) { + elementType.appendTo(arrayBlock, i, resultBuilder); + } + for (position++; position < positionCount; position++) { + if (TRUE.equals(function.apply(arrayBlock.isNull(position) ? + null : + (Block) elementType.getObject(arrayBlock, position)))) { + elementType.appendTo(arrayBlock, position, resultBuilder); + } + } + } + else { + while (position < positionCount && TRUE.equals(function.apply((Block) elementType.getObject(arrayBlock, position)))) { + position++; + } - @FunctionalInterface - public interface FilterSliceLambda - extends LambdaFunctionInterface - { - Boolean apply(Slice x); - } + if (position == positionCount) { + // Nothing filtered out. So just return the original. + return arrayBlock; + } - @FunctionalInterface - public interface FilterBlockLambda - extends LambdaFunctionInterface - { - Boolean apply(Block x); - } + resultBuilder = elementType.createBlockBuilder(null, positionCount); + for (int i = 0; i < position; i++) { + elementType.appendTo(arrayBlock, i, resultBuilder); + } + for (position++; position < positionCount; position++) { + if (TRUE.equals(function.apply((Block) elementType.getObject(arrayBlock, position)))) { + elementType.appendTo(arrayBlock, position, resultBuilder); + } + } + } - @FunctionalInterface - public interface FilterVoidLambda - extends LambdaFunctionInterface - { - Boolean apply(Void x); + return resultBuilder.build(); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFlattenFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFlattenFunction.java index d6a48526b429f..cab59b576e049 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFlattenFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFlattenFunction.java @@ -19,6 +19,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; @@ -28,8 +29,9 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; @@ -44,7 +46,7 @@ public class ArrayFlattenFunction private ArrayFlattenFunction() { - super(new Signature(FUNCTION_NAME, + super(new Signature(QualifiedFunctionName.of(DEFAULT_NAMESPACE, FUNCTION_NAME), FunctionKind.SCALAR, ImmutableList.of(typeVariable("E")), ImmutableList.of(), @@ -72,12 +74,12 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type elementType = boundVariables.getTypeVariable("E"); Type arrayType = typeManager.getParameterizedType(StandardTypes.ARRAY, ImmutableList.of(TypeSignatureParameter.of(elementType.getTypeSignature()))); MethodHandle methodHandle = METHOD_HANDLE.bindTo(elementType).bindTo(arrayType); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java index 7bf89dbb6b99a..3ffb86c80a044 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.scalar; import com.facebook.presto.operator.aggregation.TypedSet; -import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.Description; @@ -22,23 +21,16 @@ import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.function.TypeParameter; import com.facebook.presto.spi.type.Type; -import com.google.common.collect.ImmutableList; @ScalarFunction("array_intersect") @Description("Intersects elements of the two given arrays") public final class ArrayIntersectFunction { - private final PageBuilder pageBuilder; - - @TypeParameter("E") - public ArrayIntersectFunction(@TypeParameter("E") Type elementType) - { - pageBuilder = new PageBuilder(ImmutableList.of(elementType)); - } + private ArrayIntersectFunction() {} @TypeParameter("E") @SqlType("array(E)") - public Block intersect( + public static Block intersect( @TypeParameter("E") Type type, @SqlType("array(E)") Block leftArray, @SqlType("array(E)") Block rightArray) @@ -56,16 +48,12 @@ public Block intersect( return rightArray; } - if (pageBuilder.isFull()) { - pageBuilder.reset(); - } - TypedSet rightTypedSet = new TypedSet(type, rightPositionCount, "array_intersect"); for (int i = 0; i < rightPositionCount; i++) { rightTypedSet.add(rightArray, i); } - BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0); + BlockBuilder blockBuilder = type.createBlockBuilder(null, Math.min(leftPositionCount, rightPositionCount)); // The intersected set can have at most rightPositionCount elements TypedSet intersectTypedSet = new TypedSet(type, blockBuilder, rightPositionCount, "array_intersect"); @@ -75,8 +63,6 @@ public Block intersect( } } - pageBuilder.declarePositions(intersectTypedSet.size()); - - return blockBuilder.getRegion(blockBuilder.getPositionCount() - intersectTypedSet.size(), intersectTypedSet.size()); + return blockBuilder.build(); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayJoin.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayJoin.java index 32c8aab33f373..a1e88d1910866 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayJoin.java @@ -17,13 +17,14 @@ import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; @@ -40,10 +41,11 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.metadata.CastType.CAST; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.function.Signature.typeVariable; @@ -93,7 +95,8 @@ public static class ArrayJoinWithNullReplacement public ArrayJoinWithNullReplacement() { - super(new Signature(FUNCTION_NAME, + super(new Signature( + QualifiedFunctionName.of(DEFAULT_NAMESPACE, FUNCTION_NAME), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -121,7 +124,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { return specializeArrayJoin(boundVariables.getTypeVariables(), functionManager, ImmutableList.of(false, false, false), METHOD_HANDLE); } @@ -129,7 +132,8 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in public ArrayJoin() { - super(new Signature(FUNCTION_NAME, + super(new Signature( + QualifiedFunctionName.of(DEFAULT_NAMESPACE, FUNCTION_NAME), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -163,12 +167,12 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { return specializeArrayJoin(boundVariables.getTypeVariables(), functionManager, ImmutableList.of(false, false), METHOD_HANDLE); } - private static ScalarFunctionImplementation specializeArrayJoin(Map types, FunctionManager functionManager, List nullableArguments, MethodHandle methodHandle) + private static BuiltInScalarFunctionImplementation specializeArrayJoin(Map types, FunctionManager functionManager, List nullableArguments, MethodHandle methodHandle) { Type type = types.get("T"); List argumentProperties = nullableArguments.stream() @@ -178,7 +182,7 @@ private static ScalarFunctionImplementation specializeArrayJoin(Map elementType = type.getJavaType(); @@ -222,7 +226,7 @@ else if (elementType == Slice.class) { cast = MethodHandles.foldArguments(cast, getter.bindTo(type)); MethodHandle target = MethodHandles.insertArguments(methodHandle, 0, cast); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, argumentProperties, target, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayNoneMatchFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayNoneMatchFunction.java new file mode 100644 index 0000000000000..7be3f505ecf97 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayNoneMatchFunction.java @@ -0,0 +1,112 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.SqlNullable; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.function.TypeParameter; +import com.facebook.presto.spi.function.TypeParameterSpecialization; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; +import io.airlift.slice.Slice; + +@Description("Returns true if all elements of the array don't match the given predicate") +@ScalarFunction(value = "none_match") +public final class ArrayNoneMatchFunction +{ + private ArrayNoneMatchFunction() {} + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = Block.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean noneMatchBlock( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") BlockToBooleanFunction function) + { + Boolean anyMatchResult = ArrayAnyMatchFunction.anyMatchBlock(elementType, arrayBlock, function); + if (anyMatchResult == null) { + return null; + } + return !anyMatchResult; + } + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = Slice.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean noneMatchSlice( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") SliceToBooleanFunction function) + { + Boolean anyMatchResult = ArrayAnyMatchFunction.anyMatchSlice(elementType, arrayBlock, function); + if (anyMatchResult == null) { + return null; + } + return !anyMatchResult; + } + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = long.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean noneMatchLong( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") LongToBooleanFunction function) + { + Boolean anyMatchResult = ArrayAnyMatchFunction.anyMatchLong(elementType, arrayBlock, function); + if (anyMatchResult == null) { + return null; + } + return !anyMatchResult; + } + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = double.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean noneMatchDouble( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") DoubleToBooleanFunction function) + { + Boolean anyMatchResult = ArrayAnyMatchFunction.anyMatchDouble(elementType, arrayBlock, function); + if (anyMatchResult == null) { + return null; + } + return !anyMatchResult; + } + + @TypeParameter("T") + @TypeParameterSpecialization(name = "T", nativeContainerType = boolean.class) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean noneMatchBoolean( + @TypeParameter("T") Type elementType, + @SqlType("array(T)") Block arrayBlock, + @SqlType("function(T, boolean)") BooleanToBooleanFunction function) + { + Boolean anyMatchResult = ArrayAnyMatchFunction.anyMatchBoolean(elementType, arrayBlock, function); + if (anyMatchResult == null) { + return null; + } + return !anyMatchResult; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayReduceFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayReduceFunction.java index 16dda53f3baea..d6fec733041bc 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayReduceFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayReduceFunction.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.SqlScalarFunction; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -28,10 +29,11 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.spi.type.TypeUtils.readNativeValue; @@ -47,7 +49,7 @@ public final class ArrayReduceFunction private ArrayReduceFunction() { super(new Signature( - "reduce", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "reduce"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T"), typeVariable("S"), typeVariable("R")), ImmutableList.of(), @@ -81,13 +83,13 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type inputType = boundVariables.getTypeVariable("T"); Type intermediateType = boundVariables.getTypeVariable("S"); Type outputType = boundVariables.getTypeVariable("R"); MethodHandle methodHandle = METHOD_HANDLE.bindTo(inputType); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( true, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArraySortFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArraySortFunction.java index 299ad15402a91..743b797ebd57d 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArraySortFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArraySortFunction.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.operator.scalar; -import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; @@ -23,11 +22,10 @@ import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.function.TypeParameter; import com.facebook.presto.spi.type.Type; -import com.google.common.collect.ImmutableList; import com.google.common.primitives.Ints; import java.lang.invoke.MethodHandle; -import java.util.Collections; +import java.util.AbstractList; import java.util.Comparator; import java.util.List; @@ -39,72 +37,129 @@ @Description("Sorts the given array in ascending order according to the natural ordering of its elements.") public final class ArraySortFunction { - private final PageBuilder pageBuilder; - private static final int INITIAL_LENGTH = 128; - private List positions = Ints.asList(new int[INITIAL_LENGTH]); - - @TypeParameter("E") - public ArraySortFunction(@TypeParameter("E") Type elementType) - { - pageBuilder = new PageBuilder(ImmutableList.of(elementType)); - } + private ArraySortFunction() {} @TypeParameter("E") @SqlType("array(E)") - public Block sort( + public static Block sort( @OperatorDependency(operator = LESS_THAN, argumentTypes = {"E", "E"}) MethodHandle lessThanFunction, @TypeParameter("E") Type type, @SqlType("array(E)") Block block) { int arrayLength = block.getPositionCount(); - if (positions.size() < arrayLength) { - positions = Ints.asList(new int[arrayLength]); - } - for (int i = 0; i < arrayLength; i++) { - positions.set(i, i); + + if (arrayLength < 2) { + return block; } - Collections.sort(positions.subList(0, arrayLength), new Comparator() - { - @Override - public int compare(Integer p1, Integer p2) + ListOfPositions listOfPositions = new ListOfPositions(block.getPositionCount()); + if (block.mayHaveNull()) { + listOfPositions.sort(new Comparator() { - boolean nullLeft = block.isNull(p1); - boolean nullRight = block.isNull(p2); - if (nullLeft && nullRight) { - return 0; - } - if (nullLeft) { - return 1; - } - if (nullRight) { - return -1; - } + @Override + public int compare(Integer p1, Integer p2) + { + if (block.isNull(p1)) { + return block.isNull(p2) ? 0 : 1; + } + else if (block.isNull(p2)) { + return -1; + } - try { - //TODO: This could be quite slow, it should use parametric equals - return type.compareTo(block, p1, block, p2); + try { + //TODO: This could be quite slow, it should use parametric equals + return type.compareTo(block, p1, block, p2); + } + catch (PrestoException e) { + if (e.getErrorCode() == NOT_SUPPORTED.toErrorCode()) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Array contains elements not supported for comparison", e); + } + throw e; + } } - catch (PrestoException e) { - if (e.getErrorCode() == NOT_SUPPORTED.toErrorCode()) { - throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Array contains elements not supported for comparison", e); + }); + } + else { + listOfPositions.sort(new Comparator() + { + @Override + public int compare(Integer p1, Integer p2) + { + try { + //TODO: This could be quite slow, it should use parametric equals + return type.compareTo(block, p1, block, p2); + } + catch (PrestoException e) { + if (e.getErrorCode() == NOT_SUPPORTED.toErrorCode()) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Array contains elements not supported for comparison", e); + } + throw e; } - throw e; } - } - }); + }); + } - if (pageBuilder.isFull()) { - pageBuilder.reset(); + List sortedListOfPositions = listOfPositions.getSortedListOfPositions(); + if (sortedListOfPositions == listOfPositions) { + // Original array is already sorted. + return block; } - BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0); + BlockBuilder blockBuilder = type.createBlockBuilder(null, arrayLength); for (int i = 0; i < arrayLength; i++) { - type.appendTo(block, positions.get(i), blockBuilder); + type.appendTo(block, sortedListOfPositions.get(i), blockBuilder); } - pageBuilder.declarePositions(arrayLength); - return blockBuilder.getRegion(blockBuilder.getPositionCount() - arrayLength, arrayLength); + return blockBuilder.build(); + } + + private static class ListOfPositions + extends AbstractList + { + private final int size; + private List sortedListOfPositions; + + ListOfPositions(int size) + { + this.size = size; + } + + @Override + public final int size() + { + return size; + } + + @Override + public final Integer get(int i) + { + return i; + } + + @Override + public final Integer set(int index, Integer position) + { + if (index != position) { + // The element at position is out of order. + if (sortedListOfPositions == null) { + // So we need to store the entire position array in a new list. + sortedListOfPositions = Ints.asList(new int[size()]); + for (int i = 0; i < size(); i++) { + sortedListOfPositions.set(i, i); + } + } + + // Set the new position to be used for this index. + sortedListOfPositions.set(index, position); + } + + return position; + } + + List getSortedListOfPositions() + { + return sortedListOfPositions == null ? this : sortedListOfPositions; + } } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArraySubscriptOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArraySubscriptOperator.java index 135780db13128..0b192e20ff99b 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArraySubscriptOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArraySubscriptOperator.java @@ -26,8 +26,8 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.function.OperatorType.SUBSCRIPT; import static com.facebook.presto.spi.function.Signature.typeVariable; @@ -58,7 +58,7 @@ protected ArraySubscriptOperator() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { checkArgument(boundVariables.getTypeVariables().size() == 1, "Expected one type, got %s", boundVariables.getTypeVariables()); Type elementType = boundVariables.getTypeVariable("E"); @@ -81,7 +81,7 @@ else if (elementType.getJavaType() == Slice.class) { } methodHandle = methodHandle.bindTo(elementType); requireNonNull(methodHandle, "methodHandle is null"); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( true, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToArrayCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToArrayCast.java index adaf32d3ef9b0..0a8a80a3aa7b2 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToArrayCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToArrayCast.java @@ -45,8 +45,8 @@ import static com.facebook.presto.bytecode.Parameter.arg; import static com.facebook.presto.bytecode.ParameterizedType.type; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantBoolean; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.CAST; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -70,17 +70,17 @@ private ArrayToArrayCast() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { checkArgument(arity == 1, "Expected arity to be 1"); Type fromType = boundVariables.getTypeVariable("F"); Type toType = boundVariables.getTypeVariable("T"); FunctionHandle functionHandle = functionManager.lookupCast(CastType.CAST, fromType.getTypeSignature(), toType.getTypeSignature()); - ScalarFunctionImplementation function = functionManager.getScalarFunctionImplementation(functionHandle); + BuiltInScalarFunctionImplementation function = functionManager.getBuiltInScalarFunctionImplementation(functionHandle); Class castOperatorClass = generateArrayCast(typeManager, functionManager.getFunctionMetadata(functionHandle), function); MethodHandle methodHandle = methodHandle(castOperatorClass, "castArray", ConnectorSession.class, Block.class); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), @@ -88,7 +88,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in methodHandle); } - private static Class generateArrayCast(TypeManager typeManager, FunctionMetadata elementCastFunctionMetadata, ScalarFunctionImplementation elementCast) + private static Class generateArrayCast(TypeManager typeManager, FunctionMetadata elementCastFunctionMetadata, BuiltInScalarFunctionImplementation elementCast) { CallSiteBinder binder = new CallSiteBinder(); @@ -117,7 +117,7 @@ private static Class generateArrayCast(TypeManager typeManager, FunctionMetad Type fromElementType = typeManager.getType(elementCastFunctionMetadata.getArgumentTypes().get(0)); Type toElementType = typeManager.getType(elementCastFunctionMetadata.getReturnType()); CachedInstanceBinder cachedInstanceBinder = new CachedInstanceBinder(definition, binder); - ArrayMapBytecodeExpression newArray = ArrayGeneratorUtils.map(scope, cachedInstanceBinder, fromElementType, toElementType, value, elementCastFunctionMetadata.getName(), elementCast); + ArrayMapBytecodeExpression newArray = ArrayGeneratorUtils.map(scope, cachedInstanceBinder, fromElementType, toElementType, value, elementCastFunctionMetadata.getName().getFunctionName(), elementCast); // return the block body.append(newArray.ret()); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToElementConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToElementConcatFunction.java index 6611088958ec6..6f7526cb9ff62 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToElementConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToElementConcatFunction.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.SqlScalarFunction; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -26,8 +27,9 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; @@ -36,7 +38,6 @@ public class ArrayToElementConcatFunction extends SqlScalarFunction { public static final ArrayToElementConcatFunction ARRAY_TO_ELEMENT_CONCAT_FUNCTION = new ArrayToElementConcatFunction(); - private static final String FUNCTION_NAME = "concat"; private static final MethodHandle METHOD_HANDLE_BOOLEAN = methodHandle(ArrayConcatUtils.class, "appendElement", Type.class, Block.class, boolean.class); private static final MethodHandle METHOD_HANDLE_LONG = methodHandle(ArrayConcatUtils.class, "appendElement", Type.class, Block.class, long.class); @@ -46,7 +47,8 @@ public class ArrayToElementConcatFunction public ArrayToElementConcatFunction() { - super(new Signature(FUNCTION_NAME, + super(new Signature( + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "concat"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("E")), ImmutableList.of(), @@ -74,7 +76,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type type = boundVariables.getTypeVariable("E"); MethodHandle methodHandle; @@ -95,7 +97,7 @@ else if (type.getJavaType() == Slice.class) { } methodHandle = methodHandle.bindTo(type); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToJsonCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToJsonCast.java index 364b510ac12bc..f1526407e6668 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToJsonCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToJsonCast.java @@ -33,9 +33,9 @@ import java.io.IOException; import java.lang.invoke.MethodHandle; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.operator.scalar.JsonOperators.JSON_FACTORY; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -62,7 +62,7 @@ private ArrayToJsonCast() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { checkArgument(arity == 1, "Expected arity to be 1"); Type type = boundVariables.getTypeVariable("T"); @@ -71,7 +71,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in JsonGeneratorWriter writer = JsonGeneratorWriter.createJsonGeneratorWriter(type); MethodHandle methodHandle = METHOD_HANDLE.bindTo(writer); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayTransformFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayTransformFunction.java index 530e237a8cc36..9bcb66d462191 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayTransformFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayTransformFunction.java @@ -29,6 +29,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; @@ -55,9 +56,10 @@ import static com.facebook.presto.bytecode.expression.BytecodeExpressions.newInstance; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.subtract; import static com.facebook.presto.bytecode.instruction.VariableInstruction.incrementVariable; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.sql.gen.SqlTypeBytecodeExpression.constantType; @@ -74,7 +76,7 @@ public final class ArrayTransformFunction private ArrayTransformFunction() { super(new Signature( - "transform", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "transform"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T"), typeVariable("U")), ImmutableList.of(), @@ -102,12 +104,12 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type inputType = boundVariables.getTypeVariable("T"); Type outputType = boundVariables.getTypeVariable("U"); Class generatedClass = generateTransform(inputType, outputType); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/Unnester.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/BlockToBooleanFunction.java similarity index 69% rename from presto-main/src/main/java/com/facebook/presto/operator/Unnester.java rename to presto-main/src/main/java/com/facebook/presto/operator/scalar/BlockToBooleanFunction.java index d4709c49ac00c..8bf296cbd753d 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/Unnester.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/BlockToBooleanFunction.java @@ -11,18 +11,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.operator; +package com.facebook.presto.operator.scalar; -import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.sql.gen.lambda.LambdaFunctionInterface; -public interface Unnester +@FunctionalInterface +public interface BlockToBooleanFunction + extends LambdaFunctionInterface { - int getChannelCount(); - - void appendNext(PageBuilder pageBuilder, int outputChannelOffset); - - boolean hasNext(); - - void setBlock(Block block); + Boolean apply(Block value); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/BooleanToBooleanFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/BooleanToBooleanFunction.java new file mode 100644 index 0000000000000..5e7b258b52126 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/BooleanToBooleanFunction.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.sql.gen.lambda.LambdaFunctionInterface; + +@FunctionalInterface +public interface BooleanToBooleanFunction + extends LambdaFunctionInterface +{ + Boolean apply(Boolean value); +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ScalarFunctionImplementation.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/BuiltInScalarFunctionImplementation.java similarity index 94% rename from presto-main/src/main/java/com/facebook/presto/operator/scalar/ScalarFunctionImplementation.java rename to presto-main/src/main/java/com/facebook/presto/operator/scalar/BuiltInScalarFunctionImplementation.java index 2953b9e1fd260..6fe41722a9008 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ScalarFunctionImplementation.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/BuiltInScalarFunctionImplementation.java @@ -14,6 +14,7 @@ package com.facebook.presto.operator.scalar; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.function.ScalarFunctionImplementation; import com.google.common.collect.ImmutableList; import java.lang.invoke.MethodHandle; @@ -21,18 +22,19 @@ import java.util.Objects; import java.util.Optional; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentType.FUNCTION_TYPE; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentType.VALUE_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentType.FUNCTION_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentType.VALUE_TYPE; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static java.lang.String.format; import static java.util.Objects.requireNonNull; -public final class ScalarFunctionImplementation +public final class BuiltInScalarFunctionImplementation + implements ScalarFunctionImplementation { private final List choices; - public ScalarFunctionImplementation( + public BuiltInScalarFunctionImplementation( boolean nullable, List argumentProperties, MethodHandle methodHandle) @@ -44,7 +46,7 @@ public ScalarFunctionImplementation( Optional.empty()); } - public ScalarFunctionImplementation( + public BuiltInScalarFunctionImplementation( boolean nullable, List argumentProperties, MethodHandle methodHandle, @@ -68,7 +70,7 @@ public ScalarFunctionImplementation( * * @param choices the list of choices, ordered from generic to specific */ - public ScalarFunctionImplementation(List choices) + public BuiltInScalarFunctionImplementation(List choices) { checkArgument(!choices.isEmpty(), "choices is an empty list"); this.choices = ImmutableList.copyOf(choices); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/CastFromUnknownOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/CastFromUnknownOperator.java index 2071ba1907d0a..9468625b1f9ca 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/CastFromUnknownOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/CastFromUnknownOperator.java @@ -23,8 +23,8 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.CAST; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -46,13 +46,13 @@ public CastFromUnknownOperator() } @Override - public ScalarFunctionImplementation specialize( + public BuiltInScalarFunctionImplementation specialize( BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type toType = boundVariables.getTypeVariable("E"); MethodHandle methodHandle = METHOD_HANDLE_NON_NULL.asType(METHOD_HANDLE_NON_NULL.type().changeReturnType(toType.getJavaType())); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ConcatFunction.java index b1a1f77ae217e..8409aa393a032 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ConcatFunction.java @@ -27,6 +27,7 @@ import com.facebook.presto.metadata.SqlScalarFunction; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; @@ -49,8 +50,9 @@ import static com.facebook.presto.bytecode.expression.BytecodeExpressions.add; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantInt; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.invokeStatic; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; @@ -76,7 +78,7 @@ public final class ConcatFunction private ConcatFunction(TypeSignature type, String description) { super(new Signature( - "concat", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "concat"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), @@ -105,7 +107,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { if (arity < 2) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "There must be two or more concatenation arguments"); @@ -114,7 +116,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in Class clazz = generateConcat(getSignature().getReturnType(), arity); MethodHandle methodHandle = methodHandle(clazz, "concat", nCopies(arity, Slice.class).toArray(new Class[arity])); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, nCopies(arity, valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/DateTimeFunctions.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/DateTimeFunctions.java index 80719d1e69e57..d0352652e64d1 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/DateTimeFunctions.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/DateTimeFunctions.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.scalar; +import com.facebook.airlift.concurrent.ThreadLocalCache; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.Description; @@ -21,7 +22,6 @@ import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.TimeZoneKey; -import io.airlift.concurrent.ThreadLocalCache; import io.airlift.slice.Slice; import io.airlift.units.Duration; import org.joda.time.DateTime; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/DoubleToBooleanFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/DoubleToBooleanFunction.java new file mode 100644 index 0000000000000..26b74d0c69c6c --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/DoubleToBooleanFunction.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.sql.gen.lambda.LambdaFunctionInterface; + +@FunctionalInterface +public interface DoubleToBooleanFunction + extends LambdaFunctionInterface +{ + Boolean apply(Double value); +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ElementToArrayConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ElementToArrayConcatFunction.java index 1960cda236675..63a2ca140253b 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ElementToArrayConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ElementToArrayConcatFunction.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.SqlScalarFunction; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -26,8 +27,9 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; @@ -36,7 +38,6 @@ public class ElementToArrayConcatFunction extends SqlScalarFunction { public static final ElementToArrayConcatFunction ELEMENT_TO_ARRAY_CONCAT_FUNCTION = new ElementToArrayConcatFunction(); - private static final String FUNCTION_NAME = "concat"; private static final MethodHandle METHOD_HANDLE_BOOLEAN = methodHandle(ArrayConcatUtils.class, "prependElement", Type.class, boolean.class, Block.class); private static final MethodHandle METHOD_HANDLE_LONG = methodHandle(ArrayConcatUtils.class, "prependElement", Type.class, long.class, Block.class); @@ -47,7 +48,7 @@ public class ElementToArrayConcatFunction public ElementToArrayConcatFunction() { super(new Signature( - FUNCTION_NAME, + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "concat"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("E")), ImmutableList.of(), @@ -75,7 +76,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type type = boundVariables.getTypeVariable("E"); MethodHandle methodHandle; @@ -96,7 +97,7 @@ else if (type.getJavaType() == Slice.class) { } methodHandle = methodHandle.bindTo(type); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/FailureFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/FailureFunction.java index 4d84e45a346bd..ba2980717febe 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/FailureFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/FailureFunction.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.scalar; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.client.FailureInfo; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.StandardErrorCode; @@ -20,7 +21,6 @@ import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; public final class FailureFunction @@ -40,6 +40,24 @@ public static boolean failWithException(@SqlType(StandardTypes.JSON) Slice failu throw new PrestoException(StandardErrorCode.GENERIC_USER_ERROR, failureInfo.toException()); } + // This function is only used to propagate optimization failures. + @Description("Decodes json to an exception and throws it with supplied errorCode") + @ScalarFunction(value = "fail", hidden = true) + @SqlType("unknown") + public static boolean failWithException( + @SqlType(StandardTypes.INTEGER) long errorCode, + @SqlType(StandardTypes.JSON) Slice failureInfoSlice) + { + FailureInfo failureInfo = JSON_CODEC.fromJson(failureInfoSlice.getBytes()); + // wrap the failure in a new exception to append the current stack trace + for (StandardErrorCode standardErrorCode : StandardErrorCode.values()) { + if (standardErrorCode.toErrorCode().getCode() == errorCode) { + throw new PrestoException(standardErrorCode, failureInfo.toException()); + } + } + throw new PrestoException(StandardErrorCode.GENERIC_INTERNAL_ERROR, "Unable to find error for code: " + errorCode, failureInfo.toException()); + } + @Description("Throws an exception with a given message") @ScalarFunction(value = "fail", hidden = true) @SqlType("unknown") diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/Greatest.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/Greatest.java index 9a98fd869d1b0..3831dec1a9770 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/Greatest.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/Greatest.java @@ -14,6 +14,9 @@ package com.facebook.presto.operator.scalar; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; + +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; public final class Greatest extends AbstractGreatestLeast @@ -22,7 +25,7 @@ public final class Greatest public Greatest() { - super("greatest", OperatorType.GREATER_THAN); + super(QualifiedFunctionName.of(DEFAULT_NAMESPACE, "greatest"), OperatorType.GREATER_THAN); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/HyperLogLogFunctions.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/HyperLogLogFunctions.java index 36ba7a54da188..c3ee8d4b24eba 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/HyperLogLogFunctions.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/HyperLogLogFunctions.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.scalar; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.function.Description; import com.facebook.presto.spi.function.ScalarFunction; @@ -20,7 +21,6 @@ import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; import io.airlift.slice.Slice; -import io.airlift.stats.cardinality.HyperLogLog; import static com.facebook.presto.operator.aggregation.ApproximateSetAggregation.DEFAULT_STANDARD_ERROR; import static com.facebook.presto.operator.aggregation.HyperLogLogUtils.standardErrorToBuckets; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/IdentityCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/IdentityCast.java index 7fa41a329144e..a8f09bd606175 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/IdentityCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/IdentityCast.java @@ -24,8 +24,8 @@ import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.google.common.base.Preconditions.checkArgument; @@ -45,12 +45,12 @@ protected IdentityCast() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { checkArgument(boundVariables.getTypeVariables().size() == 1, "Expected only one type"); Type type = boundVariables.getTypeVariable("T"); MethodHandle identity = MethodHandles.identity(type.getJavaType()); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), identity); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/InvokeFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/InvokeFunction.java index 858589b842a03..0eb8f3817720f 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/InvokeFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/InvokeFunction.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -26,7 +27,8 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; @@ -45,7 +47,7 @@ public final class InvokeFunction private InvokeFunction() { super(new Signature( - "invoke", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "invoke"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -73,10 +75,10 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type returnType = boundVariables.getTypeVariable("T"); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( true, ImmutableList.of(functionTypeArgumentProperty(InvokeLambda.class)), METHOD_HANDLE.asType( diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/IpPrefixFunctions.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/IpPrefixFunctions.java new file mode 100644 index 0000000000000..3f8f776e7c10e --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/IpPrefixFunctions.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.LiteralParameters; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.type.StandardTypes; +import com.google.common.net.InetAddresses; +import io.airlift.slice.Slice; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.type.IpAddressOperators.castFromVarcharToIpAddress; +import static com.facebook.presto.type.IpPrefixOperators.castFromVarcharToIpPrefix; +import static com.facebook.presto.util.Failures.checkCondition; +import static io.airlift.slice.Slices.utf8Slice; + +public final class IpPrefixFunctions +{ + private IpPrefixFunctions() {} + + @Description("IP prefix for a given IP address and subnet size") + @ScalarFunction("ip_prefix") + @SqlType(StandardTypes.IPPREFIX) + public static Slice ipPrefix(@SqlType(StandardTypes.IPADDRESS) Slice value, @SqlType(StandardTypes.BIGINT) long subnetSize) + { + InetAddress address; + try { + address = InetAddress.getByAddress(value.getBytes()); + } + catch (UnknownHostException e) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Invalid IP address binary: " + value.toStringUtf8(), e); + } + + int addressLength = address.getAddress().length; + if (addressLength == 4) { + checkCondition(0 <= subnetSize && subnetSize <= 32, INVALID_FUNCTION_ARGUMENT, "IPv4 subnet size must be in range [0, 32]"); + } + else if (addressLength == 16) { + checkCondition(0 <= subnetSize && subnetSize <= 128, INVALID_FUNCTION_ARGUMENT, "IPv6 subnet size must be in range [0, 128]"); + } + else { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Invalid InetAddress length: " + addressLength); + } + + return castFromVarcharToIpPrefix(utf8Slice(InetAddresses.toAddrString(address) + "/" + subnetSize)); + } + + @Description("IP prefix for a given IP address and subnet size") + @ScalarFunction("ip_prefix") + @LiteralParameters("x") + @SqlType(StandardTypes.IPPREFIX) + public static Slice stringIpPrefix(@SqlType("varchar(x)") Slice slice, @SqlType(StandardTypes.BIGINT) long subnetSize) + { + return ipPrefix(castFromVarcharToIpAddress(slice), subnetSize); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonFunctions.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonFunctions.java index 18a626f0bd5c1..1a122754375a1 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonFunctions.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonFunctions.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.scalar; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; @@ -33,7 +34,6 @@ import com.fasterxml.jackson.databind.MappingJsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.primitives.Doubles; -import io.airlift.json.ObjectMapperProvider; import io.airlift.slice.DynamicSliceOutput; import io.airlift.slice.Slice; import io.airlift.slice.SliceOutput; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToArrayCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToArrayCast.java index 5631391303a37..504f10c2c7443 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToArrayCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToArrayCast.java @@ -16,11 +16,13 @@ import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.operator.scalar.JsonToArrayCast.JSON_TO_ARRAY; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.Signature.typeVariable; @@ -35,7 +37,7 @@ public final class JsonStringToArrayCast private JsonStringToArrayCast() { super(new Signature( - JSON_STRING_TO_ARRAY_NAME, + QualifiedFunctionName.of(DEFAULT_NAMESPACE, JSON_STRING_TO_ARRAY_NAME), SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -64,7 +66,7 @@ public final boolean isHidden() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { return JSON_TO_ARRAY.specialize(boundVariables, arity, typeManager, functionManager); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToMapCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToMapCast.java index 4dc16652a306c..bf1e710f868ec 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToMapCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToMapCast.java @@ -16,11 +16,13 @@ import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.operator.scalar.JsonToMapCast.JSON_TO_MAP; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.Signature.comparableTypeParameter; @@ -36,7 +38,7 @@ public final class JsonStringToMapCast private JsonStringToMapCast() { super(new Signature( - JSON_STRING_TO_MAP_NAME, + QualifiedFunctionName.of(DEFAULT_NAMESPACE, JSON_STRING_TO_MAP_NAME), SCALAR, ImmutableList.of(comparableTypeParameter("K"), typeVariable("V")), ImmutableList.of(), @@ -65,7 +67,7 @@ public final boolean isHidden() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { return JSON_TO_MAP.specialize(boundVariables, arity, typeManager, functionManager); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToRowCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToRowCast.java index 06e512fb5e81e..e958f5003c27e 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToRowCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToRowCast.java @@ -16,11 +16,13 @@ import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.operator.scalar.JsonToRowCast.JSON_TO_ROW; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.Signature.withVariadicBound; @@ -35,7 +37,7 @@ public final class JsonStringToRowCast private JsonStringToRowCast() { super(new Signature( - JSON_STRING_TO_ROW_NAME, + QualifiedFunctionName.of(DEFAULT_NAMESPACE, JSON_STRING_TO_ROW_NAME), SCALAR, ImmutableList.of(withVariadicBound("T", "row")), ImmutableList.of(), @@ -64,7 +66,7 @@ public final boolean isHidden() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { return JSON_TO_ROW.specialize(boundVariables, arity, typeManager, functionManager); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToArrayCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToArrayCast.java index 46baf8ac4b49e..64cd45c153492 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToArrayCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToArrayCast.java @@ -36,8 +36,8 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -67,7 +67,7 @@ private JsonToArrayCast() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { checkArgument(arity == 1, "Expected arity to be 1"); Type type = boundVariables.getTypeVariable("T"); @@ -76,7 +76,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in BlockBuilderAppender elementAppender = BlockBuilderAppender.createBlockBuilderAppender(arrayType.getElementType()); MethodHandle methodHandle = METHOD_HANDLE.bindTo(arrayType).bindTo(elementAppender); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( true, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToMapCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToMapCast.java index 574db4ec4c1a6..36829d8aaf85e 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToMapCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToMapCast.java @@ -36,8 +36,8 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; import static com.facebook.presto.spi.function.Signature.comparableTypeParameter; import static com.facebook.presto.spi.function.Signature.typeVariable; @@ -70,7 +70,7 @@ private JsonToMapCast() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { checkArgument(arity == 1, "Expected arity to be 1"); Type keyType = boundVariables.getTypeVariable("K"); @@ -81,7 +81,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in BlockBuilderAppender keyAppender = createBlockBuilderAppender(mapType.getKeyType()); BlockBuilderAppender valueAppender = createBlockBuilderAppender(mapType.getValueType()); MethodHandle methodHandle = METHOD_HANDLE.bindTo(mapType).bindTo(keyAppender).bindTo(valueAppender); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( true, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToRowCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToRowCast.java index b37370ac5f3b6..28c7cf33cde07 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToRowCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToRowCast.java @@ -39,8 +39,8 @@ import java.util.Map; import java.util.Optional; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; import static com.facebook.presto.spi.function.Signature.withVariadicBound; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -74,7 +74,7 @@ private JsonToRowCast() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { checkArgument(arity == 1, "Expected arity to be 1"); RowType rowType = (RowType) boundVariables.getTypeVariable("T"); @@ -85,7 +85,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in .map(rowField -> createBlockBuilderAppender(rowField.getType())) .toArray(BlockBuilderAppender[]::new); MethodHandle methodHandle = METHOD_HANDLE.bindTo(rowType).bindTo(fieldAppenders).bindTo(getFieldNameToIndex(rowFields)); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( true, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/Least.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/Least.java index fa88efdea438d..46b3fc48c32a7 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/Least.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/Least.java @@ -14,6 +14,9 @@ package com.facebook.presto.operator.scalar; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; + +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; public final class Least extends AbstractGreatestLeast @@ -22,7 +25,7 @@ public final class Least public Least() { - super("least", OperatorType.LESS_THAN); + super(QualifiedFunctionName.of(DEFAULT_NAMESPACE, "least"), OperatorType.LESS_THAN); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/LongToBooleanFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/LongToBooleanFunction.java new file mode 100644 index 0000000000000..830fa6cd10b12 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/LongToBooleanFunction.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.sql.gen.lambda.LambdaFunctionInterface; + +@FunctionalInterface +public interface LongToBooleanFunction + extends LambdaFunctionInterface +{ + Boolean apply(Long value); +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java index 65f97b5a242c0..6a911588d1834 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.StandardTypes; @@ -35,8 +36,9 @@ import java.lang.invoke.MethodHandle; import java.util.Optional; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -58,7 +60,8 @@ public final class MapConcatFunction private MapConcatFunction() { - super(new Signature(FUNCTION_NAME, + super(new Signature( + QualifiedFunctionName.of(DEFAULT_NAMESPACE, FUNCTION_NAME), FunctionKind.SCALAR, ImmutableList.of(typeVariable("K"), typeVariable("V")), ImmutableList.of(), @@ -86,7 +89,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { if (arity < 2) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "There must be two or more concatenation arguments to " + FUNCTION_NAME); @@ -105,7 +108,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in METHOD_HANDLE.bindTo(mapType), USER_STATE_FACTORY.bindTo(mapType)); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, nCopies(arity, valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandleAndConstructor.getMethodHandle(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConstructor.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConstructor.java index a052ff2771292..f390481fc954a 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConstructor.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConstructor.java @@ -26,6 +26,7 @@ import com.facebook.presto.spi.block.MapBlockBuilder; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.Type; @@ -37,8 +38,9 @@ import java.lang.invoke.MethodHandle; import java.util.Optional; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.function.OperatorType.INDETERMINATE; import static com.facebook.presto.spi.function.Signature.comparableTypeParameter; @@ -74,7 +76,7 @@ public final class MapConstructor public MapConstructor() { super(new Signature( - "map", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "map"), FunctionKind.SCALAR, ImmutableList.of(comparableTypeParameter("K"), typeVariable("V")), ImmutableList.of(), @@ -102,19 +104,19 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type keyType = boundVariables.getTypeVariable("K"); Type valueType = boundVariables.getTypeVariable("V"); Type mapType = typeManager.getParameterizedType(MAP, ImmutableList.of(TypeSignatureParameter.of(keyType.getTypeSignature()), TypeSignatureParameter.of(valueType.getTypeSignature()))); - MethodHandle keyHashCode = functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(OperatorType.HASH_CODE, fromTypes(keyType))).getMethodHandle(); - MethodHandle keyEqual = functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(OperatorType.EQUAL, fromTypes(keyType, keyType))).getMethodHandle(); - MethodHandle keyIndeterminate = functionManager.getScalarFunctionImplementation( + MethodHandle keyHashCode = functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(OperatorType.HASH_CODE, fromTypes(keyType))).getMethodHandle(); + MethodHandle keyEqual = functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(OperatorType.EQUAL, fromTypes(keyType, keyType))).getMethodHandle(); + MethodHandle keyIndeterminate = functionManager.getBuiltInScalarFunctionImplementation( functionManager.resolveOperator(INDETERMINATE, fromTypeSignatures((keyType.getTypeSignature())))).getMethodHandle(); MethodHandle instanceFactory = constructorMethodHandle(State.class, MapType.class).bindTo(mapType); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapElementAtFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapElementAtFunction.java index 3493736c49692..acf7438ed63e5 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapElementAtFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapElementAtFunction.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.block.SingleMapBlock; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -30,8 +31,9 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.spi.type.TypeUtils.readNativeValue; @@ -52,7 +54,7 @@ public class MapElementAtFunction protected MapElementAtFunction() { super(new Signature( - "element_at", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "element_at"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("K"), typeVariable("V")), ImmutableList.of(), @@ -80,12 +82,12 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type keyType = boundVariables.getTypeVariable("K"); Type valueType = boundVariables.getTypeVariable("V"); - MethodHandle keyEqualsMethod = functionManager.getScalarFunctionImplementation( + MethodHandle keyEqualsMethod = functionManager.getBuiltInScalarFunctionImplementation( functionManager.resolveOperator(OperatorType.EQUAL, fromTypes(keyType, keyType))).getMethodHandle(); MethodHandle methodHandle; @@ -107,7 +109,7 @@ else if (keyType.getJavaType() == Slice.class) { methodHandle = methodHandle.bindTo(keyEqualsMethod).bindTo(keyType).bindTo(valueType); methodHandle = methodHandle.asType(methodHandle.type().changeReturnType(Primitives.wrap(valueType.getJavaType()))); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( true, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapFilterFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapFilterFunction.java index 642c1b266854e..981a1c25334a5 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapFilterFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapFilterFunction.java @@ -30,6 +30,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.StandardTypes; @@ -60,9 +61,10 @@ import static com.facebook.presto.bytecode.expression.BytecodeExpressions.notEqual; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.subtract; import static com.facebook.presto.bytecode.instruction.VariableInstruction.incrementVariable; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.sql.gen.SqlTypeBytecodeExpression.constantType; @@ -80,7 +82,7 @@ public final class MapFilterFunction private MapFilterFunction() { super(new Signature( - "map_filter", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "map_filter"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("K"), typeVariable("V")), ImmutableList.of(), @@ -108,14 +110,14 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type keyType = boundVariables.getTypeVariable("K"); Type valueType = boundVariables.getTypeVariable("V"); MapType mapType = (MapType) typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of( TypeSignatureParameter.of(keyType.getTypeSignature()), TypeSignatureParameter.of(valueType.getTypeSignature()))); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapHashCodeOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapHashCodeOperator.java index e033044b3b126..f1cb90c05e9e9 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapHashCodeOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapHashCodeOperator.java @@ -25,8 +25,8 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.HASH_CODE; import static com.facebook.presto.spi.function.Signature.comparableTypeParameter; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -50,16 +50,16 @@ private MapHashCodeOperator() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type keyType = boundVariables.getTypeVariable("K"); Type valueType = boundVariables.getTypeVariable("V"); - MethodHandle keyHashCodeFunction = functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(HASH_CODE, fromTypes(keyType))).getMethodHandle(); - MethodHandle valueHashCodeFunction = functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(HASH_CODE, fromTypes(valueType))).getMethodHandle(); + MethodHandle keyHashCodeFunction = functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(HASH_CODE, fromTypes(keyType))).getMethodHandle(); + MethodHandle valueHashCodeFunction = functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(HASH_CODE, fromTypes(valueType))).getMethodHandle(); MethodHandle method = METHOD_HANDLE.bindTo(keyHashCodeFunction).bindTo(valueHashCodeFunction).bindTo(keyType).bindTo(valueType); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), method); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapSubscriptOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapSubscriptOperator.java index 4b5e1bf8dea6b..00355f4368cb0 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapSubscriptOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapSubscriptOperator.java @@ -33,8 +33,8 @@ import java.lang.invoke.MethodHandles; import static com.facebook.presto.metadata.CastType.CAST; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.function.OperatorType.SUBSCRIPT; import static com.facebook.presto.spi.function.Signature.typeVariable; @@ -66,7 +66,7 @@ public MapSubscriptOperator(boolean legacyMissingKey) } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type keyType = boundVariables.getTypeVariable("K"); Type valueType = boundVariables.getTypeVariable("V"); @@ -92,7 +92,7 @@ else if (keyType.getJavaType() == Slice.class) { methodHandle = methodHandle.bindTo(missingKeyExceptionFactory).bindTo(valueType); methodHandle = methodHandle.asType(methodHandle.type().changeReturnType(Primitives.wrap(valueType.getJavaType()))); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( true, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapToJsonCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapToJsonCast.java index 577ca3e3c89be..4cfb74e8999f7 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapToJsonCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapToJsonCast.java @@ -35,9 +35,9 @@ import java.util.Map; import java.util.TreeMap; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.operator.scalar.JsonOperators.JSON_FACTORY; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -66,7 +66,7 @@ private MapToJsonCast() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { checkArgument(arity == 1, "Expected arity to be 1"); Type keyType = boundVariables.getTypeVariable("K"); @@ -80,7 +80,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in JsonGeneratorWriter writer = JsonGeneratorWriter.createJsonGeneratorWriter(valueType); MethodHandle methodHandle = METHOD_HANDLE.bindTo(provider).bindTo(writer); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapToMapCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapToMapCast.java index 7f0fe9c5b0f34..23db25231aec3 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapToMapCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapToMapCast.java @@ -32,8 +32,8 @@ import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; import static com.facebook.presto.spi.block.MethodHandleUtil.compose; import static com.facebook.presto.spi.block.MethodHandleUtil.nativeValueGetter; @@ -79,7 +79,7 @@ public MapToMapCast() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { checkArgument(arity == 1, "Expected arity to be 1"); Type fromKeyType = boundVariables.getTypeVariable("FK"); @@ -95,7 +95,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in MethodHandle keyProcessor = buildProcessor(functionManager, fromKeyType, toKeyType, true); MethodHandle valueProcessor = buildProcessor(functionManager, fromValueType, toValueType, false); MethodHandle target = MethodHandles.insertArguments(METHOD_HANDLE, 0, keyProcessor, valueProcessor, toMapType); - return new ScalarFunctionImplementation(true, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), target); + return new BuiltInScalarFunctionImplementation(true, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), target); } /** @@ -107,7 +107,7 @@ private MethodHandle buildProcessor(FunctionManager functionManager, Type fromTy MethodHandle getter = nativeValueGetter(fromType); // Adapt cast that takes ([ConnectorSession,] ?) to one that takes (?, ConnectorSession), where ? is the return type of getter. - ScalarFunctionImplementation castImplementation = functionManager.getScalarFunctionImplementation(functionManager.lookupCast(CastType.CAST, fromType.getTypeSignature(), toType.getTypeSignature())); + BuiltInScalarFunctionImplementation castImplementation = functionManager.getBuiltInScalarFunctionImplementation(functionManager.lookupCast(CastType.CAST, fromType.getTypeSignature(), toType.getTypeSignature())); MethodHandle cast = castImplementation.getMethodHandle(); if (cast.type().parameterArray()[0] != ConnectorSession.class) { cast = MethodHandles.dropArguments(cast, 0, ConnectorSession.class); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformKeyFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformKeyFunction.java index b8a3e0076d648..9706ccb6d4c32 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformKeyFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformKeyFunction.java @@ -34,6 +34,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.StandardTypes; @@ -69,9 +70,10 @@ import static com.facebook.presto.bytecode.expression.BytecodeExpressions.newInstance; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.subtract; import static com.facebook.presto.bytecode.instruction.VariableInstruction.incrementVariable; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -90,7 +92,7 @@ public final class MapTransformKeyFunction private MapTransformKeyFunction() { super(new Signature( - "transform_keys", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "transform_keys"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("K1"), typeVariable("K2"), typeVariable("V")), ImmutableList.of(), @@ -118,7 +120,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type keyType = boundVariables.getTypeVariable("K1"); Type transformedKeyType = boundVariables.getTypeVariable("K2"); @@ -126,7 +128,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in MapType resultMapType = (MapType) typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of( TypeSignatureParameter.of(transformedKeyType.getTypeSignature()), TypeSignatureParameter.of(valueType.getTypeSignature()))); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), @@ -192,7 +194,7 @@ private static MethodHandle generateTransformKey(Type keyType, Type transformedK TypedSet.class, constantType(binder, transformedKeyType), divide(positionCount, constantInt(2)), - constantString(MAP_TRANSFORM_KEY_FUNCTION.getSignature().getName())))); + constantString(MAP_TRANSFORM_KEY_FUNCTION.getSignature().getNameSuffix())))); // throw null key exception block BytecodeNode throwNullKeyException = new BytecodeBlock() @@ -276,7 +278,8 @@ private static MethodHandle generateTransformKey(Type keyType, Type transformedK .append(new IfStatement() .condition(typedSet.invoke("contains", boolean.class, blockBuilder.cast(Block.class), position)) .ifTrue(throwDuplicatedKeyException) - .ifFalse(typedSet.invoke("add", void.class, blockBuilder.cast(Block.class), position))))); + .ifFalse(typedSet.invoke("add", boolean.class, blockBuilder.cast(Block.class), position) + .pop())))); body.append(mapBlockBuilder .invoke("closeEntry", BlockBuilder.class) diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformValueFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformValueFunction.java index 7eebb260d50db..ac3c8fcd39407 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformValueFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformValueFunction.java @@ -23,6 +23,7 @@ import com.facebook.presto.bytecode.Variable; import com.facebook.presto.bytecode.control.ForLoop; import com.facebook.presto.bytecode.control.IfStatement; +import com.facebook.presto.bytecode.control.TryCatch; import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; @@ -32,6 +33,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.StandardTypes; @@ -41,6 +43,7 @@ import com.facebook.presto.sql.gen.CallSiteBinder; import com.facebook.presto.sql.gen.SqlTypeBytecodeExpression; import com.facebook.presto.sql.gen.lambda.BinaryFunctionInterface; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Primitives; @@ -60,13 +63,15 @@ import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantString; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.equal; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.getStatic; +import static com.facebook.presto.bytecode.expression.BytecodeExpressions.invokeStatic; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.lessThan; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.newInstance; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.subtract; import static com.facebook.presto.bytecode.instruction.VariableInstruction.incrementVariable; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -85,7 +90,7 @@ public final class MapTransformValueFunction private MapTransformValueFunction() { super(new Signature( - "transform_values", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "transform_values"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("K"), typeVariable("V1"), typeVariable("V2")), ImmutableList.of(), @@ -113,7 +118,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type keyType = boundVariables.getTypeVariable("K"); Type valueType = boundVariables.getTypeVariable("V1"); @@ -121,7 +126,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in Type resultMapType = typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of( TypeSignatureParameter.of(keyType.getTypeSignature()), TypeSignatureParameter.of(transformedValueType.getTypeSignature()))); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), @@ -228,6 +233,7 @@ private static MethodHandle generateTransform(Type keyType, Type valueType, Type writeTransformedValueElement = new BytecodeBlock().append(blockBuilder.invoke("appendNull", BlockBuilder.class).pop()); } + Variable transformationException = scope.declareVariable(Throwable.class, "transformationException"); body.append(new ForLoop() .initialize(position.set(constantInt(0))) .condition(lessThan(position, positionCount)) @@ -235,7 +241,19 @@ private static MethodHandle generateTransform(Type keyType, Type valueType, Type .body(new BytecodeBlock() .append(loadKeyElement) .append(loadValueElement) - .append(transformedValueElement.set(function.invoke("apply", Object.class, keyElement.cast(Object.class), valueElement.cast(Object.class)).cast(transformedValueJavaType))) + .append( + new TryCatch( + "Close builder before throwing to avoid subsequent calls finding it in an inconsistent state if we are in in a TRY() call.", + transformedValueElement.set(function.invoke("apply", Object.class, keyElement.cast(Object.class), valueElement.cast(Object.class)) + .cast(transformedValueJavaType)), + new BytecodeBlock() + .append(mapBlockBuilder.invoke("closeEntry", BlockBuilder.class).pop()) + .append(pageBuilder.invoke("declarePosition", void.class)) + .putVariable(transformationException) + .append(invokeStatic(Throwables.class, "throwIfUnchecked", void.class, transformationException)) + .append(newInstance(RuntimeException.class, transformationException)) + .throwObject(), + type(Throwable.class))) .append(keySqlType.invoke("appendTo", void.class, block, position, blockBuilder)) .append(writeTransformedValueElement))); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapZipWithFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapZipWithFunction.java index 6dc41ac93e697..73299327da607 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapZipWithFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapZipWithFunction.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.SingleMapBlock; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.StandardTypes; @@ -33,9 +34,10 @@ import java.lang.invoke.MethodHandle; import java.util.Optional; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.spi.type.TypeUtils.readNativeValue; @@ -54,7 +56,7 @@ public final class MapZipWithFunction private MapZipWithFunction() { super(new Signature( - "map_zip_with", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "map_zip_with"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("K"), typeVariable("V1"), typeVariable("V2"), typeVariable("V3")), ImmutableList.of(), @@ -82,7 +84,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type keyType = boundVariables.getTypeVariable("K"); Type inputValueType1 = boundVariables.getTypeVariable("V1"); @@ -93,7 +95,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in ImmutableList.of( TypeSignatureParameter.of(keyType.getTypeSignature()), TypeSignatureParameter.of(outputValueType.getTypeSignature()))); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ParametricScalar.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ParametricScalar.java index 986140ad31b0d..107d5739b1d71 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ParametricScalar.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ParametricScalar.java @@ -80,19 +80,19 @@ public ParametricImplementationsGroup getImpleme } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Signature boundSignature = applyBoundVariables(getSignature(), boundVariables, arity); if (implementations.getExactImplementations().containsKey(boundSignature)) { ParametricScalarImplementation implementation = implementations.getExactImplementations().get(boundSignature); - Optional scalarFunctionImplementation = implementation.specialize(boundSignature, boundVariables, typeManager, functionManager); - checkCondition(scalarFunctionImplementation.isPresent(), FUNCTION_IMPLEMENTATION_ERROR, String.format("Exact implementation of %s do not match expected java types.", boundSignature.getName())); + Optional scalarFunctionImplementation = implementation.specialize(boundSignature, boundVariables, typeManager, functionManager); + checkCondition(scalarFunctionImplementation.isPresent(), FUNCTION_IMPLEMENTATION_ERROR, String.format("Exact implementation of %s do not match expected java types.", boundSignature.getNameSuffix())); return scalarFunctionImplementation.get(); } - ScalarFunctionImplementation selectedImplementation = null; + BuiltInScalarFunctionImplementation selectedImplementation = null; for (ParametricScalarImplementation implementation : implementations.getSpecializedImplementations()) { - Optional scalarFunctionImplementation = implementation.specialize(boundSignature, boundVariables, typeManager, functionManager); + Optional scalarFunctionImplementation = implementation.specialize(boundSignature, boundVariables, typeManager, functionManager); if (scalarFunctionImplementation.isPresent()) { checkCondition(selectedImplementation == null, AMBIGUOUS_FUNCTION_IMPLEMENTATION, "Ambiguous implementation for %s with bindings %s", getSignature(), boundVariables.getTypeVariables()); selectedImplementation = scalarFunctionImplementation.get(); @@ -102,7 +102,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in return selectedImplementation; } for (ParametricScalarImplementation implementation : implementations.getGenericImplementations()) { - Optional scalarFunctionImplementation = implementation.specialize(boundSignature, boundVariables, typeManager, functionManager); + Optional scalarFunctionImplementation = implementation.specialize(boundSignature, boundVariables, typeManager, functionManager); if (scalarFunctionImplementation.isPresent()) { checkCondition(selectedImplementation == null, AMBIGUOUS_FUNCTION_IMPLEMENTATION, "Ambiguous implementation for %s with bindings %s", getSignature(), boundVariables.getTypeVariables()); selectedImplementation = scalarFunctionImplementation.get(); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/QuantileDigestFunctions.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/QuantileDigestFunctions.java index a5e0d1c2a2619..bf0c3cbd7a694 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/QuantileDigestFunctions.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/QuantileDigestFunctions.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.scalar; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.Description; @@ -20,7 +21,6 @@ import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; import io.airlift.slice.Slice; -import io.airlift.stats.QuantileDigest; import static com.facebook.presto.operator.aggregation.FloatingPointBitsConverterUtil.sortableIntToFloat; import static com.facebook.presto.operator.aggregation.FloatingPointBitsConverterUtil.sortableLongToDouble; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/Re2JCastToRegexpFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/Re2JCastToRegexpFunction.java index 3aa79ae513a50..165ed3671952a 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/Re2JCastToRegexpFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/Re2JCastToRegexpFunction.java @@ -26,8 +26,8 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.function.OperatorType.CAST; import static com.facebook.presto.spi.type.Chars.padSpaces; @@ -64,9 +64,9 @@ private Re2JCastToRegexpFunction(String sourceType, int dfaStatesLimit, int dfaR } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), insertArguments(METHOD_HANDLE, 0, dfaStatesLimit, dfaRetries, padSpaces, boundVariables.getLongVariable("x"))); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowComparisonOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowComparisonOperator.java index 2c73c36729cd2..4d3925e2ea094 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowComparisonOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowComparisonOperator.java @@ -50,7 +50,7 @@ protected List getMethodHandles(RowType type, FunctionManager func ImmutableList.Builder argumentMethods = ImmutableList.builder(); for (Type parameterType : type.getTypeParameters()) { FunctionHandle operatorHandle = functionManager.resolveOperator(operatorType, fromTypes(parameterType, parameterType)); - argumentMethods.add(functionManager.getScalarFunctionImplementation(operatorHandle).getMethodHandle()); + argumentMethods.add(functionManager.getBuiltInScalarFunctionImplementation(operatorHandle).getMethodHandle()); } return argumentMethods.build(); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowDistinctFromOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowDistinctFromOperator.java index 98553ab3da70c..8660b80a890c6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowDistinctFromOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowDistinctFromOperator.java @@ -17,8 +17,8 @@ import com.facebook.presto.metadata.FunctionInvoker; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlOperator; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ReturnPlaceConvention; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ScalarImplementationChoice; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ReturnPlaceConvention; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ScalarImplementationChoice; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.InvocationConvention; @@ -31,9 +31,9 @@ import java.util.List; import java.util.Optional; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; import static com.facebook.presto.spi.function.InvocationConvention.InvocationArgumentConvention.NULL_FLAG; import static com.facebook.presto.spi.function.OperatorType.IS_DISTINCT_FROM; import static com.facebook.presto.spi.function.Signature.comparableWithVariadicBound; @@ -61,7 +61,7 @@ private RowDistinctFromOperator() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { ImmutableList.Builder argumentMethods = ImmutableList.builder(); Type type = boundVariables.getTypeVariable("T"); @@ -75,7 +75,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in false))); argumentMethods.add(functionInvoker.methodHandle()); } - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( ImmutableList.of( new ScalarImplementationChoice( false, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowEqualOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowEqualOperator.java index ba4dbb2519fdf..f831cd441a0fa 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowEqualOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowEqualOperator.java @@ -27,8 +27,8 @@ import java.lang.invoke.MethodHandle; import java.util.List; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.EQUAL; import static com.facebook.presto.spi.function.Signature.comparableWithVariadicBound; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -54,10 +54,10 @@ private RowEqualOperator() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { RowType type = (RowType) boundVariables.getTypeVariable("T"); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( true, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), @@ -77,7 +77,7 @@ public static List resolveFieldEqualOperators(RowType rowType, Fun private static MethodHandle resolveEqualOperator(Type type, FunctionManager functionManager) { FunctionHandle operator = functionManager.resolveOperator(EQUAL, fromTypes(type, type)); - ScalarFunctionImplementation implementation = functionManager.getScalarFunctionImplementation(operator); + BuiltInScalarFunctionImplementation implementation = functionManager.getBuiltInScalarFunctionImplementation(operator); return implementation.getMethodHandle(); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowGreaterThanOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowGreaterThanOperator.java index 822a4a10fad6f..eb7ed136c3d5e 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowGreaterThanOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowGreaterThanOperator.java @@ -24,8 +24,8 @@ import java.lang.invoke.MethodHandle; import java.util.List; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; import static com.facebook.presto.util.Reflection.methodHandle; @@ -41,10 +41,10 @@ private RowGreaterThanOperator() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type type = boundVariables.getTypeVariable("T"); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowGreaterThanOrEqualOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowGreaterThanOrEqualOperator.java index 20617fc92a96f..98abee1cd6843 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowGreaterThanOrEqualOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowGreaterThanOrEqualOperator.java @@ -24,8 +24,8 @@ import java.lang.invoke.MethodHandle; import java.util.List; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN_OR_EQUAL; import static com.facebook.presto.util.Reflection.methodHandle; @@ -42,10 +42,10 @@ private RowGreaterThanOrEqualOperator() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type type = boundVariables.getTypeVariable("T"); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowHashCodeOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowHashCodeOperator.java index 9b7b5f773973f..d0912af6612a6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowHashCodeOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowHashCodeOperator.java @@ -26,8 +26,8 @@ import java.lang.invoke.MethodHandle; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.HASH_CODE; import static com.facebook.presto.spi.function.Signature.comparableWithVariadicBound; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -49,10 +49,10 @@ private RowHashCodeOperator() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type type = boundVariables.getTypeVariable("T"); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), METHOD_HANDLE.bindTo(type)); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowIndeterminateOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowIndeterminateOperator.java index 9eb560cb2c600..89f7e2f4484b6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowIndeterminateOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowIndeterminateOperator.java @@ -43,8 +43,8 @@ import static com.facebook.presto.bytecode.ParameterizedType.type; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantFalse; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantInt; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; import static com.facebook.presto.spi.function.OperatorType.INDETERMINATE; import static com.facebook.presto.spi.function.Signature.withVariadicBound; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -69,13 +69,13 @@ private RowIndeterminateOperator() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { checkArgument(arity == 1, "Expected arity to be 1"); Type type = boundVariables.getTypeVariable("T"); Class indeterminateOperatorClass = generateIndeterminate(type, functionManager); MethodHandle indeterminateMethod = methodHandle(indeterminateOperatorClass, "indeterminate", type.getJavaType(), boolean.class); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of(valueTypeArgumentProperty(USE_NULL_FLAG)), indeterminateMethod); @@ -133,11 +133,11 @@ private static Class generateIndeterminate(Type type, FunctionManager functio FunctionHandle functionHandle = functionManager.resolveOperator(INDETERMINATE, fromTypes(fieldTypes.get(i))); - ScalarFunctionImplementation function = functionManager.getScalarFunctionImplementation(functionHandle); + BuiltInScalarFunctionImplementation function = functionManager.getBuiltInScalarFunctionImplementation(functionHandle); BytecodeExpression element = constantType(binder, fieldTypes.get(i)).getValue(value, constantInt(i)); ifNullField.ifFalse(new IfStatement("if the field is not null but indeterminate...") - .condition(invokeFunction(scope, cachedInstanceBinder, functionManager.getFunctionMetadata(functionHandle).getName(), function, element)) + .condition(invokeFunction(scope, cachedInstanceBinder, functionManager.getFunctionMetadata(functionHandle).getName().getFunctionName(), function, element)) .ifTrue(new BytecodeBlock() .push(true) .gotoLabel(end))); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowLessThanOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowLessThanOperator.java index 106c60d871bf5..532c75e7e1b8f 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowLessThanOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowLessThanOperator.java @@ -24,8 +24,8 @@ import java.lang.invoke.MethodHandle; import java.util.List; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.LESS_THAN; import static com.facebook.presto.util.Reflection.methodHandle; @@ -41,10 +41,10 @@ private RowLessThanOperator() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type type = boundVariables.getTypeVariable("T"); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowLessThanOrEqualOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowLessThanOrEqualOperator.java index e24f72af4d31e..2162d4d59958e 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowLessThanOrEqualOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowLessThanOrEqualOperator.java @@ -24,8 +24,8 @@ import java.lang.invoke.MethodHandle; import java.util.List; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.LESS_THAN; import static com.facebook.presto.spi.function.OperatorType.LESS_THAN_OR_EQUAL; import static com.facebook.presto.util.Reflection.methodHandle; @@ -42,10 +42,10 @@ private RowLessThanOrEqualOperator() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type type = boundVariables.getTypeVariable("T"); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowNotEqualOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowNotEqualOperator.java index 61d20ebf2a067..06528f94745b3 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowNotEqualOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowNotEqualOperator.java @@ -26,8 +26,8 @@ import java.lang.invoke.MethodHandle; import java.util.List; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.NOT_EQUAL; import static com.facebook.presto.spi.function.Signature.comparableWithVariadicBound; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -49,10 +49,10 @@ private RowNotEqualOperator() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { RowType type = (RowType) boundVariables.getTypeVariable("T"); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( true, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowToJsonCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowToJsonCast.java index cd49a29cf95ec..c606b94a1f1c1 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowToJsonCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowToJsonCast.java @@ -35,9 +35,9 @@ import java.util.ArrayList; import java.util.List; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.operator.scalar.JsonOperators.JSON_FACTORY; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; import static com.facebook.presto.spi.function.Signature.withVariadicBound; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -65,7 +65,7 @@ private RowToJsonCast() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { checkArgument(arity == 1, "Expected arity to be 1"); Type type = boundVariables.getTypeVariable("T"); @@ -78,7 +78,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in } MethodHandle methodHandle = METHOD_HANDLE.bindTo(fieldWriters); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowToRowCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowToRowCast.java index d16912acc0b9d..87f36a804f24c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowToRowCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/RowToRowCast.java @@ -53,8 +53,8 @@ import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantBoolean; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantInt; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantNull; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.CAST; import static com.facebook.presto.spi.function.Signature.withVariadicBound; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -77,7 +77,7 @@ private RowToRowCast() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { checkArgument(arity == 1, "Expected arity to be 1"); Type fromType = boundVariables.getTypeVariable("F"); @@ -87,7 +87,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in } Class castOperatorClass = generateRowCast(fromType, toType, functionManager); MethodHandle methodHandle = methodHandle(castOperatorClass, "castRow", ConnectorSession.class, Block.class); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); @@ -142,7 +142,7 @@ private static Class generateRowCast(Type fromType, Type toType, FunctionMana // loop through to append member blocks for (int i = 0; i < toTypes.size(); i++) { FunctionHandle functionHandle = functionManager.lookupCast(CastType.CAST, fromTypes.get(i).getTypeSignature(), toTypes.get(i).getTypeSignature()); - ScalarFunctionImplementation function = functionManager.getScalarFunctionImplementation(functionHandle); + BuiltInScalarFunctionImplementation function = functionManager.getBuiltInScalarFunctionImplementation(functionHandle); Type currentFromType = fromTypes.get(i); if (currentFromType.equals(UNKNOWN)) { body.append(singleRowBlockWriter.invoke("appendNull", BlockBuilder.class).pop()); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/SessionFunctions.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/SessionFunctions.java index 0e4151577901b..560f994f33597 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/SessionFunctions.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/SessionFunctions.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.operator.scalar; -import com.facebook.presto.FullConnectorSession; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.function.Description; import com.facebook.presto.spi.function.ScalarFunction; @@ -34,13 +33,4 @@ public static Slice currentUser(ConnectorSession session) { return utf8Slice(session.getUser()); } - - @ScalarFunction(value = "$current_path", hidden = true) - @Description("retrieve current path") - @SqlType(StandardTypes.VARCHAR) - public static Slice currentPath(ConnectorSession session) - { - // this function is a language construct and has special access to internals - return utf8Slice(((FullConnectorSession) session).getSession().getPath().toString()); - } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/SliceToBooleanFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/SliceToBooleanFunction.java new file mode 100644 index 0000000000000..2d752004ee8b4 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/SliceToBooleanFunction.java @@ -0,0 +1,24 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.sql.gen.lambda.LambdaFunctionInterface; +import io.airlift.slice.Slice; + +@FunctionalInterface +public interface SliceToBooleanFunction + extends LambdaFunctionInterface +{ + Boolean apply(Slice value); +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryCastFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryCastFunction.java index 65e3a1b83e4a3..81464d91481ee 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryCastFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryCastFunction.java @@ -16,9 +16,10 @@ import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -28,6 +29,7 @@ import java.lang.invoke.MethodHandle; import java.util.List; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.metadata.CastType.CAST; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -45,7 +47,7 @@ public class TryCastFunction public TryCastFunction() { super(new Signature( - TRY_CAST_NAME, + QualifiedFunctionName.of(DEFAULT_NAMESPACE, TRY_CAST_NAME), FunctionKind.SCALAR, ImmutableList.of(typeVariable("F"), typeVariable("T")), ImmutableList.of(), @@ -73,7 +75,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type fromType = boundVariables.getTypeVariable("F"); Type toType = boundVariables.getTypeVariable("T"); @@ -84,7 +86,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in // the resulting method needs to return a boxed type FunctionHandle functionHandle = functionManager.lookupCast(CAST, fromType.getTypeSignature(), toType.getTypeSignature()); - ScalarFunctionImplementation implementation = functionManager.getScalarFunctionImplementation(functionHandle); + BuiltInScalarFunctionImplementation implementation = functionManager.getBuiltInScalarFunctionImplementation(functionHandle); argumentProperties = ImmutableList.of(implementation.getArgumentProperty(0)); MethodHandle coercion = implementation.getMethodHandle(); coercion = coercion.asType(methodType(returnType, coercion.type())); @@ -92,6 +94,6 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in MethodHandle exceptionHandler = dropArguments(constant(returnType, null), 0, RuntimeException.class); tryCastHandle = catchException(coercion, RuntimeException.class, exceptionHandler); - return new ScalarFunctionImplementation(true, argumentProperties, tryCastHandle); + return new BuiltInScalarFunctionImplementation(true, argumentProperties, tryCastHandle); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipFunction.java index 0e7890f24c86e..8621ce30e9e9c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipFunction.java @@ -17,10 +17,11 @@ import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.Type; @@ -32,8 +33,9 @@ import java.util.List; import java.util.stream.IntStream; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -66,7 +68,8 @@ private ZipFunction(int arity) private ZipFunction(List typeParameters) { - super(new Signature("zip", + super(new Signature( + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "zip"), FunctionKind.SCALAR, typeParameters.stream().map(Signature::typeVariable).collect(toImmutableList()), ImmutableList.of(), @@ -95,13 +98,13 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { List types = this.typeParameters.stream().map(boundVariables::getTypeVariable).collect(toImmutableList()); List argumentProperties = nCopies(types.size(), valueTypeArgumentProperty(RETURN_NULL_ON_NULL)); List> javaArgumentTypes = nCopies(types.size(), Block.class); MethodHandle methodHandle = METHOD_HANDLE.bindTo(types).asVarargsCollector(Block[].class).asType(methodType(Block.class, javaArgumentTypes)); - return new ScalarFunctionImplementation(false, argumentProperties, methodHandle); + return new BuiltInScalarFunctionImplementation(false, argumentProperties, methodHandle); } @UsedByGeneratedCode diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipWithFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipWithFunction.java index 613090012922c..55ae92ed6b42a 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipWithFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipWithFunction.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; @@ -30,9 +31,10 @@ import java.lang.invoke.MethodHandle; import java.util.Optional; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.spi.type.TypeUtils.readNativeValue; @@ -52,7 +54,7 @@ public final class ZipWithFunction private ZipWithFunction() { super(new Signature( - "zip_with", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "zip_with"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T"), typeVariable("U"), typeVariable("R")), ImmutableList.of(), @@ -80,13 +82,13 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type leftElementType = boundVariables.getTypeVariable("T"); Type rightElementType = boundVariables.getTypeVariable("U"); Type outputElementType = boundVariables.getTypeVariable("R"); ArrayType outputArrayType = new ArrayType(outputElementType); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ParametricScalarImplementation.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ParametricScalarImplementation.java index ae89bb723464f..3b5a4e06a7e08 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ParametricScalarImplementation.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ParametricScalarImplementation.java @@ -18,11 +18,11 @@ import com.facebook.presto.operator.ParametricImplementation; import com.facebook.presto.operator.annotations.FunctionsParserHelper; import com.facebook.presto.operator.annotations.ImplementationDependency; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ReturnPlaceConvention; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ScalarImplementationChoice; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ReturnPlaceConvention; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ScalarImplementationChoice; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.function.BlockIndex; @@ -70,12 +70,12 @@ import static com.facebook.presto.operator.annotations.ImplementationDependency.checkTypeParameters; import static com.facebook.presto.operator.annotations.ImplementationDependency.getImplementationDependencyAnnotation; import static com.facebook.presto.operator.annotations.ImplementationDependency.validateImplementationDependencyAnnotation; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentType.VALUE_TYPE; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentType.VALUE_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -118,7 +118,7 @@ private ParametricScalarImplementation( } } - public Optional specialize(Signature boundSignature, BoundVariables boundVariables, TypeManager typeManager, FunctionManager functionManager) + public Optional specialize(Signature boundSignature, BoundVariables boundVariables, TypeManager typeManager, FunctionManager functionManager) { List implementationChoices = new ArrayList<>(); for (Map.Entry> entry : specializedTypeParameters.entrySet()) { @@ -169,7 +169,7 @@ public Optional specialize(Signature boundSignatur boundMethodHandle.asType(javaMethodType(choice, boundSignature, typeManager)), boundConstructor)); } - return Optional.of(new ScalarFunctionImplementation(implementationChoices)); + return Optional.of(new BuiltInScalarFunctionImplementation(implementationChoices)); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarFromAnnotationsParser.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarFromAnnotationsParser.java index c2a2f2f474240..16801114a4cfb 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarFromAnnotationsParser.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarFromAnnotationsParser.java @@ -99,7 +99,6 @@ private static List findScalarsInFunctionSetClass(Class< private static SqlScalarFunction parseParametricScalar(ScalarHeaderAndMethods scalar, Optional> constructor) { ScalarImplementationHeader header = scalar.getHeader(); - checkArgument(!header.getName().isEmpty()); Map signatures = new HashMap<>(); for (Method method : scalar.getMethods()) { diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java index 52c95d38a07a6..dc7a3da110d5a 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java @@ -15,6 +15,7 @@ import com.facebook.presto.operator.scalar.ScalarHeader; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.ScalarOperator; import com.google.common.collect.ImmutableList; @@ -24,7 +25,7 @@ import java.util.List; import java.util.Optional; -import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.parseDescription; import static com.google.common.base.CaseFormat.LOWER_CAMEL; import static com.google.common.base.CaseFormat.LOWER_UNDERSCORE; @@ -33,20 +34,20 @@ public class ScalarImplementationHeader { - private final String name; + private final QualifiedFunctionName name; private final Optional operatorType; private final ScalarHeader header; private ScalarImplementationHeader(String name, ScalarHeader header) { - this.name = requireNonNull(name); + this.name = QualifiedFunctionName.of(DEFAULT_NAMESPACE, requireNonNull(name)); this.operatorType = Optional.empty(); this.header = requireNonNull(header); } private ScalarImplementationHeader(OperatorType operatorType, ScalarHeader header) { - this.name = mangleOperatorName(operatorType); + this.name = operatorType.getFunctionName(); this.operatorType = Optional.of(operatorType); this.header = requireNonNull(header); } @@ -94,7 +95,7 @@ public static List fromAnnotatedElement(AnnotatedEle return result; } - public String getName() + public QualifiedFunctionName getName() { return name; } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/unnest/ArrayOfRowsUnnester.java b/presto-main/src/main/java/com/facebook/presto/operator/unnest/ArrayOfRowsUnnester.java new file mode 100644 index 0000000000000..f5de3f39941bd --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/unnest/ArrayOfRowsUnnester.java @@ -0,0 +1,120 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.ColumnarArray; +import com.facebook.presto.spi.block.ColumnarRow; +import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.Iterables; + +import static com.facebook.presto.spi.block.ColumnarArray.toColumnarArray; +import static com.facebook.presto.spi.block.ColumnarRow.toColumnarRow; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +/** + * Unnester for a nested column with array type, only when array elements are of {@code RowType} type. + * It maintains {@link ColumnarArray} and {@link ColumnarRow} objects to get underlying elements. The two + * different columnar structures are required because there are two layers of translation involved. One + * from {@code ArrayBlock} to {@code RowBlock}, and then from {@code RowBlock} to individual element blocks. + * + * All protected methods implemented here assume that they are invoked when {@code columnarArray} and + * {@code columnarRow} are non-null. + */ +class ArrayOfRowsUnnester + extends Unnester +{ + private ColumnarArray columnarArray; + private ColumnarRow columnarRow; + private final int fieldCount; + + // Keeping track of null row element count is required. This count needs to be deducted + // when translating row block indexes to element block indexes. + private int nullRowsEncountered; + + public ArrayOfRowsUnnester(RowType elementType) + { + super(Iterables.toArray(requireNonNull(elementType, "elementType is null").getTypeParameters(), Type.class)); + this.fieldCount = elementType.getTypeParameters().size(); + this.nullRowsEncountered = 0; + } + + @Override + public int getChannelCount() + { + return fieldCount; + } + + @Override + int getInputEntryCount() + { + if (columnarArray == null) { + return 0; + } + return columnarArray.getPositionCount(); + } + + @Override + protected void resetColumnarStructure(Block block) + { + columnarArray = toColumnarArray(block); + columnarRow = toColumnarRow(columnarArray.getElementsBlock()); + nullRowsEncountered = 0; + } + + @Override + public void processCurrentPosition(int requireCount) + { + // Translate to row block index + int rowBlockIndex = columnarArray.getOffset(getCurrentPosition()); + + // Unnest current entry + for (int i = 0; i < getCurrentUnnestedLength(); i++) { + if (columnarRow.isNull(rowBlockIndex + i)) { + // Nulls have to be appended when Row element itself is null + for (int field = 0; field < fieldCount; field++) { + getBlockBuilder(field).appendNull(); + } + nullRowsEncountered++; + } + else { + for (int field = 0; field < fieldCount; field++) { + getBlockBuilder(field).appendElement(rowBlockIndex + i - nullRowsEncountered); + } + } + } + + // Append nulls if more output entries are needed + for (int i = 0; i < requireCount - getCurrentUnnestedLength(); i++) { + for (int field = 0; field < fieldCount; field++) { + getBlockBuilder(field).appendNull(); + } + } + } + + @Override + protected Block getElementsBlock(int channel) + { + checkState(channel >= 0 && channel < fieldCount, "Invalid channel number"); + return columnarRow.getField(channel); + } + + @Override + protected int getElementsLength(int index) + { + return columnarArray.getLength(index); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/unnest/ArrayUnnester.java b/presto-main/src/main/java/com/facebook/presto/operator/unnest/ArrayUnnester.java new file mode 100644 index 0000000000000..dc7dc6f923866 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/unnest/ArrayUnnester.java @@ -0,0 +1,86 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.ColumnarArray; +import com.facebook.presto.spi.type.Type; + +import static com.facebook.presto.spi.block.ColumnarArray.toColumnarArray; +import static com.google.common.base.Preconditions.checkState; + +/** + * Unnester for a nested column with array type, only when array elements are NOT of type {@code RowType}. + * Maintains a {@link ColumnarArray} object to get underlying elements block from the array block. + * + * All protected methods implemented here assume that they are being invoked when {@code columnarArray} is non-null. + */ +class ArrayUnnester + extends Unnester +{ + private ColumnarArray columnarArray; + + public ArrayUnnester(Type elementType) + { + super(elementType); + } + + @Override + public int getChannelCount() + { + return 1; + } + + @Override + protected int getInputEntryCount() + { + if (columnarArray == null) { + return 0; + } + return columnarArray.getPositionCount(); + } + + @Override + protected void resetColumnarStructure(Block block) + { + this.columnarArray = toColumnarArray(block); + } + + @Override + protected Block getElementsBlock(int channel) + { + checkState(channel == 0, "index is not 0"); + return columnarArray.getElementsBlock(); + } + + @Override + protected void processCurrentPosition(int requiredOutputCount) + { + // Translate indices + int startElementIndex = columnarArray.getOffset(getCurrentPosition()); + int length = columnarArray.getLength(getCurrentPosition()); + + // Append elements and nulls + getBlockBuilder(0).appendRange(startElementIndex, length); + for (int i = 0; i < requiredOutputCount - length; i++) { + getBlockBuilder(0).appendNull(); + } + } + + @Override + protected int getElementsLength(int index) + { + return columnarArray.getLength(index); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/unnest/MapUnnester.java b/presto-main/src/main/java/com/facebook/presto/operator/unnest/MapUnnester.java new file mode 100644 index 0000000000000..e133541a6f259 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/unnest/MapUnnester.java @@ -0,0 +1,95 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.ColumnarMap; +import com.facebook.presto.spi.type.Type; + +import static com.facebook.presto.spi.block.ColumnarMap.toColumnarMap; +import static com.google.common.base.Preconditions.checkState; + +/** + * Unnester for a nested column with map type. + * Maintains a {@link ColumnarMap} object to get underlying keys and values block from the map block. + * + * All protected methods implemented here assume that they are being invoked when {@code columnarMap} is non-null. + */ +class MapUnnester + extends Unnester +{ + private ColumnarMap columnarMap; + + public MapUnnester(Type keyType, Type valueType) + { + super(keyType, valueType); + } + + @Override + protected void processCurrentPosition(int requiredOutputCount) + { + // Translate indices + int mapLength = columnarMap.getEntryCount(getCurrentPosition()); + int startingOffset = columnarMap.getOffset(getCurrentPosition()); + + // Append elements and nulls for keys Block + getBlockBuilder(0).appendRange(startingOffset, mapLength); + for (int i = 0; i < requiredOutputCount - mapLength; i++) { + getBlockBuilder(0).appendNull(); + } + + // Append elements and nulls for values Block + getBlockBuilder(1).appendRange(startingOffset, mapLength); + for (int i = 0; i < requiredOutputCount - mapLength; i++) { + getBlockBuilder(1).appendNull(); + } + } + + @Override + public int getChannelCount() + { + return 2; + } + + @Override + public int getInputEntryCount() + { + if (columnarMap == null) { + return 0; + } + return columnarMap.getPositionCount(); + } + + @Override + protected void resetColumnarStructure(Block block) + { + this.columnarMap = toColumnarMap(block); + } + + @Override + protected Block getElementsBlock(int channel) + { + checkState(channel == 0 || channel == 1, "index is not 0 or 1"); + if (channel == 0) { + return columnarMap.getKeysBlock(); + } + return columnarMap.getValuesBlock(); + } + + @Override + protected int getElementsLength(int index) + { + return columnarMap.getEntryCount(index); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/unnest/ReplicatedBlockBuilder.java b/presto-main/src/main/java/com/facebook/presto/operator/unnest/ReplicatedBlockBuilder.java new file mode 100644 index 0000000000000..cd8e19d707d39 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/unnest/ReplicatedBlockBuilder.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.DictionaryBlock; + +import java.util.Arrays; + +import static com.facebook.presto.operator.unnest.UnnestOperatorBlockUtil.calculateNewArraySize; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkElementIndex; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +/** + * This class manages the details for building replicate channel blocks without copying input data + */ +class ReplicatedBlockBuilder +{ + private Block source; + private int[] ids; + private int positionCount; + + public void resetInputBlock(Block block) + { + this.source = requireNonNull(block, "block is null"); + } + + public void startNewOutput(int expectedEntries) + { + checkState(source != null, "source is null"); + this.ids = new int[expectedEntries]; + this.positionCount = 0; + } + + /** + * Repeat the source element at position {@code index} for {@code count} times in the output + */ + public void appendRepeated(int index, int count) + { + checkState(source != null, "source is null"); + checkElementIndex(index, source.getPositionCount()); + checkArgument(count >= 0, "count should be >= 0"); + + if (positionCount + count > ids.length) { + // Grow capacity + int newSize = Math.max(calculateNewArraySize(ids.length), positionCount + count); + ids = Arrays.copyOf(ids, newSize); + } + + Arrays.fill(ids, positionCount, positionCount + count, index); + positionCount += count; + } + + public Block buildOutputAndFlush() + { + Block outputBlock = new DictionaryBlock(positionCount, source, ids); + + // Flush stored state, so that ids can not be modified after the dictionary has been constructed + ids = new int[0]; + positionCount = 0; + + return outputBlock; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/unnest/UnnestBlockBuilder.java b/presto-main/src/main/java/com/facebook/presto/operator/unnest/UnnestBlockBuilder.java new file mode 100644 index 0000000000000..4f405018398e8 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/unnest/UnnestBlockBuilder.java @@ -0,0 +1,265 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.block.PageBuilderStatus; +import com.facebook.presto.spi.type.Type; +import com.google.common.annotations.VisibleForTesting; + +import java.util.Arrays; + +import static com.facebook.presto.operator.unnest.UnnestOperatorBlockUtil.calculateNewArraySize; +import static com.google.common.base.Preconditions.checkElementIndex; +import static com.google.common.base.Preconditions.checkPositionIndexes; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +/** + * This class manages the low level details of building unnested blocks with a goal of minimizing data copying + */ +class UnnestBlockBuilder +{ + private final Type type; + private Block source; + + // checks for existence of null element in the source when required + private final NullElementFinder nullFinder = new NullElementFinder(); + + // flag indicating whether we are using copied block or a dictionary block + private boolean usingCopiedBlock; + + // State for output dictionary block + private int[] ids; + private int positionCount; + + // State for output copied block + private BlockBuilder outputBlockBuilder; + private PageBuilderStatus pageBuilderStatus; + + public UnnestBlockBuilder(Type type) + { + this.type = requireNonNull(type, "type is null"); + } + + /** + * Replaces input source block with {@code block}. The old data structures for output have to be + * reset as well, because they are based on the source. + */ + public void resetInputBlock(Block block) + { + this.source = requireNonNull(block, "block is null"); + // output and null-check have to be cleared because the source has changed + clearCurrentOutput(); + nullFinder.resetCheck(); + } + + /** + * Resets all data structures used for producing current output block, except for the source block. + */ + @VisibleForTesting + void clearCurrentOutput() + { + // clear dictionary block state + ids = new int[0]; + positionCount = 0; + usingCopiedBlock = false; + + // clear copied block state + outputBlockBuilder = null; + pageBuilderStatus = null; + } + + /** + * Prepares for a new output block. + * The caller has to ensure that the source is not null when this method is invoked. + */ + public void startNewOutput(PageBuilderStatus pageBuilderStatus, int expectedEntries) + { + checkState(source != null, "source is null"); + this.pageBuilderStatus = pageBuilderStatus; + + // Prepare for a new dictionary block + this.ids = new int[expectedEntries]; + this.positionCount = 0; + this.usingCopiedBlock = false; + } + + public Block buildOutputAndFlush() + { + Block outputBlock; + if (usingCopiedBlock) { + outputBlock = outputBlockBuilder.build(); + } + else { + outputBlock = new DictionaryBlock(positionCount, source, ids); + } + + // Flush stored state, so that ids can not be modified after the dictionary has been constructed + clearCurrentOutput(); + + return outputBlock; + } + + /** + * Append element at position {@code index} in the source block to output block + */ + public void appendElement(int index) + { + checkState(source != null, "source is null"); + checkElementIndex(index, source.getPositionCount()); + + if (usingCopiedBlock) { + type.appendTo(source, index, outputBlockBuilder); + } + else { + appendId(index); + } + } + + /** + * Append range of elements starting at {@code startPosition} and length {@code length} + * from the source block to output block. + * + * Purpose of this method is to avoid repeated range checks for every invocation of {@link #appendElement}. + */ + public void appendRange(int startPosition, int length) + { + // check range validity + checkState(source != null, "source is null"); + checkPositionIndexes(startPosition, startPosition + length, source.getPositionCount()); + + if (usingCopiedBlock) { + for (int i = 0; i < length; i++) { + type.appendTo(source, startPosition + i, outputBlockBuilder); + } + } + else { + for (int i = 0; i < length; i++) { + appendId(startPosition + i); + } + } + } + + public void appendNull() + { + if (usingCopiedBlock) { + outputBlockBuilder.appendNull(); + } + else { + int nullIndex = nullFinder.getNullElementIndex(); + if (nullIndex == -1) { + startUsingCopiedBlock(); + appendNull(); + } + else { + appendId(nullIndex); + } + } + } + + /** + * This method is invoked when transition from dictionary block to copied block happens. + * - outputBlockBuilder is created and existing elements as indicated by ids[] are copied over. + * - ids[] and positionCount are reset, as they are not used anymore. + */ + private void startUsingCopiedBlock() + { + requireNonNull(pageBuilderStatus, "pageBuilderStatus is null"); + // initialize state for copied block + outputBlockBuilder = type.createBlockBuilder(pageBuilderStatus.createBlockBuilderStatus(), ids.length); + + // Transfer elements from ids[] to output page builder + for (int i = 0; i < positionCount; i++) { + type.appendTo(source, ids[i], outputBlockBuilder); + } + + // reset state for dictionary block + ids = new int[0]; + positionCount = 0; + + usingCopiedBlock = true; + } + + /** + * This method appends {@code sourcePosition} to ids array. + * + * The caller of this method has to ensure the following: + * - {@code sourcePosition} is a valid index in the source + * - {@code usingCopiedBlock} is false + */ + private void appendId(int sourcePosition) + { + if (positionCount == ids.length) { + // grow capacity + int newSize = calculateNewArraySize(ids.length); + ids = Arrays.copyOf(ids, newSize); + } + ids[positionCount] = sourcePosition; + positionCount++; + } + + /** + * Get number of positions in the output block being prepared + */ + public int getPositionCount() + { + if (usingCopiedBlock) { + return outputBlockBuilder.getPositionCount(); + } + return positionCount; + } + + /** + * This class checks for the presence of a non-null element in {@code source}, and stores its position. + * The result is cached with the first invocation of {@link #getNullElementIndex} after the reset. The + * cache can be invalidated by invoking {@link #resetCheck}. + */ + private class NullElementFinder + { + private boolean checkedForNull; + private int nullElementPosition = -1; + + void resetCheck() + { + this.checkedForNull = false; + this.nullElementPosition = -1; + } + + private int getNullElementIndex() + { + if (checkedForNull) { + return nullElementPosition; + } + checkForNull(); + return nullElementPosition; + } + + private void checkForNull() + { + nullElementPosition = -1; + + for (int i = 0; i < source.getPositionCount(); i++) { + if (source.isNull(i)) { + nullElementPosition = i; + break; + } + } + + checkedForNull = true; + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/unnest/UnnestOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/unnest/UnnestOperator.java new file mode 100644 index 0000000000000..a0677ec5e3458 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/unnest/UnnestOperator.java @@ -0,0 +1,306 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.operator.DriverContext; +import com.facebook.presto.operator.Operator; +import com.facebook.presto.operator.OperatorContext; +import com.facebook.presto.operator.OperatorFactory; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.block.PageBuilderStatus; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.MapType; +import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +public class UnnestOperator + implements Operator +{ + public static class UnnestOperatorFactory + implements OperatorFactory + { + private final int operatorId; + private final PlanNodeId planNodeId; + private final List replicateChannels; + private final List replicateTypes; + private final List unnestChannels; + private final List unnestTypes; + private final boolean withOrdinality; + private boolean closed; + + public UnnestOperatorFactory(int operatorId, PlanNodeId planNodeId, List replicateChannels, List replicateTypes, List unnestChannels, List unnestTypes, boolean withOrdinality) + { + this.operatorId = operatorId; + this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); + this.replicateChannels = ImmutableList.copyOf(requireNonNull(replicateChannels, "replicateChannels is null")); + this.replicateTypes = ImmutableList.copyOf(requireNonNull(replicateTypes, "replicateTypes is null")); + checkArgument(replicateChannels.size() == replicateTypes.size(), "replicateChannels and replicateTypes do not match"); + this.unnestChannels = ImmutableList.copyOf(requireNonNull(unnestChannels, "unnestChannels is null")); + this.unnestTypes = ImmutableList.copyOf(requireNonNull(unnestTypes, "unnestTypes is null")); + checkArgument(unnestChannels.size() == unnestTypes.size(), "unnestChannels and unnestTypes do not match"); + this.withOrdinality = withOrdinality; + } + + @Override + public Operator createOperator(DriverContext driverContext) + { + checkState(!closed, "Factory is already closed"); + OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, UnnestOperator.class.getSimpleName()); + return new UnnestOperator(operatorContext, replicateChannels, replicateTypes, unnestChannels, unnestTypes, withOrdinality, SystemSessionProperties.isLegacyUnnest(driverContext.getSession())); + } + + @Override + public void noMoreOperators() + { + closed = true; + } + + @Override + public OperatorFactory duplicate() + { + return new UnnestOperator.UnnestOperatorFactory(operatorId, planNodeId, replicateChannels, replicateTypes, unnestChannels, unnestTypes, withOrdinality); + } + } + + private static final int MAX_ROWS_PER_BLOCK = 1000; + private static final int MAX_BYTES_PER_PAGE = 1024 * 1024; + + // Output row count is checked *after* processing every input row. For this reason, estimated rows per + // block are always going to be slightly greater than {@code maxRowsPerBlock}. Accounting for this skew + // helps avoid array copies in blocks. + private static final double OVERFLOW_SKEW = 1.25; + private static final int estimatedMaxRowsPerBlock = (int) Math.ceil(MAX_ROWS_PER_BLOCK * OVERFLOW_SKEW); + + private final OperatorContext operatorContext; + private final List replicateChannels; + private final List replicateTypes; + private final List unnestChannels; + private final List unnestTypes; + private final boolean withOrdinality; + + private boolean finishing; + private Page currentPage; + private int currentPosition; + + private final List unnesters; + private final int unnestOutputChannelCount; + + private final List replicatedBlockBuilders; + + private BlockBuilder ordinalityBlockBuilder; + + private int outputChannelCount; + + public UnnestOperator(OperatorContext operatorContext, List replicateChannels, List replicateTypes, List unnestChannels, List unnestTypes, boolean withOrdinality, boolean isLegacyUnnest) + { + this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); + + this.replicateChannels = ImmutableList.copyOf(requireNonNull(replicateChannels, "replicateChannels is null")); + this.replicateTypes = ImmutableList.copyOf(requireNonNull(replicateTypes, "replicateTypes is null")); + checkArgument(replicateChannels.size() == replicateTypes.size(), "replicate channels or types has wrong size"); + this.replicatedBlockBuilders = replicateTypes.stream() + .map(type -> new ReplicatedBlockBuilder()) + .collect(toImmutableList()); + + this.unnestChannels = ImmutableList.copyOf(requireNonNull(unnestChannels, "unnestChannels is null")); + this.unnestTypes = ImmutableList.copyOf(requireNonNull(unnestTypes, "unnestTypes is null")); + checkArgument(unnestChannels.size() == unnestTypes.size(), "unnest channels or types has wrong size"); + this.unnesters = unnestTypes.stream() + .map((Type nestedType) -> createUnnester(nestedType, isLegacyUnnest)) + .collect(toImmutableList()); + this.unnestOutputChannelCount = unnesters.stream().mapToInt(Unnester::getChannelCount).sum(); + + this.withOrdinality = withOrdinality; + + this.outputChannelCount = unnestOutputChannelCount + replicateTypes.size() + (withOrdinality ? 1 : 0); + } + + @Override + public OperatorContext getOperatorContext() + { + return operatorContext; + } + + @Override + public void finish() + { + finishing = true; + } + + @Override + public boolean isFinished() + { + return finishing && currentPage == null; + } + + @Override + public boolean needsInput() + { + return !finishing && currentPage == null; + } + + @Override + public void addInput(Page page) + { + checkState(!finishing, "Operator is already finishing"); + requireNonNull(page, "page is null"); + checkState(currentPage == null, "currentPage is not null"); + + currentPage = page; + currentPosition = 0; + resetBlockBuilders(); + } + + private void resetBlockBuilders() + { + for (int i = 0; i < replicateTypes.size(); i++) { + Block newInputBlock = currentPage.getBlock(replicateChannels.get(i)); + replicatedBlockBuilders.get(i).resetInputBlock(newInputBlock); + } + + for (int i = 0; i < unnestTypes.size(); i++) { + int inputChannel = unnestChannels.get(i); + Block unnestChannelInputBlock = currentPage.getBlock(inputChannel); + unnesters.get(i).resetInput(unnestChannelInputBlock); + } + } + + @Override + public Page getOutput() + { + if (currentPage == null) { + return null; + } + + PageBuilderStatus pageBuilderStatus = new PageBuilderStatus(MAX_BYTES_PER_PAGE); + prepareForNewOutput(pageBuilderStatus); + + int outputRowCount = 0; + + while (currentPosition < currentPage.getPositionCount()) { + outputRowCount += processCurrentPosition(); + currentPosition++; + + if (outputRowCount >= MAX_ROWS_PER_BLOCK || pageBuilderStatus.isFull()) { + break; + } + } + + Block[] outputBlocks = buildOutputBlocks(); + + if (currentPosition == currentPage.getPositionCount()) { + currentPage = null; + currentPosition = 0; + } + + return new Page(outputBlocks); + } + + private int processCurrentPosition() + { + // Determine number of output rows for this input position + int maxEntries = getCurrentMaxEntries(); + + // Append elements repeatedly to replicate output columns + replicatedBlockBuilders.forEach(blockBuilder -> blockBuilder.appendRepeated(currentPosition, maxEntries)); + + // Process this position in unnesters + unnesters.forEach(unnester -> unnester.processCurrentAndAdvance(maxEntries)); + + if (withOrdinality) { + for (long ordinalityCount = 1; ordinalityCount <= maxEntries; ordinalityCount++) { + BIGINT.writeLong(ordinalityBlockBuilder, ordinalityCount); + } + } + + return maxEntries; + } + + private int getCurrentMaxEntries() + { + return unnesters.stream() + .mapToInt(Unnester::getCurrentUnnestedLength) + .max() + .orElse(0); + } + + private void prepareForNewOutput(PageBuilderStatus pageBuilderStatus) + { + unnesters.forEach(unnester -> unnester.startNewOutput(pageBuilderStatus, estimatedMaxRowsPerBlock)); + replicatedBlockBuilders.forEach(replicatedBlockBuilder -> replicatedBlockBuilder.startNewOutput(estimatedMaxRowsPerBlock)); + + if (withOrdinality) { + ordinalityBlockBuilder = BIGINT.createBlockBuilder(pageBuilderStatus.createBlockBuilderStatus(), estimatedMaxRowsPerBlock); + } + } + + private Block[] buildOutputBlocks() + { + Block[] outputBlocks = new Block[outputChannelCount]; + int offset = 0; + + for (int replicateIndex = 0; replicateIndex < replicateTypes.size(); replicateIndex++) { + outputBlocks[offset++] = replicatedBlockBuilders.get(replicateIndex).buildOutputAndFlush(); + } + + for (int unnestIndex = 0; unnestIndex < unnesters.size(); unnestIndex++) { + Unnester unnester = unnesters.get(unnestIndex); + Block[] block = unnester.buildOutputBlocksAndFlush(); + for (int j = 0; j < unnester.getChannelCount(); j++) { + outputBlocks[offset++] = block[j]; + } + } + + if (withOrdinality) { + outputBlocks[offset] = ordinalityBlockBuilder.build(); + } + + return outputBlocks; + } + + private static Unnester createUnnester(Type nestedType, boolean isLegacyUnnest) + { + if (nestedType instanceof ArrayType) { + Type elementType = ((ArrayType) nestedType).getElementType(); + + if (!isLegacyUnnest && elementType instanceof RowType) { + return new ArrayOfRowsUnnester(((RowType) elementType)); + } + else { + return new ArrayUnnester(elementType); + } + } + else if (nestedType instanceof MapType) { + Type keyType = ((MapType) nestedType).getKeyType(); + Type valueType = ((MapType) nestedType).getValueType(); + return new MapUnnester(keyType, valueType); + } + else { + throw new IllegalArgumentException("Cannot unnest type: " + nestedType); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/unnest/UnnestOperatorBlockUtil.java b/presto-main/src/main/java/com/facebook/presto/operator/unnest/UnnestOperatorBlockUtil.java new file mode 100644 index 0000000000000..7a666b278694e --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/unnest/UnnestOperatorBlockUtil.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import static java.lang.String.format; + +final class UnnestOperatorBlockUtil +{ + private UnnestOperatorBlockUtil() {} + + private static final int DEFAULT_CAPACITY = 64; + // See java.util.ArrayList for an explanation + static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8; + + // Copied from io.prestosql.spi.block.BlockUtil#calculateNewArraySize + static int calculateNewArraySize(int currentSize) + { + // grow array by 50% + long newSize = (long) currentSize + (currentSize >> 1); + + // verify new size is within reasonable bounds + if (newSize < DEFAULT_CAPACITY) { + newSize = DEFAULT_CAPACITY; + } + else if (newSize > MAX_ARRAY_SIZE) { + newSize = MAX_ARRAY_SIZE; + if (newSize == currentSize) { + throw new IllegalArgumentException(format("Can not grow array beyond '%s'", MAX_ARRAY_SIZE)); + } + } + return (int) newSize; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/unnest/Unnester.java b/presto-main/src/main/java/com/facebook/presto/operator/unnest/Unnester.java new file mode 100644 index 0000000000000..42ba77fe2fa43 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/unnest/Unnester.java @@ -0,0 +1,125 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.PageBuilderStatus; +import com.facebook.presto.spi.type.Type; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +/** + * Generic Unnester implementation for nested columns. + * + * This is a layer of abstraction between {@link UnnestOperator} and {@link UnnestBlockBuilder} to enable + * translation of indices from input nested blocks to underlying element blocks. + */ +abstract class Unnester +{ + private final UnnestBlockBuilder[] unnestBlockBuilders; + private int currentPosition; + + protected Unnester(Type... types) + { + requireNonNull(types, "types is null"); + this.unnestBlockBuilders = new UnnestBlockBuilder[types.length]; + this.currentPosition = 0; + for (int i = 0; i < types.length; i++) { + requireNonNull(types[i], "type is null"); + unnestBlockBuilders[i] = new UnnestBlockBuilder(types[i]); + } + } + + /** + * Update {@code unnestBlockBuilders} with a new input. + */ + public final void resetInput(Block block) + { + requireNonNull(block, "block is null"); + resetColumnarStructure(block); + for (int i = 0; i < getChannelCount(); i++) { + unnestBlockBuilders[i].resetInputBlock(getElementsBlock(i)); + } + currentPosition = 0; + } + + public final int getCurrentUnnestedLength() + { + return getElementsLength(currentPosition); + } + + /** + * Prepare for a new output page + */ + public final void startNewOutput(PageBuilderStatus status, int expectedEntries) + { + for (int i = 0; i < getChannelCount(); i++) { + unnestBlockBuilders[i].startNewOutput(status, expectedEntries); + } + } + + public final void processCurrentAndAdvance(int requiredOutputCount) + { + checkState(currentPosition >= 0 && currentPosition < getInputEntryCount(), "position out of bounds"); + processCurrentPosition(requiredOutputCount); + currentPosition++; + } + + /** + * Build output and flush output state for the @{code unnestBlockBuilders}. + */ + public final Block[] buildOutputBlocksAndFlush() + { + Block[] outputBlocks = new Block[getChannelCount()]; + for (int i = 0; i < getChannelCount(); i++) { + outputBlocks[i] = unnestBlockBuilders[i].buildOutputAndFlush(); + } + return outputBlocks; + } + + protected final int getCurrentPosition() + { + return currentPosition; + } + + protected final UnnestBlockBuilder getBlockBuilder(int index) + { + return unnestBlockBuilders[index]; + } + + public abstract int getChannelCount(); + + abstract int getInputEntryCount(); + + /** + * Process current position for all {@code unnestBlockBuilders}. This method has to + * (1) translate {@code currentPosition} to indexes for the block builders, using columnar structures. + * (2) invoke {@link UnnestBlockBuilder} methods to produce {@code requiredOutputCount} number of rows + * for every output block. + */ + protected abstract void processCurrentPosition(int requiredOutputCount); + + protected abstract void resetColumnarStructure(Block block); + + /** + * Get underlying elements block corresponding to {@code channel} + */ + protected abstract Block getElementsBlock(int channel); + + /** + * Get the length of the nested element at position {@code index} + */ + protected abstract int getElementsLength(int index); +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/AbstractWindowFunctionSupplier.java b/presto-main/src/main/java/com/facebook/presto/operator/window/AbstractWindowFunctionSupplier.java index a97f0d0759593..09734d3590233 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/AbstractWindowFunctionSupplier.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/AbstractWindowFunctionSupplier.java @@ -46,21 +46,21 @@ public final String getDescription() } @Override - public final WindowFunction createWindowFunction(List argumentChannels) + public final WindowFunction createWindowFunction(List argumentChannels, boolean ignoreNulls) { requireNonNull(argumentChannels, "inputs is null"); checkArgument(argumentChannels.size() == signature.getArgumentTypes().size(), "Expected %s arguments for function %s, but got %s", signature.getArgumentTypes().size(), - signature.getName(), + signature.getNameSuffix(), argumentChannels.size()); - return newWindowFunction(argumentChannels); + return newWindowFunction(argumentChannels, ignoreNulls); } /** * Create window function instance using the supplied arguments. The * inputs have already validated. */ - protected abstract WindowFunction newWindowFunction(List inputs); + protected abstract WindowFunction newWindowFunction(List inputs, boolean ignoreNulls); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/AggregateWindowFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/window/AggregateWindowFunction.java index 447a1ce53c15c..25ed08c4b6ac0 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/AggregateWindowFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/AggregateWindowFunction.java @@ -94,7 +94,7 @@ public static WindowFunctionSupplier supplier(Signature signature, final Interna return new AbstractWindowFunctionSupplier(signature, null) { @Override - protected WindowFunction newWindowFunction(List inputs) + protected WindowFunction newWindowFunction(List inputs, boolean ignoreNulls) { return new AggregateWindowFunction(function, inputs); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/FirstValueFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/window/FirstValueFunction.java index 16aac611bb62c..03bedcdcc301e 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/FirstValueFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/FirstValueFunction.java @@ -40,6 +40,23 @@ public void processRow(BlockBuilder output, int frameStart, int frameEnd, int cu return; } - windowIndex.appendTo(argumentChannel, frameStart, output); + int valuePosition = frameStart; + + if (ignoreNulls) { + while (valuePosition >= 0 && valuePosition <= frameEnd) { + if (!windowIndex.isNull(argumentChannel, valuePosition)) { + break; + } + + valuePosition++; + } + + if (valuePosition > frameEnd) { + output.appendNull(); + return; + } + } + + windowIndex.appendTo(argumentChannel, valuePosition, output); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/LagFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/window/LagFunction.java index 19059ede26ac9..b0d593d1a5bfa 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/LagFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/LagFunction.java @@ -50,9 +50,26 @@ public void processRow(BlockBuilder output, int frameStart, int frameEnd, int cu long offset = (offsetChannel < 0) ? 1 : windowIndex.getLong(offsetChannel, currentPosition); checkCondition(offset >= 0, INVALID_FUNCTION_ARGUMENT, "Offset must be at least 0"); - long valuePosition = currentPosition - offset; + long valuePosition; - if ((valuePosition >= 0) && (valuePosition <= currentPosition)) { + if (ignoreNulls && (offset > 0)) { + long count = 0; + valuePosition = currentPosition - 1; + while (withinPartition(valuePosition, currentPosition)) { + if (!windowIndex.isNull(valueChannel, toIntExact(valuePosition))) { + count++; + if (count == offset) { + break; + } + } + valuePosition--; + } + } + else { + valuePosition = currentPosition - offset; + } + + if (withinPartition(valuePosition, currentPosition)) { windowIndex.appendTo(valueChannel, toIntExact(valuePosition), output); } else if (defaultChannel >= 0) { @@ -63,4 +80,9 @@ else if (defaultChannel >= 0) { } } } + + private boolean withinPartition(long valuePosition, long currentPosition) + { + return valuePosition >= 0 && valuePosition <= currentPosition; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/LastValueFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/window/LastValueFunction.java index 98a3746d82df2..a3a82bab72122 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/LastValueFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/LastValueFunction.java @@ -40,6 +40,23 @@ public void processRow(BlockBuilder output, int frameStart, int frameEnd, int cu return; } - windowIndex.appendTo(argumentChannel, frameEnd, output); + int valuePosition = frameEnd; + + if (ignoreNulls) { + while (valuePosition >= frameStart) { + if (!windowIndex.isNull(argumentChannel, valuePosition)) { + break; + } + + valuePosition--; + } + + if (valuePosition < frameStart) { + output.appendNull(); + return; + } + } + + windowIndex.appendTo(argumentChannel, valuePosition, output); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/LeadFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/window/LeadFunction.java index e0d36d3868615..cc80648974032 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/LeadFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/LeadFunction.java @@ -50,9 +50,26 @@ public void processRow(BlockBuilder output, int frameStart, int frameEnd, int cu long offset = (offsetChannel < 0) ? 1 : windowIndex.getLong(offsetChannel, currentPosition); checkCondition(offset >= 0, INVALID_FUNCTION_ARGUMENT, "Offset must be at least 0"); - long valuePosition = currentPosition + offset; + long valuePosition; - if ((valuePosition >= 0) && (valuePosition < windowIndex.size())) { + if (ignoreNulls && (offset > 0)) { + long count = 0; + valuePosition = currentPosition + 1; + while (withinPartition(valuePosition)) { + if (!windowIndex.isNull(valueChannel, toIntExact(valuePosition))) { + count++; + if (count == offset) { + break; + } + } + valuePosition++; + } + } + else { + valuePosition = currentPosition + offset; + } + + if (withinPartition(valuePosition)) { windowIndex.appendTo(valueChannel, toIntExact(valuePosition), output); } else if (defaultChannel >= 0) { @@ -63,4 +80,9 @@ else if (defaultChannel >= 0) { } } } + + private boolean withinPartition(long valuePosition) + { + return (valuePosition >= 0) && (valuePosition < windowIndex.size()); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/NthValueFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/window/NthValueFunction.java index 2265b59cc08e6..5ee7bec67b6e9 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/NthValueFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/NthValueFunction.java @@ -46,8 +46,25 @@ public void processRow(BlockBuilder output, int frameStart, int frameEnd, int cu long offset = windowIndex.getLong(offsetChannel, currentPosition); checkCondition(offset >= 1, INVALID_FUNCTION_ARGUMENT, "Offset must be at least 1"); - // offset is base 1 - long valuePosition = frameStart + (offset - 1); + long valuePosition; + + if (ignoreNulls) { + long count = 0; + valuePosition = frameStart; + while (valuePosition >= 0 && valuePosition <= frameEnd) { + if (!windowIndex.isNull(valueChannel, toIntExact(valuePosition))) { + count++; + if (count == offset) { + break; + } + } + valuePosition++; + } + } + else { + // offset is base 1 + valuePosition = frameStart + (offset - 1); + } if ((valuePosition >= frameStart) && (valuePosition <= frameEnd)) { windowIndex.appendTo(valueChannel, toIntExact(valuePosition), output); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/ReflectionWindowFunctionSupplier.java b/presto-main/src/main/java/com/facebook/presto/operator/window/ReflectionWindowFunctionSupplier.java index 7f81f05da38b1..ea4728ed7a433 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/ReflectionWindowFunctionSupplier.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/ReflectionWindowFunctionSupplier.java @@ -14,7 +14,9 @@ package com.facebook.presto.operator.window; import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.ValueWindowFunction; import com.facebook.presto.spi.function.WindowFunction; import com.facebook.presto.spi.type.Type; import com.google.common.collect.Lists; @@ -23,6 +25,7 @@ import java.lang.reflect.Constructor; import java.util.List; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.spi.function.FunctionKind.WINDOW; import static java.util.Objects.requireNonNull; @@ -33,7 +36,7 @@ public class ReflectionWindowFunctionSupplier public ReflectionWindowFunctionSupplier(String name, Type returnType, List argumentTypes, Class type) { - this(new Signature(name, WINDOW, returnType.getTypeSignature(), Lists.transform(argumentTypes, Type::getTypeSignature)), type); + this(new Signature(QualifiedFunctionName.of(DEFAULT_NAMESPACE, name), WINDOW, returnType.getTypeSignature(), Lists.transform(argumentTypes, Type::getTypeSignature)), type); } public ReflectionWindowFunctionSupplier(Signature signature, Class type) @@ -53,15 +56,23 @@ public ReflectionWindowFunctionSupplier(Signature signature, Class type) } @Override - protected T newWindowFunction(List inputs) + protected T newWindowFunction(List inputs, boolean ignoreNulls) { try { + T windowFunction; + if (getSignature().getArgumentTypes().isEmpty()) { - return constructor.newInstance(); + windowFunction = constructor.newInstance(); } else { - return constructor.newInstance(inputs); + windowFunction = constructor.newInstance(inputs); + } + + if (windowFunction instanceof ValueWindowFunction) { + ((ValueWindowFunction) windowFunction).setIgnoreNulls(ignoreNulls); } + + return windowFunction; } catch (ReflectiveOperationException e) { throw new RuntimeException(e); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/SqlWindowFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/window/SqlWindowFunction.java index 52802eb898279..4795c2e64674d 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/SqlWindowFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/SqlWindowFunction.java @@ -14,15 +14,15 @@ package com.facebook.presto.operator.window; import com.facebook.presto.metadata.BoundVariables; +import com.facebook.presto.metadata.BuiltInFunction; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.SqlFunction; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.TypeManager; import static java.util.Objects.requireNonNull; public class SqlWindowFunction - implements SqlFunction + extends BuiltInFunction { private final WindowFunctionSupplier supplier; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/WindowAnnotationsParser.java b/presto-main/src/main/java/com/facebook/presto/operator/window/WindowAnnotationsParser.java index 174662a0c4d45..f312ab2eb72fc 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/WindowAnnotationsParser.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/WindowAnnotationsParser.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.window; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.function.TypeVariableConstraint; import com.facebook.presto.spi.function.WindowFunction; @@ -23,6 +24,7 @@ import java.util.List; import java.util.stream.Stream; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.spi.function.FunctionKind.WINDOW; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -42,7 +44,9 @@ public static List parseFunctionDefinition(Class clazz, WindowFunctionSignature window) + private static SqlWindowFunction parse( + Class clazz, + WindowFunctionSignature window) { List typeVariables = ImmutableList.of(); if (!window.typeVariable().isEmpty()) { @@ -54,7 +58,7 @@ private static SqlWindowFunction parse(Class clazz, Wi .collect(toImmutableList()); Signature signature = new Signature( - window.name(), + QualifiedFunctionName.of(DEFAULT_NAMESPACE, window.name()), WINDOW, typeVariables, ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/WindowFunctionSupplier.java b/presto-main/src/main/java/com/facebook/presto/operator/window/WindowFunctionSupplier.java index 4237415c010b1..ed64d5e478b3c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/WindowFunctionSupplier.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/WindowFunctionSupplier.java @@ -24,5 +24,5 @@ public interface WindowFunctionSupplier String getDescription(); - WindowFunction createWindowFunction(List argumentChannels); + WindowFunction createWindowFunction(List argumentChannels, boolean ignoreNulls); } diff --git a/presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java b/presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java index 8dc1bbd32ff8e..b7612c8b41f98 100644 --- a/presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java +++ b/presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.security; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.spi.CatalogSchemaName; import com.facebook.presto.spi.ConnectorId; @@ -30,8 +32,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-main/src/main/java/com/facebook/presto/security/FileBasedSystemAccessControl.java b/presto-main/src/main/java/com/facebook/presto/security/FileBasedSystemAccessControl.java index f88b81b4436e9..bf311700090c0 100644 --- a/presto-main/src/main/java/com/facebook/presto/security/FileBasedSystemAccessControl.java +++ b/presto-main/src/main/java/com/facebook/presto/security/FileBasedSystemAccessControl.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.security; +import com.facebook.airlift.log.Logger; import com.facebook.presto.plugin.base.security.ForwardingSystemAccessControl; import com.facebook.presto.spi.CatalogSchemaName; import com.facebook.presto.spi.CatalogSchemaTableName; @@ -25,7 +26,6 @@ import com.facebook.presto.spi.security.SystemAccessControlFactory; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; import io.airlift.units.Duration; import java.nio.file.Paths; diff --git a/presto-main/src/main/java/com/facebook/presto/server/AsyncHttpExecutionMBean.java b/presto-main/src/main/java/com/facebook/presto/server/AsyncHttpExecutionMBean.java index c4f14b7bd35ca..811b8fcfd34be 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/AsyncHttpExecutionMBean.java +++ b/presto-main/src/main/java/com/facebook/presto/server/AsyncHttpExecutionMBean.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.server; -import io.airlift.concurrent.ThreadPoolExecutorMBean; +import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java index 96379cdea20ff..77a0c435fd092 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.airlift.discovery.server.EmbeddedDiscoveryModule; import com.facebook.presto.client.QueryResults; import com.facebook.presto.cost.CostCalculator; import com.facebook.presto.cost.CostCalculator.EstimatedExchanges; @@ -27,6 +30,7 @@ import com.facebook.presto.execution.CallTask; import com.facebook.presto.execution.ClusterSizeMonitor; import com.facebook.presto.execution.CommitTask; +import com.facebook.presto.execution.CreateFunctionTask; import com.facebook.presto.execution.CreateRoleTask; import com.facebook.presto.execution.CreateSchemaTask; import com.facebook.presto.execution.CreateTableTask; @@ -34,6 +38,7 @@ import com.facebook.presto.execution.DataDefinitionTask; import com.facebook.presto.execution.DeallocateTask; import com.facebook.presto.execution.DropColumnTask; +import com.facebook.presto.execution.DropFunctionTask; import com.facebook.presto.execution.DropRoleTask; import com.facebook.presto.execution.DropSchemaTask; import com.facebook.presto.execution.DropTableTask; @@ -58,7 +63,6 @@ import com.facebook.presto.execution.RevokeRolesTask; import com.facebook.presto.execution.RevokeTask; import com.facebook.presto.execution.RollbackTask; -import com.facebook.presto.execution.SetPathTask; import com.facebook.presto.execution.SetRoleTask; import com.facebook.presto.execution.SetSessionTask; import com.facebook.presto.execution.SqlQueryManager; @@ -96,12 +100,14 @@ import com.facebook.presto.sql.tree.AddColumn; import com.facebook.presto.sql.tree.Call; import com.facebook.presto.sql.tree.Commit; +import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateRole; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateView; import com.facebook.presto.sql.tree.Deallocate; import com.facebook.presto.sql.tree.DropColumn; +import com.facebook.presto.sql.tree.DropFunction; import com.facebook.presto.sql.tree.DropRole; import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; @@ -116,7 +122,6 @@ import com.facebook.presto.sql.tree.Revoke; import com.facebook.presto.sql.tree.RevokeRoles; import com.facebook.presto.sql.tree.Rollback; -import com.facebook.presto.sql.tree.SetPath; import com.facebook.presto.sql.tree.SetRole; import com.facebook.presto.sql.tree.SetSession; import com.facebook.presto.sql.tree.StartTransaction; @@ -132,9 +137,6 @@ import com.google.inject.Scopes; import com.google.inject.TypeLiteral; import com.google.inject.multibindings.MapBinder; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.airlift.discovery.server.EmbeddedDiscoveryModule; import io.airlift.units.Duration; import javax.annotation.PreDestroy; @@ -145,21 +147,21 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.discovery.client.DiscoveryBinder.discoveryBinder; +import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; +import static com.facebook.airlift.http.server.HttpServerBinder.httpServerBinder; +import static com.facebook.airlift.jaxrs.JaxrsBinder.jaxrsBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.presto.execution.DataDefinitionExecution.DataDefinitionExecutionFactory; import static com.facebook.presto.execution.QueryExecution.QueryExecutionFactory; import static com.facebook.presto.execution.SqlQueryExecution.SqlQueryExecutionFactory; import static com.facebook.presto.util.StatementUtils.getAllQueryTypes; import static com.google.common.base.Verify.verify; import static com.google.inject.multibindings.MapBinder.newMapBinder; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.concurrent.Threads.threadsNamed; -import static io.airlift.configuration.ConditionalModule.installModuleIf; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.discovery.client.DiscoveryBinder.discoveryBinder; -import static io.airlift.http.client.HttpClientBinder.httpClientBinder; -import static io.airlift.http.server.HttpServerBinder.httpServerBinder; -import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; @@ -306,6 +308,8 @@ protected void setup(Binder binder) bindDataDefinitionTask(binder, executionBinder, DropTable.class, DropTableTask.class); bindDataDefinitionTask(binder, executionBinder, CreateView.class, CreateViewTask.class); bindDataDefinitionTask(binder, executionBinder, DropView.class, DropViewTask.class); + bindDataDefinitionTask(binder, executionBinder, CreateFunction.class, CreateFunctionTask.class); + bindDataDefinitionTask(binder, executionBinder, DropFunction.class, DropFunctionTask.class); bindDataDefinitionTask(binder, executionBinder, Use.class, UseTask.class); bindDataDefinitionTask(binder, executionBinder, SetSession.class, SetSessionTask.class); bindDataDefinitionTask(binder, executionBinder, ResetSession.class, ResetSessionTask.class); @@ -322,7 +326,6 @@ protected void setup(Binder binder) bindDataDefinitionTask(binder, executionBinder, Revoke.class, RevokeTask.class); bindDataDefinitionTask(binder, executionBinder, Prepare.class, PrepareTask.class); bindDataDefinitionTask(binder, executionBinder, Deallocate.class, DeallocateTask.class); - bindDataDefinitionTask(binder, executionBinder, SetPath.class, SetPathTask.class); MapBinder executionPolicyBinder = newMapBinder(binder, String.class, ExecutionPolicy.class); executionPolicyBinder.addBinding("all-at-once").to(AllAtOnceExecutionPolicy.class); diff --git a/presto-main/src/main/java/com/facebook/presto/server/EmbeddedDiscoveryConfig.java b/presto-main/src/main/java/com/facebook/presto/server/EmbeddedDiscoveryConfig.java index 9a8d7807a69ef..5692050b4b8af 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/EmbeddedDiscoveryConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/EmbeddedDiscoveryConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.server; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; public class EmbeddedDiscoveryConfig { diff --git a/presto-main/src/main/java/com/facebook/presto/server/ExchangeExecutionMBean.java b/presto-main/src/main/java/com/facebook/presto/server/ExchangeExecutionMBean.java index 57b0585fd37ec..a6e13e247d9bd 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ExchangeExecutionMBean.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ExchangeExecutionMBean.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; import com.facebook.presto.operator.ForExchange; -import io.airlift.concurrent.ThreadPoolExecutorMBean; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-main/src/main/java/com/facebook/presto/server/GenerateTraceTokenRequestFilter.java b/presto-main/src/main/java/com/facebook/presto/server/GenerateTraceTokenRequestFilter.java index 4e95e37fa1f3f..7721e10897eca 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/GenerateTraceTokenRequestFilter.java +++ b/presto-main/src/main/java/com/facebook/presto/server/GenerateTraceTokenRequestFilter.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.http.client.HttpRequestFilter; +import com.facebook.airlift.http.client.Request; import com.google.common.annotations.VisibleForTesting; -import io.airlift.http.client.HttpRequestFilter; -import io.airlift.http.client.Request; import java.util.concurrent.atomic.AtomicLong; -import static io.airlift.http.client.Request.Builder.fromRequest; -import static io.airlift.http.client.TraceTokenRequestFilter.TRACETOKEN_HEADER; +import static com.facebook.airlift.http.client.Request.Builder.fromRequest; +import static com.facebook.airlift.http.client.TraceTokenRequestFilter.TRACETOKEN_HEADER; import static java.lang.String.format; import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/server/GracefulShutdownHandler.java b/presto-main/src/main/java/com/facebook/presto/server/GracefulShutdownHandler.java index 958a9b969e10c..bf6148321cffb 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/GracefulShutdownHandler.java +++ b/presto-main/src/main/java/com/facebook/presto/server/GracefulShutdownHandler.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.execution.TaskInfo; import com.facebook.presto.execution.TaskManager; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import io.airlift.units.Duration; import javax.annotation.concurrent.GuardedBy; @@ -30,9 +30,9 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeoutException; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; -import static io.airlift.concurrent.Threads.threadsNamed; import static java.lang.Thread.currentThread; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newSingleThreadExecutor; diff --git a/presto-main/src/main/java/com/facebook/presto/server/GracefulShutdownModule.java b/presto-main/src/main/java/com/facebook/presto/server/GracefulShutdownModule.java index 8f7874ff3c708..6565aa6f71d6b 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/GracefulShutdownModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/GracefulShutdownModule.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.google.inject.Binder; import com.google.inject.Scopes; -import io.airlift.configuration.AbstractConfigurationAwareModule; public class GracefulShutdownModule extends AbstractConfigurationAwareModule diff --git a/presto-main/src/main/java/com/facebook/presto/server/HttpRequestSessionContext.java b/presto-main/src/main/java/com/facebook/presto/server/HttpRequestSessionContext.java index fde6e47eb9660..22c2234c8f66b 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/HttpRequestSessionContext.java +++ b/presto-main/src/main/java/com/facebook/presto/server/HttpRequestSessionContext.java @@ -46,12 +46,10 @@ import java.util.Set; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CATALOG; -import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_CAPABILITIES; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_INFO; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_TAGS; import static com.facebook.presto.client.PrestoHeaders.PRESTO_EXTRA_CREDENTIAL; import static com.facebook.presto.client.PrestoHeaders.PRESTO_LANGUAGE; -import static com.facebook.presto.client.PrestoHeaders.PRESTO_PATH; import static com.facebook.presto.client.PrestoHeaders.PRESTO_PREPARED_STATEMENT; import static com.facebook.presto.client.PrestoHeaders.PRESTO_RESOURCE_ESTIMATE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_ROLE; @@ -64,10 +62,12 @@ import static com.facebook.presto.client.PrestoHeaders.PRESTO_USER; import static com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DOUBLE; import static com.google.common.base.Strings.emptyToNull; +import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.net.HttpHeaders.USER_AGENT; +import static com.google.common.net.HttpHeaders.X_FORWARDED_FOR; import static java.lang.String.format; public final class HttpRequestSessionContext @@ -77,7 +77,6 @@ public final class HttpRequestSessionContext private final String catalog; private final String schema; - private final String path; private final Identity identity; @@ -88,7 +87,6 @@ public final class HttpRequestSessionContext private final String timeZoneId; private final String language; private final Set clientTags; - private final Set clientCapabilities; private final ResourceEstimates resourceEstimates; private final Map systemProperties; @@ -105,7 +103,6 @@ public HttpRequestSessionContext(HttpServletRequest servletRequest) { catalog = trimEmptyToNull(servletRequest.getHeader(PRESTO_CATALOG)); schema = trimEmptyToNull(servletRequest.getHeader(PRESTO_SCHEMA)); - path = trimEmptyToNull(servletRequest.getHeader(PRESTO_PATH)); assertRequest((catalog != null) || (schema == null), "Schema is set but catalog is not"); String user = trimEmptyToNull(servletRequest.getHeader(PRESTO_USER)); @@ -119,12 +116,11 @@ public HttpRequestSessionContext(HttpServletRequest servletRequest) source = servletRequest.getHeader(PRESTO_SOURCE); traceToken = Optional.ofNullable(trimEmptyToNull(servletRequest.getHeader(PRESTO_TRACE_TOKEN))); userAgent = servletRequest.getHeader(USER_AGENT); - remoteUserAddress = servletRequest.getRemoteAddr(); + remoteUserAddress = !isNullOrEmpty(servletRequest.getHeader(X_FORWARDED_FOR)) ? servletRequest.getHeader(X_FORWARDED_FOR) : servletRequest.getRemoteAddr(); timeZoneId = servletRequest.getHeader(PRESTO_TIME_ZONE); language = servletRequest.getHeader(PRESTO_LANGUAGE); clientInfo = servletRequest.getHeader(PRESTO_CLIENT_INFO); clientTags = parseClientTags(servletRequest); - clientCapabilities = parseClientCapabilities(servletRequest); resourceEstimates = parseResourceEstimate(servletRequest); // parse session properties @@ -167,6 +163,134 @@ else if (nameParts.size() == 2) { transactionId = parseTransactionId(transactionIdHeader); } + private static List splitSessionHeader(Enumeration headers) + { + Splitter splitter = Splitter.on(',').trimResults().omitEmptyStrings(); + return Collections.list(headers).stream() + .map(splitter::splitToList) + .flatMap(Collection::stream) + .collect(toImmutableList()); + } + + private static Map parseSessionHeaders(HttpServletRequest servletRequest) + { + return parseProperty(servletRequest, PRESTO_SESSION); + } + + private static Map parseRoleHeaders(HttpServletRequest servletRequest) + { + ImmutableMap.Builder roles = ImmutableMap.builder(); + for (String header : splitSessionHeader(servletRequest.getHeaders(PRESTO_ROLE))) { + List nameValue = Splitter.on('=').limit(2).trimResults().splitToList(header); + assertRequest(nameValue.size() == 2, "Invalid %s header", PRESTO_ROLE); + roles.put(nameValue.get(0), SelectedRole.valueOf(urlDecode(nameValue.get(1)))); + } + return roles.build(); + } + + private static Map parseExtraCredentials(HttpServletRequest servletRequest) + { + return parseCredentialProperty(servletRequest, PRESTO_EXTRA_CREDENTIAL); + } + + private static Map parseProperty(HttpServletRequest servletRequest, String headerName) + { + Map properties = new HashMap<>(); + for (String header : splitSessionHeader(servletRequest.getHeaders(headerName))) { + List nameValue = Splitter.on('=').trimResults().splitToList(header); + assertRequest(nameValue.size() == 2, "Invalid %s header", headerName); + properties.put(nameValue.get(0), nameValue.get(1)); + } + return properties; + } + + private static Map parseCredentialProperty(HttpServletRequest servletRequest, String headerName) + { + Map properties = new HashMap<>(); + for (String header : splitSessionHeader(servletRequest.getHeaders(headerName))) { + List nameValue = Splitter.on('=').limit(2).trimResults().splitToList(header); + assertRequest(nameValue.size() == 2, "Invalid %s header", headerName); + properties.put(nameValue.get(0), urlDecode(nameValue.get(1))); + } + return properties; + } + + private static void assertRequest(boolean expression, String format, Object... args) + { + if (!expression) { + throw badRequest(format(format, args)); + } + } + + private static Map parsePreparedStatementsHeaders(HttpServletRequest servletRequest) + { + ImmutableMap.Builder preparedStatements = ImmutableMap.builder(); + for (String header : splitSessionHeader(servletRequest.getHeaders(PRESTO_PREPARED_STATEMENT))) { + List nameValue = Splitter.on('=').limit(2).trimResults().splitToList(header); + assertRequest(nameValue.size() == 2, "Invalid %s header", PRESTO_PREPARED_STATEMENT); + + String statementName; + String sqlString; + try { + statementName = urlDecode(nameValue.get(0)); + sqlString = urlDecode(nameValue.get(1)); + } + catch (IllegalArgumentException e) { + throw badRequest(format("Invalid %s header: %s", PRESTO_PREPARED_STATEMENT, e.getMessage())); + } + + // Validate statement + SqlParser sqlParser = new SqlParser(); + try { + sqlParser.createStatement(sqlString, new ParsingOptions(AS_DOUBLE /* anything */)); + } + catch (ParsingException e) { + throw badRequest(format("Invalid %s header: %s", PRESTO_PREPARED_STATEMENT, e.getMessage())); + } + + preparedStatements.put(statementName, sqlString); + } + return preparedStatements.build(); + } + + private static Optional parseTransactionId(String transactionId) + { + transactionId = trimEmptyToNull(transactionId); + if (transactionId == null || transactionId.equalsIgnoreCase("none")) { + return Optional.empty(); + } + try { + return Optional.of(TransactionId.valueOf(transactionId)); + } + catch (Exception e) { + throw badRequest(e.getMessage()); + } + } + + private static WebApplicationException badRequest(String message) + { + throw new WebApplicationException(Response + .status(Status.BAD_REQUEST) + .type(MediaType.TEXT_PLAIN) + .entity(message) + .build()); + } + + private static String trimEmptyToNull(String value) + { + return emptyToNull(nullToEmpty(value).trim()); + } + + private static String urlDecode(String value) + { + try { + return URLDecoder.decode(value, "UTF-8"); + } + catch (UnsupportedEncodingException e) { + throw new AssertionError(e); + } + } + @Override public Identity getIdentity() { @@ -185,12 +309,6 @@ public String getSchema() return schema; } - @Override - public String getPath() - { - return path; - } - @Override public String getSource() { @@ -221,12 +339,6 @@ public Set getClientTags() return clientTags; } - @Override - public Set getClientCapabilities() - { - return clientCapabilities; - } - @Override public ResourceEstimates getResourceEstimates() { @@ -281,59 +393,12 @@ public Optional getTraceToken() return traceToken; } - private static List splitSessionHeader(Enumeration headers) - { - Splitter splitter = Splitter.on(',').trimResults().omitEmptyStrings(); - return Collections.list(headers).stream() - .map(splitter::splitToList) - .flatMap(Collection::stream) - .collect(toImmutableList()); - } - - private static Map parseSessionHeaders(HttpServletRequest servletRequest) - { - return parseProperty(servletRequest, PRESTO_SESSION); - } - - private static Map parseRoleHeaders(HttpServletRequest servletRequest) - { - ImmutableMap.Builder roles = ImmutableMap.builder(); - for (String header : splitSessionHeader(servletRequest.getHeaders(PRESTO_ROLE))) { - List nameValue = Splitter.on('=').limit(2).trimResults().splitToList(header); - assertRequest(nameValue.size() == 2, "Invalid %s header", PRESTO_ROLE); - roles.put(nameValue.get(0), SelectedRole.valueOf(urlDecode(nameValue.get(1)))); - } - return roles.build(); - } - - private static Map parseExtraCredentials(HttpServletRequest servletRequest) - { - return parseProperty(servletRequest, PRESTO_EXTRA_CREDENTIAL); - } - - private static Map parseProperty(HttpServletRequest servletRequest, String headerName) - { - Map properties = new HashMap<>(); - for (String header : splitSessionHeader(servletRequest.getHeaders(headerName))) { - List nameValue = Splitter.on('=').trimResults().splitToList(header); - assertRequest(nameValue.size() == 2, "Invalid %s header", headerName); - properties.put(nameValue.get(0), nameValue.get(1)); - } - return properties; - } - private Set parseClientTags(HttpServletRequest servletRequest) { Splitter splitter = Splitter.on(',').trimResults().omitEmptyStrings(); return ImmutableSet.copyOf(splitter.split(nullToEmpty(servletRequest.getHeader(PRESTO_CLIENT_TAGS)))); } - private Set parseClientCapabilities(HttpServletRequest servletRequest) - { - Splitter splitter = Splitter.on(',').trimResults().omitEmptyStrings(); - return ImmutableSet.copyOf(splitter.split(nullToEmpty(servletRequest.getHeader(PRESTO_CLIENT_CAPABILITIES)))); - } - private ResourceEstimates parseResourceEstimate(HttpServletRequest servletRequest) { ResourceEstimateBuilder builder = new ResourceEstimateBuilder(); @@ -365,80 +430,4 @@ private ResourceEstimates parseResourceEstimate(HttpServletRequest servletReques return builder.build(); } - - private static void assertRequest(boolean expression, String format, Object... args) - { - if (!expression) { - throw badRequest(format(format, args)); - } - } - - private static Map parsePreparedStatementsHeaders(HttpServletRequest servletRequest) - { - ImmutableMap.Builder preparedStatements = ImmutableMap.builder(); - for (String header : splitSessionHeader(servletRequest.getHeaders(PRESTO_PREPARED_STATEMENT))) { - List nameValue = Splitter.on('=').limit(2).trimResults().splitToList(header); - assertRequest(nameValue.size() == 2, "Invalid %s header", PRESTO_PREPARED_STATEMENT); - - String statementName; - String sqlString; - try { - statementName = urlDecode(nameValue.get(0)); - sqlString = urlDecode(nameValue.get(1)); - } - catch (IllegalArgumentException e) { - throw badRequest(format("Invalid %s header: %s", PRESTO_PREPARED_STATEMENT, e.getMessage())); - } - - // Validate statement - SqlParser sqlParser = new SqlParser(); - try { - sqlParser.createStatement(sqlString, new ParsingOptions(AS_DOUBLE /* anything */)); - } - catch (ParsingException e) { - throw badRequest(format("Invalid %s header: %s", PRESTO_PREPARED_STATEMENT, e.getMessage())); - } - - preparedStatements.put(statementName, sqlString); - } - return preparedStatements.build(); - } - - private static Optional parseTransactionId(String transactionId) - { - transactionId = trimEmptyToNull(transactionId); - if (transactionId == null || transactionId.equalsIgnoreCase("none")) { - return Optional.empty(); - } - try { - return Optional.of(TransactionId.valueOf(transactionId)); - } - catch (Exception e) { - throw badRequest(e.getMessage()); - } - } - - private static WebApplicationException badRequest(String message) - { - throw new WebApplicationException(Response - .status(Status.BAD_REQUEST) - .type(MediaType.TEXT_PLAIN) - .entity(message) - .build()); - } - - private static String trimEmptyToNull(String value) - { - return emptyToNull(nullToEmpty(value).trim()); - } - - private static String urlDecode(String value) - { - try { - return URLDecoder.decode(value, "UTF-8"); - } - catch (UnsupportedEncodingException e) { - throw new AssertionError(e); - } - } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationConfig.java b/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationConfig.java index 4e76cec5e1e58..1c50dcafb913a 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationConfig.java @@ -13,9 +13,14 @@ */ package com.facebook.presto.server; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.ConfigSecuritySensitive; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.ConfigSecuritySensitive; +import io.airlift.units.DataSize; + +import java.util.Optional; + +import static io.airlift.units.DataSize.Unit.MEGABYTE; public class InternalCommunicationConfig { @@ -24,9 +29,13 @@ public class InternalCommunicationConfig private boolean httpsRequired; private String keyStorePath; private String keyStorePassword; + private String trustStorePath; + private Optional excludeCipherSuites = Optional.empty(); + private Optional includedCipherSuites = Optional.empty(); private boolean kerberosEnabled; private boolean kerberosUseCanonicalHostname = true; private boolean binaryTransportEnabled; + private DataSize maxTaskUpdateSize = new DataSize(16, MEGABYTE); public boolean isHttpsRequired() { @@ -65,6 +74,42 @@ public InternalCommunicationConfig setKeyStorePassword(String keyStorePassword) return this; } + public String getTrustStorePath() + { + return trustStorePath; + } + + @Config("internal-communication.https.trust-store-path") + public InternalCommunicationConfig setTrustStorePath(String trustStorePath) + { + this.trustStorePath = trustStorePath; + return this; + } + + public Optional getIncludedCipherSuites() + { + return includedCipherSuites; + } + + @Config("internal-communication.https.included-cipher") + public InternalCommunicationConfig setIncludedCipherSuites(String includedCipherSuites) + { + this.includedCipherSuites = Optional.ofNullable(includedCipherSuites); + return this; + } + + public Optional getExcludeCipherSuites() + { + return excludeCipherSuites; + } + + @Config("internal-communication.https.excluded-cipher") + public InternalCommunicationConfig setExcludeCipherSuites(String excludeCipherSuites) + { + this.excludeCipherSuites = Optional.ofNullable(excludeCipherSuites); + return this; + } + public boolean isKerberosEnabled() { return kerberosEnabled; @@ -101,4 +146,17 @@ public InternalCommunicationConfig setBinaryTransportEnabled(boolean binaryTrans this.binaryTransportEnabled = binaryTransportEnabled; return this; } + + public DataSize getMaxTaskUpdateSize() + { + return maxTaskUpdateSize; + } + + @Config("experimental.internal-communication.max-task-update-size") + @ConfigDescription("Enables limit on the size of the task update") + public InternalCommunicationConfig setMaxTaskUpdateSize(DataSize maxTaskUpdateSize) + { + this.maxTaskUpdateSize = maxTaskUpdateSize; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationModule.java b/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationModule.java index 554a9451daa50..799a77569ee79 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationModule.java @@ -13,22 +13,22 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.airlift.http.client.HttpClientConfig; +import com.facebook.airlift.http.client.spnego.KerberosConfig; import com.google.inject.Binder; import com.google.inject.Module; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.airlift.http.client.HttpClientConfig; -import io.airlift.http.client.spnego.KerberosConfig; import java.io.UncheckedIOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Locale; +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static com.facebook.presto.server.InternalCommunicationConfig.INTERNAL_COMMUNICATION_KERBEROS_ENABLED; import static com.facebook.presto.server.security.KerberosConfig.HTTP_SERVER_AUTHENTICATION_KRB5_KEYTAB; import static com.google.common.base.Verify.verify; -import static io.airlift.configuration.ConditionalModule.installModuleIf; -import static io.airlift.configuration.ConfigBinder.configBinder; public class InternalCommunicationModule extends AbstractConfigurationAwareModule @@ -40,6 +40,13 @@ protected void setup(Binder binder) configBinder(binder).bindConfigGlobalDefaults(HttpClientConfig.class, config -> { config.setKeyStorePath(internalCommunicationConfig.getKeyStorePath()); config.setKeyStorePassword(internalCommunicationConfig.getKeyStorePassword()); + config.setTrustStorePath(internalCommunicationConfig.getTrustStorePath()); + if (internalCommunicationConfig.getIncludedCipherSuites().isPresent()) { + config.setHttpsIncludedCipherSuites(internalCommunicationConfig.getIncludedCipherSuites().get()); + } + if (internalCommunicationConfig.getExcludeCipherSuites().isPresent()) { + config.setHttpsExcludedCipherSuites(internalCommunicationConfig.getExcludeCipherSuites().get()); + } }); install(installModuleIf(InternalCommunicationConfig.class, InternalCommunicationConfig::isKerberosEnabled, kerberosInternalCommunicationModule())); diff --git a/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java b/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java index b25ae65a3f093..312f8fbdbfd2e 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.connector.ConnectorManager; import com.facebook.presto.eventlistener.EventListenerManager; @@ -25,6 +27,7 @@ import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.connector.ConnectorFactory; import com.facebook.presto.spi.eventlistener.EventListenerFactory; +import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory; import com.facebook.presto.spi.resourceGroups.ResourceGroupConfigurationManagerFactory; import com.facebook.presto.spi.security.PasswordAuthenticatorFactory; import com.facebook.presto.spi.security.SystemAccessControlFactory; @@ -34,8 +37,6 @@ import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; -import io.airlift.log.Logger; -import io.airlift.node.NodeInfo; import io.airlift.resolver.ArtifactResolver; import io.airlift.resolver.DefaultArtifact; import org.sonatype.aether.artifact.Artifact; @@ -207,7 +208,12 @@ public void installPlugin(Plugin plugin) for (Class functionClass : plugin.getFunctions()) { log.info("Registering functions from %s", functionClass.getName()); - metadata.addFunctions(extractFunctions(functionClass)); + metadata.registerBuiltInFunctions(extractFunctions(functionClass)); + } + + for (FunctionNamespaceManagerFactory functionNamespaceManagerFactory : plugin.getFunctionNamespaceManagerFactories()) { + log.info("Registering function namespace manager %s", functionNamespaceManagerFactory.getName()); + metadata.getFunctionManager().addFunctionNamespaceFactory(functionNamespaceManagerFactory); } for (SessionPropertyConfigurationManagerFactory sessionConfigFactory : plugin.getSessionPropertyConfigurationManagerFactories()) { diff --git a/presto-main/src/main/java/com/facebook/presto/server/PluginManagerConfig.java b/presto-main/src/main/java/com/facebook/presto/server/PluginManagerConfig.java index 858110a396044..89d12fb29fe11 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PluginManagerConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PluginManagerConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.configuration.Config; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import io.airlift.configuration.Config; import io.airlift.resolver.ArtifactResolver; import javax.validation.constraints.NotNull; diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 2425ed965c9aa..51790449ae797 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -13,6 +13,21 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.discovery.client.Announcer; +import com.facebook.airlift.discovery.client.DiscoveryModule; +import com.facebook.airlift.discovery.client.ServiceAnnouncement; +import com.facebook.airlift.event.client.HttpEventModule; +import com.facebook.airlift.event.client.JsonEventModule; +import com.facebook.airlift.http.server.HttpServerModule; +import com.facebook.airlift.jaxrs.JaxrsModule; +import com.facebook.airlift.jmx.JmxHttpModule; +import com.facebook.airlift.jmx.JmxModule; +import com.facebook.airlift.json.JsonModule; +import com.facebook.airlift.log.LogJmxModule; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.node.NodeModule; +import com.facebook.airlift.tracetoken.TraceTokenModule; import com.facebook.presto.eventlistener.EventListenerManager; import com.facebook.presto.eventlistener.EventListenerModule; import com.facebook.presto.execution.resourceGroups.ResourceGroupManager; @@ -21,6 +36,7 @@ import com.facebook.presto.metadata.Catalog; import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.metadata.StaticCatalogStore; +import com.facebook.presto.metadata.StaticFunctionNamespaceStore; import com.facebook.presto.security.AccessControlManager; import com.facebook.presto.security.AccessControlModule; import com.facebook.presto.server.security.PasswordAuthenticatorManager; @@ -34,21 +50,6 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Injector; import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.discovery.client.Announcer; -import io.airlift.discovery.client.DiscoveryModule; -import io.airlift.discovery.client.ServiceAnnouncement; -import io.airlift.event.client.HttpEventModule; -import io.airlift.event.client.JsonEventModule; -import io.airlift.http.server.HttpServerModule; -import io.airlift.jaxrs.JaxrsModule; -import io.airlift.jmx.JmxHttpModule; -import io.airlift.jmx.JmxModule; -import io.airlift.json.JsonModule; -import io.airlift.log.LogJmxModule; -import io.airlift.log.Logger; -import io.airlift.node.NodeModule; -import io.airlift.tracetoken.TraceTokenModule; import org.weakref.jmx.guice.MBeanModule; import java.util.LinkedHashSet; @@ -56,13 +57,13 @@ import java.util.Map; import java.util.Set; +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; +import static com.facebook.airlift.discovery.client.ServiceAnnouncement.ServiceAnnouncementBuilder; +import static com.facebook.airlift.discovery.client.ServiceAnnouncement.serviceAnnouncement; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; import static com.facebook.presto.server.PrestoSystemRequirements.verifyJvmRequirements; import static com.facebook.presto.server.PrestoSystemRequirements.verifySystemTimeIsReasonable; import static com.google.common.base.Strings.nullToEmpty; -import static io.airlift.configuration.ConditionalModule.installModuleIf; -import static io.airlift.discovery.client.ServiceAnnouncement.ServiceAnnouncementBuilder; -import static io.airlift.discovery.client.ServiceAnnouncement.serviceAnnouncement; -import static io.airlift.json.JsonBinder.jsonBinder; import static java.util.Objects.requireNonNull; public class PrestoServer @@ -137,6 +138,7 @@ public void run() injector.getInstance(ServerConfig.class), injector.getInstance(NodeSchedulerConfig.class)); + injector.getInstance(StaticFunctionNamespaceStore.class).loadFunctionNamespaceManagers(); injector.getInstance(SessionPropertyDefaults.class).loadConfigurationManager(); injector.getInstance(ResourceGroupManager.class).loadConfigurationManager(); injector.getInstance(AccessControlManager.class).loadSystemAccessControl(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/QuerySessionSupplier.java b/presto-main/src/main/java/com/facebook/presto/server/QuerySessionSupplier.java index ab44a8048bc3b..36b89ef1cf54d 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/QuerySessionSupplier.java +++ b/presto-main/src/main/java/com/facebook/presto/server/QuerySessionSupplier.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.type.TimeZoneKey; import com.facebook.presto.sql.SqlEnvironmentConfig; -import com.facebook.presto.sql.SqlPath; import com.facebook.presto.transaction.TransactionManager; import javax.annotation.concurrent.ThreadSafe; @@ -42,7 +41,6 @@ public class QuerySessionSupplier private final TransactionManager transactionManager; private final AccessControl accessControl; private final SessionPropertyManager sessionPropertyManager; - private final Optional path; private final Optional forcedSessionTimeZone; @Inject @@ -56,7 +54,6 @@ public QuerySessionSupplier( this.accessControl = requireNonNull(accessControl, "accessControl is null"); this.sessionPropertyManager = requireNonNull(sessionPropertyManager, "sessionPropertyManager is null"); requireNonNull(config, "config is null"); - this.path = requireNonNull(config.getPath(), "path is null"); this.forcedSessionTimeZone = requireNonNull(config.getForcedSessionTimeZone(), "forcedSessionTimeZone is null"); } @@ -72,19 +69,13 @@ public Session createSession(QueryId queryId, SessionContext context) .setSource(context.getSource()) .setCatalog(context.getCatalog()) .setSchema(context.getSchema()) - .setPath(new SqlPath(path)) .setRemoteUserAddress(context.getRemoteUserAddress()) .setUserAgent(context.getUserAgent()) .setClientInfo(context.getClientInfo()) .setClientTags(context.getClientTags()) - .setClientCapabilities(context.getClientCapabilities()) .setTraceToken(context.getTraceToken()) .setResourceEstimates(context.getResourceEstimates()); - if (context.getPath() != null) { - sessionBuilder.setPath(new SqlPath(Optional.of(context.getPath()))); - } - if (forcedSessionTimeZone.isPresent()) { sessionBuilder.setTimeZoneKey(forcedSessionTimeZone.get()); } diff --git a/presto-main/src/main/java/com/facebook/presto/server/RequestHelpers.java b/presto-main/src/main/java/com/facebook/presto/server/RequestHelpers.java index a5b26cade0bab..e3f337b803c5c 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/RequestHelpers.java +++ b/presto-main/src/main/java/com/facebook/presto/server/RequestHelpers.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.server; -import io.airlift.http.client.Request.Builder; +import com.facebook.airlift.http.client.Request.Builder; import static com.facebook.presto.PrestoMediaTypes.APPLICATION_JACKSON_SMILE; import static com.google.common.net.HttpHeaders.ACCEPT; diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java b/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java index bbe127c6979c5..c6bf957cf069d 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.server; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import io.airlift.units.Duration; import javax.validation.constraints.NotNull; diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerInfoResource.java b/presto-main/src/main/java/com/facebook/presto/server/ServerInfoResource.java index e1baaf2df335b..a3ef179ab7d4c 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerInfoResource.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerInfoResource.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.client.NodeVersion; import com.facebook.presto.client.ServerInfo; import com.facebook.presto.metadata.StaticCatalogStore; import com.facebook.presto.spi.NodeState; -import io.airlift.node.NodeInfo; import javax.inject.Inject; import javax.ws.rs.Consumes; diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index af5a7f0624c23..e44cd282006ce 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -13,6 +13,11 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.airlift.stats.GcMonitor; +import com.facebook.airlift.stats.JmxGcMonitor; +import com.facebook.airlift.stats.PauseMeter; import com.facebook.presto.GroupByHashPageIndexerFactory; import com.facebook.presto.PagesIndexPageSorter; import com.facebook.presto.SystemSessionProperties; @@ -22,6 +27,9 @@ import com.facebook.presto.client.ServerInfo; import com.facebook.presto.connector.ConnectorManager; import com.facebook.presto.connector.system.SystemConnectorModule; +import com.facebook.presto.cost.FilterStatsCalculator; +import com.facebook.presto.cost.ScalarStatsCalculator; +import com.facebook.presto.cost.StatsNormalizer; import com.facebook.presto.event.SplitMonitor; import com.facebook.presto.execution.ExecutionFailureInfo; import com.facebook.presto.execution.ExplainAnalyzeContext; @@ -58,6 +66,7 @@ import com.facebook.presto.metadata.ColumnPropertyManager; import com.facebook.presto.metadata.DiscoveryNodeManager; import com.facebook.presto.metadata.ForNodeManager; +import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.HandleJsonModule; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Metadata; @@ -66,6 +75,8 @@ import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.metadata.StaticCatalogStore; import com.facebook.presto.metadata.StaticCatalogStoreConfig; +import com.facebook.presto.metadata.StaticFunctionNamespaceStore; +import com.facebook.presto.metadata.StaticFunctionNamespaceStoreConfig; import com.facebook.presto.metadata.TablePropertyManager; import com.facebook.presto.metadata.ViewDefinition; import com.facebook.presto.operator.ExchangeClientConfig; @@ -123,6 +134,7 @@ import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.LocalExecutionPlanner; import com.facebook.presto.sql.planner.NodePartitioningManager; +import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.facebook.presto.sql.tree.Expression; @@ -131,18 +143,14 @@ import com.facebook.presto.type.TypeDeserializer; import com.facebook.presto.type.TypeRegistry; import com.facebook.presto.util.FinalizerService; +import com.facebook.presto.util.GcStatusMonitor; import com.facebook.presto.version.EmbedVersion; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; import com.google.inject.Key; import com.google.inject.Provides; import com.google.inject.Scopes; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.slice.Slice; -import io.airlift.stats.GcMonitor; -import io.airlift.stats.JmxGcMonitor; -import io.airlift.stats.PauseMeter; import io.airlift.units.DataSize; import io.airlift.units.Duration; @@ -154,20 +162,20 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.discovery.client.DiscoveryBinder.discoveryBinder; +import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; +import static com.facebook.airlift.jaxrs.JaxrsBinder.jaxrsBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.presto.execution.scheduler.NodeSchedulerConfig.NetworkTopologyType.FLAT; import static com.facebook.presto.execution.scheduler.NodeSchedulerConfig.NetworkTopologyType.LEGACY; import static com.facebook.presto.server.smile.SmileCodecBinder.smileCodecBinder; import static com.google.common.base.Strings.nullToEmpty; import static com.google.inject.multibindings.Multibinder.newSetBinder; import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.configuration.ConditionalModule.installModuleIf; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.discovery.client.DiscoveryBinder.discoveryBinder; -import static io.airlift.http.client.HttpClientBinder.httpClientBinder; -import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; @@ -198,8 +206,6 @@ protected void setup(Binder binder) install(new WorkerModule()); } - InternalCommunicationConfig internalCommunicationConfig = buildConfigObject(InternalCommunicationConfig.class); - install(new InternalCommunicationModule()); configBinder(binder).bindConfig(FeaturesConfig.class); @@ -284,6 +290,7 @@ protected void setup(Binder binder) // Add monitoring for JVM pauses binder.bind(PauseMeter.class).in(Scopes.SINGLETON); newExporter(binder).export(PauseMeter.class).withGeneratedName(); + binder.bind(GcStatusMonitor.class).in(Scopes.SINGLETON); configBinder(binder).bindConfig(MemoryManagerConfig.class); configBinder(binder).bindConfig(NodeMemoryConfig.class); @@ -367,6 +374,9 @@ protected void setup(Binder binder) // metadata binder.bind(StaticCatalogStore.class).in(Scopes.SINGLETON); configBinder(binder).bindConfig(StaticCatalogStoreConfig.class); + binder.bind(StaticFunctionNamespaceStore.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(StaticFunctionNamespaceStoreConfig.class); + binder.bind(FunctionManager.class).in(Scopes.SINGLETON); binder.bind(MetadataManager.class).in(Scopes.SINGLETON); binder.bind(Metadata.class).to(MetadataManager.class).in(Scopes.SINGLETON); @@ -388,6 +398,9 @@ protected void setup(Binder binder) // split manager binder.bind(SplitManager.class).in(Scopes.SINGLETON); + // partitioning provider manager + binder.bind(PartitioningProviderManager.class).in(Scopes.SINGLETON); + // node partitioning manager binder.bind(NodePartitioningManager.class).in(Scopes.SINGLETON); @@ -401,6 +414,9 @@ protected void setup(Binder binder) binder.install(new HandleJsonModule()); // connector + binder.bind(ScalarStatsCalculator.class).in(Scopes.SINGLETON); + binder.bind(StatsNormalizer.class).in(Scopes.SINGLETON); + binder.bind(FilterStatsCalculator.class).in(Scopes.SINGLETON); binder.bind(ConnectorManager.class).in(Scopes.SINGLETON); // system connector diff --git a/presto-main/src/main/java/com/facebook/presto/server/SessionContext.java b/presto-main/src/main/java/com/facebook/presto/server/SessionContext.java index 419438b52b67c..39df171bb55a9 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/SessionContext.java +++ b/presto-main/src/main/java/com/facebook/presto/server/SessionContext.java @@ -33,9 +33,6 @@ public interface SessionContext @Nullable String getSchema(); - @Nullable - String getPath(); - @Nullable String getSource(); @@ -49,8 +46,6 @@ public interface SessionContext Set getClientTags(); - Set getClientCapabilities(); - ResourceEstimates getResourceEstimates(); @Nullable diff --git a/presto-main/src/main/java/com/facebook/presto/server/SessionPropertyDefaults.java b/presto-main/src/main/java/com/facebook/presto/server/SessionPropertyDefaults.java index 60fe6a212194f..140c1f84cbf9e 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/SessionPropertyDefaults.java +++ b/presto-main/src/main/java/com/facebook/presto/server/SessionPropertyDefaults.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.Session; import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import com.facebook.presto.spi.resourceGroups.SessionPropertyConfigurationManagerContext; @@ -20,8 +22,6 @@ import com.facebook.presto.spi.session.SessionPropertyConfigurationManager; import com.facebook.presto.spi.session.SessionPropertyConfigurationManagerFactory; import com.google.common.annotations.VisibleForTesting; -import io.airlift.log.Logger; -import io.airlift.node.NodeInfo; import javax.inject.Inject; diff --git a/presto-main/src/main/java/com/facebook/presto/server/StatementHttpExecutionMBean.java b/presto-main/src/main/java/com/facebook/presto/server/StatementHttpExecutionMBean.java index 56ddefdf2de10..acd7acb1022a2 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/StatementHttpExecutionMBean.java +++ b/presto-main/src/main/java/com/facebook/presto/server/StatementHttpExecutionMBean.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.server; -import io.airlift.concurrent.ThreadPoolExecutorMBean; +import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-main/src/main/java/com/facebook/presto/server/StatusResource.java b/presto-main/src/main/java/com/facebook/presto/server/StatusResource.java index ea9f56ec83618..a69a749ff7751 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/StatusResource.java +++ b/presto-main/src/main/java/com/facebook/presto/server/StatusResource.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.client.NodeVersion; import com.facebook.presto.memory.LocalMemoryManager; import com.sun.management.OperatingSystemMXBean; -import io.airlift.node.NodeInfo; import javax.inject.Inject; import javax.ws.rs.GET; diff --git a/presto-main/src/main/java/com/facebook/presto/server/TaskResource.java b/presto-main/src/main/java/com/facebook/presto/server/TaskResource.java index c5c070ac8dfc0..2cf16d8774207 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/TaskResource.java +++ b/presto-main/src/main/java/com/facebook/presto/server/TaskResource.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.stats.TimeStat; import com.facebook.presto.Session; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.TaskInfo; @@ -28,8 +30,6 @@ import com.google.common.reflect.TypeToken; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.stats.TimeStat; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.weakref.jmx.Managed; @@ -60,6 +60,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadLocalRandom; +import static com.facebook.airlift.concurrent.MoreFutures.addTimeout; +import static com.facebook.airlift.http.server.AsyncResponseHandler.bindAsyncResponse; import static com.facebook.presto.PrestoMediaTypes.APPLICATION_JACKSON_SMILE; import static com.facebook.presto.PrestoMediaTypes.PRESTO_PAGES; import static com.facebook.presto.client.PrestoHeaders.PRESTO_BUFFER_COMPLETE; @@ -71,8 +73,6 @@ import static com.facebook.presto.client.PrestoHeaders.PRESTO_TASK_INSTANCE_ID; import static com.google.common.collect.Iterables.transform; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.addTimeout; -import static io.airlift.http.server.AsyncResponseHandler.bindAsyncResponse; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; @@ -133,7 +133,8 @@ public Response createOrUpdateTask(@PathParam("taskId") TaskId taskId, TaskUpdat taskUpdateRequest.getFragment(), taskUpdateRequest.getSources(), taskUpdateRequest.getOutputIds(), - taskUpdateRequest.getTotalPartitions()); + taskUpdateRequest.getTotalPartitions(), + taskUpdateRequest.getTableWriteInfo()); if (shouldSummarize(uriInfo)) { taskInfo = taskInfo.summarize(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/TaskUpdateRequest.java b/presto-main/src/main/java/com/facebook/presto/server/TaskUpdateRequest.java index d5515954a2a81..7e5d238b33b5c 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/TaskUpdateRequest.java +++ b/presto-main/src/main/java/com/facebook/presto/server/TaskUpdateRequest.java @@ -16,6 +16,7 @@ import com.facebook.presto.SessionRepresentation; import com.facebook.presto.execution.TaskSource; import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.sql.planner.PlanFragment; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -38,6 +39,7 @@ public class TaskUpdateRequest private final List sources; private final OutputBuffers outputIds; private final OptionalInt totalPartitions; + private final Optional tableWriteInfo; @JsonCreator public TaskUpdateRequest( @@ -46,7 +48,8 @@ public TaskUpdateRequest( @JsonProperty("fragment") Optional fragment, @JsonProperty("sources") List sources, @JsonProperty("outputIds") OutputBuffers outputIds, - @JsonProperty("totalPartitions") OptionalInt totalPartitions) + @JsonProperty("totalPartitions") OptionalInt totalPartitions, + @JsonProperty("tableWriteInfo") Optional tableWriteInfo) { requireNonNull(session, "session is null"); requireNonNull(extraCredentials, "credentials is null"); @@ -54,6 +57,7 @@ public TaskUpdateRequest( requireNonNull(sources, "sources is null"); requireNonNull(outputIds, "outputIds is null"); requireNonNull(totalPartitions, "totalPartitions is null"); + requireNonNull(tableWriteInfo, "tableWriteInfo is null"); this.session = session; this.extraCredentials = extraCredentials; @@ -61,6 +65,7 @@ public TaskUpdateRequest( this.sources = ImmutableList.copyOf(sources); this.outputIds = outputIds; this.totalPartitions = totalPartitions; + this.tableWriteInfo = tableWriteInfo; } @JsonProperty @@ -99,6 +104,12 @@ public OptionalInt getTotalPartitions() return totalPartitions; } + @JsonProperty + public Optional getTableWriteInfo() + { + return tableWriteInfo; + } + @Override public String toString() { diff --git a/presto-main/src/main/java/com/facebook/presto/server/ThrowableMapper.java b/presto-main/src/main/java/com/facebook/presto/server/ThrowableMapper.java index 08f38f87094b2..02103191cab01 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ThrowableMapper.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ThrowableMapper.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.log.Logger; import com.google.common.base.Throwables; -import io.airlift.log.Logger; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; diff --git a/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java b/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java index b1a575a37e479..b283f9da51873 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java +++ b/presto-main/src/main/java/com/facebook/presto/server/WorkerResource.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.ResponseHandler; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.spi.NodeState; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.Request; -import io.airlift.http.client.ResponseHandler; import javax.inject.Inject; import javax.ws.rs.GET; @@ -31,9 +31,9 @@ import java.io.InputStream; import java.util.Set; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; import static com.google.common.net.HttpHeaders.CONTENT_TYPE; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.Request.Builder.prepareGet; import static java.util.Objects.requireNonNull; import static javax.ws.rs.core.MediaType.APPLICATION_JSON; import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE; @@ -94,7 +94,7 @@ public InputStream handleException(Request request, Exception exception) } @Override - public InputStream handle(Request request, io.airlift.http.client.Response response) + public InputStream handle(Request request, com.facebook.airlift.http.client.Response response) { try { if (APPLICATION_JSON.equals(response.getHeader(CONTENT_TYPE))) { diff --git a/presto-main/src/main/java/com/facebook/presto/server/protocol/PurgeQueriesRunnable.java b/presto-main/src/main/java/com/facebook/presto/server/protocol/PurgeQueriesRunnable.java index 07af5a3e2a1f4..5ae544198bdf9 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/protocol/PurgeQueriesRunnable.java +++ b/presto-main/src/main/java/com/facebook/presto/server/protocol/PurgeQueriesRunnable.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.server.protocol; +import com.facebook.airlift.log.Logger; import com.facebook.presto.execution.QueryManager; import com.facebook.presto.execution.QueryState; import com.facebook.presto.spi.QueryId; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; import java.util.NoSuchElementException; import java.util.concurrent.ConcurrentMap; diff --git a/presto-main/src/main/java/com/facebook/presto/server/protocol/Query.java b/presto-main/src/main/java/com/facebook/presto/server/protocol/Query.java index 5c1936cb472a9..63670f5cb6d5c 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/protocol/Query.java +++ b/presto-main/src/main/java/com/facebook/presto/server/protocol/Query.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.server.protocol; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.client.Column; import com.facebook.presto.client.FailureInfo; @@ -25,6 +26,8 @@ import com.facebook.presto.execution.QueryManager; import com.facebook.presto.execution.QueryState; import com.facebook.presto.execution.QueryStats; +import com.facebook.presto.execution.StageExecutionInfo; +import com.facebook.presto.execution.StageExecutionStats; import com.facebook.presto.execution.StageInfo; import com.facebook.presto.execution.TaskInfo; import com.facebook.presto.execution.buffer.PagesSerde; @@ -51,7 +54,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.log.Logger; import io.airlift.units.DataSize; import io.airlift.units.Duration; @@ -74,6 +76,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicLong; +import static com.facebook.airlift.concurrent.MoreFutures.addSuccessCallback; +import static com.facebook.airlift.concurrent.MoreFutures.addTimeout; import static com.facebook.presto.SystemSessionProperties.isExchangeCompressionEnabled; import static com.facebook.presto.execution.QueryState.FAILED; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; @@ -82,8 +86,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.util.concurrent.Futures.immediateFuture; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.addSuccessCallback; -import static io.airlift.concurrent.MoreFutures.addTimeout; import static java.lang.String.format; import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; @@ -134,9 +136,6 @@ class Query @GuardedBy("this") private Optional setSchema = Optional.empty(); - @GuardedBy("this") - private Optional setPath = Optional.empty(); - @GuardedBy("this") private Map setSessionProperties = ImmutableMap.of(); @@ -260,11 +259,6 @@ public synchronized Optional getSetSchema() return setSchema; } - public synchronized Optional getSetPath() - { - return setPath; - } - public synchronized Map getSetSessionProperties() { return setSessionProperties; @@ -482,10 +476,9 @@ public synchronized QueryResults getNextResult(OptionalLong token, UriInfo uriIn nextResultsUri = createNextResultsUri(scheme, uriInfo); } - // update catalog, schema, and path + // update catalog and schema setCatalog = queryInfo.getSetCatalog(); setSchema = queryInfo.getSetSchema(); - setPath = queryInfo.getSetPath(); // update setSessionProperties setSessionProperties = queryInfo.getSetSessionProperties(); @@ -579,9 +572,9 @@ private synchronized URI createNextResultsUri(String scheme, UriInfo uriInfo) .scheme(scheme) .replacePath("/v1/statement") .path(queryId.toString()) - .path(slug) .path(String.valueOf(resultId.incrementAndGet())) .replaceQuery("") + .queryParam("slug", slug) .build(); } @@ -617,7 +610,8 @@ private static StageStats toStageStats(StageInfo stageInfo) return null; } - com.facebook.presto.execution.StageStats stageStats = stageInfo.getStageStats(); + StageExecutionInfo currentStageExecutionInfo = stageInfo.getLatestAttemptExecutionInfo(); + StageExecutionStats stageExecutionStats = currentStageExecutionInfo.getStats(); ImmutableList.Builder subStages = ImmutableList.builder(); for (StageInfo subStage : stageInfo.getSubStages()) { @@ -625,7 +619,7 @@ private static StageStats toStageStats(StageInfo stageInfo) } Set uniqueNodes = new HashSet<>(); - for (TaskInfo task : stageInfo.getTasks()) { + for (TaskInfo task : currentStageExecutionInfo.getTasks()) { // todo add nodeId to TaskInfo URI uri = task.getTaskStatus().getSelf(); uniqueNodes.add(uri.getHost() + ":" + uri.getPort()); @@ -633,17 +627,17 @@ private static StageStats toStageStats(StageInfo stageInfo) return StageStats.builder() .setStageId(String.valueOf(stageInfo.getStageId().getId())) - .setState(stageInfo.getState().toString()) - .setDone(stageInfo.getState().isDone()) + .setState(currentStageExecutionInfo.getState().toString()) + .setDone(currentStageExecutionInfo.getState().isDone()) .setNodes(uniqueNodes.size()) - .setTotalSplits(stageStats.getTotalDrivers()) - .setQueuedSplits(stageStats.getQueuedDrivers()) - .setRunningSplits(stageStats.getRunningDrivers() + stageStats.getBlockedDrivers()) - .setCompletedSplits(stageStats.getCompletedDrivers()) - .setCpuTimeMillis(stageStats.getTotalCpuTime().toMillis()) - .setWallTimeMillis(stageStats.getTotalScheduledTime().toMillis()) - .setProcessedRows(stageStats.getRawInputPositions()) - .setProcessedBytes(stageStats.getRawInputDataSize().toBytes()) + .setTotalSplits(stageExecutionStats.getTotalDrivers()) + .setQueuedSplits(stageExecutionStats.getQueuedDrivers()) + .setRunningSplits(stageExecutionStats.getRunningDrivers() + stageExecutionStats.getBlockedDrivers()) + .setCompletedSplits(stageExecutionStats.getCompletedDrivers()) + .setCpuTimeMillis(stageExecutionStats.getTotalCpuTime().toMillis()) + .setWallTimeMillis(stageExecutionStats.getTotalScheduledTime().toMillis()) + .setProcessedRows(stageExecutionStats.getRawInputPositions()) + .setProcessedBytes(stageExecutionStats.getRawInputDataSize().toBytes()) .setSubStages(subStages.build()) .build(); } @@ -654,7 +648,7 @@ private static Set globalUniqueNodes(StageInfo stageInfo) return ImmutableSet.of(); } ImmutableSet.Builder nodes = ImmutableSet.builder(); - for (TaskInfo task : stageInfo.getTasks()) { + for (TaskInfo task : stageInfo.getLatestAttemptExecutionInfo().getTasks()) { // todo add nodeId to TaskInfo URI uri = task.getTaskStatus().getSelf(); nodes.add(uri.getHost() + ":" + uri.getPort()); @@ -675,7 +669,7 @@ private static URI findCancelableLeafStage(QueryInfo queryInfo) private static URI findCancelableLeafStage(StageInfo stage) { // if this stage is already done, we can't cancel it - if (stage.getState().isDone()) { + if (stage.getLatestAttemptExecutionInfo().getState().isDone()) { return null; } diff --git a/presto-main/src/main/java/com/facebook/presto/server/protocol/StatementResource.java b/presto-main/src/main/java/com/facebook/presto/server/protocol/StatementResource.java index c4c03b88aea0c..9ef4fd145ee1b 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/protocol/StatementResource.java +++ b/presto-main/src/main/java/com/facebook/presto/server/protocol/StatementResource.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.server.protocol; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.client.QueryResults; import com.facebook.presto.execution.QueryManager; import com.facebook.presto.memory.context.SimpleLocalMemoryContext; @@ -27,8 +29,6 @@ import com.google.common.collect.Ordering; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.stats.CounterStat; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.weakref.jmx.Managed; @@ -64,12 +64,13 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; +import static com.facebook.airlift.http.server.AsyncResponseHandler.bindAsyncResponse; import static com.facebook.presto.client.PrestoHeaders.PRESTO_ADDED_PREPARE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLEAR_SESSION; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLEAR_TRANSACTION_ID; import static com.facebook.presto.client.PrestoHeaders.PRESTO_DEALLOCATED_PREPARE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SET_CATALOG; -import static com.facebook.presto.client.PrestoHeaders.PRESTO_SET_PATH; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SET_ROLE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SET_SCHEMA; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SET_SESSION; @@ -78,8 +79,6 @@ import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.net.HttpHeaders.X_FORWARDED_PROTO; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.Threads.threadsNamed; -import static io.airlift.http.server.AsyncResponseHandler.bindAsyncResponse; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; @@ -172,12 +171,12 @@ public Response createQuery( } @GET - @Path("{queryId}/{slug}/{token}") + @Path("{queryId}/{token}") @Produces(MediaType.APPLICATION_JSON) public void getQueryResults( @PathParam("queryId") QueryId queryId, - @PathParam("slug") String slug, @PathParam("token") long token, + @QueryParam("slug") String slug, @QueryParam("maxWait") Duration maxWait, @QueryParam("targetResultSize") DataSize targetResultSize, @HeaderParam(X_FORWARDED_PROTO) String proto, @@ -233,9 +232,9 @@ private static Response toResponse(Query query, QueryResults queryResults) { ResponseBuilder response = Response.ok(queryResults); + // add set catalog and schema query.getSetCatalog().ifPresent(catalog -> response.header(PRESTO_SET_CATALOG, catalog)); query.getSetSchema().ifPresent(schema -> response.header(PRESTO_SET_SCHEMA, schema)); - query.getSetPath().ifPresent(path -> response.header(PRESTO_SET_PATH, path)); // add set session properties query.getSetSessionProperties().entrySet() @@ -274,12 +273,12 @@ private static Response toResponse(Query query, QueryResults queryResults) } @DELETE - @Path("{queryId}/{slug}/{token}") + @Path("{queryId}/{token}") @Produces(MediaType.APPLICATION_JSON) public Response cancelQuery( @PathParam("queryId") QueryId queryId, - @PathParam("slug") String slug, - @PathParam("token") long token) + @PathParam("token") long token, + @QueryParam("slug") String slug) { Query query = getQuery(queryId, slug); if (query == null) { diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java index cf16512be3dc0..9bf710684d5a8 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java @@ -13,6 +13,11 @@ */ package com.facebook.presto.server.remotetask; +import com.facebook.airlift.concurrent.SetThreadName; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.ResponseHandler; +import com.facebook.airlift.log.Logger; import com.facebook.presto.execution.StateMachine; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.TaskStatus; @@ -23,11 +28,6 @@ import com.facebook.presto.spi.PrestoException; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.concurrent.SetThreadName; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.Request; -import io.airlift.http.client.ResponseHandler; -import io.airlift.log.Logger; import io.airlift.units.Duration; import javax.annotation.concurrent.GuardedBy; @@ -38,6 +38,8 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CURRENT_STATE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_WAIT; import static com.facebook.presto.server.RequestHelpers.setContentTypeHeaders; @@ -47,8 +49,6 @@ import static com.facebook.presto.spi.StandardErrorCode.REMOTE_TASK_MISMATCH; import static com.facebook.presto.util.Failures.REMOTE_TASK_MISMATCH_ERROR; import static com.google.common.base.Strings.isNullOrEmpty; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.Request.Builder.prepareGet; import static io.airlift.units.Duration.nanosSince; import static java.lang.String.format; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java index ee961e63c948f..db6df492df45b 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpLocationFactory.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.server.remotetask; +import com.facebook.airlift.http.server.HttpServerInfo; import com.facebook.presto.execution.LocationFactory; import com.facebook.presto.execution.StageId; import com.facebook.presto.execution.TaskId; @@ -20,13 +21,12 @@ import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.server.InternalCommunicationConfig; import com.facebook.presto.spi.QueryId; -import io.airlift.http.server.HttpServerInfo; import javax.inject.Inject; import java.net.URI; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; import static java.util.Objects.requireNonNull; public class HttpLocationFactory diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java index 9f6f5faf08f0d..08ab4985a68b5 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java @@ -13,6 +13,13 @@ */ package com.facebook.presto.server.remotetask; +import com.facebook.airlift.concurrent.SetThreadName; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpUriBuilder; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.ResponseHandler; +import com.facebook.airlift.http.client.StatusResponseHandler.StatusResponse; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.execution.FutureStateChange; import com.facebook.presto.execution.Lifespan; @@ -28,6 +35,7 @@ import com.facebook.presto.execution.buffer.BufferInfo; import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.execution.buffer.PageBufferInfo; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.metadata.Split; import com.facebook.presto.operator.TaskStats; import com.facebook.presto.server.TaskUpdateRequest; @@ -48,13 +56,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import io.airlift.concurrent.SetThreadName; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpUriBuilder; -import io.airlift.http.client.Request; -import io.airlift.http.client.ResponseHandler; -import io.airlift.http.client.StatusResponseHandler.StatusResponse; -import io.airlift.log.Logger; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -75,12 +76,20 @@ import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; +import static com.facebook.airlift.http.client.HttpStatus.NO_CONTENT; +import static com.facebook.airlift.http.client.HttpStatus.OK; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.Request.Builder.prepareDelete; +import static com.facebook.airlift.http.client.Request.Builder.preparePost; +import static com.facebook.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; +import static com.facebook.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; import static com.facebook.presto.execution.TaskInfo.createInitialTask; import static com.facebook.presto.execution.TaskState.ABORTED; import static com.facebook.presto.execution.TaskState.FAILED; @@ -90,6 +99,7 @@ import static com.facebook.presto.server.smile.AdaptingJsonResponseHandler.createAdaptingJsonResponseHandler; import static com.facebook.presto.server.smile.FullSmileResponseHandler.createFullSmileResponseHandler; import static com.facebook.presto.server.smile.JsonCodecWrapper.unwrapJsonCodec; +import static com.facebook.presto.spi.StandardErrorCode.EXCEEDED_TASK_UPDATE_SIZE_LIMIT; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.util.Failures.toFailure; import static com.google.common.base.MoreObjects.toStringHelper; @@ -100,13 +110,7 @@ import static com.google.common.util.concurrent.Futures.addCallback; import static com.google.common.util.concurrent.Futures.immediateFuture; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.http.client.HttpStatus.NO_CONTENT; -import static io.airlift.http.client.HttpStatus.OK; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.Request.Builder.prepareDelete; -import static io.airlift.http.client.Request.Builder.preparePost; -import static io.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; -import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -116,6 +120,7 @@ public final class HttpRemoteTask implements RemoteTask { private static final Logger log = Logger.get(HttpRemoteTask.class); + private static final double UPDATE_WITHOUT_PLAN_STATS_SAMPLE_RATE = 0.01; private final TaskId taskId; private final URI taskLocation; @@ -177,6 +182,9 @@ public final class HttpRemoteTask private final AtomicBoolean aborting = new AtomicBoolean(false); private final boolean isBinaryTransportEnabled; + private final int maxTaskUpdateSizeInBytes; + + private final TableWriteInfo tableWriteInfo; public HttpRemoteTask( Session session, @@ -201,7 +209,9 @@ public HttpRemoteTask( Codec taskUpdateRequestCodec, PartitionedSplitCountTracker partitionedSplitCountTracker, RemoteTaskStats stats, - boolean isBinaryTransportEnabled) + boolean isBinaryTransportEnabled, + TableWriteInfo tableWriteInfo, + int maxTaskUpdateSizeInBytes) { requireNonNull(session, "session is null"); requireNonNull(taskId, "taskId is null"); @@ -219,6 +229,7 @@ public HttpRemoteTask( requireNonNull(maxErrorDuration, "maxErrorDuration is null"); requireNonNull(stats, "stats is null"); requireNonNull(taskInfoRefreshMaxWait, "taskInfoRefreshMaxWait is null"); + requireNonNull(tableWriteInfo, "tableWriteInfo is null"); try (SetThreadName ignored = new SetThreadName("HttpRemoteTask-%s", taskId)) { this.taskId = taskId; @@ -239,6 +250,8 @@ public HttpRemoteTask( this.maxErrorDuration = maxErrorDuration; this.stats = stats; this.isBinaryTransportEnabled = isBinaryTransportEnabled; + this.tableWriteInfo = tableWriteInfo; + this.maxTaskUpdateSizeInBytes = maxTaskUpdateSizeInBytes; this.tableScanPlanNodeIds = ImmutableSet.copyOf(planFragment.getTableScanSchedulingOrder()); this.remoteSourcePlanNodeIds = planFragment.getRemoteSourceNodes().stream() @@ -439,7 +452,8 @@ private void doRemoveRemoteSource(RequestErrorTracker errorTracker, Request requ { errorTracker.startRequest(); - FutureCallback callback = new FutureCallback() { + FutureCallback callback = new FutureCallback() + { @Override public void onSuccess(@Nullable StatusResponse response) { @@ -611,16 +625,29 @@ private synchronized void sendUpdate() List sources = getSources(); Optional fragment = sendPlan.get() ? Optional.of(planFragment) : Optional.empty(); + Optional writeInfo = sendPlan.get() ? Optional.of(tableWriteInfo) : Optional.empty(); TaskUpdateRequest updateRequest = new TaskUpdateRequest( session.toSessionRepresentation(), session.getIdentity().getExtraCredentials(), fragment, sources, outputBuffers.get(), - totalPartitions); + totalPartitions, + writeInfo); byte[] taskUpdateRequestJson = taskUpdateRequestCodec.toBytes(updateRequest); + + if (taskUpdateRequestJson.length > maxTaskUpdateSizeInBytes) { + throw new PrestoException(EXCEEDED_TASK_UPDATE_SIZE_LIMIT, format("TaskUpdate size of %d Bytes has exceeded the limit of %d Bytes", taskUpdateRequestJson.length, maxTaskUpdateSizeInBytes)); + } + if (fragment.isPresent()) { - stats.updateWithPlanBytes(taskUpdateRequestJson.length); + stats.updateWithPlanSize(taskUpdateRequestJson.length); + } + else { + if (ThreadLocalRandom.current().nextDouble() < UPDATE_WITHOUT_PLAN_STATS_SAMPLE_RATE) { + // This is to keep track of the task update size even when the plan fragment is NOT present + stats.updateWithoutPlanSize(taskUpdateRequestJson.length); + } } HttpUriBuilder uriBuilder = getHttpUriBuilder(taskStatus); diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java index 3a7705257086e..f295fd19c02b7 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java @@ -13,6 +13,11 @@ */ package com.facebook.presto.server.remotetask; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.execution.LocationFactory; import com.facebook.presto.execution.NodeTaskMap.PartitionedSplitCountTracker; @@ -24,6 +29,7 @@ import com.facebook.presto.execution.TaskManagerConfig; import com.facebook.presto.execution.TaskStatus; import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.operator.ForScheduler; @@ -34,11 +40,6 @@ import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.sql.planner.PlanFragment; import com.google.common.collect.Multimap; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.concurrent.ThreadPoolExecutorMBean; -import io.airlift.http.client.HttpClient; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import io.airlift.units.Duration; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -52,8 +53,9 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.server.smile.JsonCodecWrapper.wrapJsonCodec; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; @@ -79,6 +81,7 @@ public class HttpRemoteTaskFactory private final ScheduledExecutorService errorScheduledExecutor; private final RemoteTaskStats stats; private final boolean isBinaryTransportEnabled; + private final int maxTaskUpdateSizeInBytes; @Inject public HttpRemoteTaskFactory(QueryManagerConfig config, @@ -105,6 +108,7 @@ public HttpRemoteTaskFactory(QueryManagerConfig config, this.executorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) coreExecutor); this.stats = requireNonNull(stats, "stats is null"); isBinaryTransportEnabled = requireNonNull(communicationConfig, "communicationConfig is null").isBinaryTransportEnabled(); + this.maxTaskUpdateSizeInBytes = toIntExact(requireNonNull(communicationConfig, "communicationConfig is null").getMaxTaskUpdateSize().toBytes()); if (isBinaryTransportEnabled) { this.taskStatusCodec = taskStatusSmileCodec; @@ -145,7 +149,8 @@ public RemoteTask createRemoteTask(Session session, OptionalInt totalPartitions, OutputBuffers outputBuffers, PartitionedSplitCountTracker partitionedSplitCountTracker, - boolean summarizeTaskInfo) + boolean summarizeTaskInfo, + TableWriteInfo tableWriteInfo) { return new HttpRemoteTask(session, taskId, @@ -169,6 +174,8 @@ public RemoteTask createRemoteTask(Session session, taskUpdateRequestCodec, partitionedSplitCountTracker, stats, - isBinaryTransportEnabled); + isBinaryTransportEnabled, + tableWriteInfo, + maxTaskUpdateSizeInBytes); } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/RemoteTaskStats.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/RemoteTaskStats.java index 12e56c22eef05..4b2aaf54e5a47 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/RemoteTaskStats.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/RemoteTaskStats.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.server.remotetask; -import io.airlift.stats.DistributionStat; +import com.facebook.airlift.stats.DistributionStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -25,7 +25,8 @@ public class RemoteTaskStats private final IncrementalAverage infoRoundTripMillis = new IncrementalAverage(); private final IncrementalAverage statusRoundTripMillis = new IncrementalAverage(); private final IncrementalAverage responseSizeBytes = new IncrementalAverage(); - private final DistributionStat updateWithPlanBytes = new DistributionStat(); + private final DistributionStat updateWithPlanSize = new DistributionStat(); + private final DistributionStat updateWithoutPlanSize = new DistributionStat(); private long requestSuccess; private long requestFailure; @@ -60,33 +61,56 @@ public void updateFailure() requestFailure++; } - public void updateWithPlanBytes(long bytes) + public void updateWithPlanSize(long bytes) { - updateWithPlanBytes.add(bytes); + updateWithPlanSize.add(bytes); + } + + public void updateWithoutPlanSize(long bytes) + { + updateWithoutPlanSize.add(bytes); } @Managed public double getResponseSizeBytes() { - return responseSizeBytes.get(); + return responseSizeBytes.getAverage(); } @Managed public double getStatusRoundTripMillis() { - return statusRoundTripMillis.get(); + return statusRoundTripMillis.getAverage(); + } + + @Managed + public long getStatusRoundTripCount() + { + return statusRoundTripMillis.getCount(); } @Managed public double getUpdateRoundTripMillis() { - return updateRoundTripMillis.get(); + return updateRoundTripMillis.getAverage(); + } + + @Managed + public long getUpdateRoundTripCount() + { + return updateRoundTripMillis.getCount(); } @Managed public double getInfoRoundTripMillis() { - return infoRoundTripMillis.get(); + return infoRoundTripMillis.getAverage(); + } + + @Managed + public long getInfoRoundTripCount() + { + return infoRoundTripMillis.getCount(); } @Managed @@ -103,15 +127,22 @@ public long getRequestFailure() @Managed @Nested - public DistributionStat getUpdateWithPlanBytes() + public DistributionStat getUpdateWithPlanSize() { - return updateWithPlanBytes; + return updateWithPlanSize; + } + + @Managed + @Nested + public DistributionStat getUpdateWithoutPlanSize() + { + return updateWithoutPlanSize; } @ThreadSafe private static class IncrementalAverage { - private long count; + private volatile long count; private volatile double average; synchronized void add(long value) @@ -120,9 +151,14 @@ synchronized void add(long value) average = average + (value - average) / count; } - double get() + double getAverage() { return average; } + + long getCount() + { + return count; + } } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/RequestErrorTracker.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/RequestErrorTracker.java index 77a77dfd91109..f2ca8db6dce48 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/RequestErrorTracker.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/RequestErrorTracker.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.server.remotetask; +import com.facebook.airlift.event.client.ServiceUnavailableException; +import com.facebook.airlift.log.Logger; import com.facebook.presto.execution.TaskId; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.PrestoTransportException; @@ -20,8 +22,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFutureTask; -import io.airlift.event.client.ServiceUnavailableException; -import io.airlift.log.Logger; import io.airlift.units.Duration; import javax.annotation.concurrent.ThreadSafe; diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/SimpleHttpResponseHandler.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/SimpleHttpResponseHandler.java index e9c006450330e..03110acdf4c15 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/SimpleHttpResponseHandler.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/SimpleHttpResponseHandler.java @@ -13,17 +13,17 @@ */ package com.facebook.presto.server.remotetask; +import com.facebook.airlift.http.client.HttpStatus; import com.facebook.presto.server.smile.BaseResponse; import com.facebook.presto.server.smile.JsonResponseWrapper; import com.facebook.presto.spi.PrestoException; import com.google.common.util.concurrent.FutureCallback; -import io.airlift.http.client.HttpStatus; import java.net.URI; +import static com.facebook.airlift.http.client.HttpStatus.OK; import static com.facebook.presto.server.smile.JsonResponseWrapper.unwrapJsonResponse; import static com.facebook.presto.spi.StandardErrorCode.REMOTE_TASK_ERROR; -import static io.airlift.http.client.HttpStatus.OK; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -91,7 +91,8 @@ private String createErrorMessage(BaseResponse response) uri, OK.code(), response.getStatusCode(), - response.getStatusMessage()); + response.getStatusMessage(), + new String(response.getResponseBytes())); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/TaskInfoFetcher.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/TaskInfoFetcher.java index 9151aefd3e921..e2312fe4ef250 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/TaskInfoFetcher.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/TaskInfoFetcher.java @@ -13,6 +13,11 @@ */ package com.facebook.presto.server.remotetask; +import com.facebook.airlift.concurrent.SetThreadName; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpUriBuilder; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.ResponseHandler; import com.facebook.presto.execution.StateMachine; import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.facebook.presto.execution.TaskId; @@ -23,11 +28,6 @@ import com.facebook.presto.server.smile.SmileCodec; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.concurrent.SetThreadName; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpUriBuilder; -import io.airlift.http.client.Request; -import io.airlift.http.client.ResponseHandler; import io.airlift.units.Duration; import javax.annotation.concurrent.GuardedBy; @@ -41,14 +41,14 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CURRENT_STATE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_WAIT; import static com.facebook.presto.server.RequestHelpers.setContentTypeHeaders; import static com.facebook.presto.server.smile.AdaptingJsonResponseHandler.createAdaptingJsonResponseHandler; import static com.facebook.presto.server.smile.FullSmileResponseHandler.createFullSmileResponseHandler; import static com.facebook.presto.server.smile.JsonCodecWrapper.unwrapJsonCodec; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.Request.Builder.prepareGet; import static io.airlift.units.Duration.nanosSince; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; diff --git a/presto-main/src/main/java/com/facebook/presto/server/security/JsonWebTokenAuthenticator.java b/presto-main/src/main/java/com/facebook/presto/server/security/JsonWebTokenAuthenticator.java index 72d26ca77eddf..068494610d73a 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/security/JsonWebTokenAuthenticator.java +++ b/presto-main/src/main/java/com/facebook/presto/server/security/JsonWebTokenAuthenticator.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.server.security; +import com.facebook.airlift.security.pem.PemReader; import com.facebook.presto.spi.security.BasicPrincipal; import com.google.common.base.CharMatcher; -import io.airlift.security.pem.PemReader; import io.jsonwebtoken.Claims; import io.jsonwebtoken.Jws; import io.jsonwebtoken.JwsHeader; diff --git a/presto-main/src/main/java/com/facebook/presto/server/security/JsonWebTokenConfig.java b/presto-main/src/main/java/com/facebook/presto/server/security/JsonWebTokenConfig.java index 978b06d75c7a3..1eaddacf1a1cb 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/security/JsonWebTokenConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/security/JsonWebTokenConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.server.security; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import javax.validation.constraints.NotNull; diff --git a/presto-main/src/main/java/com/facebook/presto/server/security/KerberosAuthenticator.java b/presto-main/src/main/java/com/facebook/presto/server/security/KerberosAuthenticator.java index 684db8e9ce103..1ca05becf9525 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/security/KerberosAuthenticator.java +++ b/presto-main/src/main/java/com/facebook/presto/server/security/KerberosAuthenticator.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.server.security; +import com.facebook.airlift.log.Logger; import com.sun.security.auth.module.Krb5LoginModule; -import io.airlift.log.Logger; import org.ietf.jgss.GSSContext; import org.ietf.jgss.GSSCredential; import org.ietf.jgss.GSSException; diff --git a/presto-main/src/main/java/com/facebook/presto/server/security/KerberosConfig.java b/presto-main/src/main/java/com/facebook/presto/server/security/KerberosConfig.java index d3bfc98f8a5ee..ca0416f036eec 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/security/KerberosConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/security/KerberosConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.server.security; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import javax.validation.constraints.NotNull; diff --git a/presto-main/src/main/java/com/facebook/presto/server/security/PasswordAuthenticatorManager.java b/presto-main/src/main/java/com/facebook/presto/server/security/PasswordAuthenticatorManager.java index 7e6fa66ced4c4..9b1a3ccfe4513 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/security/PasswordAuthenticatorManager.java +++ b/presto-main/src/main/java/com/facebook/presto/server/security/PasswordAuthenticatorManager.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.server.security; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.security.PasswordAuthenticator; import com.facebook.presto.spi.security.PasswordAuthenticatorFactory; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; import java.io.File; import java.util.HashMap; diff --git a/presto-main/src/main/java/com/facebook/presto/server/security/SecurityConfig.java b/presto-main/src/main/java/com/facebook/presto/server/security/SecurityConfig.java index 68293cfcba462..31849ea562327 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/security/SecurityConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/security/SecurityConfig.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.server.security; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.DefunctConfig; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.DefunctConfig; import javax.validation.constraints.NotNull; diff --git a/presto-main/src/main/java/com/facebook/presto/server/security/ServerSecurityModule.java b/presto-main/src/main/java/com/facebook/presto/server/security/ServerSecurityModule.java index 81cfb07628828..c1c98d020737c 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/security/ServerSecurityModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/security/ServerSecurityModule.java @@ -13,26 +13,26 @@ */ package com.facebook.presto.server.security; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.airlift.http.server.TheServlet; import com.facebook.presto.server.security.SecurityConfig.AuthenticationType; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; import com.google.inject.Provides; import com.google.inject.Scopes; import com.google.inject.multibindings.Multibinder; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.airlift.http.server.TheServlet; import javax.servlet.Filter; import java.util.List; import java.util.Set; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static com.facebook.presto.server.security.SecurityConfig.AuthenticationType.CERTIFICATE; import static com.facebook.presto.server.security.SecurityConfig.AuthenticationType.JWT; import static com.facebook.presto.server.security.SecurityConfig.AuthenticationType.KERBEROS; import static com.facebook.presto.server.security.SecurityConfig.AuthenticationType.PASSWORD; import static com.google.inject.multibindings.Multibinder.newSetBinder; -import static io.airlift.configuration.ConfigBinder.configBinder; public class ServerSecurityModule extends AbstractConfigurationAwareModule diff --git a/presto-main/src/main/java/com/facebook/presto/server/smile/AdaptingJsonResponseHandler.java b/presto-main/src/main/java/com/facebook/presto/server/smile/AdaptingJsonResponseHandler.java index bd80fa8577464..befd1bdeb0f7d 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/smile/AdaptingJsonResponseHandler.java +++ b/presto-main/src/main/java/com/facebook/presto/server/smile/AdaptingJsonResponseHandler.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.server.smile; -import io.airlift.http.client.FullJsonResponseHandler; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.ResponseHandler; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.http.client.FullJsonResponseHandler; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.ResponseHandler; +import com.facebook.airlift.json.JsonCodec; +import static com.facebook.airlift.http.client.FullJsonResponseHandler.createFullJsonResponseHandler; import static com.facebook.presto.server.smile.JsonResponseWrapper.wrapJsonResponse; -import static io.airlift.http.client.FullJsonResponseHandler.createFullJsonResponseHandler; import static java.util.Objects.requireNonNull; /** diff --git a/presto-main/src/main/java/com/facebook/presto/server/smile/BaseResponse.java b/presto-main/src/main/java/com/facebook/presto/server/smile/BaseResponse.java index fffd365d42785..bb5dd0f23adda 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/smile/BaseResponse.java +++ b/presto-main/src/main/java/com/facebook/presto/server/smile/BaseResponse.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.server.smile; +import com.facebook.airlift.http.client.HeaderName; import com.google.common.collect.ListMultimap; -import io.airlift.http.client.HeaderName; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/server/smile/FullSmileResponseHandler.java b/presto-main/src/main/java/com/facebook/presto/server/smile/FullSmileResponseHandler.java index 2baf9424c0594..6df868249db50 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/smile/FullSmileResponseHandler.java +++ b/presto-main/src/main/java/com/facebook/presto/server/smile/FullSmileResponseHandler.java @@ -13,22 +13,22 @@ */ package com.facebook.presto.server.smile; +import com.facebook.airlift.http.client.HeaderName; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.ResponseHandler; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ListMultimap; import com.google.common.net.MediaType; -import io.airlift.http.client.HeaderName; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.ResponseHandler; import java.io.IOException; import java.io.UncheckedIOException; import java.util.List; +import static com.facebook.airlift.http.client.ResponseHandlerUtils.propagate; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.io.ByteStreams.toByteArray; import static com.google.common.net.HttpHeaders.CONTENT_TYPE; -import static io.airlift.http.client.ResponseHandlerUtils.propagate; import static java.util.Objects.requireNonNull; public class FullSmileResponseHandler diff --git a/presto-main/src/main/java/com/facebook/presto/server/smile/JsonCodecWrapper.java b/presto-main/src/main/java/com/facebook/presto/server/smile/JsonCodecWrapper.java index 8efee7c0518fe..b6afe8680f920 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/smile/JsonCodecWrapper.java +++ b/presto-main/src/main/java/com/facebook/presto/server/smile/JsonCodecWrapper.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.server.smile; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import static com.google.common.base.Verify.verify; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/server/smile/JsonResponseWrapper.java b/presto-main/src/main/java/com/facebook/presto/server/smile/JsonResponseWrapper.java index b52cdab2ca6f9..0f44aa33a3584 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/smile/JsonResponseWrapper.java +++ b/presto-main/src/main/java/com/facebook/presto/server/smile/JsonResponseWrapper.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.server.smile; +import com.facebook.airlift.http.client.FullJsonResponseHandler.JsonResponse; +import com.facebook.airlift.http.client.HeaderName; import com.google.common.collect.ListMultimap; -import io.airlift.http.client.FullJsonResponseHandler.JsonResponse; -import io.airlift.http.client.HeaderName; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/server/smile/SmileBodyGenerator.java b/presto-main/src/main/java/com/facebook/presto/server/smile/SmileBodyGenerator.java index 2ada78e896638..8c9a0dd169540 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/smile/SmileBodyGenerator.java +++ b/presto-main/src/main/java/com/facebook/presto/server/smile/SmileBodyGenerator.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.server.smile; +import com.facebook.airlift.http.client.StaticBodyGenerator; import com.google.common.annotations.Beta; -import io.airlift.http.client.StaticBodyGenerator; @Beta public class SmileBodyGenerator diff --git a/presto-main/src/main/java/com/facebook/presto/server/smile/SmileObjectMapperProvider.java b/presto-main/src/main/java/com/facebook/presto/server/smile/SmileObjectMapperProvider.java index 71ee905ee7d2f..132bcaca0852f 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/smile/SmileObjectMapperProvider.java +++ b/presto-main/src/main/java/com/facebook/presto/server/smile/SmileObjectMapperProvider.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.server.smile; +import com.facebook.airlift.json.ObjectMapperProvider; import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.google.inject.Inject; -import io.airlift.json.ObjectMapperProvider; public class SmileObjectMapperProvider extends ObjectMapperProvider diff --git a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java index eb22455406b6d..d83fc24b00ec4 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java @@ -13,6 +13,22 @@ */ package com.facebook.presto.server.testing; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.discovery.client.Announcer; +import com.facebook.airlift.discovery.client.DiscoveryModule; +import com.facebook.airlift.discovery.client.ServiceAnnouncement; +import com.facebook.airlift.discovery.client.ServiceSelectorManager; +import com.facebook.airlift.discovery.client.testing.TestingDiscoveryModule; +import com.facebook.airlift.event.client.EventModule; +import com.facebook.airlift.http.server.TheServlet; +import com.facebook.airlift.http.server.testing.TestingHttpServer; +import com.facebook.airlift.http.server.testing.TestingHttpServerModule; +import com.facebook.airlift.jaxrs.JaxrsModule; +import com.facebook.airlift.jmx.testing.TestingJmxModule; +import com.facebook.airlift.json.JsonModule; +import com.facebook.airlift.node.testing.TestingNodeModule; +import com.facebook.airlift.tracetoken.TraceTokenModule; import com.facebook.presto.connector.ConnectorManager; import com.facebook.presto.cost.StatsCalculator; import com.facebook.presto.eventlistener.EventListenerManager; @@ -62,24 +78,15 @@ import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.Scopes; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.discovery.client.Announcer; -import io.airlift.discovery.client.DiscoveryModule; -import io.airlift.discovery.client.ServiceAnnouncement; -import io.airlift.discovery.client.ServiceSelectorManager; -import io.airlift.discovery.client.testing.TestingDiscoveryModule; -import io.airlift.event.client.EventModule; -import io.airlift.http.server.testing.TestingHttpServer; -import io.airlift.http.server.testing.TestingHttpServerModule; -import io.airlift.jaxrs.JaxrsModule; -import io.airlift.jmx.testing.TestingJmxModule; -import io.airlift.json.JsonModule; -import io.airlift.node.testing.TestingNodeModule; -import io.airlift.tracetoken.TraceTokenModule; import org.weakref.jmx.guice.MBeanModule; import javax.annotation.concurrent.GuardedBy; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; import java.io.Closeable; import java.io.IOException; @@ -95,14 +102,15 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; +import static com.facebook.airlift.discovery.client.ServiceAnnouncement.serviceAnnouncement; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.base.Throwables.throwIfUnchecked; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.configuration.ConditionalModule.installModuleIf; -import static io.airlift.discovery.client.ServiceAnnouncement.serviceAnnouncement; -import static io.airlift.json.JsonBinder.jsonBinder; +import static com.google.inject.multibindings.Multibinder.newSetBinder; import static java.lang.Integer.parseInt; import static java.nio.file.Files.createTempDirectory; import static java.nio.file.Files.isDirectory; @@ -139,6 +147,7 @@ public class TestingPrestoServer private final TaskManager taskManager; private final GracefulShutdownHandler gracefulShutdownHandler; private final ShutdownAction shutdownAction; + private final RequestBlocker requestBlocker; private final boolean coordinator; public static class TestShutdownAction @@ -252,6 +261,9 @@ public TestingPrestoServer( binder.bind(ShutdownAction.class).to(TestShutdownAction.class).in(Scopes.SINGLETON); binder.bind(GracefulShutdownHandler.class).in(Scopes.SINGLETON); binder.bind(ProcedureTester.class).in(Scopes.SINGLETON); + binder.bind(RequestBlocker.class).in(Scopes.SINGLETON); + newSetBinder(binder, Filter.class, TheServlet.class).addBinding() + .to(RequestBlocker.class).in(Scopes.SINGLETON); }); if (discoveryUri != null) { @@ -324,6 +336,7 @@ public TestingPrestoServer( taskManager = injector.getInstance(TaskManager.class); shutdownAction = injector.getInstance(ShutdownAction.class); announcer = injector.getInstance(Announcer.class); + requestBlocker = injector.getInstance(RequestBlocker.class); announcer.forceAnnounce(); @@ -512,6 +525,16 @@ public T getInstance(Key key) return injector.getInstance(key); } + public void stopResponding() + { + requestBlocker.block(); + } + + public void startResponding() + { + requestBlocker.unblock(); + } + private static void updateConnectorIdAnnouncement(Announcer announcer, ConnectorId connectorId, InternalNodeManager nodeManager) { // @@ -555,4 +578,51 @@ private static Path tempDirectory() throw new UncheckedIOException(e); } } + + private static class RequestBlocker + implements Filter + { + private static final Object monitor = new Object(); + private volatile boolean blocked; + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException + { + synchronized (monitor) { + while (blocked) { + try { + monitor.wait(); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } + + chain.doFilter(request, response); + } + + public void block() + { + synchronized (monitor) { + blocked = true; + } + } + + public void unblock() + { + synchronized (monitor) { + blocked = false; + monitor.notifyAll(); + } + } + + @Override + public void init(FilterConfig filterConfig) {} + + @Override + public void destroy() {} + } } diff --git a/presto-main/src/main/java/com/facebook/presto/spiller/FileSingleStreamSpillerFactory.java b/presto-main/src/main/java/com/facebook/presto/spiller/FileSingleStreamSpillerFactory.java index b3301e413ab47..63cc32b2205e9 100644 --- a/presto-main/src/main/java/com/facebook/presto/spiller/FileSingleStreamSpillerFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/spiller/FileSingleStreamSpillerFactory.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spiller; +import com.facebook.airlift.log.Logger; import com.facebook.presto.execution.buffer.PagesSerde; import com.facebook.presto.execution.buffer.PagesSerdeFactory; import com.facebook.presto.memory.context.LocalMemoryContext; @@ -25,7 +26,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.inject.Inject; -import io.airlift.log.Logger; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -37,9 +37,9 @@ import java.util.List; import java.util.Optional; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.spi.StandardErrorCode.OUT_OF_SPILL_SPACE; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.format; import static java.nio.file.Files.createDirectories; import static java.nio.file.Files.delete; diff --git a/presto-main/src/main/java/com/facebook/presto/spiller/GenericPartitioningSpiller.java b/presto-main/src/main/java/com/facebook/presto/spiller/GenericPartitioningSpiller.java index 83c2dcb28278a..01ef6665fd140 100644 --- a/presto-main/src/main/java/com/facebook/presto/spiller/GenericPartitioningSpiller.java +++ b/presto-main/src/main/java/com/facebook/presto/spiller/GenericPartitioningSpiller.java @@ -37,10 +37,10 @@ import java.util.function.IntPredicate; import java.util.function.Predicate; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static java.util.Objects.requireNonNull; @ThreadSafe diff --git a/presto-main/src/main/java/com/facebook/presto/spiller/NodeSpillConfig.java b/presto-main/src/main/java/com/facebook/presto/spiller/NodeSpillConfig.java index 6aeaf32986ccc..d36d16a5ebb9d 100644 --- a/presto-main/src/main/java/com/facebook/presto/spiller/NodeSpillConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/spiller/NodeSpillConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.spiller; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import io.airlift.units.DataSize; import javax.validation.constraints.NotNull; diff --git a/presto-main/src/main/java/com/facebook/presto/split/CloseableSplitSourceProvider.java b/presto-main/src/main/java/com/facebook/presto/split/CloseableSplitSourceProvider.java index 455e0b5101b34..8ad553d5a3707 100644 --- a/presto-main/src/main/java/com/facebook/presto/split/CloseableSplitSourceProvider.java +++ b/presto-main/src/main/java/com/facebook/presto/split/CloseableSplitSourceProvider.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.split; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy; -import io.airlift.log.Logger; import javax.annotation.concurrent.GuardedBy; diff --git a/presto-main/src/main/java/com/facebook/presto/split/ConnectorAwareSplitSource.java b/presto-main/src/main/java/com/facebook/presto/split/ConnectorAwareSplitSource.java index a324df4f21ab9..f9cd0fe8f38c0 100644 --- a/presto-main/src/main/java/com/facebook/presto/split/ConnectorAwareSplitSource.java +++ b/presto-main/src/main/java/com/facebook/presto/split/ConnectorAwareSplitSource.java @@ -25,8 +25,8 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import static com.facebook.airlift.concurrent.MoreFutures.toListenableFuture; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.toListenableFuture; import static java.util.Objects.requireNonNull; public class ConnectorAwareSplitSource diff --git a/presto-main/src/main/java/com/facebook/presto/split/EmptySplitPageSource.java b/presto-main/src/main/java/com/facebook/presto/split/EmptySplitPageSource.java index 5515751ccd379..e14888d78008b 100644 --- a/presto-main/src/main/java/com/facebook/presto/split/EmptySplitPageSource.java +++ b/presto-main/src/main/java/com/facebook/presto/split/EmptySplitPageSource.java @@ -45,6 +45,12 @@ public long getCompletedBytes() return 0; } + @Override + public long getCompletedPositions() + { + return 0; + } + @Override public long getReadTimeNanos() { diff --git a/presto-main/src/main/java/com/facebook/presto/split/MappedPageSource.java b/presto-main/src/main/java/com/facebook/presto/split/MappedPageSource.java index c0c4aef54051f..5ea400f07ad12 100644 --- a/presto-main/src/main/java/com/facebook/presto/split/MappedPageSource.java +++ b/presto-main/src/main/java/com/facebook/presto/split/MappedPageSource.java @@ -42,6 +42,12 @@ public long getCompletedBytes() return delegate.getCompletedBytes(); } + @Override + public long getCompletedPositions() + { + return delegate.getCompletedPositions(); + } + @Override public long getReadTimeNanos() { diff --git a/presto-main/src/main/java/com/facebook/presto/split/PageSourceManager.java b/presto-main/src/main/java/com/facebook/presto/split/PageSourceManager.java index 5ee36435480e2..4291faba5f3e4 100644 --- a/presto-main/src/main/java/com/facebook/presto/split/PageSourceManager.java +++ b/presto-main/src/main/java/com/facebook/presto/split/PageSourceManager.java @@ -19,6 +19,7 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; import java.util.List; @@ -47,12 +48,15 @@ public void removeConnectorPageSourceProvider(ConnectorId connectorId) } @Override - public ConnectorPageSource createPageSource(Session session, Split split, List columns) + public ConnectorPageSource createPageSource(Session session, Split split, TableHandle table, List columns) { requireNonNull(split, "split is null"); requireNonNull(columns, "columns is null"); ConnectorSession connectorSession = session.toConnectorSession(split.getConnectorId()); + if (table.getLayout().isPresent()) { + return getPageSourceProvider(split).createPageSource(split.getTransactionHandle(), connectorSession, split.getConnectorSplit(), table.getLayout().get(), columns); + } return getPageSourceProvider(split).createPageSource(split.getTransactionHandle(), connectorSession, split.getConnectorSplit(), columns); } diff --git a/presto-main/src/main/java/com/facebook/presto/split/PageSourceProvider.java b/presto-main/src/main/java/com/facebook/presto/split/PageSourceProvider.java index 4aa7ba37a6438..1fb802f239332 100644 --- a/presto-main/src/main/java/com/facebook/presto/split/PageSourceProvider.java +++ b/presto-main/src/main/java/com/facebook/presto/split/PageSourceProvider.java @@ -17,10 +17,11 @@ import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorPageSource; +import com.facebook.presto.spi.TableHandle; import java.util.List; public interface PageSourceProvider { - ConnectorPageSource createPageSource(Session session, Split split, List columns); + ConnectorPageSource createPageSource(Session session, Split split, TableHandle table, List columns); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/InterpretedFunctionInvoker.java b/presto-main/src/main/java/com/facebook/presto/sql/InterpretedFunctionInvoker.java index b929cf65dd2ae..0460c34416a58 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/InterpretedFunctionInvoker.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/InterpretedFunctionInvoker.java @@ -14,8 +14,8 @@ package com.facebook.presto.sql; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.function.FunctionHandle; import com.google.common.base.Defaults; @@ -25,9 +25,9 @@ import java.util.Arrays; import java.util.List; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentType.VALUE_TYPE; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentType.VALUE_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; import static com.google.common.base.Throwables.throwIfUnchecked; import static java.lang.invoke.MethodHandleProxies.asInterfaceInstance; import static java.util.Objects.requireNonNull; @@ -48,7 +48,7 @@ public Object invoke(FunctionHandle functionHandle, ConnectorSession session, Ob public Object invoke(FunctionHandle functionHandle, ConnectorSession session, List arguments) { - return invoke(functionManager.getScalarFunctionImplementation(functionHandle), session, arguments); + return invoke(functionManager.getBuiltInScalarFunctionImplementation(functionHandle), session, arguments); } /** @@ -56,7 +56,7 @@ public Object invoke(FunctionHandle functionHandle, ConnectorSession session, Li *

* Returns a value in the native container type corresponding to the declared SQL return type */ - private Object invoke(ScalarFunctionImplementation function, ConnectorSession session, List arguments) + private Object invoke(BuiltInScalarFunctionImplementation function, ConnectorSession session, List arguments) { MethodHandle method = function.getMethodHandle(); @@ -103,7 +103,7 @@ else if (function.getArgumentProperty(i).getNullConvention() == USE_NULL_FLAG) { } } - private static MethodHandle bindInstanceFactory(MethodHandle method, ScalarFunctionImplementation implementation) + private static MethodHandle bindInstanceFactory(MethodHandle method, BuiltInScalarFunctionImplementation implementation) { if (!implementation.getInstanceFactory().isPresent()) { return method; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/SqlEnvironmentConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/SqlEnvironmentConfig.java index cfd3a4ff3cfea..133b653fa473e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/SqlEnvironmentConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/SqlEnvironmentConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.sql; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import com.facebook.presto.spi.type.TimeZoneKey; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; @@ -24,22 +24,8 @@ public class SqlEnvironmentConfig { - private Optional path = Optional.empty(); private Optional forcedSessionTimeZone = Optional.empty(); - @NotNull - public Optional getPath() - { - return path; - } - - @Config("sql.path") - public SqlEnvironmentConfig setPath(String path) - { - this.path = Optional.ofNullable(path); - return this; - } - @NotNull public Optional getForcedSessionTimeZone() { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/SqlPath.java b/presto-main/src/main/java/com/facebook/presto/sql/SqlPath.java deleted file mode 100644 index 4d9889b6d095e..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/sql/SqlPath.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sql; - -import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.tree.PathElement; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Joiner; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.util.Objects.requireNonNull; - -public final class SqlPath -{ - private List parsedPath; - private final Optional rawPath; - - @JsonCreator - public SqlPath(@JsonProperty("rawPath") Optional path) - { - requireNonNull(path, "path is null"); - this.rawPath = path; - } - - @JsonProperty - public Optional getRawPath() - { - return rawPath; - } - - public List getParsedPath() - { - if (parsedPath == null) { - parsePath(); - } - - return parsedPath; - } - - private void parsePath() - { - checkState(rawPath.isPresent(), "rawPath must be present to parse"); - - SqlParser parser = new SqlParser(); - List pathSpecification = parser.createPathSpecification(rawPath.get()).getPath(); - - this.parsedPath = pathSpecification.stream() - .map(pathElement -> new SqlPathElement(pathElement.getCatalog(), pathElement.getSchema())) - .collect(toImmutableList()); - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - SqlPath that = (SqlPath) obj; - return Objects.equals(parsedPath, that.parsedPath); - } - - @Override - public int hashCode() - { - return Objects.hash(parsedPath); - } - - @Override - public String toString() - { - if (rawPath.isPresent()) { - return Joiner.on(", ").join(getParsedPath()); - } - //empty string is only used for an uninitialized path, as an empty path would be \"\" - return ""; - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java index 9337631cd4695..e96d7dd3c94c9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java @@ -14,7 +14,6 @@ package com.facebook.presto.sql.analyzer; import com.facebook.presto.Session; -import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; @@ -27,6 +26,7 @@ import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.DecimalParseResult; import com.facebook.presto.spi.type.Decimals; @@ -50,7 +50,6 @@ import com.facebook.presto.sql.tree.CharLiteral; import com.facebook.presto.sql.tree.CoalesceExpression; import com.facebook.presto.sql.tree.ComparisonExpression; -import com.facebook.presto.sql.tree.CurrentPath; import com.facebook.presto.sql.tree.CurrentTime; import com.facebook.presto.sql.tree.CurrentUser; import com.facebook.presto.sql.tree.DecimalLiteral; @@ -97,6 +96,7 @@ import com.facebook.presto.sql.tree.TryExpression; import com.facebook.presto.sql.tree.WhenClause; import com.facebook.presto.sql.tree.WindowFrame; +import com.facebook.presto.transaction.TransactionId; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -115,7 +115,6 @@ import java.util.Set; import java.util.function.Function; -import static com.facebook.presto.SystemSessionProperties.isLegacyRowFieldOrdinalAccessEnabled; import static com.facebook.presto.metadata.CastType.CAST; import static com.facebook.presto.spi.function.OperatorType.SUBSCRIPT; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -162,6 +161,7 @@ import static com.google.common.collect.Iterables.getOnlyElement; import static java.lang.Math.toIntExact; import static java.lang.String.format; +import static java.util.Collections.emptyList; import static java.util.Collections.unmodifiableMap; import static java.util.Collections.unmodifiableSet; import static java.util.Objects.requireNonNull; @@ -176,7 +176,6 @@ public class ExpressionAnalyzer private final Function statementAnalyzerFactory; private final TypeProvider symbolTypes; private final boolean isDescribe; - private final boolean legacyRowFieldOrdinalAccess; private final Map, FunctionHandle> resolvedFunctions = new LinkedHashMap<>(); private final Set> scalarSubqueries = new LinkedHashSet<>(); @@ -192,15 +191,17 @@ public class ExpressionAnalyzer private final Set> windowFunctions = new LinkedHashSet<>(); private final Multimap tableColumnReferences = HashMultimap.create(); - private final Session session; + private final Optional transactionId; + private final SqlFunctionProperties sqlFunctionProperties; private final List parameters; private final WarningCollector warningCollector; - public ExpressionAnalyzer( + private ExpressionAnalyzer( FunctionManager functionManager, TypeManager typeManager, Function statementAnalyzerFactory, - Session session, + Optional transactionId, + SqlFunctionProperties sqlFunctionProperties, TypeProvider symbolTypes, List parameters, WarningCollector warningCollector, @@ -209,11 +210,11 @@ public ExpressionAnalyzer( this.functionManager = requireNonNull(functionManager, "functionManager is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.statementAnalyzerFactory = requireNonNull(statementAnalyzerFactory, "statementAnalyzerFactory is null"); - this.session = requireNonNull(session, "session is null"); + this.transactionId = requireNonNull(transactionId, "transactionId is null"); + this.sqlFunctionProperties = requireNonNull(sqlFunctionProperties, "sqlFunctionProperties is null"); this.symbolTypes = requireNonNull(symbolTypes, "symbolTypes is null"); this.parameters = requireNonNull(parameters, "parameters is null"); this.isDescribe = isDescribe; - this.legacyRowFieldOrdinalAccess = isLegacyRowFieldOrdinalAccessEnabled(session); this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); } @@ -454,7 +455,7 @@ protected Type visitDereferenceExpression(DereferenceExpression node, StackableA } } - if (legacyRowFieldOrdinalAccess && rowFieldType == null) { + if (sqlFunctionProperties.isLegacyRowFieldOrdinalAccessEnabled() && rowFieldType == null) { OptionalInt rowIndex = parseAnonymousRowFieldOrdinalAccess(fieldName, rowType.getFields()); if (rowIndex.isPresent()) { rowFieldType = rowType.getFields().get(rowIndex.getAsInt()).getType(); @@ -785,8 +786,8 @@ protected Type visitTimeLiteral(TimeLiteral node, StackableAstVisitorContext context) { try { - if (SystemSessionProperties.isLegacyTimestamp(session)) { - parseTimestampLiteral(session.getTimeZoneKey(), node.getValue()); + if (sqlFunctionProperties.isLegacyTimestamp()) { + parseTimestampLiteral(sqlFunctionProperties.getTimeZoneKey(), node.getValue()); } else { parseTimestampLiteral(node.getValue()); @@ -880,7 +881,8 @@ protected Type visitFunctionCall(FunctionCall node, StackableAstVisitorContext argumentTypes = argumentTypesBuilder.build(); - FunctionHandle function = resolveFunction(session, node, argumentTypes, functionManager); + FunctionHandle function = resolveFunction(transactionId, node, argumentTypes, functionManager); FunctionMetadata functionMetadata = functionManager.getFunctionMetadata(function); if (node.getOrderBy().isPresent()) { @@ -962,12 +964,6 @@ protected Type visitCurrentUser(CurrentUser node, StackableAstVisitorContext context) - { - return setExpressionType(node, VARCHAR); - } - @Override protected Type visitParameter(Parameter node, StackableAstVisitorContext context) { @@ -1456,10 +1452,10 @@ public List getFunctionInputTypes() } } - public static FunctionHandle resolveFunction(Session session, FunctionCall node, List argumentTypes, FunctionManager functionManager) + public static FunctionHandle resolveFunction(Optional transactionId, FunctionCall node, List argumentTypes, FunctionManager functionManager) { try { - return functionManager.resolveFunction(session, node.getName(), argumentTypes); + return functionManager.resolveFunction(transactionId, node.getName(), argumentTypes); } catch (PrestoException e) { if (e.getErrorCode().getCode() == StandardErrorCode.FUNCTION_NOT_FOUND.toErrorCode().getCode()) { @@ -1579,6 +1575,44 @@ public static ExpressionAnalysis analyzeExpression( analyzer.getWindowFunctions()); } + public static ExpressionAnalysis analyzeSqlFunctionExpression( + Metadata metadata, + SqlFunctionProperties sqlFunctionProperties, + Expression expression, + Map argumentTypes) + { + ExpressionAnalyzer analyzer = ExpressionAnalyzer.createWithoutSubqueries( + metadata.getFunctionManager(), + metadata.getTypeManager(), + Optional.empty(), + sqlFunctionProperties, + TypeProvider.copyOf(argumentTypes), + emptyList(), + node -> new SemanticException(NOT_SUPPORTED, node, "SQL function does not support subquery"), + WarningCollector.NOOP, + false); + + analyzer.analyze( + expression, + Scope.builder() + .withRelationType( + RelationId.anonymous(), + new RelationType(argumentTypes.entrySet().stream() + .map(entry -> Field.newUnqualified(entry.getKey(), entry.getValue())) + .collect(toImmutableList()))).build()); + return new ExpressionAnalysis( + analyzer.getExpressionTypes(), + analyzer.getExpressionCoercions(), + analyzer.getSubqueryInPredicates(), + analyzer.getScalarSubqueries(), + analyzer.getExistsSubqueries(), + analyzer.getColumnReferences(), + analyzer.getTypeOnlyCoercions(), + analyzer.getQuantifiedComparisons(), + analyzer.getLambdaArgumentReferences(), + analyzer.getWindowFunctions()); + } + private static ExpressionAnalyzer create( Analysis analysis, Session session, @@ -1592,7 +1626,8 @@ private static ExpressionAnalyzer create( metadata.getFunctionManager(), metadata.getTypeManager(), node -> new StatementAnalyzer(analysis, metadata, sqlParser, accessControl, session, warningCollector), - session, + session.getTransactionId(), + session.getSqlFunctionProperties(), types, analysis.getParameters(), warningCollector, @@ -1655,6 +1690,29 @@ public static ExpressionAnalyzer createWithoutSubqueries( Function statementAnalyzerRejection, WarningCollector warningCollector, boolean isDescribe) + { + return createWithoutSubqueries( + functionManager, + typeManager, + session.getTransactionId(), + session.getSqlFunctionProperties(), + symbolTypes, + parameters, + statementAnalyzerRejection, + warningCollector, + isDescribe); + } + + public static ExpressionAnalyzer createWithoutSubqueries( + FunctionManager functionManager, + TypeManager typeManager, + Optional transactionId, + SqlFunctionProperties sqlFunctionProperties, + TypeProvider symbolTypes, + List parameters, + Function statementAnalyzerRejection, + WarningCollector warningCollector, + boolean isDescribe) { return new ExpressionAnalyzer( functionManager, @@ -1662,7 +1720,8 @@ public static ExpressionAnalyzer createWithoutSubqueries( node -> { throw statementAnalyzerRejection.apply(node); }, - session, + transactionId, + sqlFunctionProperties, symbolTypes, parameters, warningCollector, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index 71798ed5f87d0..249f58e0fe5fe 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -13,15 +13,15 @@ */ package com.facebook.presto.sql.analyzer; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.DefunctConfig; import com.facebook.presto.operator.aggregation.arrayagg.ArrayAggGroupImplementation; import com.facebook.presto.operator.aggregation.histogram.HistogramGroupImplementation; import com.facebook.presto.operator.aggregation.multimapagg.MultimapAggGroupImplementation; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.DefunctConfig; import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.airlift.units.MaxDataSize; @@ -43,6 +43,7 @@ import static io.airlift.units.DataSize.Unit.KILOBYTE; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; @DefunctConfig({ "resource-group-manager", @@ -123,6 +124,7 @@ public class FeaturesConfig private boolean preferPartialAggregation = true; private boolean optimizeTopNRowNumber = true; private boolean pushLimitThroughOuterJoin = true; + private boolean optimizeFullOuterJoinWithCoalesce = true; private Duration iterativeOptimizerTimeout = new Duration(3, MINUTES); // by default let optimizer wait a long time in case it retrieves some data from ConnectorMetadata @@ -132,10 +134,17 @@ public class FeaturesConfig private boolean legacyUnnestArrayRows; private boolean jsonSerdeCodeGenerationEnabled; - private int maxConcurrentMaterializations = 10; + private int maxConcurrentMaterializations = 3; + private boolean optimizedRepartitioningEnabled; private boolean pushdownSubfieldsEnabled; + private boolean tableWriterMergeOperatorEnabled = true; + + private Duration indexLoaderTimeout = new Duration(20, SECONDS); + + private boolean listNonBuiltInFunctions; + public enum JoinReorderingStrategy { NONE, @@ -1044,4 +1053,66 @@ public boolean isPushdownSubfieldsEnabled() { return pushdownSubfieldsEnabled; } + + public boolean isTableWriterMergeOperatorEnabled() + { + return tableWriterMergeOperatorEnabled; + } + + @Config("experimental.table-writer-merge-operator-enabled") + public FeaturesConfig setTableWriterMergeOperatorEnabled(boolean tableWriterMergeOperatorEnabled) + { + this.tableWriterMergeOperatorEnabled = tableWriterMergeOperatorEnabled; + return this; + } + + @Config("optimizer.optimize-full-outer-join-with-coalesce") + public FeaturesConfig setOptimizeFullOuterJoinWithCoalesce(boolean optimizeFullOuterJoinWithCoalesce) + { + this.optimizeFullOuterJoinWithCoalesce = optimizeFullOuterJoinWithCoalesce; + return this; + } + + public Boolean isOptimizeFullOuterJoinWithCoalesce() + { + return this.optimizeFullOuterJoinWithCoalesce; + } + + @Config("index-loader-timeout") + @ConfigDescription("Time limit for loading indexes for index joins") + public FeaturesConfig setIndexLoaderTimeout(Duration indexLoaderTimeout) + { + this.indexLoaderTimeout = indexLoaderTimeout; + return this; + } + + public Duration getIndexLoaderTimeout() + { + return this.indexLoaderTimeout; + } + + public boolean isOptimizedRepartitioningEnabled() + { + return optimizedRepartitioningEnabled; + } + + @Config("experimental.optimized-repartitioning") + @ConfigDescription("Experimental: Use optimized repartitioning") + public FeaturesConfig setOptimizedRepartitioningEnabled(boolean optimizedRepartitioningEnabled) + { + this.optimizedRepartitioningEnabled = optimizedRepartitioningEnabled; + return this; + } + + public boolean isListNonBuiltInFunctions() + { + return listNonBuiltInFunctions; + } + + @Config("list-non-built-in-functions") + public FeaturesConfig setListNonBuiltInFunctions(boolean listNonBuiltInFunctions) + { + this.listNonBuiltInFunctions = listNonBuiltInFunctions; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/SemanticErrorCode.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/SemanticErrorCode.java index 6cac18ea5aa5b..012f766445ec2 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/SemanticErrorCode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/SemanticErrorCode.java @@ -49,6 +49,8 @@ public enum SemanticErrorCode INVALID_LITERAL, FUNCTION_NOT_FOUND, + INVALID_FUNCTION_NAME, + DUPLICATE_PARAMETER_NAME, ORDER_BY_MUST_BE_IN_SELECT, ORDER_BY_MUST_BE_IN_AGGREGATE, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java index d8f51d82e532a..c56874247f887 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java @@ -49,6 +49,7 @@ import com.facebook.presto.sql.tree.Analyze; import com.facebook.presto.sql.tree.Call; import com.facebook.presto.sql.tree.Commit; +import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateTableAsSelect; @@ -59,6 +60,7 @@ import com.facebook.presto.sql.tree.Delete; import com.facebook.presto.sql.tree.DereferenceExpression; import com.facebook.presto.sql.tree.DropColumn; +import com.facebook.presto.sql.tree.DropFunction; import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; import com.facebook.presto.sql.tree.DropView; @@ -113,6 +115,7 @@ import com.facebook.presto.sql.tree.SimpleGroupBy; import com.facebook.presto.sql.tree.SingleColumn; import com.facebook.presto.sql.tree.SortItem; +import com.facebook.presto.sql.tree.SqlParameterDeclaration; import com.facebook.presto.sql.tree.StartTransaction; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.sql.tree.Table; @@ -125,6 +128,7 @@ import com.facebook.presto.sql.tree.With; import com.facebook.presto.sql.tree.WithQuery; import com.facebook.presto.sql.util.AstUtils; +import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; @@ -139,6 +143,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import static com.facebook.presto.SystemSessionProperties.getMaxGroupingSets; @@ -149,6 +154,7 @@ import static com.facebook.presto.spi.function.FunctionKind.WINDOW; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.sql.NodeUtils.getSortItemsFromOrderBy; import static com.facebook.presto.sql.NodeUtils.mapFromProperties; @@ -165,8 +171,10 @@ import static com.facebook.presto.sql.analyzer.SemanticErrorCode.COLUMN_NAME_NOT_SPECIFIED; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.COLUMN_TYPE_UNKNOWN; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.DUPLICATE_COLUMN_NAME; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.DUPLICATE_PARAMETER_NAME; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.DUPLICATE_PROPERTY; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.DUPLICATE_RELATION; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_FUNCTION_NAME; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_ORDINAL; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_PROCEDURE_ARGUMENTS; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_WINDOW_FRAME; @@ -212,7 +220,10 @@ import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.Locale.ENGLISH; +import static java.util.Map.Entry; import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.counting; +import static java.util.stream.Collectors.groupingBy; class StatementAnalyzer { @@ -538,6 +549,55 @@ protected Scope visitCreateView(CreateView node, Optional scope) return createAndAssignScope(node, scope); } + @Override + protected Scope visitCreateFunction(CreateFunction node, Optional scope) + { + analysis.setUpdateType("CREATE FUNCTION"); + + // Check function name + checkFunctionName(node, node.getFunctionName()); + + // Check parameter + List duplicateParameters = node.getParameters().stream() + .map(SqlParameterDeclaration::getName) + .map(Identifier::getValue) + .collect(groupingBy(Function.identity(), counting())) + .entrySet() + .stream() + .filter(entry -> entry.getValue() > 1) + .map(Entry::getKey) + .collect(toImmutableList()); + if (!duplicateParameters.isEmpty()) { + throw new SemanticException(DUPLICATE_PARAMETER_NAME, node, "Duplicate function parameter name: %s", Joiner.on(", ").join(duplicateParameters)); + } + + // Check return type + Type returnType = metadata.getType(parseTypeSignature(node.getReturnType())); + List fields = node.getParameters().stream() + .map(parameter -> Field.newUnqualified(parameter.getName().getValue(), metadata.getType(parseTypeSignature(parameter.getType())))) + .collect(toImmutableList()); + Scope functionScope = Scope.builder() + .withRelationType(RelationId.anonymous(), new RelationType(fields)) + .build(); + Type bodyType = analyzeExpression(node.getBody(), functionScope).getExpressionTypes().get(NodeRef.of(node.getBody())); + if (!bodyType.equals(returnType)) { + throw new SemanticException(TYPE_MISMATCH, node, "Function implementation type '%s' does not match declared return type '%s'", bodyType, returnType); + } + + Analyzer.verifyNoAggregateWindowOrGroupingFunctions(analysis.getFunctionHandles(), metadata.getFunctionManager(), node.getBody(), "CREATE FUNCTION body"); + + // TODO: Check body contains no SQL invoked functions + + return createAndAssignScope(node, scope); + } + + @Override + protected Scope visitDropFunction(DropFunction node, Optional scope) + { + checkFunctionName(node, node.getFunctionName()); + return createAndAssignScope(node, scope); + } + @Override protected Scope visitSetSession(SetSession node, Optional scope) { @@ -1495,6 +1555,13 @@ else if (column.getExpression() instanceof Identifier) { return assignments.build(); } + private void checkFunctionName(Statement node, QualifiedName functionName) + { + if (functionName.getParts().size() != 3) { + throw new SemanticException(INVALID_FUNCTION_NAME, node, format("Function name should be in the form of catalog.schema.function_name, found: %s", functionName)); + } + } + private class OrderByExpressionRewriter extends ExpressionRewriter { @@ -1925,7 +1992,6 @@ private RelationType analyzeView(Query query, QualifiedObjectName name, Optional viewAccessControl = accessControl; } - // TODO: record path in view definition (?) (check spec) and feed it into the session object we use to evaluate the query defined by the view Session viewSession = Session.builder(metadata.getSessionPropertyManager()) .setQueryId(session.getQueryId()) .setTransactionId(session.getTransactionId().orElse(null)) @@ -1933,7 +1999,6 @@ private RelationType analyzeView(Query query, QualifiedObjectName name, Optional .setSource(session.getSource().orElse(null)) .setCatalog(catalog.orElse(null)) .setSchema(schema.orElse(null)) - .setPath(session.getPath()) .setTimeZoneKey(session.getTimeZoneKey()) .setLocale(session.getLocale()) .setRemoteUserAddress(session.getRemoteUserAddress().orElse(null)) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/ArrayGeneratorUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/ArrayGeneratorUtils.java index b02ab4ef89708..ced4714312f4c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/ArrayGeneratorUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/ArrayGeneratorUtils.java @@ -16,7 +16,7 @@ import com.facebook.presto.bytecode.Scope; import com.facebook.presto.bytecode.Variable; import com.facebook.presto.bytecode.expression.BytecodeExpression; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.spi.type.Type; import java.util.function.Function; @@ -29,7 +29,7 @@ private ArrayGeneratorUtils() { } - public static ArrayMapBytecodeExpression map(Scope scope, CachedInstanceBinder cachedInstanceBinder, Type fromElementType, Type toElementType, Variable array, String elementFunctionName, ScalarFunctionImplementation elementFunction) + public static ArrayMapBytecodeExpression map(Scope scope, CachedInstanceBinder cachedInstanceBinder, Type fromElementType, Type toElementType, Variable array, String elementFunctionName, BuiltInScalarFunctionImplementation elementFunction) { return map( scope, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/BodyCompiler.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/BodyCompiler.java index c80883264870b..225a726f38c3d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/BodyCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/BodyCompiler.java @@ -14,11 +14,12 @@ package com.facebook.presto.sql.gen; import com.facebook.presto.bytecode.ClassDefinition; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.relation.RowExpression; import java.util.List; public interface BodyCompiler { - void generateMethods(ClassDefinition classDefinition, CallSiteBinder callSiteBinder, RowExpression filter, List projections); + void generateMethods(SqlFunctionProperties sqlFunctionProperties, ClassDefinition classDefinition, CallSiteBinder callSiteBinder, RowExpression filter, List projections); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/BytecodeGeneratorContext.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/BytecodeGeneratorContext.java index 96115e1772d40..2b7cb0116f8d1 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/BytecodeGeneratorContext.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/BytecodeGeneratorContext.java @@ -18,7 +18,7 @@ import com.facebook.presto.bytecode.Scope; import com.facebook.presto.bytecode.Variable; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.sql.gen.BytecodeUtils.OutputBlockVariableAndType; @@ -83,7 +83,7 @@ public FunctionManager getFunctionManager() return manager; } - public BytecodeNode generateCall(String name, ScalarFunctionImplementation function, List arguments) + public BytecodeNode generateCall(String name, BuiltInScalarFunctionImplementation function, List arguments) { return generateCall(name, function, arguments, Optional.empty()); } @@ -93,7 +93,7 @@ public BytecodeNode generateCall(String name, ScalarFunctionImplementation funct */ public BytecodeNode generateCall( String name, - ScalarFunctionImplementation function, + BuiltInScalarFunctionImplementation function, List arguments, Optional outputBlockVariableAndType) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/BytecodeUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/BytecodeUtils.java index 25b122b60f976..f3780b654b715 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/BytecodeUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/BytecodeUtils.java @@ -20,10 +20,10 @@ import com.facebook.presto.bytecode.control.IfStatement; import com.facebook.presto.bytecode.expression.BytecodeExpression; import com.facebook.presto.bytecode.instruction.LabelNode; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ScalarImplementationChoice; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ScalarImplementationChoice; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.type.Type; @@ -44,8 +44,8 @@ import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantFalse; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantTrue; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.invokeDynamic; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentType.VALUE_TYPE; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ReturnPlaceConvention.PROVIDED_BLOCKBUILDER; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentType.VALUE_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ReturnPlaceConvention.PROVIDED_BLOCKBUILDER; import static com.facebook.presto.sql.gen.Bootstrap.BOOTSTRAP_METHOD; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -182,7 +182,7 @@ public static BytecodeExpression loadConstant(Binding binding) public static BytecodeNode generateInvocation( Scope scope, String name, - ScalarFunctionImplementation function, + BuiltInScalarFunctionImplementation function, Optional instance, List arguments, CallSiteBinder binder) @@ -200,7 +200,7 @@ public static BytecodeNode generateInvocation( public static BytecodeNode generateInvocation( Scope scope, String name, - ScalarFunctionImplementation function, + BuiltInScalarFunctionImplementation function, Optional instance, List arguments, CallSiteBinder binder, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/CompilerOperations.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/CompilerOperations.java index 0f91ebca5f947..af118cac0a04d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/CompilerOperations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/CompilerOperations.java @@ -67,7 +67,8 @@ public static boolean in(Object value, Set set) public static boolean testMask(@Nullable Block masks, int index) { if (masks != null) { - return BOOLEAN.getBoolean(masks, index); + boolean isNull = masks.mayHaveNull() && masks.isNull(index); + return !isNull && BOOLEAN.getBoolean(masks, index); } return true; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/CursorProcessorCompiler.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/CursorProcessorCompiler.java index f824eaa3d8605..5dc633b209d95 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/CursorProcessorCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/CursorProcessorCompiler.java @@ -30,6 +30,7 @@ import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.RecordCursor; import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.InputReferenceExpression; @@ -40,15 +41,12 @@ import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.gen.LambdaBytecodeGenerator.CompiledLambda; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.primitives.Primitives; import io.airlift.slice.Slice; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import static com.facebook.presto.bytecode.Access.PUBLIC; import static com.facebook.presto.bytecode.Access.a; @@ -58,7 +56,7 @@ import static com.facebook.presto.bytecode.expression.BytecodeExpressions.newInstance; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.or; import static com.facebook.presto.bytecode.instruction.JumpInstruction.jump; -import static com.facebook.presto.sql.gen.LambdaExpressionExtractor.extractLambdaExpressions; +import static com.facebook.presto.sql.gen.LambdaBytecodeGenerator.generateMethodsForLambda; import static java.lang.String.format; public class CursorProcessorCompiler @@ -72,19 +70,19 @@ public CursorProcessorCompiler(Metadata metadata) } @Override - public void generateMethods(ClassDefinition classDefinition, CallSiteBinder callSiteBinder, RowExpression filter, List projections) + public void generateMethods(SqlFunctionProperties sqlFunctionProperties, ClassDefinition classDefinition, CallSiteBinder callSiteBinder, RowExpression filter, List projections) { CachedInstanceBinder cachedInstanceBinder = new CachedInstanceBinder(classDefinition, callSiteBinder); generateProcessMethod(classDefinition, projections.size()); - Map filterCompiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, filter, "filter"); - generateFilterMethod(classDefinition, callSiteBinder, cachedInstanceBinder, filterCompiledLambdaMap, filter); + Map filterCompiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, filter, metadata, sqlFunctionProperties, "filter"); + generateFilterMethod(sqlFunctionProperties, classDefinition, callSiteBinder, cachedInstanceBinder, filterCompiledLambdaMap, filter); for (int i = 0; i < projections.size(); i++) { String methodName = "project_" + i; - Map projectCompiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, projections.get(i), methodName); - generateProjectMethod(classDefinition, callSiteBinder, cachedInstanceBinder, projectCompiledLambdaMap, methodName, projections.get(i)); + Map projectCompiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, projections.get(i), metadata, sqlFunctionProperties, methodName); + generateProjectMethod(sqlFunctionProperties, classDefinition, callSiteBinder, cachedInstanceBinder, projectCompiledLambdaMap, methodName, projections.get(i)); } MethodDefinition constructorDefinition = classDefinition.declareConstructor(a(PUBLIC)); @@ -191,36 +189,8 @@ private static IfStatement createProjectIfStatement( return ifStatement; } - private Map generateMethodsForLambda( - ClassDefinition containerClassDefinition, - CallSiteBinder callSiteBinder, - CachedInstanceBinder cachedInstanceBinder, - RowExpression projection, - String methodPrefix) - { - Set lambdaExpressions = ImmutableSet.copyOf(extractLambdaExpressions(projection)); - - ImmutableMap.Builder compiledLambdaMap = ImmutableMap.builder(); - - int counter = 0; - for (LambdaDefinitionExpression lambdaExpression : lambdaExpressions) { - String methodName = methodPrefix + "_lambda_" + counter; - CompiledLambda compiledLambda = LambdaBytecodeGenerator.preGenerateLambdaExpression( - lambdaExpression, - methodName, - containerClassDefinition, - compiledLambdaMap.build(), - callSiteBinder, - cachedInstanceBinder, - metadata.getFunctionManager()); - compiledLambdaMap.put(lambdaExpression, compiledLambda); - counter++; - } - - return compiledLambdaMap.build(); - } - private void generateFilterMethod( + SqlFunctionProperties sqlFunctionProperties, ClassDefinition classDefinition, CallSiteBinder callSiteBinder, CachedInstanceBinder cachedInstanceBinder, @@ -237,10 +207,12 @@ private void generateFilterMethod( Variable wasNullVariable = scope.declareVariable(type(boolean.class), "wasNull"); RowExpressionCompiler compiler = new RowExpressionCompiler( + classDefinition, callSiteBinder, cachedInstanceBinder, fieldReferenceCompiler(cursor), - metadata.getFunctionManager(), + metadata, + sqlFunctionProperties, compiledLambdaMap); LabelNode end = new LabelNode("end"); @@ -259,6 +231,7 @@ private void generateFilterMethod( } private void generateProjectMethod( + SqlFunctionProperties sqlFunctionProperties, ClassDefinition classDefinition, CallSiteBinder callSiteBinder, CachedInstanceBinder cachedInstanceBinder, @@ -277,10 +250,12 @@ private void generateProjectMethod( Variable wasNullVariable = scope.declareVariable(type(boolean.class), "wasNull"); RowExpressionCompiler compiler = new RowExpressionCompiler( + classDefinition, callSiteBinder, cachedInstanceBinder, fieldReferenceCompiler(cursor), - metadata.getFunctionManager(), + metadata, + sqlFunctionProperties, compiledLambdaMap); method.getBody() diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/ExpressionCompiler.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/ExpressionCompiler.java index 09ce4e4b533e2..b524d298bd9fb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/ExpressionCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/ExpressionCompiler.java @@ -21,6 +21,7 @@ import com.facebook.presto.operator.project.PageProcessor; import com.facebook.presto.operator.project.PageProjection; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.relation.RowExpression; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.CacheBuilder; @@ -66,7 +67,7 @@ public ExpressionCompiler(Metadata metadata, PageFunctionCompiler pageFunctionCo this.cursorProcessors = CacheBuilder.newBuilder() .recordStats() .maximumSize(1000) - .build(CacheLoader.from(key -> compile(key.getFilter(), key.getProjections(), new CursorProcessorCompiler(metadata), CursorProcessor.class))); + .build(CacheLoader.from(key -> compile(key.getSqlFunctionProperties(), key.getFilter(), key.getProjections(), new CursorProcessorCompiler(metadata), CursorProcessor.class))); this.cacheStatsMBean = new CacheStatsMBean(cursorProcessors); } @@ -77,9 +78,9 @@ public CacheStatsMBean getCursorProcessorCache() return cacheStatsMBean; } - public Supplier compileCursorProcessor(Optional filter, List projections, Object uniqueKey) + public Supplier compileCursorProcessor(SqlFunctionProperties sqlFunctionProperties, Optional filter, List projections, Object uniqueKey) { - Class cursorProcessor = cursorProcessors.getUnchecked(new CacheKey(filter, projections, uniqueKey)); + Class cursorProcessor = cursorProcessors.getUnchecked(new CacheKey(sqlFunctionProperties, filter, projections, uniqueKey)); return () -> { try { return cursorProcessor.getConstructor().newInstance(); @@ -90,20 +91,21 @@ public Supplier compileCursorProcessor(Optional }; } - public Supplier compilePageProcessor(Optional filter, List projections, Optional classNameSuffix) + public Supplier compilePageProcessor(SqlFunctionProperties sqlFunctionProperties, Optional filter, List projections, Optional classNameSuffix) { - return compilePageProcessor(filter, projections, classNameSuffix, OptionalInt.empty()); + return compilePageProcessor(sqlFunctionProperties, filter, projections, classNameSuffix, OptionalInt.empty()); } private Supplier compilePageProcessor( + SqlFunctionProperties sqlFunctionProperties, Optional filter, List projections, Optional classNameSuffix, OptionalInt initialBatchSize) { - Optional> filterFunctionSupplier = filter.map(expression -> pageFunctionCompiler.compileFilter(expression, classNameSuffix)); + Optional> filterFunctionSupplier = filter.map(expression -> pageFunctionCompiler.compileFilter(sqlFunctionProperties, expression, classNameSuffix)); List> pageProjectionSuppliers = projections.stream() - .map(projection -> pageFunctionCompiler.compileProjection(projection, classNameSuffix)) + .map(projection -> pageFunctionCompiler.compileProjection(sqlFunctionProperties, projection, classNameSuffix)) .collect(toImmutableList()); return () -> { @@ -115,22 +117,22 @@ private Supplier compilePageProcessor( }; } - public Supplier compilePageProcessor(Optional filter, List projections) + public Supplier compilePageProcessor(SqlFunctionProperties sqlFunctionProperties, Optional filter, List projections) { - return compilePageProcessor(filter, projections, Optional.empty()); + return compilePageProcessor(sqlFunctionProperties, filter, projections, Optional.empty()); } @VisibleForTesting - public Supplier compilePageProcessor(Optional filter, List projections, int initialBatchSize) + public Supplier compilePageProcessor(SqlFunctionProperties sqlFunctionProperties, Optional filter, List projections, int initialBatchSize) { - return compilePageProcessor(filter, projections, Optional.empty(), OptionalInt.of(initialBatchSize)); + return compilePageProcessor(sqlFunctionProperties, filter, projections, Optional.empty(), OptionalInt.of(initialBatchSize)); } - private Class compile(Optional filter, List projections, BodyCompiler bodyCompiler, Class superType) + private Class compile(SqlFunctionProperties sqlFunctionProperties, Optional filter, List projections, BodyCompiler bodyCompiler, Class superType) { // create filter and project page iterator class try { - return compileProcessor(filter.orElse(constant(true, BOOLEAN)), projections, bodyCompiler, superType); + return compileProcessor(sqlFunctionProperties, filter.orElse(constant(true, BOOLEAN)), projections, bodyCompiler, superType); } catch (CompilationException e) { throw new PrestoException(COMPILER_ERROR, e.getCause()); @@ -138,6 +140,7 @@ private Class compile(Optional filter, List Class compileProcessor( + SqlFunctionProperties sqlFunctionProperties, RowExpression filter, List projections, BodyCompiler bodyCompiler, @@ -150,7 +153,7 @@ private Class compileProcessor( type(superType)); CallSiteBinder callSiteBinder = new CallSiteBinder(); - bodyCompiler.generateMethods(classDefinition, callSiteBinder, filter, projections); + bodyCompiler.generateMethods(sqlFunctionProperties, classDefinition, callSiteBinder, filter, projections); // // toString method @@ -177,17 +180,24 @@ private static void generateToString(ClassDefinition classDefinition, CallSiteBi private static final class CacheKey { + private final SqlFunctionProperties sqlFunctionProperties; private final Optional filter; private final List projections; private final Object uniqueKey; - private CacheKey(Optional filter, List projections, Object uniqueKey) + private CacheKey(SqlFunctionProperties sqlFunctionProperties, Optional filter, List projections, Object uniqueKey) { + this.sqlFunctionProperties = sqlFunctionProperties; this.filter = filter; this.uniqueKey = uniqueKey; this.projections = ImmutableList.copyOf(projections); } + public SqlFunctionProperties getSqlFunctionProperties() + { + return sqlFunctionProperties; + } + private Optional getFilter() { return filter; @@ -201,7 +211,7 @@ private List getProjections() @Override public int hashCode() { - return Objects.hash(filter, projections, uniqueKey); + return Objects.hash(sqlFunctionProperties, filter, projections, uniqueKey); } @Override @@ -214,7 +224,8 @@ public boolean equals(Object obj) return false; } CacheKey other = (CacheKey) obj; - return Objects.equals(this.filter, other.filter) && + return Objects.equals(this.sqlFunctionProperties, other.sqlFunctionProperties) && + Objects.equals(this.filter, other.filter) && Objects.equals(this.projections, other.projections) && Objects.equals(this.uniqueKey, other.uniqueKey); } @@ -223,6 +234,7 @@ public boolean equals(Object obj) public String toString() { return toStringHelper(this) + .add("sqlFunctionProperties", sqlFunctionProperties) .add("filter", filter) .add("projections", projections) .add("uniqueKey", uniqueKey) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/FunctionCallCodeGenerator.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/FunctionCallCodeGenerator.java index 2203a2b5719b5..2d12903176509 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/FunctionCallCodeGenerator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/FunctionCallCodeGenerator.java @@ -16,7 +16,7 @@ import com.facebook.presto.bytecode.BytecodeNode; import com.facebook.presto.bytecode.Variable; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.type.Type; @@ -26,7 +26,7 @@ import java.util.List; import java.util.Optional; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentType.VALUE_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentType.VALUE_TYPE; public class FunctionCallCodeGenerator { @@ -34,12 +34,12 @@ public BytecodeNode generateCall(FunctionHandle functionHandle, BytecodeGenerato { FunctionManager functionManager = context.getFunctionManager(); - ScalarFunctionImplementation function = functionManager.getScalarFunctionImplementation(functionHandle); + BuiltInScalarFunctionImplementation function = functionManager.getBuiltInScalarFunctionImplementation(functionHandle); List argumentsBytecode = new ArrayList<>(); for (int i = 0; i < arguments.size(); i++) { RowExpression argument = arguments.get(i); - ScalarFunctionImplementation.ArgumentProperty argumentProperty = function.getArgumentProperty(i); + BuiltInScalarFunctionImplementation.ArgumentProperty argumentProperty = function.getArgumentProperty(i); if (argumentProperty.getArgumentType() == VALUE_TYPE) { argumentsBytecode.add(context.generate(argument, Optional.empty())); } @@ -49,7 +49,7 @@ public BytecodeNode generateCall(FunctionHandle functionHandle, BytecodeGenerato } return context.generateCall( - functionManager.getFunctionMetadata(functionHandle).getName(), + functionManager.getFunctionMetadata(functionHandle).getName().getFunctionName(), function, argumentsBytecode, outputBlockVariable.map(variable -> new OutputBlockVariableAndType(variable, returnType))); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/InCodeGenerator.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/InCodeGenerator.java index 649168576a11a..1a794f69e0e66 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/InCodeGenerator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/InCodeGenerator.java @@ -21,7 +21,7 @@ import com.facebook.presto.bytecode.control.SwitchStatement.SwitchBuilder; import com.facebook.presto.bytecode.instruction.LabelNode; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.RowExpression; @@ -123,9 +123,9 @@ public BytecodeNode generateExpression(BytecodeGeneratorContext generatorContext SwitchGenerationCase switchGenerationCase = checkSwitchGenerationCase(type, values); FunctionHandle hashCodeHandle = generatorContext.getFunctionManager().resolveOperator(HASH_CODE, fromTypes(type)); - MethodHandle hashCodeFunction = generatorContext.getFunctionManager().getScalarFunctionImplementation(hashCodeHandle).getMethodHandle(); + MethodHandle hashCodeFunction = generatorContext.getFunctionManager().getBuiltInScalarFunctionImplementation(hashCodeHandle).getMethodHandle(); FunctionHandle isIndeterminateHandle = generatorContext.getFunctionManager().resolveOperator(INDETERMINATE, fromTypes(type)); - ScalarFunctionImplementation isIndeterminateFunction = generatorContext.getFunctionManager().getScalarFunctionImplementation(isIndeterminateHandle); + BuiltInScalarFunctionImplementation isIndeterminateFunction = generatorContext.getFunctionManager().getBuiltInScalarFunctionImplementation(isIndeterminateHandle); ImmutableListMultimap.Builder hashBucketsBuilder = ImmutableListMultimap.builder(); ImmutableList.Builder defaultBucket = ImmutableList.builder(); @@ -295,7 +295,7 @@ private static BytecodeBlock buildInCase( Variable value, Collection testValues, boolean checkForNulls, - ScalarFunctionImplementation isIndeterminateFunction) + BuiltInScalarFunctionImplementation isIndeterminateFunction) { Variable caseWasNull = null; // caseWasNull is set to true the first time a null in `testValues` is encountered if (checkForNulls) { @@ -332,7 +332,7 @@ private static BytecodeBlock buildInCase( elseBlock.gotoLabel(noMatchLabel); FunctionHandle equalsHandle = generatorContext.getFunctionManager().resolveOperator(EQUAL, fromTypes(type, type)); - ScalarFunctionImplementation equalsFunction = generatorContext.getFunctionManager().getScalarFunctionImplementation(equalsHandle); + BuiltInScalarFunctionImplementation equalsFunction = generatorContext.getFunctionManager().getBuiltInScalarFunctionImplementation(equalsHandle); BytecodeNode elseNode = elseBlock; for (BytecodeNode testNode : testValues) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/InvokeFunctionBytecodeExpression.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/InvokeFunctionBytecodeExpression.java index 54bad4747a41f..75be5120427db 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/InvokeFunctionBytecodeExpression.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/InvokeFunctionBytecodeExpression.java @@ -18,7 +18,7 @@ import com.facebook.presto.bytecode.MethodGenerationContext; import com.facebook.presto.bytecode.Scope; import com.facebook.presto.bytecode.expression.BytecodeExpression; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Primitives; @@ -34,12 +34,12 @@ public class InvokeFunctionBytecodeExpression extends BytecodeExpression { - public static BytecodeExpression invokeFunction(Scope scope, CachedInstanceBinder cachedInstanceBinder, String name, ScalarFunctionImplementation function, BytecodeExpression... parameters) + public static BytecodeExpression invokeFunction(Scope scope, CachedInstanceBinder cachedInstanceBinder, String name, BuiltInScalarFunctionImplementation function, BytecodeExpression... parameters) { return invokeFunction(scope, cachedInstanceBinder, name, function, ImmutableList.copyOf(parameters)); } - public static BytecodeExpression invokeFunction(Scope scope, CachedInstanceBinder cachedInstanceBinder, String name, ScalarFunctionImplementation function, List parameters) + public static BytecodeExpression invokeFunction(Scope scope, CachedInstanceBinder cachedInstanceBinder, String name, BuiltInScalarFunctionImplementation function, List parameters) { requireNonNull(scope, "scope is null"); requireNonNull(function, "function is null"); @@ -59,7 +59,7 @@ private InvokeFunctionBytecodeExpression( Scope scope, CallSiteBinder binder, String name, - ScalarFunctionImplementation function, + BuiltInScalarFunctionImplementation function, Optional instance, List parameters) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/JoinCompiler.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/JoinCompiler.java index 108d41e3e93b8..62adc995b5010 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/JoinCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/JoinCompiler.java @@ -35,7 +35,7 @@ import com.facebook.presto.operator.LookupSourceSupplier; import com.facebook.presto.operator.PagesHash; import com.facebook.presto.operator.PagesHashStrategy; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.block.Block; @@ -707,7 +707,7 @@ private void generatePositionNotDistinctFromRowWithPageMethod( continue; } } - ScalarFunctionImplementation operator = functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(OperatorType.IS_DISTINCT_FROM, fromTypes(type, type))); + BuiltInScalarFunctionImplementation operator = functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(OperatorType.IS_DISTINCT_FROM, fromTypes(type, type))); callSiteBinder.bind(operator.getMethodHandle()); List argumentsBytecode = new ArrayList<>(); argumentsBytecode.add(generateInputReference(callSiteBinder, scope, type, leftBlock, leftBlockPosition)); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/JoinFilterFunctionCompiler.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/JoinFilterFunctionCompiler.java index 541b67f7caab4..904b0f6cd5388 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/JoinFilterFunctionCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/JoinFilterFunctionCompiler.java @@ -30,6 +30,7 @@ import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionVisitor; @@ -38,8 +39,6 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import it.unimi.dsi.fastutil.longs.LongArrayList; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -51,7 +50,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import static com.facebook.presto.bytecode.Access.FINAL; import static com.facebook.presto.bytecode.Access.PRIVATE; @@ -62,7 +60,7 @@ import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantFalse; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantInt; import static com.facebook.presto.sql.gen.BytecodeUtils.invoke; -import static com.facebook.presto.sql.gen.LambdaExpressionExtractor.extractLambdaExpressions; +import static com.facebook.presto.sql.gen.LambdaBytecodeGenerator.generateMethodsForLambda; import static com.facebook.presto.util.CompilerUtils.defineClass; import static com.facebook.presto.util.CompilerUtils.makeClassName; import static com.google.common.base.MoreObjects.toStringHelper; @@ -81,7 +79,7 @@ public JoinFilterFunctionCompiler(Metadata metadata) private final LoadingCache joinFilterFunctionFactories = CacheBuilder.newBuilder() .recordStats() .maximumSize(1000) - .build(CacheLoader.from(key -> internalCompileFilterFunctionFactory(key.getFilter(), key.getLeftBlocksSize()))); + .build(CacheLoader.from(key -> internalCompileFilterFunctionFactory(key.getSqlFunctionProperties(), key.getFilter(), key.getLeftBlocksSize()))); @Managed @Nested @@ -90,18 +88,18 @@ public CacheStatsMBean getJoinFilterFunctionFactoryStats() return new CacheStatsMBean(joinFilterFunctionFactories); } - public JoinFilterFunctionFactory compileJoinFilterFunction(RowExpression filter, int leftBlocksSize) + public JoinFilterFunctionFactory compileJoinFilterFunction(SqlFunctionProperties sqlFunctionProperties, RowExpression filter, int leftBlocksSize) { - return joinFilterFunctionFactories.getUnchecked(new JoinFilterCacheKey(filter, leftBlocksSize)); + return joinFilterFunctionFactories.getUnchecked(new JoinFilterCacheKey(sqlFunctionProperties, filter, leftBlocksSize)); } - private JoinFilterFunctionFactory internalCompileFilterFunctionFactory(RowExpression filterExpression, int leftBlocksSize) + private JoinFilterFunctionFactory internalCompileFilterFunctionFactory(SqlFunctionProperties sqlFunctionProperties, RowExpression filterExpression, int leftBlocksSize) { - Class internalJoinFilterFunction = compileInternalJoinFilterFunction(filterExpression, leftBlocksSize); + Class internalJoinFilterFunction = compileInternalJoinFilterFunction(sqlFunctionProperties, filterExpression, leftBlocksSize); return new IsolatedJoinFilterFunctionFactory(internalJoinFilterFunction); } - private Class compileInternalJoinFilterFunction(RowExpression filterExpression, int leftBlocksSize) + private Class compileInternalJoinFilterFunction(SqlFunctionProperties sqlFunctionProperties, RowExpression filterExpression, int leftBlocksSize) { ClassDefinition classDefinition = new ClassDefinition( a(PUBLIC, FINAL), @@ -111,7 +109,7 @@ private Class compileInternalJoinFilterFun CallSiteBinder callSiteBinder = new CallSiteBinder(); - new JoinFilterFunctionCompiler(metadata).generateMethods(classDefinition, callSiteBinder, filterExpression, leftBlocksSize); + new JoinFilterFunctionCompiler(metadata).generateMethods(sqlFunctionProperties, classDefinition, callSiteBinder, filterExpression, leftBlocksSize); // // toString method @@ -127,14 +125,14 @@ private Class compileInternalJoinFilterFun return defineClass(classDefinition, InternalJoinFilterFunction.class, callSiteBinder.getBindings(), getClass().getClassLoader()); } - private void generateMethods(ClassDefinition classDefinition, CallSiteBinder callSiteBinder, RowExpression filter, int leftBlocksSize) + private void generateMethods(SqlFunctionProperties sqlFunctionProperties, ClassDefinition classDefinition, CallSiteBinder callSiteBinder, RowExpression filter, int leftBlocksSize) { CachedInstanceBinder cachedInstanceBinder = new CachedInstanceBinder(classDefinition, callSiteBinder); FieldDefinition sessionField = classDefinition.declareField(a(PRIVATE, FINAL), "session", ConnectorSession.class); - Map compiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, leftBlocksSize, filter); - generateFilterMethod(classDefinition, callSiteBinder, cachedInstanceBinder, compiledLambdaMap, filter, leftBlocksSize, sessionField); + Map compiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, filter, metadata, sqlFunctionProperties); + generateFilterMethod(sqlFunctionProperties, classDefinition, callSiteBinder, cachedInstanceBinder, compiledLambdaMap, filter, leftBlocksSize, sessionField); generateConstructor(classDefinition, sessionField, cachedInstanceBinder); } @@ -160,6 +158,7 @@ private static void generateConstructor( } private void generateFilterMethod( + SqlFunctionProperties sqlFunctionProperties, ClassDefinition classDefinition, CallSiteBinder callSiteBinder, CachedInstanceBinder cachedInstanceBinder, @@ -193,10 +192,12 @@ private void generateFilterMethod( scope.declareVariable("session", body, method.getThis().getField(sessionField)); RowExpressionCompiler compiler = new RowExpressionCompiler( + classDefinition, callSiteBinder, cachedInstanceBinder, fieldReferenceCompiler(callSiteBinder, leftPosition, leftPage, rightPosition, rightPage, leftBlocksSize), - metadata.getFunctionManager(), + metadata, + sqlFunctionProperties, compiledLambdaMap); BytecodeNode visitorBody = compiler.compile(filter, scope, Optional.empty()); @@ -210,33 +211,6 @@ private void generateFilterMethod( .ifFalse(result.ret())); } - private Map generateMethodsForLambda( - ClassDefinition containerClassDefinition, - CallSiteBinder callSiteBinder, - CachedInstanceBinder cachedInstanceBinder, - int leftBlocksSize, - RowExpression filter) - { - Set lambdaExpressions = ImmutableSet.copyOf(extractLambdaExpressions(filter)); - ImmutableMap.Builder compiledLambdaMap = ImmutableMap.builder(); - - int counter = 0; - for (LambdaDefinitionExpression lambdaExpression : lambdaExpressions) { - CompiledLambda compiledLambda = LambdaBytecodeGenerator.preGenerateLambdaExpression( - lambdaExpression, - "lambda_" + counter, - containerClassDefinition, - compiledLambdaMap.build(), - callSiteBinder, - cachedInstanceBinder, - metadata.getFunctionManager()); - compiledLambdaMap.put(lambdaExpression, compiledLambda); - counter++; - } - - return compiledLambdaMap.build(); - } - private static void generateToString(ClassDefinition classDefinition, CallSiteBinder callSiteBinder, String string) { // bind constant via invokedynamic to avoid constant pool issues due to large strings @@ -272,15 +246,22 @@ private static RowExpressionVisitor fieldReferenceCompiler( private static final class JoinFilterCacheKey { + private final SqlFunctionProperties sqlFunctionProperties; private final RowExpression filter; private final int leftBlocksSize; - public JoinFilterCacheKey(RowExpression filter, int leftBlocksSize) + public JoinFilterCacheKey(SqlFunctionProperties sqlFunctionProperties, RowExpression filter, int leftBlocksSize) { + this.sqlFunctionProperties = requireNonNull(sqlFunctionProperties, "sqlFunctionProperties is null"); this.filter = requireNonNull(filter, "filter can not be null"); this.leftBlocksSize = leftBlocksSize; } + public SqlFunctionProperties getSqlFunctionProperties() + { + return sqlFunctionProperties; + } + public RowExpression getFilter() { return filter; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/LambdaBytecodeGenerator.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/LambdaBytecodeGenerator.java index 2ca6db1124320..3b32891ca8bb4 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/LambdaBytecodeGenerator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/LambdaBytecodeGenerator.java @@ -24,10 +24,11 @@ import com.facebook.presto.bytecode.Scope; import com.facebook.presto.bytecode.Variable; import com.facebook.presto.bytecode.expression.BytecodeExpression; -import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.Metadata; import com.facebook.presto.operator.aggregation.AccumulatorCompiler; import com.facebook.presto.operator.aggregation.LambdaProvider; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.InputReferenceExpression; @@ -84,7 +85,20 @@ public static Map generateMethodsFor CallSiteBinder callSiteBinder, CachedInstanceBinder cachedInstanceBinder, RowExpression expression, - FunctionManager functionManager) + Metadata metadata, + SqlFunctionProperties sqlFunctionProperties) + { + return generateMethodsForLambda(containerClassDefinition, callSiteBinder, cachedInstanceBinder, expression, metadata, sqlFunctionProperties, ""); + } + + public static Map generateMethodsForLambda( + ClassDefinition containerClassDefinition, + CallSiteBinder callSiteBinder, + CachedInstanceBinder cachedInstanceBinder, + RowExpression expression, + Metadata metadata, + SqlFunctionProperties sqlFunctionProperties, + String methodNamePrefix) { Set lambdaExpressions = ImmutableSet.copyOf(extractLambdaExpressions(expression)); ImmutableMap.Builder compiledLambdaMap = ImmutableMap.builder(); @@ -93,12 +107,13 @@ public static Map generateMethodsFor for (LambdaDefinitionExpression lambdaExpression : lambdaExpressions) { CompiledLambda compiledLambda = LambdaBytecodeGenerator.preGenerateLambdaExpression( lambdaExpression, - "lambda_" + counter, + methodNamePrefix + "lambda_" + counter, containerClassDefinition, compiledLambdaMap.build(), callSiteBinder, cachedInstanceBinder, - functionManager); + metadata, + sqlFunctionProperties); compiledLambdaMap.put(lambdaExpression, compiledLambda); counter++; } @@ -116,7 +131,8 @@ public static CompiledLambda preGenerateLambdaExpression( Map compiledLambdaMap, CallSiteBinder callSiteBinder, CachedInstanceBinder cachedInstanceBinder, - FunctionManager functionManager) + Metadata metadata, + SqlFunctionProperties sqlFunctionProperties) { ImmutableList.Builder parameters = ImmutableList.builder(); ImmutableMap.Builder parameterMapBuilder = ImmutableMap.builder(); @@ -131,10 +147,12 @@ public static CompiledLambda preGenerateLambdaExpression( } RowExpressionCompiler innerExpressionCompiler = new RowExpressionCompiler( + classDefinition, callSiteBinder, cachedInstanceBinder, variableReferenceCompiler(parameterMapBuilder.build()), - functionManager, + metadata, + sqlFunctionProperties, compiledLambdaMap); return defineLambdaMethod( @@ -231,7 +249,7 @@ public static BytecodeNode generateLambda( return block; } - public static Class compileLambdaProvider(LambdaDefinitionExpression lambdaExpression, FunctionManager functionManager, Class lambdaInterface) + public static Class compileLambdaProvider(LambdaDefinitionExpression lambdaExpression, Metadata metadata, SqlFunctionProperties sqlFunctionProperties, Class lambdaInterface) { ClassDefinition lambdaProviderClassDefinition = new ClassDefinition( a(PUBLIC, Access.FINAL), @@ -249,7 +267,8 @@ public static Class compileLambdaProvider(LambdaDefini callSiteBinder, cachedInstanceBinder, lambdaExpression, - functionManager); + metadata, + sqlFunctionProperties); MethodDefinition method = lambdaProviderClassDefinition.declareMethod( a(PUBLIC), @@ -263,10 +282,12 @@ public static Class compileLambdaProvider(LambdaDefini scope.declareVariable("session", body, method.getThis().getField(sessionField)); RowExpressionCompiler rowExpressionCompiler = new RowExpressionCompiler( + lambdaProviderClassDefinition, callSiteBinder, cachedInstanceBinder, variableReferenceCompiler(ImmutableMap.of()), - functionManager, + metadata, + sqlFunctionProperties, compiledLambdaMap); BytecodeGeneratorContext generatorContext = new BytecodeGeneratorContext( @@ -274,7 +295,7 @@ public static Class compileLambdaProvider(LambdaDefini scope, callSiteBinder, cachedInstanceBinder, - functionManager); + metadata.getFunctionManager()); body.append( generateLambda( diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/NullIfCodeGenerator.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/NullIfCodeGenerator.java index 8b5c2af157231..72921aa3353c9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/NullIfCodeGenerator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/NullIfCodeGenerator.java @@ -21,7 +21,7 @@ import com.facebook.presto.bytecode.instruction.LabelNode; import com.facebook.presto.metadata.CastType; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.relation.RowExpression; @@ -68,7 +68,7 @@ public BytecodeNode generateExpression(BytecodeGeneratorContext generatorContext FunctionManager functionManager = generatorContext.getFunctionManager(); FunctionHandle equalFunction = functionManager.resolveOperator(EQUAL, fromTypes(firstType, secondType)); FunctionMetadata equalFunctionMetadata = functionManager.getFunctionMetadata(equalFunction); - ScalarFunctionImplementation equalsFunction = generatorContext.getFunctionManager().getScalarFunctionImplementation(equalFunction); + BuiltInScalarFunctionImplementation equalsFunction = generatorContext.getFunctionManager().getBuiltInScalarFunctionImplementation(equalFunction); BytecodeNode equalsCall = generatorContext.generateCall( EQUAL.name(), equalsFunction, @@ -111,6 +111,6 @@ private static BytecodeNode cast( .lookupCast(CastType.CAST, actualType.getTypeSignature(), requiredType); // TODO: do we need a full function call? (nullability checks, etc) - return generatorContext.generateCall(CAST.name(), generatorContext.getFunctionManager().getScalarFunctionImplementation(functionHandle), ImmutableList.of(argument)); + return generatorContext.generateCall(CAST.name(), generatorContext.getFunctionManager().getBuiltInScalarFunctionImplementation(functionHandle), ImmutableList.of(argument)); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/OrderingCompiler.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/OrderingCompiler.java index aa9b98415b3c7..004898aa46c9f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/OrderingCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/OrderingCompiler.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.gen; +import com.facebook.airlift.log.Logger; import com.facebook.presto.bytecode.BytecodeBlock; import com.facebook.presto.bytecode.ClassDefinition; import com.facebook.presto.bytecode.MethodDefinition; @@ -37,7 +38,6 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.objects.ObjectArrayList; import org.weakref.jmx.Managed; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/PageFunctionCompiler.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/PageFunctionCompiler.java index a0230ccf01c09..a197d408eb5b2 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/PageFunctionCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/PageFunctionCompiler.java @@ -39,6 +39,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.InputReferenceExpression; @@ -53,8 +54,6 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -63,8 +62,8 @@ import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.TreeSet; import java.util.function.Consumer; import java.util.function.Supplier; @@ -88,7 +87,7 @@ import static com.facebook.presto.operator.project.PageFieldsToInputParametersRewriter.rewritePageFieldsToInputParameters; import static com.facebook.presto.spi.StandardErrorCode.COMPILER_ERROR; import static com.facebook.presto.sql.gen.BytecodeUtils.invoke; -import static com.facebook.presto.sql.gen.LambdaExpressionExtractor.extractLambdaExpressions; +import static com.facebook.presto.sql.gen.LambdaBytecodeGenerator.generateMethodsForLambda; import static com.facebook.presto.util.CompilerUtils.defineClass; import static com.facebook.presto.util.CompilerUtils.makeClassName; import static com.facebook.presto.util.Reflection.constructorMethodHandle; @@ -100,8 +99,8 @@ public class PageFunctionCompiler private final Metadata metadata; private final DeterminismEvaluator determinismEvaluator; - private final LoadingCache> projectionCache; - private final LoadingCache> filterCache; + private final LoadingCache> projectionCache; + private final LoadingCache> filterCache; private final CacheStatsMBean projectionCacheStats; private final CacheStatsMBean filterCacheStats; @@ -121,7 +120,7 @@ public PageFunctionCompiler(Metadata metadata, int expressionCacheSize) projectionCache = CacheBuilder.newBuilder() .recordStats() .maximumSize(expressionCacheSize) - .build(CacheLoader.from(projection -> compileProjectionInternal(projection, Optional.empty()))); + .build(CacheLoader.from(cacheKey -> compileProjectionInternal(cacheKey.sqlFunctionProperties, cacheKey.rowExpression, Optional.empty()))); projectionCacheStats = new CacheStatsMBean(projectionCache); } else { @@ -133,7 +132,7 @@ public PageFunctionCompiler(Metadata metadata, int expressionCacheSize) filterCache = CacheBuilder.newBuilder() .recordStats() .maximumSize(expressionCacheSize) - .build(CacheLoader.from(filter -> compileFilterInternal(filter, Optional.empty()))); + .build(CacheLoader.from(cacheKey -> compileFilterInternal(cacheKey.sqlFunctionProperties, cacheKey.rowExpression, Optional.empty()))); filterCacheStats = new CacheStatsMBean(filterCache); } else { @@ -158,15 +157,15 @@ public CacheStatsMBean getFilterCache() return filterCacheStats; } - public Supplier compileProjection(RowExpression projection, Optional classNameSuffix) + public Supplier compileProjection(SqlFunctionProperties sqlFunctionProperties, RowExpression projection, Optional classNameSuffix) { if (projectionCache == null) { - return compileProjectionInternal(projection, classNameSuffix); + return compileProjectionInternal(sqlFunctionProperties, projection, classNameSuffix); } - return projectionCache.getUnchecked(projection); + return projectionCache.getUnchecked(new CacheKey(sqlFunctionProperties, projection)); } - private Supplier compileProjectionInternal(RowExpression projection, Optional classNameSuffix) + private Supplier compileProjectionInternal(SqlFunctionProperties sqlFunctionProperties, RowExpression projection, Optional classNameSuffix) { requireNonNull(projection, "projection is null"); @@ -187,7 +186,7 @@ private Supplier compileProjectionInternal(RowExpression project CallSiteBinder callSiteBinder = new CallSiteBinder(); // generate Work - ClassDefinition pageProjectionWorkDefinition = definePageProjectWorkClass(result.getRewrittenExpression(), callSiteBinder, classNameSuffix); + ClassDefinition pageProjectionWorkDefinition = definePageProjectWorkClass(sqlFunctionProperties, result.getRewrittenExpression(), callSiteBinder, classNameSuffix); Class pageProjectionWorkClass; try { @@ -209,7 +208,7 @@ private static ParameterizedType generateProjectionWorkClassName(Optional classNameSuffix) + private ClassDefinition definePageProjectWorkClass(SqlFunctionProperties sqlFunctionProperties, RowExpression projection, CallSiteBinder callSiteBinder, Optional classNameSuffix) { ClassDefinition classDefinition = new ClassDefinition( a(PUBLIC, FINAL), @@ -234,8 +233,8 @@ private ClassDefinition definePageProjectWorkClass(RowExpression projection, Cal method.getBody().append(method.getThis().getField(resultField)).ret(Object.class); // evaluate - Map compiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, projection); - generateEvaluateMethod(classDefinition, callSiteBinder, cachedInstanceBinder, compiledLambdaMap, projection, blockBuilderField); + Map compiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, projection, metadata, sqlFunctionProperties); + generateEvaluateMethod(sqlFunctionProperties, classDefinition, callSiteBinder, cachedInstanceBinder, compiledLambdaMap, projection, blockBuilderField); // constructor Parameter blockBuilder = arg("blockBuilder", BlockBuilder.class); @@ -313,6 +312,7 @@ private static MethodDefinition generateProcessMethod( } private MethodDefinition generateEvaluateMethod( + SqlFunctionProperties sqlFunctionProperties, ClassDefinition classDefinition, CallSiteBinder callSiteBinder, CachedInstanceBinder cachedInstanceBinder, @@ -344,10 +344,12 @@ private MethodDefinition generateEvaluateMethod( scope.declareVariable("wasNull", body, constantFalse()); RowExpressionCompiler compiler = new RowExpressionCompiler( + classDefinition, callSiteBinder, cachedInstanceBinder, fieldReferenceCompiler(callSiteBinder), - metadata.getFunctionManager(), + metadata, + sqlFunctionProperties, compiledLambdaMap); Variable outputBlockVariable = scope.createTempVariable(BlockBuilder.class); @@ -357,22 +359,22 @@ private MethodDefinition generateEvaluateMethod( return method; } - public Supplier compileFilter(RowExpression filter, Optional classNameSuffix) + public Supplier compileFilter(SqlFunctionProperties sqlFunctionProperties, RowExpression filter, Optional classNameSuffix) { if (filterCache == null) { - return compileFilterInternal(filter, classNameSuffix); + return compileFilterInternal(sqlFunctionProperties, filter, classNameSuffix); } - return filterCache.getUnchecked(filter); + return filterCache.getUnchecked(new CacheKey(sqlFunctionProperties, filter)); } - private Supplier compileFilterInternal(RowExpression filter, Optional classNameSuffix) + private Supplier compileFilterInternal(SqlFunctionProperties sqlFunctionProperties, RowExpression filter, Optional classNameSuffix) { requireNonNull(filter, "filter is null"); PageFieldsToInputParametersRewriter.Result result = rewritePageFieldsToInputParameters(filter); CallSiteBinder callSiteBinder = new CallSiteBinder(); - ClassDefinition classDefinition = defineFilterClass(result.getRewrittenExpression(), result.getInputChannels(), callSiteBinder, classNameSuffix); + ClassDefinition classDefinition = defineFilterClass(sqlFunctionProperties, result.getRewrittenExpression(), result.getInputChannels(), callSiteBinder, classNameSuffix); Class functionClass; try { @@ -397,7 +399,7 @@ private static ParameterizedType generateFilterClassName(Optional classN return makeClassName(PageFilter.class.getSimpleName(), classNameSuffix); } - private ClassDefinition defineFilterClass(RowExpression filter, InputChannels inputChannels, CallSiteBinder callSiteBinder, Optional classNameSuffix) + private ClassDefinition defineFilterClass(SqlFunctionProperties sqlFunctionProperties, RowExpression filter, InputChannels inputChannels, CallSiteBinder callSiteBinder, Optional classNameSuffix) { ClassDefinition classDefinition = new ClassDefinition( a(PUBLIC, FINAL), @@ -407,8 +409,8 @@ private ClassDefinition defineFilterClass(RowExpression filter, InputChannels in CachedInstanceBinder cachedInstanceBinder = new CachedInstanceBinder(classDefinition, callSiteBinder); - Map compiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, filter); - generateFilterMethod(classDefinition, callSiteBinder, cachedInstanceBinder, compiledLambdaMap, filter); + Map compiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, filter, metadata, sqlFunctionProperties); + generateFilterMethod(sqlFunctionProperties, classDefinition, callSiteBinder, cachedInstanceBinder, compiledLambdaMap, filter); FieldDefinition selectedPositions = classDefinition.declareField(a(PRIVATE), "selectedPositions", boolean[].class); generatePageFilterMethod(classDefinition, selectedPositions); @@ -489,6 +491,7 @@ private static MethodDefinition generatePageFilterMethod(ClassDefinition classDe } private MethodDefinition generateFilterMethod( + SqlFunctionProperties sqlFunctionProperties, ClassDefinition classDefinition, CallSiteBinder callSiteBinder, CachedInstanceBinder cachedInstanceBinder, @@ -518,10 +521,12 @@ private MethodDefinition generateFilterMethod( Variable wasNullVariable = scope.declareVariable("wasNull", body, constantFalse()); RowExpressionCompiler compiler = new RowExpressionCompiler( + classDefinition, callSiteBinder, cachedInstanceBinder, fieldReferenceCompiler(callSiteBinder), - metadata.getFunctionManager(), + metadata, + sqlFunctionProperties, compiledLambdaMap); Variable result = scope.declareVariable(boolean.class, "result"); @@ -532,32 +537,6 @@ private MethodDefinition generateFilterMethod( return method; } - private Map generateMethodsForLambda( - ClassDefinition containerClassDefinition, - CallSiteBinder callSiteBinder, - CachedInstanceBinder cachedInstanceBinder, - RowExpression expression) - { - Set lambdaExpressions = ImmutableSet.copyOf(extractLambdaExpressions(expression)); - ImmutableMap.Builder compiledLambdaMap = ImmutableMap.builder(); - - int counter = 0; - for (LambdaDefinitionExpression lambdaExpression : lambdaExpressions) { - CompiledLambda compiledLambda = LambdaBytecodeGenerator.preGenerateLambdaExpression( - lambdaExpression, - "lambda_" + counter, - containerClassDefinition, - compiledLambdaMap.build(), - callSiteBinder, - cachedInstanceBinder, - metadata.getFunctionManager()); - compiledLambdaMap.put(lambdaExpression, compiledLambda); - counter++; - } - - return compiledLambdaMap.build(); - } - private static void generateConstructor( ClassDefinition classDefinition, CachedInstanceBinder cachedInstanceBinder, @@ -618,4 +597,36 @@ private static RowExpressionVisitor fieldReferenceCompiler( (scope, field) -> scope.getVariable("position"), callSiteBinder); } + + private static final class CacheKey + { + private final SqlFunctionProperties sqlFunctionProperties; + private final RowExpression rowExpression; + + private CacheKey(SqlFunctionProperties sqlFunctionProperties, RowExpression rowExpression) + { + this.sqlFunctionProperties = requireNonNull(sqlFunctionProperties, "sqlFunctionProperties is null"); + this.rowExpression = requireNonNull(rowExpression, "rowExpression is null"); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof CacheKey)) { + return false; + } + CacheKey that = (CacheKey) o; + return Objects.equals(sqlFunctionProperties, that.sqlFunctionProperties) && + Objects.equals(rowExpression, that.rowExpression); + } + + @Override + public int hashCode() + { + return Objects.hash(sqlFunctionProperties, rowExpression); + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/RowExpressionCompiler.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/RowExpressionCompiler.java index 3350eddcfcf86..6a9e922df6c13 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/RowExpressionCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/RowExpressionCompiler.java @@ -15,9 +15,14 @@ import com.facebook.presto.bytecode.BytecodeBlock; import com.facebook.presto.bytecode.BytecodeNode; +import com.facebook.presto.bytecode.ClassDefinition; import com.facebook.presto.bytecode.Scope; import com.facebook.presto.bytecode.Variable; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.SqlFunctionProperties; +import com.facebook.presto.spi.function.SqlInvokedScalarFunctionImplementation; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.InputReferenceExpression; @@ -29,7 +34,9 @@ import com.facebook.presto.sql.gen.LambdaBytecodeGenerator.CompiledLambda; import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -40,31 +47,43 @@ import static com.facebook.presto.bytecode.instruction.Constant.loadInt; import static com.facebook.presto.bytecode.instruction.Constant.loadLong; import static com.facebook.presto.bytecode.instruction.Constant.loadString; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IS_NULL; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.OR; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.sql.gen.BytecodeUtils.generateWrite; import static com.facebook.presto.sql.gen.BytecodeUtils.loadConstant; import static com.facebook.presto.sql.gen.LambdaBytecodeGenerator.generateLambda; +import static com.facebook.presto.sql.gen.LambdaBytecodeGenerator.generateMethodsForLambda; +import static com.facebook.presto.sql.relational.SqlFunctionUtils.getSqlFunctionRowExpression; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.format; public class RowExpressionCompiler { + private final ClassDefinition classDefinition; private final CallSiteBinder callSiteBinder; private final CachedInstanceBinder cachedInstanceBinder; private final RowExpressionVisitor fieldReferenceCompiler; - private final FunctionManager functionManager; + private final Metadata metadata; + private final SqlFunctionProperties sqlFunctionProperties; private final Map compiledLambdaMap; RowExpressionCompiler( + ClassDefinition classDefinition, CallSiteBinder callSiteBinder, CachedInstanceBinder cachedInstanceBinder, RowExpressionVisitor fieldReferenceCompiler, - FunctionManager functionManager, + Metadata metadata, + SqlFunctionProperties sqlFunctionProperties, Map compiledLambdaMap) { + this.classDefinition = classDefinition; this.callSiteBinder = callSiteBinder; this.cachedInstanceBinder = cachedInstanceBinder; this.fieldReferenceCompiler = fieldReferenceCompiler; - this.functionManager = functionManager; + this.metadata = metadata; + this.sqlFunctionProperties = sqlFunctionProperties; this.compiledLambdaMap = compiledLambdaMap; } @@ -86,14 +105,62 @@ private class Visitor @Override public BytecodeNode visitCall(CallExpression call, Context context) { - BytecodeGeneratorContext generatorContext = new BytecodeGeneratorContext( - RowExpressionCompiler.this, - context.getScope(), - callSiteBinder, - cachedInstanceBinder, - functionManager); + FunctionManager functionManager = metadata.getFunctionManager(); + FunctionMetadata functionMetadata = functionManager.getFunctionMetadata(call.getFunctionHandle()); + BytecodeGeneratorContext generatorContext; + switch (functionMetadata.getImplementationType()) { + case BUILTIN: + generatorContext = new BytecodeGeneratorContext( + RowExpressionCompiler.this, + context.getScope(), + callSiteBinder, + cachedInstanceBinder, + functionManager); + return (new FunctionCallCodeGenerator()).generateCall(call.getFunctionHandle(), generatorContext, call.getType(), call.getArguments(), context.getOutputBlockVariable()); + case SQL: + SqlInvokedScalarFunctionImplementation functionImplementation = (SqlInvokedScalarFunctionImplementation) functionManager.getScalarFunctionImplementation(call.getFunctionHandle()); + RowExpression function = getSqlFunctionRowExpression(functionMetadata, functionImplementation, metadata, sqlFunctionProperties, call.getArguments()); + + // Pre-compile lambda bytecode + // When we inline the input parameters, if the parameter contains lambda, that lambda has already been pre-compiled and exist in compiledLambdaMap. When we pre-compile the function + // it would be compiled again. Do not put these in the new map in this case. + Map newCompiledLambdaMap = new HashMap<>(compiledLambdaMap); + generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, function, metadata, sqlFunctionProperties, "sql") + .forEach(newCompiledLambdaMap::putIfAbsent); - return (new FunctionCallCodeGenerator()).generateCall(call.getFunctionHandle(), generatorContext, call.getType(), call.getArguments(), context.getOutputBlockVariable()); + // generate bytecode for SQL function + RowExpressionCompiler newRowExpressionCompiler = new RowExpressionCompiler(classDefinition, callSiteBinder, cachedInstanceBinder, fieldReferenceCompiler, metadata, sqlFunctionProperties, ImmutableMap.copyOf(newCompiledLambdaMap)); + // If called on null input, directly use the generated bytecode + if (functionMetadata.isCalledOnNullInput() || call.getArguments().isEmpty()) { + return newRowExpressionCompiler.compile( + function, + context.getScope(), + context.getOutputBlockVariable(), + context.getLambdaInterface()); + } + + // If returns null on null input, generate if(any input is null, null, generated bytecode) + generatorContext = new BytecodeGeneratorContext( + newRowExpressionCompiler, + context.getScope(), + callSiteBinder, + cachedInstanceBinder, + functionManager); + + return (new IfCodeGenerator()).generateExpression( + generatorContext, + call.getType(), + ImmutableList.of( + call.getArguments().stream() + .map(argument -> new SpecialFormExpression(IS_NULL, BOOLEAN, argument)) + .reduce((a, b) -> new SpecialFormExpression(OR, BOOLEAN, a, b)).get(), + new ConstantExpression(null, call.getType()), + function), + context.getOutputBlockVariable()); + + default: + throw new IllegalArgumentException(format("Unsupported function implementation type: %s", functionMetadata.getImplementationType())); + } } @Override @@ -185,7 +252,7 @@ public BytecodeNode visitLambda(LambdaDefinitionExpression lambda, Context conte context.getScope(), callSiteBinder, cachedInstanceBinder, - functionManager); + metadata.getFunctionManager()); return generateLambda( generatorContext, @@ -237,7 +304,7 @@ public BytecodeNode visitSpecialForm(SpecialFormExpression specialForm, Context break; // functions that require varargs and/or complex types (e.g., lists) case IN: - generator = new InCodeGenerator(functionManager); + generator = new InCodeGenerator(metadata.getFunctionManager()); break; // optimized implementations (shortcircuiting behavior) case AND: @@ -263,7 +330,7 @@ public BytecodeNode visitSpecialForm(SpecialFormExpression specialForm, Context context.getScope(), callSiteBinder, cachedInstanceBinder, - functionManager); + metadata.getFunctionManager()); return generator.generateExpression(generatorContext, specialForm.getType(), specialForm.getArguments(), context.getOutputBlockVariable()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/RowExpressionPredicateCompiler.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/RowExpressionPredicateCompiler.java index 88287242bff8a..6fa3be0440009 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/RowExpressionPredicateCompiler.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/RowExpressionPredicateCompiler.java @@ -20,13 +20,13 @@ import com.facebook.presto.bytecode.Parameter; import com.facebook.presto.bytecode.Scope; import com.facebook.presto.bytecode.Variable; -import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.operator.project.PageFieldsToInputParametersRewriter; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.relation.InputReferenceExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.Predicate; @@ -38,15 +38,13 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import javax.inject.Inject; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.TreeSet; import java.util.function.Supplier; @@ -62,7 +60,7 @@ import static com.facebook.presto.operator.project.PageFieldsToInputParametersRewriter.rewritePageFieldsToInputParameters; import static com.facebook.presto.spi.StandardErrorCode.COMPILER_ERROR; import static com.facebook.presto.sql.gen.BytecodeUtils.invoke; -import static com.facebook.presto.sql.gen.LambdaExpressionExtractor.extractLambdaExpressions; +import static com.facebook.presto.sql.gen.LambdaBytecodeGenerator.generateMethodsForLambda; import static com.facebook.presto.util.CompilerUtils.defineClass; import static com.facebook.presto.util.CompilerUtils.makeClassName; import static java.util.Objects.requireNonNull; @@ -70,25 +68,25 @@ public class RowExpressionPredicateCompiler implements PredicateCompiler { - private final FunctionManager functionManager; + private final Metadata metadata; - private final LoadingCache> predicateCache; + private final LoadingCache> predicateCache; @Inject public RowExpressionPredicateCompiler(Metadata metadata) { - this(requireNonNull(metadata, "metadata is null").getFunctionManager(), 10_000); + this(requireNonNull(metadata, "metadata is null"), 10_000); } - public RowExpressionPredicateCompiler(FunctionManager functionManager, int predicateCacheSize) + public RowExpressionPredicateCompiler(Metadata metadata, int predicateCacheSize) { - this.functionManager = requireNonNull(functionManager, "functionManager is null"); + this.metadata = requireNonNull(metadata, "metadata is null"); if (predicateCacheSize > 0) { predicateCache = CacheBuilder.newBuilder() .recordStats() .maximumSize(predicateCacheSize) - .build(CacheLoader.from(this::compilePredicateInternal)); + .build(CacheLoader.from(cacheKey -> compilePredicateInternal(cacheKey.sqlFunctionProperties, cacheKey.rowExpression))); } else { predicateCache = null; @@ -96,15 +94,15 @@ public RowExpressionPredicateCompiler(FunctionManager functionManager, int predi } @Override - public Supplier compilePredicate(RowExpression predicate) + public Supplier compilePredicate(SqlFunctionProperties sqlFunctionProperties, RowExpression predicate) { if (predicateCache == null) { - return compilePredicateInternal(predicate); + return compilePredicateInternal(sqlFunctionProperties, predicate); } - return predicateCache.getUnchecked(predicate); + return predicateCache.getUnchecked(new CacheKey(sqlFunctionProperties, predicate)); } - private Supplier compilePredicateInternal(RowExpression predicate) + private Supplier compilePredicateInternal(SqlFunctionProperties sqlFunctionProperties, RowExpression predicate) { requireNonNull(predicate, "predicate is null"); @@ -114,7 +112,7 @@ private Supplier compilePredicateInternal(RowExpression predicate) .toArray(); CallSiteBinder callSiteBinder = new CallSiteBinder(); - ClassDefinition classDefinition = definePredicateClass(result.getRewrittenExpression(), inputChannels, callSiteBinder); + ClassDefinition classDefinition = definePredicateClass(sqlFunctionProperties, result.getRewrittenExpression(), inputChannels, callSiteBinder); Class predicateClass; try { @@ -134,7 +132,7 @@ private Supplier compilePredicateInternal(RowExpression predicate) }; } - private ClassDefinition definePredicateClass(RowExpression predicate, int[] inputChannels, CallSiteBinder callSiteBinder) + private ClassDefinition definePredicateClass(SqlFunctionProperties sqlFunctionProperties, RowExpression predicate, int[] inputChannels, CallSiteBinder callSiteBinder) { ClassDefinition classDefinition = new ClassDefinition( a(PUBLIC, FINAL), @@ -144,7 +142,7 @@ private ClassDefinition definePredicateClass(RowExpression predicate, int[] inpu CachedInstanceBinder cachedInstanceBinder = new CachedInstanceBinder(classDefinition, callSiteBinder); - generatePredicateMethod(classDefinition, callSiteBinder, cachedInstanceBinder, predicate); + generatePredicateMethod(sqlFunctionProperties, classDefinition, callSiteBinder, cachedInstanceBinder, predicate); // getInputChannels classDefinition.declareMethod(a(PUBLIC), "getInputChannels", type(int[].class)) @@ -159,12 +157,13 @@ private ClassDefinition definePredicateClass(RowExpression predicate, int[] inpu } private MethodDefinition generatePredicateMethod( + SqlFunctionProperties sqlFunctionProperties, ClassDefinition classDefinition, CallSiteBinder callSiteBinder, CachedInstanceBinder cachedInstanceBinder, RowExpression predicate) { - Map compiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, predicate); + Map compiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, predicate, metadata, sqlFunctionProperties); Parameter session = arg("session", ConnectorSession.class); Parameter page = arg("page", Page.class); @@ -189,10 +188,12 @@ private MethodDefinition generatePredicateMethod( Variable wasNullVariable = scope.declareVariable("wasNull", body, constantFalse()); RowExpressionCompiler compiler = new RowExpressionCompiler( + classDefinition, callSiteBinder, cachedInstanceBinder, fieldReferenceCompiler(callSiteBinder), - functionManager, + metadata, + sqlFunctionProperties, compiledLambdaMap); Variable result = scope.declareVariable(boolean.class, "result"); @@ -203,32 +204,6 @@ private MethodDefinition generatePredicateMethod( return method; } - private Map generateMethodsForLambda( - ClassDefinition containerClassDefinition, - CallSiteBinder callSiteBinder, - CachedInstanceBinder cachedInstanceBinder, - RowExpression expression) - { - Set lambdaExpressions = ImmutableSet.copyOf(extractLambdaExpressions(expression)); - ImmutableMap.Builder compiledLambdaMap = ImmutableMap.builder(); - - int counter = 0; - for (LambdaDefinitionExpression lambdaExpression : lambdaExpressions) { - LambdaBytecodeGenerator.CompiledLambda compiledLambda = LambdaBytecodeGenerator.preGenerateLambdaExpression( - lambdaExpression, - "lambda_" + counter, - containerClassDefinition, - compiledLambdaMap.build(), - callSiteBinder, - cachedInstanceBinder, - functionManager); - compiledLambdaMap.put(lambdaExpression, compiledLambda); - counter++; - } - - return compiledLambdaMap.build(); - } - private static void declareBlockVariables(RowExpression expression, Parameter page, Scope scope, BytecodeBlock body) { for (int channel : getInputChannels(expression)) { @@ -276,4 +251,36 @@ private static void generateConstructor( cachedInstanceBinder.generateInitializations(thisVariable, body); body.ret(); } + + private static final class CacheKey + { + private final SqlFunctionProperties sqlFunctionProperties; + private final RowExpression rowExpression; + + private CacheKey(SqlFunctionProperties sqlFunctionProperties, RowExpression rowExpression) + { + this.sqlFunctionProperties = requireNonNull(sqlFunctionProperties, "sqlFunctionProperties is null"); + this.rowExpression = requireNonNull(rowExpression, "rowExpression is null"); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof CacheKey)) { + return false; + } + CacheKey that = (CacheKey) o; + return Objects.equals(sqlFunctionProperties, that.sqlFunctionProperties) && + Objects.equals(rowExpression, that.rowExpression); + } + + @Override + public int hashCode() + { + return Objects.hash(sqlFunctionProperties, rowExpression); + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/SqlTypeBytecodeExpression.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/SqlTypeBytecodeExpression.java index 41914fb87f7b8..9d98e112a47e8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/SqlTypeBytecodeExpression.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/SqlTypeBytecodeExpression.java @@ -40,6 +40,16 @@ public static SqlTypeBytecodeExpression constantType(CallSiteBinder callSiteBind return new SqlTypeBytecodeExpression(type, binding, BOOTSTRAP_METHOD); } + private static String generateName(Type type) + { + String name = type.getTypeSignature().toString(); + if (name.length() > 20) { + // Use type base to reduce the identifier size in generated code + name = type.getTypeSignature().getBase(); + } + return name.replaceAll("\\W+", "_"); + } + private final Type type; private final Binding binding; private final Method bootstrapMethod; @@ -47,7 +57,6 @@ public static SqlTypeBytecodeExpression constantType(CallSiteBinder callSiteBind private SqlTypeBytecodeExpression(Type type, Binding binding, Method bootstrapMethod) { super(type(Type.class)); - this.type = requireNonNull(type, "type is null"); this.binding = requireNonNull(binding, "binding is null"); this.bootstrapMethod = requireNonNull(bootstrapMethod, "bootstrapMethod is null"); @@ -56,7 +65,7 @@ private SqlTypeBytecodeExpression(Type type, Binding binding, Method bootstrapMe @Override public BytecodeNode getBytecode(MethodGenerationContext generationContext) { - return InvokeInstruction.invokeDynamic(type.getTypeSignature().toString().replaceAll("\\W+", "_"), binding.getType(), bootstrapMethod, binding.getBindingId()); + return InvokeInstruction.invokeDynamic(generateName(type), binding.getType(), bootstrapMethod, binding.getBindingId()); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/gen/SwitchCodeGenerator.java b/presto-main/src/main/java/com/facebook/presto/sql/gen/SwitchCodeGenerator.java index 917dc42b2046b..fde1a4a1ea560 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/gen/SwitchCodeGenerator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/gen/SwitchCodeGenerator.java @@ -126,7 +126,7 @@ else if ( == ) { // check if wasNull is true BytecodeNode equalsCall = generatorContext.generateCall( EQUAL.name(), - generatorContext.getFunctionManager().getScalarFunctionImplementation(equalsFunction), + generatorContext.getFunctionManager().getBuiltInScalarFunctionImplementation(equalsFunction), ImmutableList.of(generatorContext.generate(operand, Optional.empty()), getTempVariableNode)); BytecodeBlock condition = new BytecodeBlock() diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/CompilerConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/CompilerConfig.java index 8d43d7b49c2a7..1856cc2ffa0aa 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/CompilerConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/CompilerConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.DefunctConfig; import com.facebook.presto.spi.function.Description; -import io.airlift.configuration.Config; -import io.airlift.configuration.DefunctConfig; import javax.validation.constraints.Min; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/ConnectorPlanOptimizerManager.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/ConnectorPlanOptimizerManager.java index 5754e8ad2f6df..6be197f8e92b2 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/ConnectorPlanOptimizerManager.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/ConnectorPlanOptimizerManager.java @@ -15,6 +15,7 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorPlanOptimizer; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.connector.ConnectorPlanOptimizerProvider; import com.google.common.collect.ImmutableMap; @@ -24,6 +25,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.Maps.transformValues; import static java.util.Objects.requireNonNull; @@ -43,8 +45,20 @@ public void addPlanOptimizerProvider(ConnectorId connectorId, ConnectorPlanOptim "ConnectorPlanOptimizerProvider for connector '%s' is already registered", connectorId); } - public Map> getOptimizers() + public Map> getOptimizers(PlanPhase phase) { - return ImmutableMap.copyOf(transformValues(planOptimizerProviders, ConnectorPlanOptimizerProvider::getConnectorPlanOptimizers)); + switch (phase) { + case LOGICAL: + return ImmutableMap.copyOf(transformValues(planOptimizerProviders, ConnectorPlanOptimizerProvider::getLogicalPlanOptimizers)); + case PHYSICAL: + return ImmutableMap.copyOf(transformValues(planOptimizerProviders, ConnectorPlanOptimizerProvider::getPhysicalPlanOptimizers)); + default: + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unknown plan phase " + phase); + } + } + + public enum PlanPhase + { + LOGICAL, PHYSICAL } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/EffectivePredicateExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/EffectivePredicateExtractor.java index 633885aea4f6d..a60f661fc971e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/EffectivePredicateExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/EffectivePredicateExtractor.java @@ -14,24 +14,24 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.optimizations.JoinNodeUtils; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; -import com.facebook.presto.sql.planner.plan.TopNNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.tree.ComparisonExpression; @@ -58,6 +58,7 @@ import static com.facebook.presto.sql.ExpressionUtils.extractConjuncts; import static com.facebook.presto.sql.ExpressionUtils.filterDeterministicConjuncts; import static com.facebook.presto.sql.planner.EqualityInference.createEqualityInference; +import static com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils.outputMap; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; import static com.facebook.presto.sql.tree.BooleanLiteral.TRUE_LITERAL; import static com.google.common.base.Predicates.in; @@ -70,6 +71,7 @@ *

* Note: non-deterministic predicates can not be pulled up (so they will be ignored) */ +@Deprecated public class EffectivePredicateExtractor { private static final Predicate> VARIABLE_MATCHES_EXPRESSION = @@ -224,7 +226,7 @@ public Expression visitWindow(WindowNode node, Void context) @Override public Expression visitUnion(UnionNode node, Void context) { - return deriveCommonPredicates(node, source -> Multimaps.transformValues(node.outputMap(source), variable -> new SymbolReference(variable.getName())).entries()); + return deriveCommonPredicates(node, source -> Multimaps.transformValues(outputMap(node, source), variable -> new SymbolReference(variable.getName())).entries()); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/EqualityInference.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/EqualityInference.java index e4c52c7506916..74229f75dae2a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/EqualityInference.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/EqualityInference.java @@ -51,6 +51,7 @@ /** * Makes equality based inferences to rewrite Expressions and generate equality sets in terms of specified symbol scopes */ +@Deprecated public class EqualityInference { // Ordering used to determine Expression preference when determining canonicals diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionDomainTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionDomainTranslator.java index 65d853d055a55..8f261af9eb79b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionDomainTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionDomainTranslator.java @@ -450,8 +450,8 @@ else if (symbolExpression instanceof Cast) { private Optional toNormalizedSimpleComparison(ComparisonExpression comparison) { Map, Type> expressionTypes = analyzeExpression(comparison); - Object left = ExpressionInterpreter.expressionOptimizer(comparison.getLeft(), metadata, session, expressionTypes).optimize(NoOpSymbolResolver.INSTANCE); - Object right = ExpressionInterpreter.expressionOptimizer(comparison.getRight(), metadata, session, expressionTypes).optimize(NoOpSymbolResolver.INSTANCE); + Object left = ExpressionInterpreter.expressionOptimizer(comparison.getLeft(), metadata, session, expressionTypes).optimize(NoOpVariableResolver.INSTANCE); + Object right = ExpressionInterpreter.expressionOptimizer(comparison.getRight(), metadata, session, expressionTypes).optimize(NoOpVariableResolver.INSTANCE); Type leftType = expressionTypes.get(NodeRef.of(comparison.getLeft())); Type rightType = expressionTypes.get(NodeRef.of(comparison.getRight())); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionExtractor.java index 4483bc5a1836e..0feaea4d2c63d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/ExpressionExtractor.java @@ -13,16 +13,17 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.sql.planner.iterative.GroupReference; import com.facebook.presto.sql.planner.iterative.Lookup; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.google.common.collect.ImmutableList; import java.util.List; @@ -94,7 +95,7 @@ public Void visitAggregation(AggregationNode node, ImmutableList.Builder { + private final Map, Type> generatedExpressionTypes = new HashMap<>(); + @Override public Object visitFieldReference(FieldReference node, Object context) { @@ -330,7 +340,7 @@ protected Object visitIdentifier(Identifier node, Object context) // ExpressionInterpreter should only be invoked after planning. // As a result, this method should be unreachable. // However, RelationPlanner.visitUnnest and visitValues invokes evaluateConstantExpression. - return ((SymbolResolver) context).getValue(new Symbol(node.getValue())); + return ((VariableResolver) context).getValue(variable(node.getValue(), type(node))); } @Override @@ -342,7 +352,7 @@ protected Object visitParameter(Parameter node, Object context) @Override protected Object visitSymbolReference(SymbolReference node, Object context) { - return ((SymbolResolver) context).getValue(Symbol.from(node)); + return ((VariableResolver) context).getValue(variable(node.getName(), type(node))); } @Override @@ -491,8 +501,17 @@ private boolean isEqual(Object operand1, Type type1, Object operand2, Type type2 return Boolean.TRUE.equals(invokeOperator(OperatorType.EQUAL, ImmutableList.of(type1, type2), ImmutableList.of(operand1, operand2))); } + private void addGeneratedExpressionType(Expression expression, Type type) + { + generatedExpressionTypes.put(NodeRef.of(expression), type); + } + private Type type(Expression expression) { + Type type = generatedExpressionTypes.get(NodeRef.of(expression)); + if (type != null) { + return type; + } return expressionTypes.get(NodeRef.of(expression)); } @@ -662,10 +681,10 @@ protected Object visitArithmeticUnary(ArithmeticUnaryExpression node, Object con return value; case MINUS: FunctionHandle operatorHandle = metadata.getFunctionManager().resolveOperator(OperatorType.NEGATION, fromTypes(types(node.getValue()))); - MethodHandle handle = metadata.getFunctionManager().getScalarFunctionImplementation(operatorHandle).getMethodHandle(); + MethodHandle handle = metadata.getFunctionManager().getBuiltInScalarFunctionImplementation(operatorHandle).getMethodHandle(); if (handle.type().parameterCount() > 0 && handle.type().parameterType(0) == ConnectorSession.class) { - handle = handle.bindTo(session); + handle = handle.bindTo(connectorSession); } try { return handle.invokeWithArguments(value); @@ -879,20 +898,46 @@ protected Object visitFunctionCall(FunctionCall node, Object context) argumentValues.add(value); argumentTypes.add(type); } - FunctionHandle functionHandle = metadata.getFunctionManager().resolveFunction(session, node.getName(), fromTypes(argumentTypes)); + FunctionHandle functionHandle = metadata.getFunctionManager().resolveFunction(session.getTransactionId(), node.getName(), fromTypes(argumentTypes)); FunctionMetadata functionMetadata = metadata.getFunctionManager().getFunctionMetadata(functionHandle); - for (int i = 0; i < argumentValues.size(); i++) { - Object value = argumentValues.get(i); - if (value == null && !functionMetadata.isCalledOnNullInput()) { - return null; + if (!functionMetadata.isCalledOnNullInput()) { + for (int i = 0; i < argumentValues.size(); i++) { + Object value = argumentValues.get(i); + if (value == null) { + return null; + } } } // do not optimize non-deterministic functions if (optimize && (!functionMetadata.isDeterministic() || hasUnresolvedValue(argumentValues) || node.getName().equals(QualifiedName.of("fail")))) { - return new FunctionCall(node.getName(), node.getWindow(), node.isDistinct(), toExpressions(argumentValues, argumentTypes)); + return new FunctionCall(node.getName(), node.getWindow(), node.isDistinct(), node.isIgnoreNulls(), toExpressions(argumentValues, argumentTypes)); + } + + Object result; + + switch (functionMetadata.getImplementationType()) { + case BUILTIN: + result = functionInvoker.invoke(functionHandle, connectorSession, argumentValues); + break; + case SQL: + Expression function = getSqlFunctionExpression(functionMetadata, (SqlInvokedScalarFunctionImplementation) metadata.getFunctionManager().getScalarFunctionImplementation(functionHandle), session.getSqlFunctionProperties(), node.getArguments()); + ExpressionInterpreter functionInterpreter = new ExpressionInterpreter( + function, + metadata, + session, + getExpressionTypes(session, metadata, new SqlParser(), TypeProvider.empty(), function, emptyList(), WarningCollector.NOOP), + optimize); + result = functionInterpreter.visitor.process(function, context); + break; + default: + throw new IllegalArgumentException(format("Unsupported function implementation type: %s", functionMetadata.getImplementationType())); } - return functionInvoker.invoke(functionHandle, connectorSession, argumentValues); + + if (optimize && !isSerializable(result, type(node))) { + return new FunctionCall(node.getName(), node.getWindow(), node.isDistinct(), node.isIgnoreNulls(), toExpressions(argumentValues, argumentTypes)); + } + return result; } @Override @@ -919,7 +964,7 @@ protected Object visitLambdaExpression(LambdaExpression node, Object context) .map(Primitives::wrap) .collect(toImmutableList()), argumentNames, - map -> process(body, new LambdaSymbolResolver(map))); + map -> process(body, new LambdaVariableResolver(map))); } @Override @@ -1067,16 +1112,19 @@ public Object visitCast(Cast node, Object context) return value; } - // hack!!! don't optimize CASTs for types that cannot be represented in the SQL AST - // TODO: this will not be an issue when we migrate to RowExpression tree for this, which allows arbitrary literals. if (optimize && !isSupportedLiteralType(type(node))) { + // do not invoke cast function if it can produce unserializable type return new Cast(toExpression(value, sourceType), node.getType(), node.isSafe(), node.isTypeOnly()); } FunctionHandle operator = metadata.getFunctionManager().lookupCast(CAST, sourceType.getTypeSignature(), targetType.getTypeSignature()); try { - return functionInvoker.invoke(operator, connectorSession, ImmutableList.of(value)); + Object castedValue = functionInvoker.invoke(operator, connectorSession, ImmutableList.of(value)); + if (optimize && !isSerializable(castedValue, type(node))) { + return new Cast(toExpression(value, sourceType), node.getType(), node.isSafe(), node.isTypeOnly()); + } + return castedValue; } catch (RuntimeException e) { if (node.isSafe()) { @@ -1095,7 +1143,9 @@ protected Object visitArrayConstructor(ArrayConstructor node, Object context) for (Expression expression : node.getValues()) { Object value = process(expression, context); if (value instanceof Expression) { - return visitFunctionCall(new FunctionCall(QualifiedName.of(ArrayConstructor.ARRAY_CONSTRUCTOR), node.getValues()), context); + FunctionCall functionCall = new FunctionCall(QualifiedName.of(ArrayConstructor.ARRAY_CONSTRUCTOR), node.getValues()); + addGeneratedExpressionType(functionCall, type(node)); + return visitFunctionCall(functionCall, context); } writeNativeValue(elementType, arrayBlockBuilder, value); } @@ -1106,13 +1156,9 @@ protected Object visitArrayConstructor(ArrayConstructor node, Object context) @Override protected Object visitCurrentUser(CurrentUser node, Object context) { - return visitFunctionCall(DesugarCurrentUser.getCall(node), context); - } - - @Override - protected Object visitCurrentPath(CurrentPath node, Object context) - { - return visitFunctionCall(DesugarCurrentPath.getCall(node), context); + FunctionCall functionCall = DesugarCurrentUser.getCall(node); + addGeneratedExpressionType(functionCall, type(node)); + return visitFunctionCall(functionCall, context); } @Override @@ -1225,17 +1271,31 @@ private Expression toExpression(Object base, Type type) return literalEncoder.toExpression(base, type); } + private boolean isSerializable(Object value, Type type) + { + requireNonNull(type, "type is null"); + // If value is already Expression, literal values contained inside should already have been made serializable. Otherwise, we make sure the object is small and serializable. + return value instanceof Expression || (isSupportedLiteralType(type) && estimatedSizeInBytes(value) <= MAX_SERIALIZABLE_OBJECT_SIZE); + } + private List toExpressions(List values, List types) { return literalEncoder.toExpressions(values, types); } } - private static Expression createFailureFunction(RuntimeException exception, Type type) + private Expression createFailureFunction(RuntimeException exception, Type type) { requireNonNull(exception, "Exception is null"); String failureInfo = JsonCodec.jsonCodec(FailureInfo.class).toJson(Failures.toFailure(exception).toFailureInfo()); + if (exception instanceof PrestoException) { + long errorCode = ((PrestoException) exception).getErrorCode().getCode(); + FunctionCall jsonParse = new FunctionCall(QualifiedName.of("json_parse"), ImmutableList.of(new StringLiteral(failureInfo))); + FunctionCall failureFunction = new FunctionCall(QualifiedName.of("fail"), ImmutableList.of(literalEncoder.toExpression(errorCode, INTEGER), jsonParse)); + + return new Cast(failureFunction, type.getTypeSignature().toString()); + } FunctionCall jsonParse = new FunctionCall(QualifiedName.of("json_parse"), ImmutableList.of(new StringLiteral(failureInfo))); FunctionCall failureFunction = new FunctionCall(QualifiedName.of("fail"), ImmutableList.of(jsonParse)); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/InputExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/InputExtractor.java index 16d5d89f50021..cdeb072cddf3e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/InputExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/InputExtractor.java @@ -25,7 +25,6 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.TableScanNode; -import com.facebook.presto.spi.statistics.ColumnStatistics; import com.facebook.presto.spi.statistics.TableStatistics; import com.facebook.presto.sql.planner.plan.IndexSourceNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; @@ -37,13 +36,10 @@ import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static java.util.Map.Entry; +import static com.google.common.collect.ImmutableList.toImmutableList; public class InputExtractor { @@ -120,14 +116,13 @@ public Void visitTableScan(TableScanNode node, Context context) columns.add(createColumn(metadata.getColumnMetadata(session, tableHandle, columnHandle))); } - Set desiredColumns = node.getAssignments().values().stream().collect(toImmutableSet()); + List desiredColumns = node.getAssignments().values().stream().collect(toImmutableList()); Optional statistics = Optional.empty(); if (context.isExtractStatistics()) { Constraint constraint = new Constraint<>(node.getCurrentConstraint()); - TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, constraint); - statistics = Optional.of(pruneStatistics(tableStatistics, desiredColumns)); + statistics = Optional.of(metadata.getTableStatistics(session, tableHandle, desiredColumns, constraint)); } inputs.add(createInput(metadata.getTableMetadata(session, tableHandle), tableHandle, columns, statistics)); @@ -145,14 +140,13 @@ public Void visitIndexSource(IndexSourceNode node, Context context) columns.add(createColumn(metadata.getColumnMetadata(session, tableHandle, columnHandle))); } - Set desiredColumns = node.getAssignments().values().stream().collect(toImmutableSet()); + List desiredColumns = node.getAssignments().values().stream().collect(toImmutableList()); Optional statistics = Optional.empty(); if (context.isExtractStatistics()) { Constraint constraint = new Constraint<>(node.getCurrentConstraint()); - TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, constraint); - statistics = Optional.of(pruneStatistics(tableStatistics, desiredColumns)); + statistics = Optional.of(metadata.getTableStatistics(session, tableHandle, desiredColumns, constraint)); } inputs.add(createInput(metadata.getTableMetadata(session, tableHandle), tableHandle, columns, statistics)); @@ -168,14 +162,6 @@ public Void visitPlan(PlanNode node, Context context) } return null; } - - private TableStatistics pruneStatistics(TableStatistics tableStatistics, Set desiredColumns) - { - Map columnStatistics = tableStatistics.getColumnStatistics().entrySet().stream() - .filter(entry -> desiredColumns.contains(entry.getKey())) - .collect(toImmutableMap(Entry::getKey, Entry::getValue)); - return new TableStatistics(tableStatistics.getRowCount(), columnStatistics); - } } private static class Context @@ -184,11 +170,6 @@ private static class Context public Context() {} - public Context(boolean extractStatistics) - { - this.extractStatistics = extractStatistics; - } - public boolean isExtractStatistics() { return extractStatistics; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/Interpreters.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/Interpreters.java index 165bdc73fb958..c800be2918293 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/Interpreters.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/Interpreters.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.VarcharType; @@ -67,21 +68,21 @@ static boolean interpretLikePredicate(Type valueType, Slice value, Regex regex) return LikeFunctions.likeChar((long) ((CharType) valueType).getLength(), value, regex); } - public static class LambdaSymbolResolver - implements SymbolResolver + public static class LambdaVariableResolver + implements VariableResolver { private final Map values; - public LambdaSymbolResolver(Map values) + public LambdaVariableResolver(Map values) { this.values = requireNonNull(values, "values is null"); } @Override - public Object getValue(Symbol symbol) + public Object getValue(VariableReferenceExpression variable) { - checkState(values.containsKey(symbol.getName()), "values does not contain %s", symbol); - return values.get(symbol.getName()); + checkState(values.containsKey(variable.getName()), "values does not contain %s", variable); + return values.get(variable.getName()); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralEncoder.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralEncoder.java index 873dc41518f9a..a7c4ce1eea20f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralEncoder.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralEncoder.java @@ -17,12 +17,15 @@ import com.facebook.presto.operator.scalar.VarbinaryFunctions; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockEncodingSerde; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.Decimals; +import com.facebook.presto.spi.type.FunctionType; +import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.SqlDate; import com.facebook.presto.spi.type.StandardTypes; @@ -52,6 +55,7 @@ import java.util.List; import java.util.Set; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -233,15 +237,19 @@ public Expression toExpression(Object object, Type type) // able to encode it in the plan that gets sent to workers. // We do this by transforming the in-memory varbinary into a call to from_base64() FunctionCall fromBase64 = new FunctionCall(QualifiedName.of("from_base64"), ImmutableList.of(new StringLiteral(VarbinaryFunctions.toBase64((Slice) object).toStringUtf8()))); - return new FunctionCall(QualifiedName.of(signature.getName()), ImmutableList.of(fromBase64)); + return new FunctionCall(QualifiedName.of(signature.getNameSuffix()), ImmutableList.of(fromBase64)); } Expression rawLiteral = toExpression(object, typeForMagicLiteral(type)); - return new FunctionCall(QualifiedName.of(signature.getName()), ImmutableList.of(rawLiteral)); + return new FunctionCall(QualifiedName.of(signature.getNameSuffix()), ImmutableList.of(rawLiteral)); } public static boolean isSupportedLiteralType(Type type) { + if (type instanceof FunctionType) { + // FunctionType contains compiled lambda thus not serializable. + return false; + } if (type instanceof ArrayType) { return isSupportedLiteralType(((ArrayType) type).getElementType()); } @@ -250,14 +258,43 @@ else if (type instanceof RowType) { return rowType.getTypeParameters().stream() .allMatch(LiteralEncoder::isSupportedLiteralType); } + else if (type instanceof MapType) { + MapType mapType = (MapType) type; + return isSupportedLiteralType(mapType.getKeyType()) && isSupportedLiteralType(mapType.getValueType()); + } return SUPPORTED_LITERAL_TYPES.contains(type.getJavaType()); } + public static long estimatedSizeInBytes(Object object) + { + if (object == null) { + return 1; + } + Class javaType = object.getClass(); + if (javaType == Long.class) { + return Long.BYTES; + } + else if (javaType == Double.class) { + return Double.BYTES; + } + else if (javaType == Boolean.class) { + return 1; + } + else if (object instanceof Block) { + return ((Block) object).getSizeInBytes(); + } + else if (object instanceof Slice) { + return ((Slice) object).length(); + } + // unknown for rest of types + return Integer.MAX_VALUE; + } + public static Signature getMagicLiteralFunctionSignature(Type type) { TypeSignature argumentType = typeForMagicLiteral(type).getTypeSignature(); - return new Signature(MAGIC_LITERAL_FUNCTION_PREFIX + type.getTypeSignature(), + return new Signature(QualifiedFunctionName.of(DEFAULT_NAMESPACE, MAGIC_LITERAL_FUNCTION_PREFIX + type.getTypeSignature()), SCALAR, type.getTypeSignature(), argumentType); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralInterpreter.java index 786996729dd70..90368cfa8df13 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralInterpreter.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.operator.scalar.VarbinaryFunctions; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.FunctionHandle; @@ -158,6 +159,10 @@ public static Object evaluate(ConnectorSession session, ConstantExpression node) if (type instanceof IntervalYearMonthType) { return new SqlIntervalYearMonth(((Long) node.getValue()).intValue()); } + if (type.getJavaType().equals(Slice.class)) { + // DO NOT ever remove toBase64. Calling toString directly on Slice whose base is not byte[] will cause JVM to crash. + return "'" + VarbinaryFunctions.toBase64((Slice) node.getValue()).toStringUtf8() + "'"; + } // We should not fail at the moment; just return the raw value (block, regex, etc) to the user return node.getValue(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java index 4ad8a75cd1326..51a5dfa4f2b61 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java @@ -13,14 +13,22 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.Session; import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.execution.ExplainAnalyzeContext; -import com.facebook.presto.execution.StageId; +import com.facebook.presto.execution.StageExecutionId; import com.facebook.presto.execution.TaskManagerConfig; import com.facebook.presto.execution.buffer.OutputBuffer; import com.facebook.presto.execution.buffer.PagesSerdeFactory; +import com.facebook.presto.execution.scheduler.ExecutionWriterTarget; +import com.facebook.presto.execution.scheduler.ExecutionWriterTarget.CreateHandle; +import com.facebook.presto.execution.scheduler.ExecutionWriterTarget.DeleteHandle; +import com.facebook.presto.execution.scheduler.ExecutionWriterTarget.InsertHandle; +import com.facebook.presto.execution.scheduler.TableWriteInfo; +import com.facebook.presto.execution.scheduler.TableWriteInfo.DeleteScanInfo; import com.facebook.presto.index.IndexManager; +import com.facebook.presto.metadata.AnalyzeTableHandle; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.operator.AggregationOperator.AggregationOperatorFactory; @@ -56,7 +64,6 @@ import com.facebook.presto.operator.PagesSpatialIndexFactory; import com.facebook.presto.operator.PartitionFunction; import com.facebook.presto.operator.PartitionedLookupSourceFactory; -import com.facebook.presto.operator.PartitionedOutputOperator.PartitionedOutputFactory; import com.facebook.presto.operator.PipelineExecutionStrategy; import com.facebook.presto.operator.RowNumberOperator; import com.facebook.presto.operator.ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory; @@ -72,6 +79,7 @@ import com.facebook.presto.operator.TableCommitContext; import com.facebook.presto.operator.TableFinishOperator.LifespanCommitter; import com.facebook.presto.operator.TableScanOperator.TableScanOperatorFactory; +import com.facebook.presto.operator.TableWriterMergeOperator.TableWriterMergeOperatorFactory; import com.facebook.presto.operator.TaskContext; import com.facebook.presto.operator.TaskExchangeClientManager; import com.facebook.presto.operator.TaskOutputOperator.TaskOutputFactory; @@ -96,6 +104,8 @@ import com.facebook.presto.operator.index.IndexSourceOperator; import com.facebook.presto.operator.project.CursorProcessor; import com.facebook.presto.operator.project.PageProcessor; +import com.facebook.presto.operator.repartition.OptimizedPartitionedOutputOperator.OptimizedPartitionedOutputFactory; +import com.facebook.presto.operator.repartition.PartitionedOutputOperator.PartitionedOutputFactory; import com.facebook.presto.operator.window.FrameInfo; import com.facebook.presto.operator.window.WindowFunctionSupplier; import com.facebook.presto.spi.ColumnHandle; @@ -105,15 +115,27 @@ import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.RecordSet; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.block.SortOrder; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.function.SqlFunctionProperties; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.AggregationNode.Step; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; @@ -134,13 +156,8 @@ import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler.JoinFilterFunctionFactory; import com.facebook.presto.sql.gen.OrderingCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.optimizations.IndexJoinOptimizer; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; @@ -151,11 +168,9 @@ import com.facebook.presto.sql.planner.plan.IndexSourceNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.MetadataDeleteNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; @@ -165,11 +180,9 @@ import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode.DeleteHandle; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.planner.plan.WindowNode.Frame; @@ -189,7 +202,6 @@ import com.google.common.collect.Multimap; import com.google.common.collect.SetMultimap; import com.google.common.primitives.Ints; -import io.airlift.json.JsonCodec; import io.airlift.units.DataSize; import javax.inject.Inject; @@ -200,7 +212,6 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; @@ -214,10 +225,15 @@ import static com.facebook.presto.SystemSessionProperties.getAggregationOperatorUnspillMemoryLimit; import static com.facebook.presto.SystemSessionProperties.getFilterAndProjectMinOutputPageRowCount; import static com.facebook.presto.SystemSessionProperties.getFilterAndProjectMinOutputPageSize; +import static com.facebook.presto.SystemSessionProperties.getIndexLoaderTimeout; import static com.facebook.presto.SystemSessionProperties.getTaskConcurrency; +import static com.facebook.presto.SystemSessionProperties.getTaskPartitionedWriterCount; import static com.facebook.presto.SystemSessionProperties.getTaskWriterCount; import static com.facebook.presto.SystemSessionProperties.isExchangeCompressionEnabled; +import static com.facebook.presto.SystemSessionProperties.isOptimizedRepartitioningEnabled; import static com.facebook.presto.SystemSessionProperties.isSpillEnabled; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression; import static com.facebook.presto.operator.DistinctLimitOperator.DistinctLimitOperatorFactory; import static com.facebook.presto.operator.NestedLoopBuildOperator.NestedLoopBuildOperatorFactory; import static com.facebook.presto.operator.NestedLoopJoinOperator.NestedLoopJoinOperatorFactory; @@ -225,16 +241,18 @@ import static com.facebook.presto.operator.PipelineExecutionStrategy.UNGROUPED_EXECUTION; import static com.facebook.presto.operator.TableFinishOperator.TableFinishOperatorFactory; import static com.facebook.presto.operator.TableFinishOperator.TableFinisher; -import static com.facebook.presto.operator.TableWriterOperator.CONTEXT_CHANNEL; -import static com.facebook.presto.operator.TableWriterOperator.FRAGMENT_CHANNEL; -import static com.facebook.presto.operator.TableWriterOperator.ROW_COUNT_CHANNEL; -import static com.facebook.presto.operator.TableWriterOperator.STATS_START_CHANNEL; import static com.facebook.presto.operator.TableWriterOperator.TableWriterOperatorFactory; -import static com.facebook.presto.operator.UnnestOperator.UnnestOperatorFactory; +import static com.facebook.presto.operator.TableWriterUtils.CONTEXT_CHANNEL; +import static com.facebook.presto.operator.TableWriterUtils.FRAGMENT_CHANNEL; +import static com.facebook.presto.operator.TableWriterUtils.ROW_COUNT_CHANNEL; +import static com.facebook.presto.operator.TableWriterUtils.STATS_START_CHANNEL; import static com.facebook.presto.operator.WindowFunctionDefinition.window; +import static com.facebook.presto.operator.unnest.UnnestOperator.UnnestOperatorFactory; import static com.facebook.presto.spi.StandardErrorCode.COMPILER_ERROR; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; -import static com.facebook.presto.spi.relation.RowExpressionNodeInliner.replaceExpression; +import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.INTERMEDIATE; +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.TypeUtils.writeNativeValue; import static com.facebook.presto.sql.gen.LambdaBytecodeGenerator.compileLambdaProvider; @@ -244,20 +262,19 @@ import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_BROADCAST_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SCALED_WRITER_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.FINAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignments; import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; -import static com.facebook.presto.sql.planner.plan.TableWriterNode.CreateHandle; -import static com.facebook.presto.sql.planner.plan.TableWriterNode.InsertHandle; -import static com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.util.Reflection.constructorMethodHandle; import static com.facebook.presto.util.SpatialJoinUtils.ST_CONTAINS; +import static com.facebook.presto.util.SpatialJoinUtils.ST_CROSSES; import static com.facebook.presto.util.SpatialJoinUtils.ST_DISTANCE; +import static com.facebook.presto.util.SpatialJoinUtils.ST_EQUALS; import static com.facebook.presto.util.SpatialJoinUtils.ST_INTERSECTS; +import static com.facebook.presto.util.SpatialJoinUtils.ST_OVERLAPS; +import static com.facebook.presto.util.SpatialJoinUtils.ST_TOUCHES; import static com.facebook.presto.util.SpatialJoinUtils.ST_WITHIN; import static com.facebook.presto.util.SpatialJoinUtils.extractSupportedSpatialComparisons; import static com.facebook.presto.util.SpatialJoinUtils.extractSupportedSpatialFunctions; @@ -270,16 +287,17 @@ import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.collect.Range.closedOpen; import static io.airlift.units.DataSize.Unit.BYTE; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.stream.IntStream.range; public class LocalExecutionPlanner { private final Metadata metadata; - private final SqlParser sqlParser; private final Optional explainAnalyzeContext; private final PageSourceProvider pageSourceProvider; private final IndexManager indexManager; + private final PartitioningProviderManager partitioningProviderManager; private final NodePartitioningManager nodePartitioningManager; private final PageSinkManager pageSinkManager; private final ExpressionCompiler expressionCompiler; @@ -303,10 +321,10 @@ public class LocalExecutionPlanner @Inject public LocalExecutionPlanner( Metadata metadata, - SqlParser sqlParser, Optional explainAnalyzeContext, PageSourceProvider pageSourceProvider, IndexManager indexManager, + PartitioningProviderManager partitioningProviderManager, NodePartitioningManager nodePartitioningManager, PageSinkManager pageSinkManager, ExpressionCompiler expressionCompiler, @@ -327,9 +345,9 @@ public LocalExecutionPlanner( this.explainAnalyzeContext = requireNonNull(explainAnalyzeContext, "explainAnalyzeContext is null"); this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); this.indexManager = requireNonNull(indexManager, "indexManager is null"); + this.partitioningProviderManager = requireNonNull(partitioningProviderManager, "partitioningProviderManager is null"); this.nodePartitioningManager = requireNonNull(nodePartitioningManager, "nodePartitioningManager is null"); this.metadata = requireNonNull(metadata, "metadata is null"); - this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.pageSinkManager = requireNonNull(pageSinkManager, "pageSinkManager is null"); this.expressionCompiler = requireNonNull(expressionCompiler, "compiler is null"); this.pageFunctionCompiler = requireNonNull(pageFunctionCompiler, "pageFunctionCompiler is null"); @@ -358,7 +376,8 @@ public LocalExecutionPlan plan( StageExecutionDescriptor stageExecutionDescriptor, List partitionedSourceOrder, OutputBuffer outputBuffer, - TaskExchangeClientManager taskExchangeClientManager) + TaskExchangeClientManager taskExchangeClientManager, + TableWriteInfo tableWriteInfo) { List outputLayout = partitioningScheme.getOutputLayout(); @@ -375,7 +394,8 @@ public LocalExecutionPlan plan( types, partitionedSourceOrder, new TaskOutputFactory(outputBuffer), - taskExchangeClientManager); + taskExchangeClientManager, + tableWriteInfo); } // We can convert the variables directly into channels, because the root must be a sink and therefore the layout is fixed @@ -388,29 +408,27 @@ public LocalExecutionPlan plan( partitionChannelTypes = ImmutableList.of(BIGINT); } else { + checkArgument( + partitioningScheme.getPartitioning().getArguments().stream().allMatch(argument -> argument instanceof ConstantExpression || argument instanceof VariableReferenceExpression), + format("Expect all partitioning arguments to be either ConstantExpression or VariableReferenceExpression, but get %s", partitioningScheme.getPartitioning().getArguments())); partitionChannels = partitioningScheme.getPartitioning().getArguments().stream() .map(argument -> { - if (argument.isConstant()) { + if (argument instanceof ConstantExpression) { return -1; } - return outputLayout.indexOf(argument.getVariableReference()); + return outputLayout.indexOf((VariableReferenceExpression) argument); }) .collect(toImmutableList()); partitionConstants = partitioningScheme.getPartitioning().getArguments().stream() .map(argument -> { - if (argument.isConstant()) { - return Optional.of(argument.getConstant()); + if (argument instanceof ConstantExpression) { + return Optional.of((ConstantExpression) argument); } return Optional.empty(); }) .collect(toImmutableList()); partitionChannelTypes = partitioningScheme.getPartitioning().getArguments().stream() - .map(argument -> { - if (argument.isConstant()) { - return argument.getConstant().getType(); - } - return argument.getVariableReference().getType(); - }) + .map(RowExpression::getType) .collect(toImmutableList()); } @@ -424,6 +442,28 @@ public LocalExecutionPlan plan( nullChannel = OptionalInt.of(outputLayout.indexOf(getOnlyElement(partitioningColumns))); } + OutputFactory outputFactory; + if (isOptimizedRepartitioningEnabled(taskContext.getSession())) { + outputFactory = new OptimizedPartitionedOutputFactory( + partitionFunction, + partitionChannels, + partitionConstants, + partitioningScheme.isReplicateNullsAndAny(), + nullChannel, + outputBuffer, + maxPagePartitioningBufferSize); + } + else { + outputFactory = new PartitionedOutputFactory( + partitionFunction, + partitionChannels, + partitionConstants, + partitioningScheme.isReplicateNullsAndAny(), + nullChannel, + outputBuffer, + maxPagePartitioningBufferSize); + } + return plan( taskContext, stageExecutionDescriptor, @@ -431,15 +471,9 @@ public LocalExecutionPlan plan( outputLayout, types, partitionedSourceOrder, - new PartitionedOutputFactory( - partitionFunction, - partitionChannels, - partitionConstants, - partitioningScheme.isReplicateNullsAndAny(), - nullChannel, - outputBuffer, - maxPagePartitioningBufferSize), - taskExchangeClientManager); + outputFactory, + taskExchangeClientManager, + tableWriteInfo); } public LocalExecutionPlan plan( @@ -450,10 +484,11 @@ public LocalExecutionPlan plan( TypeProvider types, List partitionedSourceOrder, OutputFactory outputOperatorFactory, - TaskExchangeClientManager taskExchangeClientManager) + TaskExchangeClientManager taskExchangeClientManager, + TableWriteInfo tableWriteInfo) { Session session = taskContext.getSession(); - LocalExecutionPlanContext context = new LocalExecutionPlanContext(taskContext, types, taskExchangeClientManager); + LocalExecutionPlanContext context = new LocalExecutionPlanContext(taskContext, taskExchangeClientManager, tableWriteInfo); PhysicalOperation physicalOperation = plan.accept(new Visitor(session, stageExecutionDescriptor), context); @@ -524,37 +559,40 @@ private static void addLookupOuterDrivers(LocalExecutionPlanContext context) private static class LocalExecutionPlanContext { private final TaskContext taskContext; - private final TypeProvider types; private final TaskExchangeClientManager taskExchangeClientManager; private final List driverFactories; private final Optional indexSourceContext; // this is shared with all subContexts private final AtomicInteger nextPipelineId; + private final TableWriteInfo tableWriteInfo; private int nextOperatorId; private boolean inputDriver = true; private OptionalInt driverInstanceCount = OptionalInt.empty(); - public LocalExecutionPlanContext(TaskContext taskContext, TypeProvider types, TaskExchangeClientManager taskExchangeClientManager) + public LocalExecutionPlanContext( + TaskContext taskContext, + TaskExchangeClientManager taskExchangeClientManager, + TableWriteInfo tableWriteInfo) { - this(taskContext, types, taskExchangeClientManager, new ArrayList<>(), Optional.empty(), new AtomicInteger(0)); + this(taskContext, taskExchangeClientManager, new ArrayList<>(), Optional.empty(), new AtomicInteger(0), tableWriteInfo); } private LocalExecutionPlanContext( TaskContext taskContext, - TypeProvider types, TaskExchangeClientManager taskExchangeClientManager, List driverFactories, Optional indexSourceContext, - AtomicInteger nextPipelineId) + AtomicInteger nextPipelineId, + TableWriteInfo tableWriteInfo) { this.taskContext = taskContext; - this.types = types; this.taskExchangeClientManager = taskExchangeClientManager; this.driverFactories = driverFactories; this.indexSourceContext = indexSourceContext; this.nextPipelineId = nextPipelineId; + this.tableWriteInfo = tableWriteInfo; } public void addDriverFactory(boolean inputDriver, boolean outputDriver, List operatorFactories, OptionalInt driverInstances, PipelineExecutionStrategy pipelineExecutionStrategy) @@ -581,14 +619,9 @@ public Session getSession() return taskContext.getSession(); } - public StageId getStageId() + public StageExecutionId getStageExecutionId() { - return taskContext.getTaskId().getStageId(); - } - - public TypeProvider getTypes() - { - return types; + return taskContext.getTaskId().getStageExecutionId(); } public TaskExchangeClientManager getTaskExchangeClientManager() @@ -621,15 +654,20 @@ private void setInputDriver(boolean inputDriver) this.inputDriver = inputDriver; } + public TableWriteInfo getTableWriteInfo() + { + return tableWriteInfo; + } + public LocalExecutionPlanContext createSubContext() { checkState(!indexSourceContext.isPresent(), "index build plan can not have sub-contexts"); - return new LocalExecutionPlanContext(taskContext, types, taskExchangeClientManager, driverFactories, indexSourceContext, nextPipelineId); + return new LocalExecutionPlanContext(taskContext, taskExchangeClientManager, driverFactories, indexSourceContext, nextPipelineId, tableWriteInfo); } public LocalExecutionPlanContext createIndexSourceSubContext(IndexSourceContext indexSourceContext) { - return new LocalExecutionPlanContext(taskContext, types, taskExchangeClientManager, driverFactories, Optional.of(indexSourceContext), nextPipelineId); + return new LocalExecutionPlanContext(taskContext, taskExchangeClientManager, driverFactories, Optional.of(indexSourceContext), nextPipelineId, tableWriteInfo); } public OptionalInt getDriverInstanceCount() @@ -722,10 +760,10 @@ private PhysicalOperation createMergeSource(RemoteSourceNode node, LocalExecutio OrderingScheme orderingScheme = node.getOrderingScheme().get(); ImmutableMap layout = makeLayout(node); - List sortChannels = getChannelsForVariables(orderingScheme.getOrderBy(), layout); - List sortOrder = orderingScheme.getOrderingList(); + List sortChannels = getChannelsForVariables(orderingScheme.getOrderByVariables(), layout); + List sortOrder = getOrderingList(orderingScheme); - List types = getSourceOperatorTypes(node, context.getTypes()); + List types = getSourceOperatorTypes(node); ImmutableList outputChannels = IntStream.range(0, types.size()) .boxed() .collect(toImmutableList()); @@ -746,7 +784,10 @@ private PhysicalOperation createMergeSource(RemoteSourceNode node, LocalExecutio private PhysicalOperation createRemoteSource(RemoteSourceNode node, LocalExecutionPlanContext context) { - if (!context.getDriverInstanceCount().isPresent()) { + if (node.isEnsureSourceOrdering()) { + context.setDriverInstanceCount(1); + } + else if (!context.getDriverInstanceCount().isPresent()) { context.setDriverInstanceCount(getTaskConcurrency(session)); } @@ -829,7 +870,7 @@ public PhysicalOperation visitTopNRowNumber(TopNRowNumberNode node, LocalExecuti .map(channel -> source.getTypes().get(channel)) .collect(toImmutableList()); - List orderByVariables = node.getOrderingScheme().getOrderBy(); + List orderByVariables = node.getOrderingScheme().getOrderByVariables(); List sortChannels = getChannelsForVariables(orderByVariables, source.getLayout()); List sortOrder = orderByVariables.stream() .map(variable -> node.getOrderingScheme().getOrdering(variable)) @@ -882,8 +923,8 @@ public PhysicalOperation visitWindow(WindowNode node, LocalExecutionPlanContext if (node.getOrderingScheme().isPresent()) { OrderingScheme orderingScheme = node.getOrderingScheme().get(); - sortChannels = getChannelsForVariables(orderingScheme.getOrderBy(), source.getLayout()); - sortOrder = orderingScheme.getOrderingList(); + sortChannels = getChannelsForVariables(orderingScheme.getOrderByVariables(), source.getLayout()); + sortOrder = getOrderingList(orderingScheme); } ImmutableList.Builder outputChannels = ImmutableList.builder(); @@ -907,8 +948,9 @@ public PhysicalOperation visitWindow(WindowNode node, LocalExecutionPlanContext FrameInfo frameInfo = new FrameInfo(frame.getType(), frame.getStartType(), frameStartChannel, frame.getEndType(), frameEndChannel); - CallExpression call = entry.getValue().getFunctionCall(); - FunctionHandle functionHandle = entry.getValue().getFunctionHandle(); + WindowNode.Function function = entry.getValue(); + CallExpression call = function.getFunctionCall(); + FunctionHandle functionHandle = function.getFunctionHandle(); ImmutableList.Builder arguments = ImmutableList.builder(); for (RowExpression argument : call.getArguments()) { checkState(argument instanceof VariableReferenceExpression); @@ -918,7 +960,7 @@ public PhysicalOperation visitWindow(WindowNode node, LocalExecutionPlanContext FunctionManager functionManager = metadata.getFunctionManager(); WindowFunctionSupplier windowFunctionSupplier = functionManager.getWindowFunctionImplementation(functionHandle); Type type = metadata.getType(functionManager.getFunctionMetadata(functionHandle).getReturnType()); - windowFunctionsBuilder.add(window(windowFunctionSupplier, type, frameInfo, arguments.build())); + windowFunctionsBuilder.add(window(windowFunctionSupplier, type, frameInfo, function.isIgnoreNulls(), arguments.build())); windowFunctionOutputVariablesBuilder.add(variable); } @@ -959,7 +1001,7 @@ public PhysicalOperation visitTopN(TopNNode node, LocalExecutionPlanContext cont { PhysicalOperation source = node.getSource().accept(this, context); - List orderByVariables = node.getOrderingScheme().getOrderBy(); + List orderByVariables = node.getOrderingScheme().getOrderByVariables(); List sortChannels = new ArrayList<>(); List sortOrders = new ArrayList<>(); @@ -984,7 +1026,7 @@ public PhysicalOperation visitSort(SortNode node, LocalExecutionPlanContext cont { PhysicalOperation source = node.getSource().accept(this, context); - List orderByVariables = node.getOrderingScheme().getOrderBy(); + List orderByVariables = node.getOrderingScheme().getOrderByVariables(); List orderByChannels = getChannelsForVariables(orderByVariables, source.getLayout()); @@ -1166,10 +1208,18 @@ private PhysicalOperation visitScanFilterAndProject( // if source is a table scan we fold it directly into the filter and project // otherwise we plan it as a normal operator Map sourceLayout; + TableHandle table = null; List columns = null; PhysicalOperation source = null; if (sourceNode instanceof TableScanNode) { TableScanNode tableScanNode = (TableScanNode) sourceNode; + Optional deleteScanInfo = context.getTableWriteInfo().getDeleteScanInfo(); + if (deleteScanInfo.isPresent() && deleteScanInfo.get().getId() == tableScanNode.getId()) { + table = deleteScanInfo.get().getTableHandle(); + } + else { + table = tableScanNode.getTable(); + } // extract the column handles and channel to type mapping sourceLayout = new LinkedHashMap<>(); @@ -1212,8 +1262,8 @@ private PhysicalOperation visitScanFilterAndProject( try { if (columns != null) { - Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(filterExpression, projections, sourceNode.getId()); - Supplier pageProcessor = expressionCompiler.compilePageProcessor(filterExpression, projections, Optional.of(context.getStageId() + "_" + planNodeId)); + Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(session.getSqlFunctionProperties(), filterExpression, projections, sourceNode.getId()); + Supplier pageProcessor = expressionCompiler.compilePageProcessor(session.getSqlFunctionProperties(), filterExpression, projections, Optional.of(context.getStageExecutionId() + "_" + planNodeId)); SourceOperatorFactory operatorFactory = new ScanFilterAndProjectOperatorFactory( context.getNextOperatorId(), @@ -1222,6 +1272,7 @@ private PhysicalOperation visitScanFilterAndProject( pageSourceProvider, cursorProcessor, pageProcessor, + table, columns, projections.stream().map(RowExpression::getType).collect(toImmutableList()), getFilterAndProjectMinOutputPageSize(session), @@ -1230,7 +1281,7 @@ private PhysicalOperation visitScanFilterAndProject( return new PhysicalOperation(operatorFactory, outputMappings, context, stageExecutionDescriptor.isScanGroupedExecution(sourceNode.getId()) ? GROUPED_EXECUTION : UNGROUPED_EXECUTION); } else { - Supplier pageProcessor = expressionCompiler.compilePageProcessor(filterExpression, projections, Optional.of(context.getStageId() + "_" + planNodeId)); + Supplier pageProcessor = expressionCompiler.compilePageProcessor(session.getSqlFunctionProperties(), filterExpression, projections, Optional.of(context.getStageExecutionId() + "_" + planNodeId)); OperatorFactory operatorFactory = new FilterAndProjectOperator.FilterAndProjectOperatorFactory( context.getNextOperatorId(), @@ -1243,6 +1294,9 @@ private PhysicalOperation visitScanFilterAndProject( return new PhysicalOperation(operatorFactory, outputMappings, context, source); } } + catch (PrestoException e) { + throw e; + } catch (RuntimeException e) { throw new PrestoException(COMPILER_ERROR, "Compiler failed", e); } @@ -1251,7 +1305,7 @@ private PhysicalOperation visitScanFilterAndProject( private RowExpression bindChannels(RowExpression expression, Map sourceLayout) { Type type = expression.getType(); - Object value = new RowExpressionInterpreter(expression, metadata, session.toConnectorSession(), true).optimize(); + Object value = new RowExpressionInterpreter(expression, metadata, session.toConnectorSession(), OPTIMIZED).optimize(); if (value instanceof RowExpression) { RowExpression optimized = (RowExpression) value; // building channel info @@ -1271,7 +1325,15 @@ public PhysicalOperation visitTableScan(TableScanNode node, LocalExecutionPlanCo columns.add(node.getAssignments().get(variable)); } - OperatorFactory operatorFactory = new TableScanOperatorFactory(context.getNextOperatorId(), node.getId(), pageSourceProvider, columns); + TableHandle tableHandle; + Optional deleteScanInfo = context.getTableWriteInfo().getDeleteScanInfo(); + if (deleteScanInfo.isPresent() && deleteScanInfo.get().getId() == node.getId()) { + tableHandle = deleteScanInfo.get().getTableHandle(); + } + else { + tableHandle = node.getTable(); + } + OperatorFactory operatorFactory = new TableScanOperatorFactory(context.getNextOperatorId(), node.getId(), pageSourceProvider, tableHandle, columns); return new PhysicalOperation(operatorFactory, makeLayout(node), context, stageExecutionDescriptor.isScanGroupedExecution(node.getId()) ? GROUPED_EXECUTION : UNGROUPED_EXECUTION); } @@ -1501,6 +1563,7 @@ public PhysicalOperation visitIndexJoin(IndexJoinNode node, LocalExecutionPlanCo nonLookupInputChannels, nonLookupOutputChannels, indexSource.getTypes(), + session.getSqlFunctionProperties(), pageFunctionCompiler)); } @@ -1524,7 +1587,8 @@ public PhysicalOperation visitIndexJoin(IndexJoinNode node, LocalExecutionPlanCo indexJoinLookupStats, SystemSessionProperties.isShareIndexLoading(session), pagesIndexFactory, - joinCompiler); + joinCompiler, + getIndexLoaderTimeout(session)); verify(probeSource.getPipelineExecutionStrategy() == UNGROUPED_EXECUTION); verify(indexSource.getPipelineExecutionStrategy() == UNGROUPED_EXECUTION); @@ -1680,37 +1744,50 @@ private Optional removeExpressionFromFilter(RowExpression filter, private SpatialPredicate spatialTest(CallExpression functionCall, boolean probeFirst, Optional comparisonOperator) { - String functionName = metadata.getFunctionManager().getFunctionMetadata(functionCall.getFunctionHandle()).getName(); - switch (functionName.toLowerCase(Locale.ENGLISH)) { - case ST_CONTAINS: - if (probeFirst) { - return (buildGeometry, probeGeometry, radius) -> probeGeometry.contains(buildGeometry); - } - else { - return (buildGeometry, probeGeometry, radius) -> buildGeometry.contains(probeGeometry); - } - case ST_WITHIN: - if (probeFirst) { - return (buildGeometry, probeGeometry, radius) -> probeGeometry.within(buildGeometry); - } - else { - return (buildGeometry, probeGeometry, radius) -> buildGeometry.within(probeGeometry); - } - case ST_INTERSECTS: - return (buildGeometry, probeGeometry, radius) -> buildGeometry.intersects(probeGeometry); - case ST_DISTANCE: - if (comparisonOperator.get() == OperatorType.LESS_THAN) { - return (buildGeometry, probeGeometry, radius) -> buildGeometry.distance(probeGeometry) < radius.getAsDouble(); - } - else if (comparisonOperator.get() == OperatorType.LESS_THAN_OR_EQUAL) { - return (buildGeometry, probeGeometry, radius) -> buildGeometry.distance(probeGeometry) <= radius.getAsDouble(); - } - else { - throw new UnsupportedOperationException("Unsupported comparison operator: " + comparisonOperator.get()); - } - default: - throw new UnsupportedOperationException("Unsupported spatial function: " + functionName); + QualifiedFunctionName functionName = metadata.getFunctionManager().getFunctionMetadata(functionCall.getFunctionHandle()).getName(); + if (functionName.equals(ST_CONTAINS)) { + if (probeFirst) { + return (buildGeometry, probeGeometry, radius) -> probeGeometry.contains(buildGeometry); + } + else { + return (buildGeometry, probeGeometry, radius) -> buildGeometry.contains(probeGeometry); + } } + if (functionName.equals(ST_WITHIN)) { + if (probeFirst) { + return (buildGeometry, probeGeometry, radius) -> probeGeometry.within(buildGeometry); + } + else { + return (buildGeometry, probeGeometry, radius) -> buildGeometry.within(probeGeometry); + } + } + if (functionName.equals(ST_CROSSES)) { + return (buildGeometry, probeGeometry, radius) -> buildGeometry.crosses(probeGeometry); + } + if (functionName.equals(ST_EQUALS)) { + return (buildGeometry, probeGeometry, radius) -> buildGeometry.Equals(probeGeometry); + } + if (functionName.equals(ST_INTERSECTS)) { + return (buildGeometry, probeGeometry, radius) -> buildGeometry.intersects(probeGeometry); + } + if (functionName.equals(ST_OVERLAPS)) { + return (buildGeometry, probeGeometry, radius) -> buildGeometry.overlaps(probeGeometry); + } + if (functionName.equals(ST_TOUCHES)) { + return (buildGeometry, probeGeometry, radius) -> buildGeometry.touches(probeGeometry); + } + if (functionName.equals(ST_DISTANCE)) { + if (comparisonOperator.get() == OperatorType.LESS_THAN) { + return (buildGeometry, probeGeometry, radius) -> buildGeometry.distance(probeGeometry) < radius.getAsDouble(); + } + else if (comparisonOperator.get() == OperatorType.LESS_THAN_OR_EQUAL) { + return (buildGeometry, probeGeometry, radius) -> buildGeometry.distance(probeGeometry) <= radius.getAsDouble(); + } + else { + throw new UnsupportedOperationException("Unsupported comparison operator: " + comparisonOperator.get()); + } + } + throw new UnsupportedOperationException("Unsupported spatial function: " + functionName); } private Set getSymbolReferences(Collection variables) @@ -1852,6 +1929,7 @@ private PagesSpatialIndexFactory createPagesSpatialIndexFactory( Optional filterFunctionFactory = joinFilter .map(filterExpression -> compileJoinFilterFunction( + session.getSqlFunctionProperties(), filterExpression, probeLayout, buildLayout)); @@ -1943,6 +2021,7 @@ private JoinBridgeManager createLookupSourceFact Optional filterFunctionFactory = node.getFilter() .map(filterExpression -> compileJoinFilterFunction( + session.getSqlFunctionProperties(), filterExpression, probeSource.getLayout(), buildSource.getLayout())); @@ -1960,6 +2039,7 @@ private JoinBridgeManager createLookupSourceFact .map(SortExpressionContext::getSearchExpressions) .map(searchExpressions -> searchExpressions.stream() .map(searchExpression -> compileJoinFilterFunction( + session.getSqlFunctionProperties(), searchExpression, probeSource.getLayout(), buildSource.getLayout())) @@ -2012,12 +2092,13 @@ private JoinBridgeManager createLookupSourceFact } private JoinFilterFunctionFactory compileJoinFilterFunction( + SqlFunctionProperties sqlFunctionProperties, RowExpression filterExpression, Map probeLayout, Map buildLayout) { Map joinSourcesLayout = createJoinSourcesLayout(buildLayout, probeLayout); - return joinFilterFunctionCompiler.compileJoinFilterFunction(bindChannels(filterExpression, joinSourcesLayout), buildLayout.size()); + return joinFilterFunctionCompiler.compileJoinFilterFunction(sqlFunctionProperties, bindChannels(filterExpression, joinSourcesLayout), buildLayout.size()); } private int sortExpressionAsSortChannel( @@ -2133,7 +2214,7 @@ public PhysicalOperation visitTableWriter(TableWriterNode node, LocalExecutionPl { // Set table writer count if (node.getPartitioningScheme().isPresent()) { - context.setDriverInstanceCount(1); + context.setDriverInstanceCount(getTaskPartitionedWriterCount(session)); } else { context.setDriverInstanceCount(getTaskWriterCount(session)); @@ -2194,7 +2275,7 @@ public PhysicalOperation visitTableWriter(TableWriterNode node, LocalExecutionPl context.getNextOperatorId(), node.getId(), pageSinkManager, - node.getTarget(), + context.getTableWriteInfo().getWriterTarget().orElseThrow(() -> new VerifyException("writerTarget is absent")), inputChannels, session, statisticsAggregation, @@ -2212,15 +2293,71 @@ public PhysicalOperation visitStatisticsWriterNode(StatisticsWriterNode node, Lo StatisticAggregationsDescriptor descriptor = node.getDescriptor().map(source.getLayout()::get); + AnalyzeTableHandle analyzeTableHandle = context.getTableWriteInfo().getAnalyzeTableHandle().orElseThrow(() -> new VerifyException("analyzeTableHandle is absent")); OperatorFactory operatorFactory = new StatisticsWriterOperatorFactory( context.getNextOperatorId(), node.getId(), - computedStatistics -> metadata.finishStatisticsCollection(session, ((StatisticsWriterNode.WriteStatisticsHandle) node.getTarget()).getHandle(), computedStatistics), + computedStatistics -> metadata.finishStatisticsCollection(session, analyzeTableHandle, computedStatistics), node.isRowCountEnabled(), descriptor); return new PhysicalOperation(operatorFactory, makeLayout(node), context, source); } + @Override + public PhysicalOperation visitTableWriteMerge(TableWriterMergeNode node, LocalExecutionPlanContext context) + { + PhysicalOperation source = node.getSource().accept(this, context); + + ImmutableMap.Builder outputMapping = ImmutableMap.builder(); + outputMapping.put(node.getRowCountVariable(), ROW_COUNT_CHANNEL); + outputMapping.put(node.getFragmentVariable(), FRAGMENT_CHANNEL); + outputMapping.put(node.getTableCommitContextVariable(), CONTEXT_CHANNEL); + + OperatorFactory statisticsAggregation = node.getStatisticsAggregation().map(aggregation -> { + List groupingVariables = aggregation.getGroupingVariables(); + if (groupingVariables.isEmpty()) { + return createAggregationOperatorFactory( + node.getId(), + aggregation.getAggregations(), + INTERMEDIATE, + STATS_START_CHANNEL, + outputMapping, + source, + context, + true); + } + return createHashAggregationOperatorFactory( + node.getId(), + aggregation.getAggregations(), + ImmutableSet.of(), + groupingVariables, + INTERMEDIATE, + Optional.empty(), + Optional.empty(), + source, + false, + false, + false, + new DataSize(0, BYTE), + context, + STATS_START_CHANNEL, + outputMapping, + 200, + Optional.empty(), + true); + }).orElse(new DevNullOperatorFactory(context.getNextOperatorId(), node.getId())); + + OperatorFactory operatorFactory = new TableWriterMergeOperatorFactory( + context.getNextOperatorId(), + node.getId(), + statisticsAggregation, + tableCommitContextCodec, + session, + getVariableTypes(node.getOutputVariables())); + + return new PhysicalOperation(operatorFactory, outputMapping.build(), context, source); + } + @Override public PhysicalOperation visitTableFinish(TableFinishNode node, LocalExecutionPlanContext context) { @@ -2268,11 +2405,13 @@ public PhysicalOperation visitTableFinish(TableFinishNode node, LocalExecutionPl .map(desc -> desc.map(aggregationOutput::get)) .orElse(StatisticAggregationsDescriptor.empty()); + ExecutionWriterTarget writerTarget = context.getTableWriteInfo().getWriterTarget().orElseThrow(() -> new VerifyException("writerTarget is absent")); + OperatorFactory operatorFactory = new TableFinishOperatorFactory( context.getNextOperatorId(), node.getId(), - createTableFinisher(session, node, metadata), - createLifespanCommitter(session, node, metadata), + createTableFinisher(session, metadata, writerTarget), + createLifespanCommitter(session, metadata, writerTarget), statisticsAggregation, descriptor, session, @@ -2300,7 +2439,7 @@ public PhysicalOperation visitDelete(DeleteNode node, LocalExecutionPlanContext @Override public PhysicalOperation visitMetadataDelete(MetadataDeleteNode node, LocalExecutionPlanContext context) { - OperatorFactory operatorFactory = new MetadataDeleteOperatorFactory(context.getNextOperatorId(), node.getId(), metadata, session, node.getTarget().getHandle()); + OperatorFactory operatorFactory = new MetadataDeleteOperatorFactory(context.getNextOperatorId(), node.getId(), metadata, session, node.getTableHandle()); return new PhysicalOperation(operatorFactory, makeLayout(node), context, UNGROUPED_EXECUTION); } @@ -2356,8 +2495,10 @@ private PhysicalOperation createLocalMerge(ExchangeNode node, LocalExecutionPlan PhysicalOperation source = sourceNode.accept(this, subContext); int operatorsCount = subContext.getDriverInstanceCount().orElse(1); - List types = getSourceOperatorTypes(node, context.getTypes()); + List types = getSourceOperatorTypes(node); LocalExchangeFactory exchangeFactory = new LocalExchangeFactory( + partitioningProviderManager, + session, node.getPartitioningScheme().getPartitioning().getHandle(), operatorsCount, types, @@ -2381,8 +2522,8 @@ private PhysicalOperation createLocalMerge(ExchangeNode node, LocalExecutionPlan OrderingScheme orderingScheme = node.getOrderingScheme().get(); ImmutableMap layout = makeLayout(node); - List sortChannels = getChannelsForVariables(orderingScheme.getOrderBy(), layout); - List orderings = orderingScheme.getOrderingList(); + List sortChannels = getChannelsForVariables(orderingScheme.getOrderByVariables(), layout); + List orderings = getOrderingList(orderingScheme); OperatorFactory operatorFactory = new LocalMergeSourceOperatorFactory( context.getNextOperatorId(), node.getId(), @@ -2409,9 +2550,12 @@ else if (context.getDriverInstanceCount().isPresent()) { context.setDriverInstanceCount(driverInstanceCount); } - List types = getSourceOperatorTypes(node, context.getTypes()); + List types = getSourceOperatorTypes(node); List channels = node.getPartitioningScheme().getPartitioning().getArguments().stream() - .map(argument -> node.getOutputVariables().indexOf(argument.getVariableReference())) + .map(argument -> { + checkArgument(argument instanceof VariableReferenceExpression, format("Expect VariableReferenceExpression but get %s", argument)); + return node.getOutputVariables().indexOf(argument); + }) .collect(toImmutableList()); Optional hashChannel = node.getPartitioningScheme().getHashColumn() .map(variable -> node.getOutputVariables().indexOf(variable)); @@ -2431,6 +2575,8 @@ else if (context.getDriverInstanceCount().isPresent()) { } LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory( + partitioningProviderManager, + session, node.getPartitioningScheme().getPartitioning().getHandle(), driverInstanceCount, types, @@ -2477,7 +2623,7 @@ public PhysicalOperation visitPlan(PlanNode node, LocalExecutionPlanContext cont throw new UnsupportedOperationException("not yet implemented"); } - private List getSourceOperatorTypes(PlanNode node, TypeProvider types) + private List getSourceOperatorTypes(PlanNode node) { return getVariableTypes(node.getOutputVariables()); } @@ -2491,8 +2637,7 @@ private List getVariableTypes(List variables) private AccumulatorFactory buildAccumulatorFactory( PhysicalOperation source, - Aggregation aggregation, - TypeProvider types) + Aggregation aggregation) { FunctionManager functionManager = metadata.getFunctionManager(); InternalAggregationFunction internalAggregationFunction = functionManager.getAggregateFunctionImplementation(aggregation.getFunctionHandle()); @@ -2512,7 +2657,7 @@ private AccumulatorFactory buildAccumulatorFactory( .collect(toImmutableList()); for (int i = 0; i < lambdas.size(); i++) { List lambdaInterfaces = internalAggregationFunction.getLambdaInterfaces(); - Class lambdaProviderClass = compileLambdaProvider(lambdas.get(i), metadata.getFunctionManager(), lambdaInterfaces.get(i)); + Class lambdaProviderClass = compileLambdaProvider(lambdas.get(i), metadata, session.getSqlFunctionProperties(), lambdaInterfaces.get(i)); try { lambdaProviders.add((LambdaProvider) constructorMethodHandle(lambdaProviderClass, ConnectorSession.class).invoke(session.toConnectorSession())); } @@ -2526,8 +2671,8 @@ private AccumulatorFactory buildAccumulatorFactory( List sortKeys = ImmutableList.of(); if (aggregation.getOrderBy().isPresent()) { OrderingScheme orderBy = aggregation.getOrderBy().get(); - sortKeys = orderBy.getOrderBy(); - sortOrders = orderBy.getOrderingList(); + sortKeys = orderBy.getOrderByVariables(); + sortOrders = getOrderingList(orderBy); } return internalAggregationFunction.bind( @@ -2573,7 +2718,7 @@ private AggregationOperatorFactory createAggregationOperatorFactory( for (Map.Entry entry : aggregations.entrySet()) { VariableReferenceExpression variable = entry.getKey(); Aggregation aggregation = entry.getValue(); - accumulatorFactories.add(buildAccumulatorFactory(source, aggregation, context.getTypes())); + accumulatorFactories.add(buildAccumulatorFactory(source, aggregation)); outputMappings.put(variable, outputChannel); // one aggregation per channel outputChannel++; } @@ -2636,7 +2781,7 @@ private OperatorFactory createHashAggregationOperatorFactory( VariableReferenceExpression variable = entry.getKey(); Aggregation aggregation = entry.getValue(); - accumulatorFactories.add(buildAccumulatorFactory(source, aggregation, context.getTypes())); + accumulatorFactories.add(buildAccumulatorFactory(source, aggregation)); aggregationOutputVariables.add(variable); } @@ -2702,9 +2847,8 @@ private OperatorFactory createHashAggregationOperatorFactory( } } - private static TableFinisher createTableFinisher(Session session, TableFinishNode node, Metadata metadata) + private static TableFinisher createTableFinisher(Session session, Metadata metadata, ExecutionWriterTarget target) { - WriterTarget target = node.getTarget(); return (fragments, statistics) -> { if (target instanceof CreateHandle) { return metadata.finishCreateTable(session, ((CreateHandle) target).getHandle(), fragments, statistics); @@ -2722,15 +2866,14 @@ else if (target instanceof DeleteHandle) { }; } - private static LifespanCommitter createLifespanCommitter(Session session, TableFinishNode node, Metadata metadata) + private static LifespanCommitter createLifespanCommitter(Session session, Metadata metadata, ExecutionWriterTarget target) { - WriterTarget target = node.getTarget(); return fragments -> { if (target instanceof CreateHandle) { - metadata.commitPartition(session, ((CreateHandle) target).getHandle(), fragments); + return metadata.commitPartitionAsync(session, ((CreateHandle) target).getHandle(), fragments); } else if (target instanceof InsertHandle) { - metadata.commitPartition(session, ((InsertHandle) target).getHandle(), fragments); + return metadata.commitPartitionAsync(session, ((InsertHandle) target).getHandle(), fragments); } else { throw new AssertionError("Unhandled target type: " + target.getClass().getName()); @@ -2771,6 +2914,15 @@ private static Function variableChannelGet }; } + /** + * List of sort orders in the same order as the list of variables returned from `getOrderByVariables()`. This means for + * index i, variable `getOrderByVariables().get(i)` has order `getOrderingList().get(i)`. + */ + private static List getOrderingList(OrderingScheme orderingScheme) + { + return orderingScheme.getOrderByVariables().stream().map(orderingScheme.getOrderingsMap()::get).collect(toImmutableList()); + } + /** * Encapsulates an physical operator plus the mapping of logical variables to channel/field */ diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java index 47edf7ff8b961..8eb5e3af4a9b2 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java @@ -32,8 +32,12 @@ import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.predicate.TupleDomain; @@ -49,19 +53,16 @@ import com.facebook.presto.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation; import com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; +import com.facebook.presto.sql.planner.plan.TableWriterNode.DeleteHandle; import com.facebook.presto.sql.planner.sanity.PlanSanityChecker; import com.facebook.presto.sql.tree.Analyze; import com.facebook.presto.sql.tree.Cast; @@ -92,10 +93,11 @@ import static com.facebook.presto.SystemSessionProperties.isPrintStatsForNonJoinQuery; import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; +import static com.facebook.presto.spi.plan.LimitNode.Step.FINAL; import static com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.plan.TableWriterNode.CreateName; import static com.facebook.presto.sql.planner.plan.TableWriterNode.InsertReference; import static com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget; @@ -202,7 +204,7 @@ private StatsAndCosts computeStats(PlanNode root, TypeProvider types) PlanNodeSearcher.searchFrom(root).where(node -> (node instanceof JoinNode) || (node instanceof SemiJoinNode)).matches()) { StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, types); - CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.empty(), session, types); + CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.empty(), session); return StatsAndCosts.create(root, statsProvider, costProvider); } return StatsAndCosts.empty(); @@ -298,7 +300,7 @@ private RelationPlan createAnalyzePlan(Analysis analysis, Analyze analyzeStateme AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty()), - new StatisticsWriterNode.WriteStatisticsReference(targetTable), + targetTable, variableAllocator.newVariable("rows", BIGINT), tableStatisticsMetadata.getTableStatistics().contains(ROW_COUNT), tableStatisticAggregation.getDescriptor()); @@ -329,7 +331,7 @@ private RelationPlan createTableCreationPlan(Analysis analysis, Query query) return createTableWriterPlan( analysis, plan, - new CreateName(destination.getCatalogName(), tableMetadata, newTableLayout), + new CreateName(new ConnectorId(destination.getCatalogName()), tableMetadata, newTableLayout), columnNames, newTableLayout, statisticsMetadata); @@ -392,7 +394,7 @@ private RelationPlan createInsertPlan(Analysis analysis, Insert insertStatement) return createTableWriterPlan( analysis, plan, - new InsertReference(insert.getTarget()), + new InsertReference(insert.getTarget(), metadata.getTableMetadata(session, insert.getTarget()).getTable()), visibleTableColumnNames, newTableLayout, statisticsMetadata); @@ -409,7 +411,7 @@ private RelationPlan createTableWriterPlan( PlanNode source = plan.getRoot(); if (!analysis.isCreateTableAsSelectWithData()) { - source = new LimitNode(idAllocator.getNextId(), source, 0L, false); + source = new LimitNode(idAllocator.getNextId(), source, 0L, FINAL); } // todo this should be checked in analysis @@ -443,32 +445,27 @@ private RelationPlan createTableWriterPlan( TableStatisticAggregation result = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnToVariableMap); - StatisticAggregations.Parts aggregations = result.getAggregations().createPartialAggregations(variableAllocator, metadata.getFunctionManager()); - - // partial aggregation is run within the TableWriteOperator to calculate the statistics for - // the data consumed by the TableWriteOperator - // final aggregation is run within the TableFinishOperator to summarize collected statistics - // by the partial aggregation from all of the writer nodes - StatisticAggregations partialAggregation = aggregations.getPartialAggregation(); - - PlanNode writerNode = new TableWriterNode( - idAllocator.getNextId(), - source, - target, - variableAllocator.newVariable("partialrows", BIGINT), - variableAllocator.newVariable("fragment", VARBINARY), - variableAllocator.newVariable("tablecommitcontext", VARBINARY), - plan.getFieldMappings(), - columnNames, - partitioningScheme, - Optional.of(partialAggregation), - Optional.of(result.getDescriptor().map(aggregations.getMappings()::get))); + StatisticAggregations.Parts aggregations = result.getAggregations().splitIntoPartialAndFinal(variableAllocator, metadata.getFunctionManager()); TableFinishNode commitNode = new TableFinishNode( idAllocator.getNextId(), - writerNode, - target, + new TableWriterNode( + idAllocator.getNextId(), + source, + Optional.of(target), + variableAllocator.newVariable("rows", BIGINT), + variableAllocator.newVariable("fragments", VARBINARY), + variableAllocator.newVariable("commitcontext", VARBINARY), + plan.getFieldMappings(), + columnNames, + partitioningScheme, + // partial aggregation is run within the TableWriteOperator to calculate the statistics for + // the data consumed by the TableWriteOperator + Optional.of(aggregations.getPartialAggregation())), + Optional.of(target), variableAllocator.newVariable("rows", BIGINT), + // final aggregation is run within the TableFinishOperator to summarize collected statistics + // by the partial aggregation from all of the writer nodes Optional.of(aggregations.getFinalAggregation()), Optional.of(result.getDescriptor())); @@ -480,16 +477,15 @@ private RelationPlan createTableWriterPlan( new TableWriterNode( idAllocator.getNextId(), source, - target, - variableAllocator.newVariable("partialrows", BIGINT), - variableAllocator.newVariable("fragment", VARBINARY), - variableAllocator.newVariable("tablecommitcontext", VARBINARY), + Optional.of(target), + variableAllocator.newVariable("rows", BIGINT), + variableAllocator.newVariable("fragments", VARBINARY), + variableAllocator.newVariable("commitcontext", VARBINARY), plan.getFieldMappings(), columnNames, partitioningScheme, - Optional.empty(), Optional.empty()), - target, + Optional.of(target), variableAllocator.newVariable("rows", BIGINT), Optional.empty(), Optional.empty()); @@ -501,10 +497,12 @@ private RelationPlan createDeletePlan(Analysis analysis, Delete node) DeleteNode deleteNode = new QueryPlanner(analysis, variableAllocator, idAllocator, buildLambdaDeclarationToVariableMap(analysis, variableAllocator), metadata, session) .plan(node); + TableHandle handle = analysis.getTableHandle(node.getTable()); + DeleteHandle deleteHandle = new DeleteHandle(handle, metadata.getTableMetadata(session, handle).getTable()); TableFinishNode commitNode = new TableFinishNode( idAllocator.getNextId(), deleteNode, - deleteNode.getTarget(), + Optional.of(deleteHandle), variableAllocator.newVariable("rows", BIGINT), Optional.empty(), Optional.empty()); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LookupSymbolResolver.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LookupSymbolResolver.java deleted file mode 100644 index 3b5a0e976fbf9..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LookupSymbolResolver.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sql.planner; - -import com.facebook.presto.spi.ColumnHandle; -import com.facebook.presto.spi.predicate.NullableValue; -import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.google.common.collect.ImmutableMap; - -import java.util.Map; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static java.util.Objects.requireNonNull; - -public class LookupSymbolResolver - implements SymbolResolver -{ - private final Map assignments; - private final Map bindings; - - public LookupSymbolResolver(Map assignments, Map bindings) - { - requireNonNull(assignments, "assignments is null"); - requireNonNull(bindings, "bindings is null"); - - this.assignments = assignments.entrySet().stream().collect(toImmutableMap(entry -> new Symbol(entry.getKey().getName()), Map.Entry::getValue)); - this.bindings = ImmutableMap.copyOf(bindings); - } - - @Override - public Object getValue(Symbol symbol) - { - ColumnHandle column = assignments.get(symbol); - checkArgument(column != null, "Missing column assignment for %s", symbol); - - if (!bindings.containsKey(column)) { - return symbol.toSymbolReference(); - } - - return bindings.get(column).getValue(); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/NoOpVariableResolver.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/NoOpVariableResolver.java new file mode 100644 index 0000000000000..83b38912665d8 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/NoOpVariableResolver.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner; + +import com.facebook.presto.spi.relation.VariableReferenceExpression; + +public class NoOpVariableResolver + implements VariableResolver +{ + public static final NoOpVariableResolver INSTANCE = new NoOpVariableResolver(); + + @Override + public Object getValue(VariableReferenceExpression variable) + { + return new Symbol(variable.getName()).toSymbolReference(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java index 1bb708a772e23..9148e38920d34 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/NodePartitioningManager.java @@ -40,8 +40,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.function.ToIntFunction; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -54,25 +52,13 @@ public class NodePartitioningManager { private final NodeScheduler nodeScheduler; - private final ConcurrentMap partitioningProviders = new ConcurrentHashMap<>(); + private final PartitioningProviderManager partitioningProviderManager; @Inject - public NodePartitioningManager(NodeScheduler nodeScheduler) + public NodePartitioningManager(NodeScheduler nodeScheduler, PartitioningProviderManager partitioningProviderManager) { this.nodeScheduler = requireNonNull(nodeScheduler, "nodeScheduler is null"); - } - - public void addPartitioningProvider(ConnectorId connectorId, ConnectorNodePartitioningProvider nodePartitioningProvider) - { - requireNonNull(connectorId, "connectorId is null"); - requireNonNull(nodePartitioningProvider, "nodePartitioningProvider is null"); - checkArgument(partitioningProviders.putIfAbsent(connectorId, nodePartitioningProvider) == null, - "NodePartitioningProvider for connector '%s' is already registered", connectorId); - } - - public void removePartitioningProvider(ConnectorId connectorId) - { - partitioningProviders.remove(connectorId); + this.partitioningProviderManager = requireNonNull(partitioningProviderManager, "partitioningProviderManager is null"); } public PartitionFunction getPartitionFunction( @@ -94,8 +80,7 @@ public PartitionFunction getPartitionFunction( partitioningScheme.getBucketToPartition().get()); } else { - ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); - checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); + ConnectorNodePartitioningProvider partitioningProvider = partitioningProviderManager.getPartitioningProvider(partitioningHandle.getConnectorId().get()); bucketFunction = partitioningProvider.getBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), @@ -113,7 +98,7 @@ public List listPartitionHandles( Session session, PartitioningHandle partitioningHandle) { - ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); + ConnectorNodePartitioningProvider partitioningProvider = partitioningProviderManager.getPartitioningProvider(partitioningHandle.getConnectorId().get()); return partitioningProvider.listPartitionHandles( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), @@ -129,11 +114,6 @@ public NodePartitionMap getNodePartitioningMap(Session session, PartitioningHand return ((SystemPartitioningHandle) partitioningHandle.getConnectorHandle()).getNodePartitionMap(session, nodeScheduler); } - ConnectorId connectorId = partitioningHandle.getConnectorId() - .orElseThrow(() -> new IllegalArgumentException("No connector ID for partitioning handle: " + partitioningHandle)); - ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(connectorId); - checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", connectorId); - ConnectorBucketNodeMap connectorBucketNodeMap = getConnectorBucketNodeMap(session, partitioningHandle); // safety check for crazy partitioning checkArgument(connectorBucketNodeMap.getBucketCount() < 1_000_000, "Too many buckets in partitioning: %s", connectorBucketNodeMap.getBucketCount()); @@ -143,6 +123,8 @@ public NodePartitionMap getNodePartitioningMap(Session session, PartitioningHand bucketToNode = getFixedMapping(connectorBucketNodeMap); } else { + ConnectorId connectorId = partitioningHandle.getConnectorId() + .orElseThrow(() -> new IllegalArgumentException("No connector ID for partitioning handle: " + partitioningHandle)); bucketToNode = createArbitraryBucketToNode( nodeScheduler.createNodeSelector(connectorId).selectRandomNodes(getMaxTasksPerStage(session)), connectorBucketNodeMap.getBucketCount()); @@ -198,8 +180,7 @@ private ConnectorBucketNodeMap getConnectorBucketNodeMap(Session session, Partit { checkArgument(!(partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle)); - ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); - checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); + ConnectorNodePartitioningProvider partitioningProvider = partitioningProviderManager.getPartitioningProvider(partitioningHandle.getConnectorId().get()); ConnectorBucketNodeMap connectorBucketNodeMap = partitioningProvider.getBucketNodeMap( partitioningHandle.getTransactionHandle().orElse(null), @@ -212,8 +193,7 @@ private ConnectorBucketNodeMap getConnectorBucketNodeMap(Session session, Partit private ToIntFunction getSplitToBucket(Session session, PartitioningHandle partitioningHandle) { - ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); - checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); + ConnectorNodePartitioningProvider partitioningProvider = partitioningProviderManager.getPartitioningProvider(partitioningHandle.getConnectorId().get()); ToIntFunction splitBucketFunction = partitioningProvider.getSplitBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/NullabilityAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/NullabilityAnalyzer.java index 7a968c44a47d8..1438a4de6f2d3 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/NullabilityAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/NullabilityAnalyzer.java @@ -13,6 +13,15 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.expressions.DefaultRowExpressionTraversalVisitor; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.tree.Cast; import com.facebook.presto.sql.tree.DefaultExpressionTraversalVisitor; import com.facebook.presto.sql.tree.DereferenceExpression; @@ -26,13 +35,22 @@ import com.facebook.presto.sql.tree.SubscriptExpression; import com.facebook.presto.sql.tree.TryExpression; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; public final class NullabilityAnalyzer { - private NullabilityAnalyzer() {} + private final FunctionManager functionManager; + private final TypeManager typeManager; + + public NullabilityAnalyzer(FunctionManager functionManager, TypeManager typeManager) + { + this.functionManager = requireNonNull(functionManager, "functionManager is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + } /** * TODO: this currently produces a very conservative estimate. @@ -40,6 +58,7 @@ private NullabilityAnalyzer() {} * can return null (e.g., if(a, b, c) might return null for non-null a * only if b or c can be null. */ + @Deprecated public static boolean mayReturnNullOnNonNullInput(Expression expression) { requireNonNull(expression, "expression is null"); @@ -49,6 +68,15 @@ public static boolean mayReturnNullOnNonNullInput(Expression expression) return result.get(); } + public boolean mayReturnNullOnNonNullInput(RowExpression expression) + { + requireNonNull(expression, "expression is null"); + + AtomicBoolean result = new AtomicBoolean(false); + expression.accept(new RowExpressionVisitor(functionManager, typeManager), result); + return result.get(); + } + private static class Visitor extends DefaultExpressionTraversalVisitor { @@ -128,4 +156,66 @@ protected Void visitFunctionCall(FunctionCall node, AtomicBoolean result) return null; } } + + private static class RowExpressionVisitor + extends DefaultRowExpressionTraversalVisitor + { + private final FunctionManager functionManager; + private final TypeManager typeManager; + + public RowExpressionVisitor(FunctionManager functionManager, TypeManager typeManager) + { + this.functionManager = functionManager; + this.typeManager = typeManager; + } + + @Override + public Void visitCall(CallExpression call, AtomicBoolean result) + { + FunctionMetadata function = functionManager.getFunctionMetadata(call.getFunctionHandle()); + Optional operator = function.getOperatorType(); + if (operator.isPresent()) { + switch (operator.get()) { + case SATURATED_FLOOR_CAST: + case CAST: { + checkArgument(call.getArguments().size() == 1); + Type sourceType = call.getArguments().get(0).getType(); + Type targetType = call.getType(); + if (!typeManager.isTypeOnlyCoercion(sourceType, targetType)) { + result.set(true); + } + } + case SUBSCRIPT: + result.set(true); + } + } + else if (!functionReturnsNullForNotNullInput(function)) { + // TODO: use function annotation instead of assume all function can return NULL + result.set(true); + } + call.getArguments().forEach(argument -> argument.accept(this, result)); + return null; + } + + private boolean functionReturnsNullForNotNullInput(FunctionMetadata function) + { + return (function.getName().getFunctionName().equalsIgnoreCase("like")); + } + + @Override + public Void visitSpecialForm(SpecialFormExpression specialForm, AtomicBoolean result) + { + switch (specialForm.getForm()) { + case IN: + case IF: + case SWITCH: + case WHEN: + case NULL_IF: + case DEREFERENCE: + result.set(true); + } + specialForm.getArguments().forEach(argument -> argument.accept(this, result)); + return null; + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/OrderingScheme.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/OrderingScheme.java deleted file mode 100644 index 0d8989a4b2006..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/OrderingScheme.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sql.planner; - -import com.facebook.presto.spi.block.SortOrder; -import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; - -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.Iterables.getOnlyElement; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -public class OrderingScheme -{ - private final List orderBy; - private final Map orderings; - - @JsonCreator - public OrderingScheme(@JsonProperty("orderBy") List orderBy, @JsonProperty("orderings") Map orderings) - { - requireNonNull(orderBy, "orderBy is null"); - requireNonNull(orderings, "orderings is null"); - checkArgument(!orderBy.isEmpty(), "orderBy is empty"); - checkArgument(orderings.keySet().equals(ImmutableSet.copyOf(orderBy)), "orderBy keys and orderings don't match"); - this.orderBy = ImmutableList.copyOf(orderBy); - this.orderings = ImmutableMap.copyOf(orderings); - } - - @JsonProperty - public List getOrderBy() - { - return orderBy; - } - - @JsonProperty - public Map getOrderings() - { - return orderings; - } - - public List getOrderingList() - { - return orderBy.stream() - .map(orderings::get) - .collect(toImmutableList()); - } - - public SortOrder getOrdering(VariableReferenceExpression variable) - { - checkArgument(orderings.containsKey(variable), format("No ordering for variable: %s", variable)); - return orderings.get(variable); - } - - @VisibleForTesting - public SortOrder getOrdering(Symbol symbol) - { - return getOnlyElement(Maps.filterKeys(orderings, variable -> variable.getName().equals(symbol.getName())).values()); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - OrderingScheme that = (OrderingScheme) o; - return Objects.equals(orderBy, that.orderBy) && - Objects.equals(orderings, that.orderings); - } - - @Override - public int hashCode() - { - return Objects.hash(orderBy, orderings); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("orderBy", orderBy) - .add("orderings", orderings) - .toString(); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java index 0e82e29701c8c..e8a7f1b2d9b30 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java @@ -19,6 +19,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.TableWriterNode; +import com.google.common.base.VerifyException; import java.util.Optional; @@ -50,26 +51,11 @@ private class Visitor @Override public Void visitTableWriter(TableWriterNode node, Void context) { - TableWriterNode.WriterTarget writerTarget = node.getTarget(); - - if (writerTarget instanceof TableWriterNode.CreateHandle) { - connectorId = ((TableWriterNode.CreateHandle) writerTarget).getHandle().getConnectorId(); - checkState(schemaTableName == null || schemaTableName.equals(((TableWriterNode.CreateHandle) writerTarget).getSchemaTableName()), - "cannot have more than a single create, insert or delete in a query"); - schemaTableName = ((TableWriterNode.CreateHandle) writerTarget).getSchemaTableName(); - } - if (writerTarget instanceof TableWriterNode.InsertHandle) { - connectorId = ((TableWriterNode.InsertHandle) writerTarget).getHandle().getConnectorId(); - checkState(schemaTableName == null || schemaTableName.equals(((TableWriterNode.InsertHandle) writerTarget).getSchemaTableName()), - "cannot have more than a single create, insert or delete in a query"); - schemaTableName = ((TableWriterNode.InsertHandle) writerTarget).getSchemaTableName(); - } - if (writerTarget instanceof TableWriterNode.DeleteHandle) { - connectorId = ((TableWriterNode.DeleteHandle) writerTarget).getHandle().getConnectorId(); - checkState(schemaTableName == null || schemaTableName.equals(((TableWriterNode.DeleteHandle) writerTarget).getSchemaTableName()), - "cannot have more than a single create, insert or delete in a query"); - schemaTableName = ((TableWriterNode.DeleteHandle) writerTarget).getSchemaTableName(); - } + TableWriterNode.WriterTarget writerTarget = node.getTarget().orElseThrow(() -> new VerifyException("target is absent")); + connectorId = writerTarget.getConnectorId(); + checkState(schemaTableName == null || schemaTableName.equals(writerTarget.getSchemaTableName()), + "cannot have more than a single create, insert or delete in a query"); + schemaTableName = writerTarget.getSchemaTableName(); return null; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/Partitioning.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/Partitioning.java index 3dae1ccc7668a..8445d65ab6598 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/Partitioning.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/Partitioning.java @@ -17,7 +17,11 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.tree.CoalesceExpression; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.SymbolReference; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -27,34 +31,38 @@ import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.COALESCE; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.isExpression; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; @Immutable public final class Partitioning { private final PartitioningHandle handle; - private final List arguments; + private final List arguments; - private Partitioning(PartitioningHandle handle, List arguments) + private Partitioning(PartitioningHandle handle, List arguments) { this.handle = requireNonNull(handle, "handle is null"); this.arguments = ImmutableList.copyOf(requireNonNull(arguments, "arguments is null")); } - public static Partitioning create(PartitioningHandle handle, List columns) + public static Partitioning create(PartitioningHandle handle, List columns) { return new Partitioning(handle, columns.stream() - .map(ArgumentBinding::new) + .map(RowExpression.class::cast) .collect(toImmutableList())); } @@ -62,7 +70,7 @@ public static Partitioning create(PartitioningHandle handle, List arguments) + @JsonProperty("arguments") List arguments) { return new Partitioning(handle, arguments); } @@ -74,7 +82,7 @@ public PartitioningHandle getHandle() } @JsonProperty - public List getArguments() + public List getArguments() { return arguments; } @@ -82,8 +90,8 @@ public List getArguments() public Set getVariableReferences() { return arguments.stream() - .filter(ArgumentBinding::isVariable) - .map(ArgumentBinding::getVariableReference) + .filter(VariableReferenceExpression.class::isInstance) + .map(VariableReferenceExpression.class::cast) .collect(toImmutableSet()); } @@ -118,8 +126,8 @@ public boolean isCompatibleWith( } for (int i = 0; i < arguments.size(); i++) { - ArgumentBinding leftArgument = arguments.get(i); - ArgumentBinding rightArgument = right.arguments.get(i); + RowExpression leftArgument = arguments.get(i); + RowExpression rightArgument = right.arguments.get(i); if (!isPartitionedWith(leftArgument, leftConstantMapping, rightArgument, rightConstantMapping, leftToRightMappings)) { return false; @@ -158,8 +166,8 @@ public boolean isRefinedPartitioningOver( } for (int i = 0; i < arguments.size(); i++) { - ArgumentBinding leftArgument = arguments.get(i); - ArgumentBinding rightArgument = right.arguments.get(i); + RowExpression leftArgument = arguments.get(i); + RowExpression rightArgument = right.arguments.get(i); if (!isPartitionedWith(leftArgument, leftConstantMapping, rightArgument, rightConstantMapping, leftToRightMappings)) { return false; @@ -169,49 +177,68 @@ public boolean isRefinedPartitioningOver( } private static boolean isPartitionedWith( - ArgumentBinding leftArgument, + RowExpression leftArgument, Function> leftConstantMapping, - ArgumentBinding rightArgument, + RowExpression rightArgument, Function> rightConstantMapping, Function> leftToRightMappings) { - if (leftArgument.isVariable()) { - if (rightArgument.isVariable()) { + if (leftArgument instanceof VariableReferenceExpression) { + if (rightArgument instanceof VariableReferenceExpression) { // variable == variable - Set mappedColumns = leftToRightMappings.apply(leftArgument.getVariableReference()); - return mappedColumns.contains(rightArgument.getVariableReference()); + Set mappedColumns = leftToRightMappings.apply((VariableReferenceExpression) leftArgument); + return mappedColumns.contains(rightArgument); } - else { + else if (rightArgument instanceof ConstantExpression) { // variable == constant // Normally, this would be a false condition, but if we happen to have an external // mapping from the variable to a constant value and that constant value matches the // right value, then we are co-partitioned. - Optional leftConstant = leftConstantMapping.apply(leftArgument.getVariableReference()); - return leftConstant.isPresent() && leftConstant.get().equals(rightArgument.getConstant()); + Optional leftConstant = leftConstantMapping.apply((VariableReferenceExpression) leftArgument); + return leftConstant.isPresent() && leftConstant.get().equals(rightArgument); + } + else { + // variable == coalesce + return false; } } - else { - if (rightArgument.isConstant()) { + else if (leftArgument instanceof ConstantExpression) { + if (rightArgument instanceof ConstantExpression) { // constant == constant - return leftArgument.getConstant().equals(rightArgument.getConstant()); + return leftArgument.equals(rightArgument); } - else { + else if (rightArgument instanceof VariableReferenceExpression) { // constant == variable - Optional rightConstant = rightConstantMapping.apply(rightArgument.getVariableReference()); - return rightConstant.isPresent() && rightConstant.get().equals(leftArgument.getConstant()); + Optional rightConstant = rightConstantMapping.apply((VariableReferenceExpression) rightArgument); + return rightConstant.isPresent() && rightConstant.get().equals(leftArgument); } + else { + // constant == coalesce + return false; + } + } + else { + // coalesce == ? + return false; } } public boolean isPartitionedOn(Collection columns, Set knownConstants) { - // partitioned on (k_1, k_2, ..., k_n) => partitioned on (k_1, k_2, ..., k_n, k_n+1, ...) - // can safely ignore all constant columns when comparing partition properties - return arguments.stream() - .filter(ArgumentBinding::isVariable) - .map(ArgumentBinding::getVariableReference) - .filter(variable -> !knownConstants.contains(variable)) - .allMatch(columns::contains); + for (RowExpression argument : arguments) { + // partitioned on (k_1, k_2, ..., k_n) => partitioned on (k_1, k_2, ..., k_n, k_n+1, ...) + // can safely ignore all constant columns when comparing partition properties + if (argument instanceof ConstantExpression) { + continue; + } + if (!(argument instanceof VariableReferenceExpression)) { + return false; + } + if (!knownConstants.contains(argument) && !columns.contains(argument)) { + return false; + } + } + return true; } public boolean isEffectivelySinglePartition(Set knownConstants) @@ -225,29 +252,133 @@ public boolean isRepartitionEffective(Collection ke .filter(variable -> !knownConstants.contains(variable)) .collect(toImmutableSet()); Set nonConstantArgs = arguments.stream() - .filter(ArgumentBinding::isVariable) - .map(ArgumentBinding::getVariableReference) + .filter(VariableReferenceExpression.class::isInstance) + .map(VariableReferenceExpression.class::cast) .filter(variable -> !knownConstants.contains(variable)) .collect(toImmutableSet()); return !nonConstantArgs.equals(keysWithoutConstants); } - public Partitioning translate(Function translator) + // Translates VariableReferenceExpression in arguments according to translator, keeps other arguments unchanged. + public Partitioning translateVariable(Function translator) { return new Partitioning(handle, arguments.stream() - .map(argument -> argument.translate(translator)) + .map(argument -> { + if (argument instanceof VariableReferenceExpression) { + return translator.apply((VariableReferenceExpression) argument); + } + return argument; + }) .collect(toImmutableList())); } - public Optional translate(Function> translator, Function> constants) + // Tries to translate VariableReferenceExpression in arguments according to translator, keeps constant arguments unchanged. If any arguments failed to translate, return empty partitioning. + public Optional translateVariableToRowExpression(Function> translator) { - ImmutableList.Builder newArguments = ImmutableList.builder(); - for (ArgumentBinding argument : arguments) { - Optional newArgument = argument.translate(translator, constants); - if (!newArgument.isPresent()) { + ImmutableList.Builder newArguments = ImmutableList.builder(); + for (RowExpression argument : arguments) { + if (argument instanceof ConstantExpression) { + newArguments.add(argument); + } + else if (argument instanceof VariableReferenceExpression) { + Optional newArgument = translator.apply((VariableReferenceExpression) argument); + if (!newArgument.isPresent()) { + return Optional.empty(); + } + newArguments.add(newArgument.get()); + } + else { return Optional.empty(); } - newArguments.add(newArgument.get()); + } + + return Optional.of(new Partitioning(handle, newArguments.build())); + } + + // Maps VariableReferenceExpression in both partitions to an COALESCE expression, keeps constant arguments unchanged. + public Optional translateToCoalesce(Partitioning other) + { + checkArgument(this.handle.equals(other.handle), "incompatible partitioning handles: %s != %s", this.handle, other.handle); + checkArgument(this.arguments.size() == other.arguments.size(), "incompatible number of partitioning arguments: %s != %s", this.arguments.size(), other.arguments.size()); + ImmutableList.Builder arguments = ImmutableList.builder(); + for (int i = 0; i < this.arguments.size(); i++) { + RowExpression leftArgument = this.arguments.get(i); + RowExpression rightArgument = other.arguments.get(i); + if (leftArgument instanceof ConstantExpression) { + arguments.add(leftArgument); + } + else if (rightArgument instanceof ConstantExpression) { + arguments.add(rightArgument); + } + else if (leftArgument instanceof VariableReferenceExpression && rightArgument instanceof VariableReferenceExpression) { + VariableReferenceExpression leftVariable = (VariableReferenceExpression) leftArgument; + VariableReferenceExpression rightVariable = (VariableReferenceExpression) rightArgument; + checkArgument(leftVariable.getType().equals(rightVariable.getType()), "incompatible types: %s != %s", leftVariable.getType(), rightVariable.getType()); + arguments.add(new SpecialFormExpression(COALESCE, leftVariable.getType(), ImmutableList.of(leftVariable, rightVariable))); + } + else { + return Optional.empty(); + } + } + return Optional.of(new Partitioning(this.handle, arguments.build())); + } + + public Optional translateRowExpression(Map inputToOutputMappings, Map assignments, TypeProvider types) + { + ImmutableList.Builder newArguments = ImmutableList.builder(); + for (RowExpression argument : arguments) { + if (argument instanceof ConstantExpression) { + newArguments.add(argument); + } + else if (argument instanceof VariableReferenceExpression) { + if (!inputToOutputMappings.containsKey(argument)) { + return Optional.empty(); + } + newArguments.add(inputToOutputMappings.get(argument)); + } + else { + checkArgument(argument instanceof SpecialFormExpression && ((SpecialFormExpression) argument).getForm().equals(COALESCE), format("Expect argument to be COALESCE but get %s", argument)); + Set coalesceArguments = ImmutableSet.copyOf(((SpecialFormExpression) argument).getArguments()); + checkArgument(coalesceArguments.stream().allMatch(VariableReferenceExpression.class::isInstance), format("Expect arguments of COALESCE to be VariableReferenceExpression but get %s", coalesceArguments)); + // We are using the property that the result of coalesce from full outer join keys would not be null despite of the order + // of the arguments. Thus we extract and compare the variables of the COALESCE as a set rather than compare COALESCE directly. + VariableReferenceExpression translated = null; + for (Map.Entry entry : assignments.entrySet()) { + if (isExpression(entry.getValue())) { + if (castToExpression(entry.getValue()) instanceof CoalesceExpression) { + Set coalesceOperands = ImmutableSet.copyOf(((CoalesceExpression) castToExpression(entry.getValue())).getOperands()); + if (!coalesceOperands.stream().allMatch(SymbolReference.class::isInstance)) { + continue; + } + + if (coalesceOperands.stream() + .map(operand -> new VariableReferenceExpression(((SymbolReference) operand).getName(), types.get(operand))) + .collect(toImmutableSet()) + .equals(coalesceArguments)) { + translated = entry.getKey(); + break; + } + } + } + else { + if (entry.getValue() instanceof SpecialFormExpression && ((SpecialFormExpression) entry.getValue()).getForm().equals(COALESCE)) { + Set assignmentArguments = ImmutableSet.copyOf(((SpecialFormExpression) entry.getValue()).getArguments()); + if (!assignmentArguments.stream().allMatch(VariableReferenceExpression.class::isInstance)) { + continue; + } + + if (assignmentArguments.equals(coalesceArguments)) { + translated = entry.getKey(); + break; + } + } + } + } + if (translated == null) { + return Optional.empty(); + } + newArguments.add(translated); + } } return Optional.of(new Partitioning(handle, newArguments.build())); @@ -286,100 +417,4 @@ public String toString() .add("arguments", arguments) .toString(); } - - @Immutable - public static final class ArgumentBinding - { - private final RowExpression rowExpression; - - @JsonCreator - public ArgumentBinding(@JsonProperty("rowExpression") RowExpression rowExpression) - { - checkArgument(rowExpression instanceof VariableReferenceExpression || rowExpression instanceof ConstantExpression, "Expect either VariableReferenceExpression or ConstantExpression"); - this.rowExpression = requireNonNull(rowExpression, "rowExpression is null"); - } - - @JsonProperty - public RowExpression getRowExpression() - { - return rowExpression; - } - - public boolean isConstant() - { - return rowExpression instanceof ConstantExpression; - } - - public boolean isVariable() - { - return rowExpression instanceof VariableReferenceExpression; - } - - public VariableReferenceExpression getVariableReference() - { - verify(rowExpression instanceof VariableReferenceExpression, "Expect the rowExpression to be a VariableReferenceExpression"); - return (VariableReferenceExpression) rowExpression; - } - - public ConstantExpression getConstant() - { - verify(rowExpression instanceof ConstantExpression, "Expect the rowExpression to be a ConstantExpression"); - return (ConstantExpression) rowExpression; - } - - public ArgumentBinding translate(Function translator) - { - if (isConstant()) { - return this; - } - return new ArgumentBinding(translator.apply((VariableReferenceExpression) rowExpression)); - } - - public Optional translate(Function> translator, Function> constants) - { - if (isConstant()) { - return Optional.of(this); - } - - Optional newColumn = translator.apply((VariableReferenceExpression) rowExpression) - .map(ArgumentBinding::new); - if (newColumn.isPresent()) { - return newColumn; - } - - // As a last resort, check for a constant mapping for the variable - // Note: this MUST be last because we want to favor the variable representation - // as it makes further optimizations possible. - return constants.apply((VariableReferenceExpression) rowExpression) - .map(ArgumentBinding::new); - } - - @Override - public String toString() - { - if (rowExpression instanceof ConstantExpression) { - return rowExpression.toString(); - } - return "\"" + rowExpression.toString() + "\""; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ArgumentBinding that = (ArgumentBinding) o; - return Objects.equals(rowExpression, that.rowExpression); - } - - @Override - public int hashCode() - { - return Objects.hash(rowExpression); - } - } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PartitioningProviderManager.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PartitioningProviderManager.java new file mode 100644 index 0000000000000..17d0ec89890e6 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PartitioningProviderManager.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner; + +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class PartitioningProviderManager +{ + private final ConcurrentMap partitioningProviders = new ConcurrentHashMap<>(); + + public ConnectorNodePartitioningProvider getPartitioningProvider(ConnectorId connectorId) + { + ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(connectorId); + checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", connectorId); + return partitioningProvider; + } + + public void addPartitioningProvider(ConnectorId connectorId, ConnectorNodePartitioningProvider nodePartitioningProvider) + { + requireNonNull(connectorId, "connectorId is null"); + requireNonNull(nodePartitioningProvider, "nodePartitioningProvider is null"); + checkArgument(partitioningProviders.putIfAbsent(connectorId, nodePartitioningProvider) == null, + "NodePartitioningProvider for connector '%s' is already registered", connectorId); + } + + public void removePartitioningProvider(ConnectorId connectorId) + { + partitioningProviders.remove(connectorId); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PartitioningScheme.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PartitioningScheme.java index e2091170a97e5..92591b818afcd 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PartitioningScheme.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PartitioningScheme.java @@ -121,7 +121,7 @@ public PartitioningScheme translateOutputLayout(List newOutputLayout.get(outputLayout.indexOf(variable))); + Partitioning newPartitioning = partitioning.translateVariable(variable -> newOutputLayout.get(outputLayout.indexOf(variable))); Optional newHashSymbol = hashColumn .map(outputLayout::indexOf) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanBuilder.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanBuilder.java index 03331590ae2c6..8af0abfe66353 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanBuilder.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanBuilder.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.analyzer.Analysis; -import com.facebook.presto.sql.planner.plan.Assignments; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.collect.ImmutableMap; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java index 80a3a53e17b1c..5baed803f3ce2 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenter.java @@ -19,7 +19,6 @@ import com.facebook.presto.execution.QueryManagerConfig; import com.facebook.presto.execution.scheduler.BucketNodeMap; import com.facebook.presto.execution.warnings.WarningCollector; -import com.facebook.presto.metadata.InsertTableHandle; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.NewTableLayout; import com.facebook.presto.metadata.PartitioningMetadata; @@ -36,9 +35,12 @@ import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.predicate.TupleDomain; @@ -46,9 +48,6 @@ import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.planner.Partitioning.ArgumentBinding; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; @@ -57,18 +56,20 @@ import com.facebook.presto.sql.planner.plan.MetadataDeleteNode; import com.facebook.presto.sql.planner.plan.OutputNode; import com.facebook.presto.sql.planner.plan.PlanFragmentId; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode.CreateHandle; -import com.facebook.presto.sql.planner.plan.TableWriterNode.InsertHandle; +import com.facebook.presto.sql.planner.plan.TableWriterNode.CreateName; +import com.facebook.presto.sql.planner.plan.TableWriterNode.InsertReference; +import com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.planner.sanity.PlanSanityChecker; +import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -85,11 +86,12 @@ import java.util.Set; import static com.facebook.presto.SystemSessionProperties.getQueryMaxStageCount; -import static com.facebook.presto.SystemSessionProperties.getTaskWriterCount; +import static com.facebook.presto.SystemSessionProperties.getTaskPartitionedWriterCount; import static com.facebook.presto.SystemSessionProperties.isDynamicScheduleForGroupedExecution; import static com.facebook.presto.SystemSessionProperties.isForceSingleNodeOutput; import static com.facebook.presto.SystemSessionProperties.isGroupedExecutionForEligibleTableScansEnabled; import static com.facebook.presto.SystemSessionProperties.isRecoverableGroupedExecutionEnabled; +import static com.facebook.presto.SystemSessionProperties.isTableWriterMergeOperatorEnabled; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.StandardErrorCode.QUERY_HAS_TOO_MANY_STAGES; import static com.facebook.presto.spi.StandardWarningCode.TOO_MANY_STAGES; @@ -109,7 +111,9 @@ import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPLICATE; +import static com.facebook.presto.sql.planner.plan.ExchangeNode.ensureSourceOrderingGatheringExchange; import static com.facebook.presto.sql.planner.plan.ExchangeNode.gatheringExchange; +import static com.facebook.presto.sql.planner.plan.ExchangeNode.partitionedExchange; import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.jsonFragmentPlan; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -130,7 +134,7 @@ */ public class PlanFragmenter { - private static final String TOO_MANY_STAGES_MESSAGE = "If the query contains multiple DISTINCTs, please set the 'use_mark_distinct' session property to false. " + + public static final String TOO_MANY_STAGES_MESSAGE = "If the query contains multiple DISTINCTs, please set the 'use_mark_distinct' session property to false. " + "If the query contains multiple CTEs that are referenced more than once, please create temporary table(s) for one or more of the CTEs."; private final Metadata metadata; @@ -217,7 +221,7 @@ private SubPlan analyzeGroupedExecution(Session session, SubPlan subPlan, boolea * We currently only support recoverable grouped execution if the following statements hold true: * - Current session enables recoverable grouped execution * - Parent sub plan contains TableFinishNode - * - Current sub plan's root is TableWriterNode + * - Current sub plan's root is TableWriterMergeNode or TableWriterNode * - Input connectors supports split source rewind * - Output connectors supports partition commit * - Bucket node map uses dynamic scheduling @@ -225,7 +229,7 @@ private SubPlan analyzeGroupedExecution(Session session, SubPlan subPlan, boolea */ boolean recoverable = isRecoverableGroupedExecutionEnabled(session) && parentContainsTableFinish && - fragment.getRoot() instanceof TableWriterNode && + (fragment.getRoot() instanceof TableWriterMergeNode || fragment.getRoot() instanceof TableWriterNode) && properties.isRecoveryEligible(); if (recoverable) { fragment = fragment.withRecoverableGroupedExecution(properties.getCapableTableScanNodes(), properties.getTotalLifespans()); @@ -497,7 +501,7 @@ else if (exchange.getType() == ExchangeNode.Type.REPARTITION) { .map(PlanFragment::getId) .collect(toImmutableList()); - return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputVariables(), exchange.getOrderingScheme(), exchange.getType()); + return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputVariables(), exchange.isEnsureSourceOrdering(), exchange.getOrderingScheme(), exchange.getType()); } private PlanNode createRemoteMaterializedExchange(ExchangeNode exchange, RewriteContext context) @@ -536,7 +540,7 @@ private PlanNode createRemoteMaterializedExchange(ExchangeNode exchange, Rewrite if (e.getErrorCode().equals(NOT_SUPPORTED.toErrorCode())) { throw new PrestoException( NOT_SUPPORTED, - format("Catalog \"%s\" does not support temporary tables. The exchange cannot be materialized.", connectorId.getCatalogName()), + format("Temporary table cannot be created in catalog \"%s\": %s", connectorId.getCatalogName(), e.getMessage()), e); } throw e; @@ -575,15 +579,15 @@ private PartitioningVariableAssignments assignPartitioningVariables(Partitioning { ImmutableList.Builder variables = ImmutableList.builder(); ImmutableMap.Builder constants = ImmutableMap.builder(); - for (ArgumentBinding argumentBinding : partitioning.getArguments()) { + for (RowExpression argument : partitioning.getArguments()) { + checkArgument(argument instanceof ConstantExpression || argument instanceof VariableReferenceExpression, format("Expect argument to be ConstantExpression or VariableReferenceExpression, get %s (%s)", argument.getClass(), argument)); VariableReferenceExpression variable; - if (argumentBinding.isConstant()) { - ConstantExpression constant = argumentBinding.getConstant(); - variable = variableAllocator.newVariable("constant_partition", constant.getType()); - constants.put(variable, constant); + if (argument instanceof ConstantExpression) { + variable = variableAllocator.newVariable("constant_partition", argument.getType()); + constants.put(variable, argument); } else { - variable = argumentBinding.getVariableReference(); + variable = (VariableReferenceExpression) argument; } variables.add(variable); } @@ -690,56 +694,79 @@ private TableFinishNode createTemporaryTableWrite( .map(columnNameToVariable::get) .collect(toImmutableList()); - InsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle); List outputColumnNames = outputs.stream() .map(variableToColumnMap::get) .map(ColumnMetadata::getName) .collect(toImmutableList()); - SchemaTableName temporaryTableName = metadata.getTableMetadata(session, tableHandle).getTable(); - InsertHandle insertHandle = new InsertHandle(insertTableHandle, new SchemaTableName(temporaryTableName.getSchemaName(), temporaryTableName.getTableName())); + SchemaTableName schemaTableName = metadata.getTableMetadata(session, tableHandle).getTable(); + InsertReference insertReference = new InsertReference(tableHandle, schemaTableName); + + PartitioningScheme partitioningScheme = new PartitioningScheme( + Partitioning.create(partitioningHandle, partitioningVariables), + outputs, + Optional.empty(), + false, + Optional.empty()); + + ExchangeNode writerRemoteSource = new ExchangeNode( + idAllocator.getNextId(), + REPARTITION, + REMOTE_STREAMING, + partitioningScheme, + sources, + inputs, + false, + Optional.empty()); + + ExchangeNode writerSource; + if (getTaskPartitionedWriterCount(session) == 1) { + writerSource = gatheringExchange( + idAllocator.getNextId(), + LOCAL, + writerRemoteSource); + } + else { + writerSource = partitionedExchange( + idAllocator.getNextId(), + LOCAL, + writerRemoteSource, + partitioningScheme); + } + + TableWriterNode tableWriter = new TableWriterNode( + idAllocator.getNextId(), + writerSource, + Optional.of(insertReference), + variableAllocator.newVariable("partialrows", BIGINT), + variableAllocator.newVariable("partialfragments", VARBINARY), + variableAllocator.newVariable("partialtablecommitcontext", VARBINARY), + outputs, + outputColumnNames, + Optional.of(partitioningScheme), + Optional.empty()); + + PlanNode tableWriterMerge = tableWriter; + if (isTableWriterMergeOperatorEnabled(session)) { + tableWriterMerge = new TableWriterMergeNode( + idAllocator.getNextId(), + gatheringExchange( + idAllocator.getNextId(), + LOCAL, + tableWriter), + variableAllocator.newVariable("intermediaterows", BIGINT), + variableAllocator.newVariable("intermediatefragments", VARBINARY), + variableAllocator.newVariable("intermediatetablecommitcontext", VARBINARY), + Optional.empty()); + } return new TableFinishNode( idAllocator.getNextId(), - gatheringExchange( + ensureSourceOrderingGatheringExchange( idAllocator.getNextId(), - LOCAL, - gatheringExchange( - idAllocator.getNextId(), - REMOTE_STREAMING, - new TableWriterNode( - idAllocator.getNextId(), - gatheringExchange( - idAllocator.getNextId(), - LOCAL, - new ExchangeNode( - idAllocator.getNextId(), - REPARTITION, - REMOTE_STREAMING, - new PartitioningScheme( - Partitioning.create(partitioningHandle, partitioningVariables), - outputs, - Optional.empty(), - false, - Optional.empty()), - sources, - inputs, - Optional.empty())), - insertHandle, - variableAllocator.newVariable("partialrows", BIGINT), - variableAllocator.newVariable("fragment", VARBINARY), - variableAllocator.newVariable("tablecommitcontext", VARBINARY), - outputs, - outputColumnNames, - Optional.of(new PartitioningScheme( - Partitioning.create(partitioningHandle, partitioningVariables), - outputs, - Optional.empty(), - false, - Optional.empty())), - Optional.empty(), - Optional.empty()))), - insertHandle, + REMOTE_STREAMING, + tableWriterMerge), + Optional.of(insertReference), variableAllocator.newVariable("rows", BIGINT), Optional.empty(), Optional.empty()); @@ -1033,13 +1060,10 @@ public GroupedExecutionProperties visitMarkDistinct(MarkDistinctNode node, Void public GroupedExecutionProperties visitTableWriter(TableWriterNode node, Void context) { GroupedExecutionProperties properties = node.getSource().accept(this, null); - // TODO (#13098): Remove partitioning and task writer count check after we have TableWriterMergeOperator - boolean recoveryEligible = properties.isRecoveryEligible() && (node.getPartitioningScheme().isPresent() || getTaskWriterCount(session) == 1); - if (node.getTarget() instanceof CreateHandle) { - recoveryEligible &= metadata.getConnectorCapabilities(session, ((CreateHandle) node.getTarget()).getHandle().getConnectorId()).contains(SUPPORTS_PARTITION_COMMIT); - } - else if (node.getTarget() instanceof InsertHandle) { - recoveryEligible &= metadata.getConnectorCapabilities(session, ((InsertHandle) node.getTarget()).getHandle().getConnectorId()).contains(SUPPORTS_PARTITION_COMMIT); + boolean recoveryEligible = properties.isRecoveryEligible(); + WriterTarget target = node.getTarget().orElseThrow(() -> new VerifyException("target is absent")); + if (target instanceof CreateName || target instanceof InsertReference) { + recoveryEligible &= metadata.getConnectorCapabilities(session, target.getConnectorId()).contains(SUPPORTS_PARTITION_COMMIT); } else { recoveryEligible = false; @@ -1060,13 +1084,16 @@ public GroupedExecutionProperties visitTableScan(TableScanNode node, Void contex return GroupedExecutionProperties.notCapable(); } List partitionHandles = nodePartitioningManager.listPartitionHandles(session, tablePartitioning.get().getPartitioningHandle()); - boolean recoveryEligible = metadata.getConnectorCapabilities(session, node.getTable().getConnectorId()).contains(SUPPORTS_REWINDABLE_SPLIT_SOURCE); - boolean useful = isGroupedExecutionForEligibleTableScansEnabled(session); if (ImmutableList.of(NOT_PARTITIONED).equals(partitionHandles)) { - return new GroupedExecutionProperties(false, useful, ImmutableList.of(), 1, recoveryEligible); + return GroupedExecutionProperties.notCapable(); } else { - return new GroupedExecutionProperties(true, useful, ImmutableList.of(node.getId()), partitionHandles.size(), recoveryEligible); + return new GroupedExecutionProperties( + true, + isGroupedExecutionForEligibleTableScansEnabled(session), + ImmutableList.of(node.getId()), + partitionHandles.size(), + metadata.getConnectorCapabilities(session, node.getTable().getConnectorId()).contains(SUPPORTS_REWINDABLE_SPLIT_SOURCE)); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index 67378588677d5..4e6436d1bc87f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -21,7 +21,6 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; -import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.iterative.IterativeOptimizer; import com.facebook.presto.sql.planner.iterative.Rule; @@ -29,7 +28,6 @@ import com.facebook.presto.sql.planner.iterative.rule.CanonicalizeExpressions; import com.facebook.presto.sql.planner.iterative.rule.CreatePartialTopN; import com.facebook.presto.sql.planner.iterative.rule.DesugarAtTimeZone; -import com.facebook.presto.sql.planner.iterative.rule.DesugarCurrentPath; import com.facebook.presto.sql.planner.iterative.rule.DesugarCurrentUser; import com.facebook.presto.sql.planner.iterative.rule.DesugarLambdaExpression; import com.facebook.presto.sql.planner.iterative.rule.DesugarRowSubscript; @@ -90,8 +88,10 @@ import com.facebook.presto.sql.planner.iterative.rule.RemoveUnreferencedScalarLateralNodes; import com.facebook.presto.sql.planner.iterative.rule.ReorderJoins; import com.facebook.presto.sql.planner.iterative.rule.RewriteSpatialPartitioningAggregation; +import com.facebook.presto.sql.planner.iterative.rule.SimplifyArrayOperations; import com.facebook.presto.sql.planner.iterative.rule.SimplifyCountOverConstant; import com.facebook.presto.sql.planner.iterative.rule.SimplifyExpressions; +import com.facebook.presto.sql.planner.iterative.rule.SimplifyRowExpressions; import com.facebook.presto.sql.planner.iterative.rule.SingleDistinctAggregationToGroupBy; import com.facebook.presto.sql.planner.iterative.rule.TransformCorrelatedInPredicateToJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformCorrelatedLateralJoinToJoin; @@ -101,10 +101,10 @@ import com.facebook.presto.sql.planner.iterative.rule.TransformExistsApplyToLateralNode; import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToSemiJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedLateralToJoin; +import com.facebook.presto.sql.planner.iterative.rule.TranslateExpressions; import com.facebook.presto.sql.planner.optimizations.AddExchanges; import com.facebook.presto.sql.planner.optimizations.AddLocalExchanges; import com.facebook.presto.sql.planner.optimizations.ApplyConnectorOptimization; -import com.facebook.presto.sql.planner.optimizations.BeginTableWrite; import com.facebook.presto.sql.planner.optimizations.CheckSubqueryNodesAreRewritten; import com.facebook.presto.sql.planner.optimizations.HashGenerationOptimizer; import com.facebook.presto.sql.planner.optimizations.ImplementIntersectAndExceptAsUnion; @@ -118,10 +118,10 @@ import com.facebook.presto.sql.planner.optimizations.PruneUnreferencedOutputs; import com.facebook.presto.sql.planner.optimizations.PushdownSubfields; import com.facebook.presto.sql.planner.optimizations.ReplicateSemiJoinInDelete; +import com.facebook.presto.sql.planner.optimizations.RowExpressionPredicatePushDown; import com.facebook.presto.sql.planner.optimizations.SetFlatteningOptimizer; import com.facebook.presto.sql.planner.optimizations.StatsRecordingPlanOptimizer; import com.facebook.presto.sql.planner.optimizations.TransformQuantifiedComparisonApplyToLateralJoin; -import com.facebook.presto.sql.planner.optimizations.TranslateExpressions; import com.facebook.presto.sql.planner.optimizations.UnaliasSymbolReferences; import com.facebook.presto.sql.planner.optimizations.WindowFilterPushDown; import com.google.common.collect.ImmutableList; @@ -135,6 +135,9 @@ import java.util.List; import java.util.Set; +import static com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager.PlanPhase.LOGICAL; +import static com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager.PlanPhase.PHYSICAL; + public class PlanOptimizers { private final List optimizers; @@ -146,7 +149,6 @@ public class PlanOptimizers public PlanOptimizers( Metadata metadata, SqlParser sqlParser, - FeaturesConfig featuresConfig, MBeanExporter exporter, SplitManager splitManager, ConnectorPlanOptimizerManager planOptimizerManager, @@ -159,7 +161,6 @@ public PlanOptimizers( { this(metadata, sqlParser, - featuresConfig, false, exporter, splitManager, @@ -189,7 +190,6 @@ public void destroy() public PlanOptimizers( Metadata metadata, SqlParser sqlParser, - FeaturesConfig featuresConfig, boolean forceSingleNode, MBeanExporter exporter, SplitManager splitManager, @@ -232,7 +232,7 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.of( - new InlineProjections(), + new InlineProjections(metadata.getFunctionManager()), new RemoveRedundantIdentityProjections())); IterativeOptimizer projectionPushDown = new IterativeOptimizer( @@ -249,9 +249,21 @@ public PlanOptimizers( estimatedExchangesCostCalculator, new SimplifyExpressions(metadata, sqlParser).rules()); + IterativeOptimizer simplifyRowExpressionOptimizer = new IterativeOptimizer( + ruleStats, + statsCalculator, + estimatedExchangesCostCalculator, + new SimplifyRowExpressions(metadata).rules()); + PlanOptimizer predicatePushDown = new StatsRecordingPlanOptimizer(optimizerStats, new PredicatePushDown(metadata, sqlParser)); + PlanOptimizer rowExpressionPredicatePushDown = new StatsRecordingPlanOptimizer(optimizerStats, new RowExpressionPredicatePushDown(metadata, sqlParser)); builder.add( + new IterativeOptimizer( + ruleStats, + statsCalculator, + estimatedExchangesCostCalculator, + new SimplifyArrayOperations().rules()), // Clean up all the sugar in expressions, e.g. AtTimeZone, must be run before all the other optimizers new IterativeOptimizer( ruleStats, @@ -261,7 +273,6 @@ public PlanOptimizers( .addAll(new DesugarLambdaExpression().rules()) .addAll(new DesugarAtTimeZone(metadata, sqlParser).rules()) .addAll(new DesugarCurrentUser().rules()) - .addAll(new DesugarCurrentPath().rules()) .addAll(new DesugarTryExpression().rules()) .addAll(new DesugarRowSubscript(metadata, sqlParser).rules()) .build()), @@ -270,6 +281,11 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, new CanonicalizeExpressions().rules()), + new IterativeOptimizer( + ruleStats, + statsCalculator, + estimatedExchangesCostCalculator, + ImmutableSet.of(new EvaluateZeroLimit())), new IterativeOptimizer( ruleStats, statsCalculator, @@ -280,7 +296,6 @@ public PlanOptimizers( .addAll(ImmutableSet.of( new RemoveRedundantIdentityProjections(), new RemoveFullSample(), - new EvaluateZeroLimit(), new EvaluateZeroSample(), new PushLimitThroughProject(), new MergeLimits(), @@ -300,7 +315,7 @@ public PlanOptimizers( new RewriteSpatialPartitioningAggregation(metadata))) .build()), simplifyOptimizer, - new UnaliasSymbolReferences(), + new UnaliasSymbolReferences(metadata.getFunctionManager()), new IterativeOptimizer( ruleStats, statsCalculator, @@ -347,7 +362,7 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.of( - new InlineProjections(), + new InlineProjections(metadata.getFunctionManager()), new RemoveRedundantIdentityProjections(), new TransformCorrelatedSingleRowSubqueryToProject())), new CheckSubqueryNodesAreRewritten(), @@ -368,7 +383,7 @@ public PlanOptimizers( inlineProjections, simplifyOptimizer, // Re-run the SimplifyExpressions to simplify any recomposed expressions from other optimizations projectionPushDown, - new UnaliasSymbolReferences(), // Run again because predicate pushdown and projection pushdown might add more projections + new UnaliasSymbolReferences(metadata.getFunctionManager()), // Run again because predicate pushdown and projection pushdown might add more projections new PruneUnreferencedOutputs(), // Make sure to run this before index join. Filtered projections may not have all the columns. new IndexJoinOptimizer(metadata), // Run this after projections and filters have been fully simplified and pushed down new IterativeOptimizer( @@ -441,9 +456,22 @@ public PlanOptimizers( ImmutableSet.>builder() .add(new RemoveRedundantIdentityProjections()) .addAll(new ExtractSpatialJoins(metadata, splitManager, pageSourceManager, sqlParser).rules()) - .add(new InlineProjections()) + .add(new InlineProjections(metadata.getFunctionManager())) .build())); + // TODO: move this before optimization if possible!! + // Replace all expressions with row expressions + builder.add(new IterativeOptimizer( + ruleStats, + statsCalculator, + costCalculator, + new TranslateExpressions(metadata, sqlParser).rules())); + // After this point, all planNodes should not contain OriginalExpression + + // TODO: move PushdownSubfields below this rule + // Pass a supplier so that we pickup connector optimizers that are installed later + builder.add(new ApplyConnectorOptimization(() -> planOptimizerManager.getOptimizers(LOGICAL))); + if (!forceSingleNode) { builder.add(new ReplicateSemiJoinInDelete()); // Must run before AddExchanges builder.add((new IterativeOptimizer( @@ -463,6 +491,7 @@ public PlanOptimizers( ImmutableSet.of(new PushTableWriteThroughUnion()))); // Must run before AddExchanges builder.add(new StatsRecordingPlanOptimizer(optimizerStats, new AddExchanges(metadata, sqlParser))); } + //noinspection UnusedAssignment estimatedExchangesCostCalculator = null; // Prevent accidental use after AddExchanges @@ -473,13 +502,12 @@ public PlanOptimizers( costCalculator, ImmutableSet.of(new RemoveEmptyDelete()))); // Run RemoveEmptyDelete after table scan is removed by PickTableLayout/AddExchanges - builder.add(predicatePushDown); // Run predicate push down one more time in case we can leverage new information from layouts' effective predicate - builder.add(simplifyOptimizer); // Should be always run after PredicatePushDown + builder.add(rowExpressionPredicatePushDown); // Run predicate push down one more time in case we can leverage new information from layouts' effective predicate + builder.add(simplifyRowExpressionOptimizer); // Should be always run after PredicatePushDown builder.add(projectionPushDown); builder.add(inlineProjections); - builder.add(new UnaliasSymbolReferences()); // Run unalias after merging projections to simplify projections more efficiently + builder.add(new UnaliasSymbolReferences(metadata.getFunctionManager())); // Run unalias after merging projections to simplify projections more efficiently builder.add(new PruneUnreferencedOutputs()); - builder.add(new IterativeOptimizer( ruleStats, statsCalculator, @@ -487,21 +515,12 @@ public PlanOptimizers( ImmutableSet.>builder() .add(new RemoveRedundantIdentityProjections()) .add(new PushRemoteExchangeThroughAssignUniqueId()) - .add(new InlineProjections()) + .add(new InlineProjections(metadata.getFunctionManager())) .build())); // Optimizers above this don't understand local exchanges, so be careful moving this. builder.add(new AddLocalExchanges(metadata, sqlParser)); - // TODO: move this before optimization if possible!! - // Replace all expressions with row expressions - builder.add(new IterativeOptimizer( - ruleStats, - statsCalculator, - costCalculator, - new TranslateExpressions(metadata, sqlParser).rules())); - // After this point, all planNodes should not contain OriginalExpression - // Optimizers above this do not need to care about aggregations with the type other than SINGLE // This optimizer must be run after all exchange-related optimizers builder.add(new IterativeOptimizer( @@ -521,11 +540,8 @@ public PlanOptimizers( new AddIntermediateAggregations(), new RemoveRedundantIdentityProjections()))); - // TODO: Do not move other PlanNode to SPI until ApplyConnectorOptimization is moved to the end of logical planning (i.e., where AddExchanges lives) - // TODO: Run PruneUnreferencedOutputs and UnaliasSymbolReferences once we have cleaned it up - // Pass a supplier so that we pickup connector optimizers that are installed later builder.add( - new ApplyConnectorOptimization(planOptimizerManager::getOptimizers), + new ApplyConnectorOptimization(() -> planOptimizerManager.getOptimizers(PHYSICAL)), new IterativeOptimizer( ruleStats, statsCalculator, @@ -537,7 +553,6 @@ public PlanOptimizers( // Precomputed hashes - this assumes that partitioning will not change builder.add(new HashGenerationOptimizer(metadata.getFunctionManager())); builder.add(new MetadataDeleteOptimizer(metadata)); - builder.add(new BeginTableWrite(metadata)); // HACK! see comments in BeginTableWrite // TODO: consider adding a formal final plan sanitization optimizer that prepares the plan for transmission/execution/logging // TODO: figure out how to improve the set flattening optimizer so that it can run at any point diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanVariableAllocator.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanVariableAllocator.java index 079bc98d98d15..943e6e1b918b2 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanVariableAllocator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanVariableAllocator.java @@ -14,6 +14,8 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.spi.VariableAllocator; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.BigintType; import com.facebook.presto.spi.type.Type; @@ -155,7 +157,19 @@ public VariableReferenceExpression toVariableReference(Expression expression) { checkArgument(expression instanceof SymbolReference, "Unexpected expression: %s", expression); String name = ((SymbolReference) expression).getName(); - checkArgument(variables.containsKey(name), "variable map does not contain name"); + checkArgument(variables.containsKey(name), "variable map does not contain name %s", name); return new VariableReferenceExpression(name, variables.get(name)); } + + public VariableReferenceExpression newVariable(RowExpression expression) + { + String nameHint = "expr"; + if (expression instanceof VariableReferenceExpression) { + nameHint = ((VariableReferenceExpression) expression).getName(); + } + else if (expression instanceof CallExpression) { + nameHint = ((CallExpression) expression).getDisplayName(); + } + return newVariable(nameHint, expression.getType(), null); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java index 20a669a39792e..38532145f11eb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java @@ -14,21 +14,23 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.OrderBy; import com.facebook.presto.sql.tree.SortItem; import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Streams; -import java.util.LinkedHashMap; +import java.util.HashSet; import java.util.List; -import java.util.Map; +import java.util.Set; import static com.facebook.presto.sql.relational.Expressions.variable; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.Streams.forEachPair; public class PlannerUtils { @@ -64,10 +66,19 @@ public static OrderingScheme toOrderingScheme(OrderBy orderBy, TypeProvider type public static OrderingScheme toOrderingScheme(List orderingSymbols, List sortOrders) { - Map orderings = new LinkedHashMap<>(); + ImmutableList.Builder builder = ImmutableList.builder(); + // don't override existing keys, i.e. when "ORDER BY a ASC, a DESC" is specified - Streams.forEachPair(orderingSymbols.stream(), sortOrders.stream(), orderings::putIfAbsent); - return new OrderingScheme(ImmutableList.copyOf(orderings.keySet()), orderings); + Set keysSeen = new HashSet<>(); + + forEachPair(orderingSymbols.stream(), sortOrders.stream(), (variable, sortOrder) -> { + if (!keysSeen.contains(variable)) { + keysSeen.add(variable); + builder.add(new Ordering(variable, sortOrder)); + } + }); + + return new OrderingScheme(builder.build()); } public static VariableReferenceExpression toVariableReference(Expression expression, TypeProvider types) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java index ac37e55840f3e..3f778e83404c2 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java @@ -18,9 +18,16 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.predicate.TupleDomain; @@ -34,17 +41,10 @@ import com.facebook.presto.sql.analyzer.RelationId; import com.facebook.presto.sql.analyzer.RelationType; import com.facebook.presto.sql.analyzer.Scope; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.planner.plan.AssignmentUtils; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.GroupIdNode; -import com.facebook.presto.sql.planner.plan.LimitNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SortNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode.DeleteHandle; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.tree.Cast; @@ -79,6 +79,9 @@ import java.util.Set; import java.util.stream.IntStream; +import static com.facebook.presto.spi.plan.AggregationNode.groupingSets; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; +import static com.facebook.presto.spi.plan.LimitNode.Step.FINAL; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.sql.NodeUtils.getSortItemsFromOrderBy; @@ -86,8 +89,6 @@ import static com.facebook.presto.sql.planner.PlannerUtils.toSortOrder; import static com.facebook.presto.sql.planner.optimizations.WindowNodeUtil.toBoundType; import static com.facebook.presto.sql.planner.optimizations.WindowNodeUtil.toWindowType; -import static com.facebook.presto.sql.planner.plan.AggregationNode.groupingSets; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identitiesAsSymbolReferences; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.asSymbolReference; @@ -144,8 +145,8 @@ public RelationPlan plan(Query query) builder = project(builder, Iterables.concat(orderBy, outputs)); builder = sort(builder, query); - builder = project(builder, analysis.getOutputExpressions(query)); builder = limit(builder, query); + builder = project(builder, analysis.getOutputExpressions(query)); return new RelationPlan(builder.getRoot(), analysis.getScope(query), computeOutputs(builder, analysis.getOutputExpressions(query))); } @@ -191,8 +192,8 @@ public RelationPlan plan(QuerySpecification node) builder = distinct(builder, node); builder = sort(builder, node); - builder = project(builder, outputs); builder = limit(builder, node); + builder = project(builder, outputs); return new RelationPlan(builder.getRoot(), analysis.getScope(node), computeOutputs(builder, outputs)); } @@ -243,7 +244,7 @@ public DeleteNode plan(Delete node) variableAllocator.newVariable("partialrows", BIGINT), variableAllocator.newVariable("fragment", VARBINARY)); - return new DeleteNode(idAllocator.getNextId(), builder.getRoot(), new DeleteHandle(handle, metadata.getTableMetadata(session, handle).getTable()), rowId, deleteNodeOutputVariables); + return new DeleteNode(idAllocator.getNextId(), builder.getRoot(), rowId, deleteNodeOutputVariables); } private static List computeOutputs(PlanBuilder builder, List outputExpressions) @@ -824,13 +825,14 @@ private PlanBuilder window(PlanBuilder subPlan, List windowFunctio analysis.getFunctionHandle(windowFunction), returnType, ((FunctionCall) rewritten).getArguments().stream().map(OriginalExpressionUtils::castToRowExpression).collect(toImmutableList())), - frame); + frame, + windowFunction.isIgnoreNulls()); ImmutableList.Builder orderByVariables = ImmutableList.builder(); orderByVariables.addAll(orderings.keySet()); Optional orderingScheme = Optional.empty(); if (!orderings.isEmpty()) { - orderingScheme = Optional.of(new OrderingScheme(orderByVariables.build(), orderings)); + orderingScheme = Optional.of(new OrderingScheme(orderByVariables.build().stream().map(variable -> new Ordering(variable, orderings.get(variable))).collect(toImmutableList()))); } // create window node @@ -883,15 +885,15 @@ private PlanBuilder distinct(PlanBuilder subPlan, QuerySpecification node) private PlanBuilder sort(PlanBuilder subPlan, Query node) { - return sort(subPlan, node.getOrderBy(), node.getLimit(), analysis.getOrderByExpressions(node)); + return sort(subPlan, node.getOrderBy(), analysis.getOrderByExpressions(node)); } private PlanBuilder sort(PlanBuilder subPlan, QuerySpecification node) { - return sort(subPlan, node.getOrderBy(), node.getLimit(), analysis.getOrderByExpressions(node)); + return sort(subPlan, node.getOrderBy(), analysis.getOrderByExpressions(node)); } - private PlanBuilder sort(PlanBuilder subPlan, Optional orderBy, Optional limit, List orderByExpressions) + private PlanBuilder sort(PlanBuilder subPlan, Optional orderBy, List orderByExpressions) { if (!orderBy.isPresent()) { return subPlan; @@ -901,33 +903,30 @@ private PlanBuilder sort(PlanBuilder subPlan, Optional orderBy, Optiona OrderingScheme orderingScheme = toOrderingScheme( orderByExpressions.stream().map(subPlan::translate).collect(toImmutableList()), orderBy.get().getSortItems().stream().map(PlannerUtils::toSortOrder).collect(toImmutableList())); - if (limit.isPresent() && !limit.get().equalsIgnoreCase("all")) { - planNode = new TopNNode(idAllocator.getNextId(), subPlan.getRoot(), Long.parseLong(limit.get()), orderingScheme, TopNNode.Step.SINGLE); - } - else { - planNode = new SortNode(idAllocator.getNextId(), subPlan.getRoot(), orderingScheme); - } + planNode = new SortNode(idAllocator.getNextId(), subPlan.getRoot(), orderingScheme, false); return subPlan.withNewRoot(planNode); } private PlanBuilder limit(PlanBuilder subPlan, Query node) { - return limit(subPlan, node.getOrderBy(), node.getLimit()); + return limit(subPlan, node.getLimit()); } private PlanBuilder limit(PlanBuilder subPlan, QuerySpecification node) { - return limit(subPlan, node.getOrderBy(), node.getLimit()); + return limit(subPlan, node.getLimit()); } - private PlanBuilder limit(PlanBuilder subPlan, Optional orderBy, Optional limit) + private PlanBuilder limit(PlanBuilder subPlan, Optional limit) { - if (!orderBy.isPresent() && limit.isPresent()) { - if (!limit.get().equalsIgnoreCase("all")) { - long limitValue = Long.parseLong(limit.get()); - subPlan = subPlan.withNewRoot(new LimitNode(idAllocator.getNextId(), subPlan.getRoot(), limitValue, false)); - } + if (!limit.isPresent()) { + return subPlan; + } + + if (!limit.get().equalsIgnoreCase("all")) { + long limitValue = Long.parseLong(limit.get()); + subPlan = subPlan.withNewRoot(new LimitNode(idAllocator.getNextId(), subPlan.getRoot(), limitValue, FINAL)); } return subPlan; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java index a3ff429ab316d..bc15541c242df 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java @@ -18,10 +18,16 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.IntersectNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.RowExpression; @@ -38,15 +44,9 @@ import com.facebook.presto.sql.analyzer.Scope; import com.facebook.presto.sql.planner.optimizations.JoinNodeUtils; import com.facebook.presto.sql.planner.optimizations.SampleNodeUtil; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.Assignments; -import com.facebook.presto.sql.planner.plan.ExceptNode; -import com.facebook.presto.sql.planner.plan.IntersectNode; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SampleNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.tree.AliasedRelation; import com.facebook.presto.sql.tree.Cast; @@ -87,13 +87,14 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.analyzer.SemanticExceptions.notSupportedException; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identitiesAsSymbolReferences; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.asSymbolReference; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; @@ -766,7 +767,7 @@ protected RelationPlan visitUnion(Union node, Void context) SetOperationPlan setOperationPlan = process(node); - PlanNode planNode = new UnionNode(idAllocator.getNextId(), setOperationPlan.getSources(), setOperationPlan.getVariableMapping()); + PlanNode planNode = new UnionNode(idAllocator.getNextId(), setOperationPlan.getSources(), setOperationPlan.getOutputVariables(), setOperationPlan.getVariableMapping()); if (node.isDistinct()) { planNode = distinct(planNode); } @@ -780,7 +781,7 @@ protected RelationPlan visitIntersect(Intersect node, Void context) SetOperationPlan setOperationPlan = process(node); - PlanNode planNode = new IntersectNode(idAllocator.getNextId(), setOperationPlan.getSources(), setOperationPlan.getVariableMapping()); + PlanNode planNode = new IntersectNode(idAllocator.getNextId(), setOperationPlan.getSources(), setOperationPlan.getOutputVariables(), setOperationPlan.getVariableMapping()); return new RelationPlan(planNode, analysis.getScope(node), planNode.getOutputVariables()); } @@ -791,7 +792,7 @@ protected RelationPlan visitExcept(Except node, Void context) SetOperationPlan setOperationPlan = process(node); - PlanNode planNode = new ExceptNode(idAllocator.getNextId(), setOperationPlan.getSources(), setOperationPlan.getVariableMapping()); + PlanNode planNode = new ExceptNode(idAllocator.getNextId(), setOperationPlan.getSources(), setOperationPlan.getOutputVariables(), setOperationPlan.getVariableMapping()); return new RelationPlan(planNode, analysis.getScope(node), planNode.getOutputVariables()); } @@ -864,12 +865,19 @@ private PlanNode distinct(PlanNode node) private static class SetOperationPlan { private final List sources; - private final ListMultimap variableMapping; + private final List outputVariables; + private final Map> variableMapping; private SetOperationPlan(List sources, ListMultimap variableMapping) { this.sources = sources; - this.variableMapping = variableMapping; + this.outputVariables = ImmutableList.copyOf(variableMapping.keySet()); + Map> mapping = new LinkedHashMap<>(); + variableMapping.asMap().forEach((key, value) -> { + checkState(value instanceof List, "variableMapping values should be of type List"); + mapping.put(key, (List) value); + }); + this.variableMapping = mapping; } public List getSources() @@ -877,7 +885,12 @@ public List getSources() return sources; } - public ListMultimap getVariableMapping() + public List getOutputVariables() + { + return outputVariables; + } + + public Map> getVariableMapping() { return variableMapping; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionEqualityInference.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionEqualityInference.java new file mode 100644 index 0000000000000..7f04c6f189edb --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionEqualityInference.java @@ -0,0 +1,502 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner; + +import com.facebook.presto.expressions.RowExpressionNodeInliner; +import com.facebook.presto.expressions.RowExpressionTreeRewriter; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; +import com.facebook.presto.util.DisjointSet; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; +import com.google.common.collect.SetMultimap; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.expressions.LogicalRowExpressions.extractConjuncts; +import static com.facebook.presto.spi.function.OperatorType.EQUAL; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.sql.relational.Expressions.uniqueSubExpressions; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Predicates.equalTo; +import static com.google.common.base.Predicates.not; +import static com.google.common.collect.Iterables.filter; +import static java.util.Objects.requireNonNull; + +public class RowExpressionEqualityInference +{ + // Ordering used to determine Expression preference when determining canonicals + private static final Ordering CANONICAL_ORDERING = Ordering.from((expression1, expression2) -> { + // Current cost heuristic: + // 1) Prefer fewer input symbols + // 2) Prefer smaller expression trees + // 3) Sort the expressions alphabetically - creates a stable consistent ordering (extremely useful for unit testing) + // TODO: be more precise in determining the cost of an RowExpression + return ComparisonChain.start() + .compare(VariablesExtractor.extractAll(expression1).size(), VariablesExtractor.extractAll(expression2).size()) + .compare(uniqueSubExpressions(expression1).size(), uniqueSubExpressions(expression2).size()) + .compare(expression1.toString(), expression2.toString()) + .result(); + }); + + private final SetMultimap equalitySets; // Indexed by canonical RowExpression + private final Map canonicalMap; // Map each known RowExpression to canonical RowExpression + private final Set derivedExpressions; + private final RowExpressionDeterminismEvaluator determinismEvaluator; + private final FunctionManager functionManager; + + private RowExpressionEqualityInference( + Iterable> equalityGroups, + Set derivedExpressions, + RowExpressionDeterminismEvaluator determinismEvaluator, + FunctionManager functionManager) + { + this.determinismEvaluator = determinismEvaluator; + this.functionManager = functionManager; + ImmutableSetMultimap.Builder setBuilder = ImmutableSetMultimap.builder(); + for (Set equalityGroup : equalityGroups) { + if (!equalityGroup.isEmpty()) { + setBuilder.putAll(CANONICAL_ORDERING.min(equalityGroup), equalityGroup); + } + } + equalitySets = setBuilder.build(); + + ImmutableMap.Builder mapBuilder = ImmutableMap.builder(); + for (Map.Entry entry : equalitySets.entries()) { + RowExpression canonical = entry.getKey(); + RowExpression expression = entry.getValue(); + mapBuilder.put(expression, canonical); + } + canonicalMap = mapBuilder.build(); + + this.derivedExpressions = ImmutableSet.copyOf(derivedExpressions); + } + + public static RowExpressionEqualityInference createEqualityInference(Metadata metadata, RowExpression... equalityInferences) + { + return new Builder(metadata) + .addEqualityInference(equalityInferences) + .build(); + } + + /** + * Attempts to rewrite an RowExpression in terms of the symbols allowed by the symbol scope + * given the known equalities. Returns null if unsuccessful. + * This method checks if rewritten expression is non-deterministic. + */ + public RowExpression rewriteExpression(RowExpression expression, Predicate variableScope) + { + checkArgument(determinismEvaluator.isDeterministic(expression), "Only deterministic expressions may be considered for rewrite"); + return rewriteExpression(expression, variableScope, true); + } + + /** + * Attempts to rewrite an Expression in terms of the symbols allowed by the symbol scope + * given the known equalities. Returns null if unsuccessful. + * This method allows rewriting non-deterministic expressions. + */ + public RowExpression rewriteExpressionAllowNonDeterministic(RowExpression expression, Predicate variableScope) + { + return rewriteExpression(expression, variableScope, true); + } + + private RowExpression rewriteExpression(RowExpression expression, Predicate variableScope, boolean allowFullReplacement) + { + Iterable subExpressions = uniqueSubExpressions(expression); + if (!allowFullReplacement) { + subExpressions = filter(subExpressions, not(equalTo(expression))); + } + + ImmutableMap.Builder expressionRemap = ImmutableMap.builder(); + for (RowExpression subExpression : subExpressions) { + RowExpression canonical = getScopedCanonical(subExpression, variableScope); + if (canonical != null) { + expressionRemap.put(subExpression, canonical); + } + } + + // Perform a naive single-pass traversal to try to rewrite non-compliant portions of the tree. Prefers to replace + // larger subtrees over smaller subtrees + // TODO: this rewrite can probably be made more sophisticated + RowExpression rewritten = RowExpressionTreeRewriter.rewriteWith(new RowExpressionNodeInliner(expressionRemap.build()), expression); + if (!variableToExpressionPredicate(variableScope).apply(rewritten)) { + // If the rewritten is still not compliant with the symbol scope, just give up + return null; + } + return rewritten; + } + + /** + * Dumps the inference equalities as equality expressions that are partitioned by the variableScope. + * All stored equalities are returned in a compact set and will be classified into three groups as determined by the symbol scope: + *
    + *
  1. equalities that fit entirely within the symbol scope
    2. + *
  2. equalities that fit entirely outside of the symbol scope
    3. + *
  3. equalities that straddle the symbol scope
    4. + *
+ * 
+     * Example:
+     *   Stored Equalities:
+     *     a = b = c
+     *     d = e = f = g
+     *
+     *   Symbol Scope:
+     *     a, b, d, e
+     *
+     *   Output EqualityPartition:
+     *     Scope Equalities:
+     *       a = b
+     *       d = e
+     *     Complement Scope Equalities
+     *       f = g
+     *     Scope Straddling Equalities
+     *       a = c
+     *       d = f
+     *
+ */ + public EqualityPartition generateEqualitiesPartitionedBy(Predicate variableScope) + { + ImmutableSet.Builder scopeEqualities = ImmutableSet.builder(); + ImmutableSet.Builder scopeComplementEqualities = ImmutableSet.builder(); + ImmutableSet.Builder scopeStraddlingEqualities = ImmutableSet.builder(); + + for (Collection equalitySet : equalitySets.asMap().values()) { + Set scopeExpressions = new LinkedHashSet<>(); + Set scopeComplementExpressions = new LinkedHashSet<>(); + Set scopeStraddlingExpressions = new LinkedHashSet<>(); + + // Try to push each non-derived expression into one side of the scope + for (RowExpression expression : filter(equalitySet, not(derivedExpressions::contains))) { + RowExpression scopeRewritten = rewriteExpression(expression, variableScope, false); + if (scopeRewritten != null) { + scopeExpressions.add(scopeRewritten); + } + RowExpression scopeComplementRewritten = rewriteExpression(expression, not(variableScope), false); + if (scopeComplementRewritten != null) { + scopeComplementExpressions.add(scopeComplementRewritten); + } + if (scopeRewritten == null && scopeComplementRewritten == null) { + scopeStraddlingExpressions.add(expression); + } + } + // Compile the equality expressions on each side of the scope + RowExpression matchingCanonical = getCanonical(scopeExpressions); + if (scopeExpressions.size() >= 2) { + for (RowExpression expression : filter(scopeExpressions, not(equalTo(matchingCanonical)))) { + scopeEqualities.add(buildEqualsExpression(functionManager, matchingCanonical, expression)); + } + } + RowExpression complementCanonical = getCanonical(scopeComplementExpressions); + if (scopeComplementExpressions.size() >= 2) { + for (RowExpression expression : filter(scopeComplementExpressions, not(equalTo(complementCanonical)))) { + scopeComplementEqualities.add(buildEqualsExpression(functionManager, complementCanonical, expression)); + } + } + + // Compile the scope straddling equality expressions + List connectingExpressions = new ArrayList<>(); + connectingExpressions.add(matchingCanonical); + connectingExpressions.add(complementCanonical); + connectingExpressions.addAll(scopeStraddlingExpressions); + connectingExpressions = ImmutableList.copyOf(filter(connectingExpressions, Predicates.notNull())); + RowExpression connectingCanonical = getCanonical(connectingExpressions); + if (connectingCanonical != null) { + for (RowExpression expression : filter(connectingExpressions, not(equalTo(connectingCanonical)))) { + scopeStraddlingEqualities.add(buildEqualsExpression(functionManager, connectingCanonical, expression)); + } + } + } + + return new EqualityPartition(scopeEqualities.build(), scopeComplementEqualities.build(), scopeStraddlingEqualities.build()); + } + + /** + * Returns the most preferrable expression to be used as the canonical expression + */ + private static RowExpression getCanonical(Iterable expressions) + { + if (Iterables.isEmpty(expressions)) { + return null; + } + return CANONICAL_ORDERING.min(expressions); + } + + /** + * Returns a canonical expression that is fully contained by the variableScope and that is equivalent + * to the specified expression. Returns null if unable to to find a canonical. + */ + @VisibleForTesting + RowExpression getScopedCanonical(RowExpression expression, Predicate variableScope) + { + RowExpression canonicalIndex = canonicalMap.get(expression); + if (canonicalIndex == null) { + return null; + } + return getCanonical(filter(equalitySets.get(canonicalIndex), variableToExpressionPredicate(variableScope))); + } + + private static Predicate variableToExpressionPredicate(final Predicate variableScope) + { + return expression -> Iterables.all(VariablesExtractor.extractUnique(expression), variableScope); + } + + public static class EqualityPartition + { + private final List scopeEqualities; + private final List scopeComplementEqualities; + private final List scopeStraddlingEqualities; + + public EqualityPartition(Iterable scopeEqualities, Iterable scopeComplementEqualities, Iterable scopeStraddlingEqualities) + { + this.scopeEqualities = ImmutableList.copyOf(requireNonNull(scopeEqualities, "scopeEqualities is null")); + this.scopeComplementEqualities = ImmutableList.copyOf(requireNonNull(scopeComplementEqualities, "scopeComplementEqualities is null")); + this.scopeStraddlingEqualities = ImmutableList.copyOf(requireNonNull(scopeStraddlingEqualities, "scopeStraddlingEqualities is null")); + } + + public List getScopeEqualities() + { + return scopeEqualities; + } + + public List getScopeComplementEqualities() + { + return scopeComplementEqualities; + } + + public List getScopeStraddlingEqualities() + { + return scopeStraddlingEqualities; + } + } + + public static class Builder + { + private final DisjointSet equalities = new DisjointSet<>(); + private final Set derivedExpressions = new LinkedHashSet<>(); + private final FunctionManager functionManager; + private final NullabilityAnalyzer nullabilityAnalyzer; + private final RowExpressionDeterminismEvaluator determinismEvaluator; + + public Builder(FunctionManager functionManager, TypeManager typeManager) + { + this.determinismEvaluator = new RowExpressionDeterminismEvaluator(functionManager); + this.functionManager = functionManager; + this.nullabilityAnalyzer = new NullabilityAnalyzer(functionManager, typeManager); + } + + public Builder(Metadata metadata) + { + this(metadata.getFunctionManager(), metadata.getTypeManager()); + } + + /** + * Determines whether an RowExpression may be successfully applied to the equality inference + */ + public Predicate isInferenceCandidate() + { + return expression -> { + expression = normalizeInPredicateToEquality(expression); + if (isOperation(expression, EQUAL) && + determinismEvaluator.isDeterministic(expression) && + !nullabilityAnalyzer.mayReturnNullOnNonNullInput(expression)) { + // We should only consider equalities that have distinct left and right components + return !getLeft(expression).equals(getRight(expression)); + } + return false; + }; + } + + public static Predicate isInferenceCandidate(Metadata metadata) + { + return new Builder(metadata).isInferenceCandidate(); + } + + /** + * Rewrite single value InPredicates as equality if possible + */ + private RowExpression normalizeInPredicateToEquality(RowExpression expression) + { + if (isInPredicate(expression)) { + int size = ((SpecialFormExpression) expression).getArguments().size() - 1; + checkArgument(size >= 1, "InList cannot be empty"); + if (size == 1) { + RowExpression leftValue = ((SpecialFormExpression) expression).getArguments().get(0); + RowExpression rightValue = ((SpecialFormExpression) expression).getArguments().get(1); + return buildEqualsExpression(functionManager, leftValue, rightValue); + } + } + return expression; + } + + /** + * Provides a convenience Iterable of RowExpression conjuncts which have not been added to the inference + */ + public Iterable nonInferrableConjuncts(RowExpression expression) + { + return filter(extractConjuncts(expression), not(isInferenceCandidate())); + } + + public static Iterable nonInferrableConjuncts(Metadata metadata, RowExpression expression) + { + return new Builder(metadata).nonInferrableConjuncts(expression); + } + + public Builder addEqualityInference(RowExpression... expressions) + { + for (RowExpression expression : expressions) { + extractInferenceCandidates(expression); + } + return this; + } + + public Builder extractInferenceCandidates(RowExpression expression) + { + return addAllEqualities(filter(extractConjuncts(expression), isInferenceCandidate())); + } + + public RowExpressionEqualityInference.Builder addAllEqualities(Iterable expressions) + { + for (RowExpression expression : expressions) { + addEquality(expression); + } + return this; + } + + public RowExpressionEqualityInference.Builder addEquality(RowExpression expression) + { + expression = normalizeInPredicateToEquality(expression); + checkArgument(isInferenceCandidate().apply(expression), "RowExpression must be a simple equality: " + expression); + addEquality(getLeft(expression), getRight(expression)); + return this; + } + + public RowExpressionEqualityInference.Builder addEquality(RowExpression expression1, RowExpression expression2) + { + checkArgument(!expression1.equals(expression2), "Need to provide equality between different expressions"); + checkArgument(determinismEvaluator.isDeterministic(expression1), "RowExpression must be deterministic: " + expression1); + checkArgument(determinismEvaluator.isDeterministic(expression2), "RowExpression must be deterministic: " + expression2); + + equalities.findAndUnion(expression1, expression2); + return this; + } + + /** + * Performs one pass of generating more equivalences by rewriting sub-expressions in terms of known equivalences. + */ + private void generateMoreEquivalences() + { + Collection> equivalentClasses = equalities.getEquivalentClasses(); + + // Map every expression to the set of equivalent expressions + ImmutableMap.Builder> mapBuilder = ImmutableMap.builder(); + for (Set expressions : equivalentClasses) { + expressions.forEach(expression -> mapBuilder.put(expression, expressions)); + } + + // For every non-derived expression, extract the sub-expressions and see if they can be rewritten as other expressions. If so, + // use this new information to update the known equalities. + Map> map = mapBuilder.build(); + for (RowExpression expression : map.keySet()) { + if (!derivedExpressions.contains(expression)) { + for (RowExpression subExpression : filter(uniqueSubExpressions(expression), not(equalTo(expression)))) { + Set equivalentSubExpressions = map.get(subExpression); + if (equivalentSubExpressions != null) { + for (RowExpression equivalentSubExpression : filter(equivalentSubExpressions, not(equalTo(subExpression)))) { + RowExpression rewritten = RowExpressionTreeRewriter.rewriteWith(new RowExpressionNodeInliner(ImmutableMap.of(subExpression, equivalentSubExpression)), expression); + equalities.findAndUnion(expression, rewritten); + derivedExpressions.add(rewritten); + } + } + } + } + } + } + + public RowExpressionEqualityInference build() + { + generateMoreEquivalences(); + return new RowExpressionEqualityInference(equalities.getEquivalentClasses(), derivedExpressions, determinismEvaluator, functionManager); + } + + private boolean isOperation(RowExpression expression, OperatorType type) + { + if (expression instanceof CallExpression) { + CallExpression call = (CallExpression) expression; + Optional expressionOperatorType = functionManager.getFunctionMetadata(call.getFunctionHandle()).getOperatorType(); + if (expressionOperatorType.isPresent()) { + return expressionOperatorType.get() == type; + } + } + return false; + } + } + + private static RowExpression getLeft(RowExpression expression) + { + checkArgument(expression instanceof CallExpression && ((CallExpression) expression).getArguments().size() == 2, "must be binary call expression"); + return ((CallExpression) expression).getArguments().get(0); + } + + private static RowExpression getRight(RowExpression expression) + { + checkArgument(expression instanceof CallExpression && ((CallExpression) expression).getArguments().size() == 2, "must be binary call expression"); + return ((CallExpression) expression).getArguments().get(1); + } + + private static boolean isInPredicate(RowExpression expression) + { + if (expression instanceof SpecialFormExpression) { + return ((SpecialFormExpression) expression).getForm() == SpecialFormExpression.Form.IN; + } + return false; + } + + private static CallExpression buildEqualsExpression(FunctionManager functionManager, RowExpression left, RowExpression right) + { + return binaryOperation(functionManager, EQUAL, left, right); + } + + private static CallExpression binaryOperation(FunctionManager functionManager, OperatorType type, RowExpression left, RowExpression right) + { + return call( + type.getFunctionName().getFunctionName(), + functionManager.resolveOperator(type, fromTypes(left.getType(), right.getType())), + BOOLEAN, + left, + right); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java index 526674860831b..77de57f20dd56 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java @@ -13,14 +13,18 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.client.FailureInfo; +import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.RowBlockBuilder; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.SqlInvokedScalarFunctionImplementation; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.InputReferenceExpression; @@ -32,19 +36,18 @@ import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.FunctionType; import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.sql.InterpretedFunctionInvoker; -import com.facebook.presto.sql.planner.Interpreters.LambdaSymbolResolver; +import com.facebook.presto.sql.planner.Interpreters.LambdaVariableResolver; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; -import com.facebook.presto.sql.relational.optimizer.ExpressionOptimizer; import com.facebook.presto.util.Failures; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Primitives; import io.airlift.joni.Regex; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; import java.lang.invoke.MethodHandle; @@ -57,7 +60,14 @@ import java.util.stream.Stream; import static com.facebook.presto.metadata.CastType.CAST; +import static com.facebook.presto.metadata.CastType.JSON_TO_ARRAY_CAST; +import static com.facebook.presto.metadata.CastType.JSON_TO_MAP_CAST; +import static com.facebook.presto.metadata.CastType.JSON_TO_ROW_CAST; +import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.OperatorType.EQUAL; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.EVALUATED; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.SERIALIZABLE; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.BIND; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.COALESCE; @@ -71,6 +81,11 @@ import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.SWITCH; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.WHEN; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.StandardTypes.ARRAY; +import static com.facebook.presto.spi.type.StandardTypes.MAP; +import static com.facebook.presto.spi.type.StandardTypes.ROW; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.spi.type.TypeUtils.writeNativeValue; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.spi.type.VarcharType.createVarcharType; @@ -78,12 +93,13 @@ import static com.facebook.presto.sql.gen.VarArgsToMapAdapterGenerator.generateVarArgsToMapAdapter; import static com.facebook.presto.sql.planner.Interpreters.interpretDereference; import static com.facebook.presto.sql.planner.Interpreters.interpretLikePredicate; +import static com.facebook.presto.sql.planner.LiteralEncoder.estimatedSizeInBytes; import static com.facebook.presto.sql.planner.LiteralEncoder.isSupportedLiteralType; -import static com.facebook.presto.sql.planner.LiteralEncoder.toRowExpression; import static com.facebook.presto.sql.planner.RowExpressionInterpreter.SpecialCallResult.changed; import static com.facebook.presto.sql.planner.RowExpressionInterpreter.SpecialCallResult.notChanged; import static com.facebook.presto.sql.relational.Expressions.call; -import static com.facebook.presto.sql.tree.ArrayConstructor.ARRAY_CONSTRUCTOR; +import static com.facebook.presto.sql.relational.Expressions.constant; +import static com.facebook.presto.sql.relational.SqlFunctionUtils.getSqlFunctionRowExpression; import static com.facebook.presto.type.JsonType.JSON; import static com.facebook.presto.type.LikeFunctions.isLikePattern; import static com.facebook.presto.type.LikeFunctions.unescapeLiteralLikePattern; @@ -95,6 +111,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getOnlyElement; import static io.airlift.slice.Slices.utf8Slice; +import static java.lang.String.format; import static java.lang.invoke.MethodHandles.insertArguments; import static java.util.Arrays.asList; import static java.util.Objects.requireNonNull; @@ -102,12 +119,14 @@ public class RowExpressionInterpreter { + private static final long MAX_SERIALIZABLE_OBJECT_SIZE = 1000; private final RowExpression expression; private final Metadata metadata; private final ConnectorSession session; - private final boolean optimize; + private final Level optimizationLevel; private final InterpretedFunctionInvoker functionInvoker; private final RowExpressionDeterminismEvaluator determinismEvaluator; + private final FunctionManager functionManager; private final FunctionResolution resolution; private final Visitor visitor; @@ -115,25 +134,26 @@ public class RowExpressionInterpreter public static Object evaluateConstantRowExpression(RowExpression expression, Metadata metadata, ConnectorSession session) { // evaluate the expression - Object result = new RowExpressionInterpreter(expression, metadata, session, false).evaluate(); + Object result = new RowExpressionInterpreter(expression, metadata, session, EVALUATED).evaluate(); verify(!(result instanceof RowExpression), "RowExpression interpreter returned an unresolved expression"); return result; } public static RowExpressionInterpreter rowExpressionInterpreter(RowExpression expression, Metadata metadata, ConnectorSession session) { - return new RowExpressionInterpreter(expression, metadata, session, false); + return new RowExpressionInterpreter(expression, metadata, session, EVALUATED); } - public RowExpressionInterpreter(RowExpression expression, Metadata metadata, ConnectorSession session, boolean optimize) + public RowExpressionInterpreter(RowExpression expression, Metadata metadata, ConnectorSession session, Level optimizationLevel) { this.expression = requireNonNull(expression, "expression is null"); this.metadata = requireNonNull(metadata, "metadata is null"); this.session = requireNonNull(session, "session is null"); - this.optimize = optimize; + this.optimizationLevel = optimizationLevel; this.functionInvoker = new InterpretedFunctionInvoker(metadata.getFunctionManager()); this.determinismEvaluator = new RowExpressionDeterminismEvaluator(metadata.getFunctionManager()); this.resolution = new FunctionResolution(metadata.getFunctionManager()); + this.functionManager = metadata.getFunctionManager(); this.visitor = new Visitor(); } @@ -145,30 +165,23 @@ public Type getType() public Object evaluate() { - checkState(!optimize, "evaluate() not allowed for optimizer"); + checkState(optimizationLevel.ordinal() >= EVALUATED.ordinal(), "evaluate() not allowed for optimizer"); return expression.accept(visitor, null); } public Object optimize() { - checkState(optimize, "optimize() not allowed for interpreter"); + checkState(optimizationLevel.ordinal() < EVALUATED.ordinal(), "optimize() not allowed for interpreter"); return optimize(null); } /** - * For test only; convenient to replace symbol with constants. Production code should not replace any symbols; use the interface above + * Replace symbol with constants */ - @VisibleForTesting - public Object optimize(SymbolResolver inputs) + public Object optimize(VariableResolver inputs) { - checkState(optimize, "optimize(SymbolResolver) not allowed for interpreter"); - Object result = expression.accept(visitor, inputs); - - if (!(result instanceof RowExpression)) { - // constant folding - return result; - } - return new ExpressionOptimizer(metadata.getFunctionManager(), session).optimize((RowExpression) result); + checkState(optimizationLevel.ordinal() <= EVALUATED.ordinal(), "optimize(SymbolResolver) not allowed for interpreter"); + return expression.accept(visitor, inputs); } private class Visitor @@ -189,8 +202,8 @@ public Object visitConstant(ConstantExpression node, Object context) @Override public Object visitVariableReference(VariableReferenceExpression node, Object context) { - if (context instanceof SymbolResolver) { - return ((SymbolResolver) context).getValue(new Symbol(node.getName())); + if (context instanceof VariableResolver) { + return ((VariableResolver) context).getValue(node); } return node; } @@ -208,9 +221,16 @@ public Object visitCall(CallExpression node, Object context) FunctionHandle functionHandle = node.getFunctionHandle(); FunctionMetadata functionMetadata = metadata.getFunctionManager().getFunctionMetadata(node.getFunctionHandle()); + if (!functionMetadata.isCalledOnNullInput()) { + for (Object value : argumentValues) { + if (value == null) { + return null; + } + } + } // Special casing for large constant array construction - if (functionMetadata.getName().toUpperCase().equals(ARRAY_CONSTRUCTOR)) { + if (resolution.isArrayConstructor(functionHandle)) { SpecialCallResult result = tryHandleArrayConstructor(node, argumentValues); if (result.isChanged()) { return result.getValue(); @@ -233,29 +253,55 @@ public Object visitCall(CallExpression node, Object context) } } - for (int i = 0; i < argumentValues.size(); i++) { - Object value = argumentValues.get(i); - if (value == null && !functionMetadata.isCalledOnNullInput()) { - return null; - } + if (functionMetadata.getFunctionKind() != SCALAR) { + return call(node.getDisplayName(), functionHandle, node.getType(), toRowExpressions(argumentValues, node.getArguments())); } // do not optimize non-deterministic functions - if (optimize && (!functionMetadata.isDeterministic() || hasUnresolvedValue(argumentValues) || functionMetadata.getName().equals("fail"))) { - return call(node.getDisplayName(), functionHandle, node.getType(), toRowExpressions(argumentValues, argumentTypes)); + if (optimizationLevel.ordinal() < EVALUATED.ordinal() && + (!functionMetadata.isDeterministic() || hasUnresolvedValue(argumentValues) || resolution.isFailFunction(functionHandle))) { + return call(node.getDisplayName(), functionHandle, node.getType(), toRowExpressions(argumentValues, node.getArguments())); + } + + Object value; + switch (functionMetadata.getImplementationType()) { + case BUILTIN: + value = functionInvoker.invoke(functionHandle, session, argumentValues); + break; + case SQL: + SqlInvokedScalarFunctionImplementation functionImplementation = (SqlInvokedScalarFunctionImplementation) functionManager.getScalarFunctionImplementation(functionHandle); + RowExpression function = getSqlFunctionRowExpression(functionMetadata, functionImplementation, metadata, session.getSqlFunctionProperties(), node.getArguments()); + RowExpressionInterpreter rowExpressionInterpreter = new RowExpressionInterpreter(function, metadata, session, optimizationLevel); + if (optimizationLevel.ordinal() >= EVALUATED.ordinal()) { + value = rowExpressionInterpreter.evaluate(); + } + else { + value = rowExpressionInterpreter.optimize(); + } + break; + default: + throw new IllegalArgumentException(format("Unsupported function implementation type: %s", functionMetadata.getImplementationType())); } - return functionInvoker.invoke(functionHandle, session, argumentValues); + + if (optimizationLevel.ordinal() <= SERIALIZABLE.ordinal() && !isSerializable(value, node.getType())) { + return call(node.getDisplayName(), functionHandle, node.getType(), toRowExpressions(argumentValues, node.getArguments())); + } + return value; } @Override public Object visitLambda(LambdaDefinitionExpression node, Object context) { - if (optimize) { + if (optimizationLevel.ordinal() < EVALUATED.ordinal()) { // TODO: enable optimization related to lambda expression - // A mechanism to convert function type back into lambda expression need to exist to enable optimization + // Currently, we are not able to determine if lambda is deterministic. + // context is passed down as null here since lambda argument can only be resolved under the evaluation context. + RowExpression rewrittenBody = toRowExpression(processWithExceptionHandling(node.getBody(), null), node.getBody()); + if (!rewrittenBody.equals(node.getBody())) { + return new LambdaDefinitionExpression(node.getArgumentTypes(), node.getArguments(), rewrittenBody); + } return node; } - RowExpression body = node.getBody(); FunctionType functionType = (FunctionType) node.getType(); checkArgument(node.getArguments().size() == functionType.getArgumentTypes().size()); @@ -267,7 +313,7 @@ public Object visitLambda(LambdaDefinitionExpression node, Object context) .map(Primitives::wrap) .collect(toImmutableList()), node.getArguments(), - map -> body.accept(this, new LambdaSymbolResolver(map))); + map -> body.accept(this, new LambdaVariableResolver(map))); } @Override @@ -284,9 +330,9 @@ public Object visitSpecialForm(SpecialFormExpression node, Object context) return new SpecialFormExpression( IF, node.getType(), - toRowExpression(condition, node.getArguments().get(0).getType()), - toRowExpression(trueValue, node.getArguments().get(1).getType()), - toRowExpression(falseValue, node.getArguments().get(2).getType())); + toRowExpression(condition, node.getArguments().get(0)), + toRowExpression(trueValue, node.getArguments().get(1)), + toRowExpression(falseValue, node.getArguments().get(2))); } else if (Boolean.TRUE.equals(condition)) { return trueValue; @@ -309,8 +355,8 @@ else if (Boolean.TRUE.equals(condition)) { return new SpecialFormExpression( NULL_IF, node.getType(), - toRowExpression(left, node.getArguments().get(0).getType()), - toRowExpression(right, node.getArguments().get(1).getType())); + toRowExpression(left, node.getArguments().get(0)), + toRowExpression(right, node.getArguments().get(1))); } Type leftType = node.getArguments().get(0).getType(); @@ -339,7 +385,7 @@ else if (Boolean.TRUE.equals(condition)) { return new SpecialFormExpression( IS_NULL, node.getType(), - toRowExpression(value, node.getArguments().get(0).getType())); + toRowExpression(value, node.getArguments().get(0))); } return value == null; } @@ -369,7 +415,7 @@ else if (Boolean.TRUE.equals(condition)) { node.getType(), toRowExpressions( asList(left, right), - ImmutableList.of(node.getArguments().get(0).getType(), node.getArguments().get(1).getType()))); + node.getArguments().subList(0, 2))); } case OR: { Object left = node.getArguments().get(0).accept(this, context); @@ -397,18 +443,22 @@ else if (Boolean.TRUE.equals(condition)) { node.getType(), toRowExpressions( asList(left, right), - ImmutableList.of(node.getArguments().get(0).getType(), node.getArguments().get(1).getType()))); + node.getArguments().subList(0, 2))); } case ROW_CONSTRUCTOR: { RowType rowType = (RowType) node.getType(); List parameterTypes = rowType.getTypeParameters(); List arguments = node.getArguments(); + checkArgument(parameterTypes.size() == arguments.size(), "RowConstructor does not contain all fields"); + for (int i = 0; i < parameterTypes.size(); i++) { + checkArgument(parameterTypes.get(i).equals(arguments.get(i).getType()), "RowConstructor has field with incorrect type"); + } int cardinality = arguments.size(); List values = new ArrayList<>(cardinality); arguments.forEach(argument -> values.add(argument.accept(this, context))); if (hasUnresolvedValue(values)) { - return new SpecialFormExpression(ROW_CONSTRUCTOR, node.getType(), toRowExpressions(values, parameterTypes)); + return new SpecialFormExpression(ROW_CONSTRUCTOR, node.getType(), toRowExpressions(values, node.getArguments())); } else { BlockBuilder blockBuilder = new RowBlockBuilder(parameterTypes, null, 1); @@ -439,7 +489,7 @@ else if (Boolean.TRUE.equals(condition)) { ImmutableList.Builder operandsBuilder = ImmutableList.builder(); Set visitedExpression = new HashSet<>(); for (Object value : values) { - RowExpression expression = toRowExpression(value, type); + RowExpression expression = LiteralEncoder.toRowExpression(value, type); if (!determinismEvaluator.isDeterministic(expression) || visitedExpression.add(expression)) { operandsBuilder.add(expression); } @@ -479,15 +529,13 @@ else if (Boolean.TRUE.equals(condition)) { boolean hasNullValue = false; boolean found = false; - List unresolvedValues = new ArrayList<>(values.size()); - List unresolvedValueTypes = new ArrayList<>(values.size()); + List unresolvedValues = new ArrayList<>(values.size()); for (int i = 0; i < values.size(); i++) { Object value = values.get(i); Type valueType = valuesTypes.get(i); if (value instanceof RowExpression || target instanceof RowExpression) { hasUnresolvedValue = true; - unresolvedValues.add(value); - unresolvedValueTypes.add(valueType); + unresolvedValues.add(toRowExpression(value, valueExpressions.get(i))); continue; } @@ -510,12 +558,11 @@ else if (!found && result) { } if (hasUnresolvedValue) { - List expressionValues = toRowExpressions(unresolvedValues, unresolvedValueTypes); List simplifiedExpressionValues = Stream.concat( Stream.concat( - Stream.of(toRowExpression(target, targetType)), - expressionValues.stream().filter(determinismEvaluator::isDeterministic).distinct()), - expressionValues.stream().filter((expression -> !determinismEvaluator.isDeterministic(expression)))) + Stream.of(toRowExpression(target, node.getArguments().get(0))), + unresolvedValues.stream().filter(determinismEvaluator::isDeterministic).distinct()), + unresolvedValues.stream().filter((expression -> !determinismEvaluator.isDeterministic(expression)))) .collect(toImmutableList()); return new SpecialFormExpression(IN, node.getType(), simplifiedExpressionValues); } @@ -539,8 +586,8 @@ else if (!found && result) { return new SpecialFormExpression( DEREFERENCE, node.getType(), - toRowExpression(base, node.getArguments().get(0).getType()), - toRowExpression(index, node.getArguments().get(1).getType())); + toRowExpression(base, node.getArguments().get(0)), + toRowExpression((long) index, node.getArguments().get(1))); } return interpretDereference(base, node.getType(), index); } @@ -553,7 +600,7 @@ else if (!found && result) { return new SpecialFormExpression( BIND, node.getType(), - toRowExpressions(values, node.getArguments().stream().map(RowExpression::getType).collect(toImmutableList()))); + toRowExpressions(values, node.getArguments())); } return insertArguments((MethodHandle) values.get(values.size() - 1), 0, values.subList(0, values.size() - 1).toArray()); } @@ -589,7 +636,7 @@ else if (!found && result) { // call equals(value, operand) if (operandValue instanceof RowExpression || value instanceof RowExpression) { // cannot fully evaluate, add updated whenClause - simplifiedWhenClauses.add(new SpecialFormExpression(WHEN, whenClause.getType(), toRowExpression(operandValue, operand.getType()), toRowExpression(resultValue, result.getType()))); + simplifiedWhenClauses.add(new SpecialFormExpression(WHEN, whenClause.getType(), toRowExpression(operandValue, operand), toRowExpression(resultValue, result))); } else if (operandValue != null) { Boolean isEqual = (Boolean) invokeOperator( @@ -609,9 +656,9 @@ else if (operandValue != null) { } ImmutableList.Builder argumentsBuilder = ImmutableList.builder(); - argumentsBuilder.add(toRowExpression(value, node.getArguments().get(0).getType())) + argumentsBuilder.add(toRowExpression(value, node.getArguments().get(0))) .addAll(simplifiedWhenClauses) - .add(toRowExpression(elseValue, node.getArguments().get(node.getArguments().size() - 1).getType())); + .add(toRowExpression(elseValue, node.getArguments().get(node.getArguments().size() - 1))); return new SpecialFormExpression(SWITCH, node.getType(), argumentsBuilder.build()); } default: @@ -642,11 +689,15 @@ private RowExpression createFailureFunction(RuntimeException exception, Type typ String failureInfo = JsonCodec.jsonCodec(FailureInfo.class).toJson(Failures.toFailure(exception).toFailureInfo()); FunctionHandle jsonParse = metadata.getFunctionManager().lookupFunction("json_parse", fromTypes(VARCHAR)); Object json = functionInvoker.invoke(jsonParse, session, utf8Slice(failureInfo)); - - FunctionHandle failureFunction = metadata.getFunctionManager().lookupFunction("fail", fromTypes(JSON)); FunctionHandle cast = metadata.getFunctionManager().lookupCast(CAST, UNKNOWN.getTypeSignature(), type.getTypeSignature()); + if (exception instanceof PrestoException) { + long errorCode = ((PrestoException) exception).getErrorCode().getCode(); + FunctionHandle failureFunction = metadata.getFunctionManager().lookupFunction("fail", fromTypes(INTEGER, JSON)); + return call(CAST.name(), cast, type, call("fail", failureFunction, UNKNOWN, constant(errorCode, INTEGER), LiteralEncoder.toRowExpression(json, JSON))); + } - return call(CAST.name(), cast, type, call("fail", failureFunction, UNKNOWN, toRowExpression(json, JSON))); + FunctionHandle failureFunction = metadata.getFunctionManager().lookupFunction("fail", fromTypes(JSON)); + return call(CAST.name(), cast, type, call("fail", failureFunction, UNKNOWN, LiteralEncoder.toRowExpression(json, JSON))); } private boolean hasUnresolvedValue(Object... values) @@ -665,21 +716,39 @@ private Object invokeOperator(OperatorType operatorType, List ar return functionInvoker.invoke(operatorHandle, session, argumentValues); } - private List toRowExpressions(List values, List types) + private List toRowExpressions(List values, List unchangedValues) { checkArgument(values != null, "value is null"); - checkArgument(types != null, "value is null"); - checkArgument(values.size() == types.size()); + checkArgument(unchangedValues != null, "value is null"); + checkArgument(values.size() == unchangedValues.size()); ImmutableList.Builder rowExpressions = ImmutableList.builder(); for (int i = 0; i < values.size(); i++) { - rowExpressions.add(toRowExpression(values.get(i), types.get(i))); + rowExpressions.add(toRowExpression(values.get(i), unchangedValues.get(i))); } return rowExpressions.build(); } + private RowExpression toRowExpression(Object value, RowExpression originalRowExpression) + { + if (optimizationLevel.ordinal() <= SERIALIZABLE.ordinal() && !isSerializable(value, originalRowExpression.getType())) { + return originalRowExpression; + } + // handle lambda + if (optimizationLevel.ordinal() < EVALUATED.ordinal() && value instanceof MethodHandle) { + return originalRowExpression; + } + return LiteralEncoder.toRowExpression(value, originalRowExpression.getType()); + } + + private boolean isSerializable(Object value, Type type) + { + // If value is already RowExpression, constant values contained inside should already have been made serializable. Otherwise, we make sure the object is small and serializable. + return value instanceof RowExpression || (isSupportedLiteralType(type) && estimatedSizeInBytes(value) <= MAX_SERIALIZABLE_OBJECT_SIZE); + } + private SpecialCallResult tryHandleArrayConstructor(CallExpression callExpression, List argumentValues) { - checkArgument(metadata.getFunctionManager().getFunctionMetadata(callExpression.getFunctionHandle()).getName().toUpperCase().equals(ARRAY_CONSTRUCTOR)); + checkArgument(resolution.isArrayConstructor(callExpression.getFunctionHandle())); boolean allConstants = true; for (Object values : argumentValues) { if (values instanceof RowExpression) { @@ -702,7 +771,8 @@ private SpecialCallResult tryHandleCast(CallExpression callExpression, List commonSuperType = typeManager.getCommonSuperType(valueType, patternType); checkArgument(commonSuperType.isPresent(), "Missing super type when optimizing %s", callExpression); - RowExpression valueExpression = toRowExpression(value, valueType); - RowExpression patternExpression = toRowExpression(unescapedPattern, patternType); + RowExpression valueExpression = LiteralEncoder.toRowExpression(value, valueType); + RowExpression patternExpression = LiteralEncoder.toRowExpression(unescapedPattern, patternType); Type superType = commonSuperType.get(); if (!valueType.equals(superType)) { FunctionHandle cast = metadata.getFunctionManager().lookupCast(CAST, valueType.getTypeSignature(), superType.getTypeSignature()); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionPredicateExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionPredicateExtractor.java new file mode 100644 index 0000000000000..1fa379281eacf --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionPredicateExtractor.java @@ -0,0 +1,425 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner; + +import com.facebook.presto.expressions.LogicalRowExpressions; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.OperatorNotFoundException; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.sql.planner.plan.AssignUniqueId; +import com.facebook.presto.sql.planner.plan.DistinctLimitNode; +import com.facebook.presto.sql.planner.plan.ExchangeNode; +import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; +import com.facebook.presto.sql.planner.plan.JoinNode; +import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.facebook.presto.sql.planner.plan.SortNode; +import com.facebook.presto.sql.planner.plan.SpatialJoinNode; +import com.facebook.presto.sql.planner.plan.WindowNode; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; + +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.extractConjuncts; +import static com.facebook.presto.spi.function.OperatorType.EQUAL; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IS_NULL; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; +import static com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils.outputMap; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.sql.relational.Expressions.specialForm; +import static com.google.common.base.Predicates.in; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +public class RowExpressionPredicateExtractor +{ + private final RowExpressionDomainTranslator domainTranslator; + private final FunctionManager functionManager; + private final TypeManager typeManager; + + public RowExpressionPredicateExtractor(RowExpressionDomainTranslator domainTranslator, FunctionManager functionManager, TypeManager typeManager) + { + this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); + this.functionManager = functionManager; + this.typeManager = typeManager; + } + + public RowExpression extract(PlanNode node) + { + return node.accept(new Visitor(domainTranslator, functionManager, typeManager), null); + } + + private static class Visitor + extends InternalPlanVisitor + { + private final RowExpressionDomainTranslator domainTranslator; + private final LogicalRowExpressions logicalRowExpressions; + private final RowExpressionDeterminismEvaluator determinismEvaluator; + private final TypeManager typeManager; + private final FunctionManager functionManger; + + public Visitor(RowExpressionDomainTranslator domainTranslator, FunctionManager functionManager, TypeManager typeManager) + { + this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); + this.typeManager = requireNonNull(typeManager); + this.functionManger = requireNonNull(functionManager); + this.determinismEvaluator = new RowExpressionDeterminismEvaluator(functionManager); + this.logicalRowExpressions = new LogicalRowExpressions(determinismEvaluator, new FunctionResolution(functionManager), functionManager); + } + + @Override + public RowExpression visitPlan(PlanNode node, Void context) + { + return TRUE_CONSTANT; + } + + @Override + public RowExpression visitAggregation(AggregationNode node, Void context) + { + // GROUP BY () always produces a group, regardless of whether there's any + // input (unlike the case where there are group by keys, which produce + // no output if there's no input). + // Therefore, we can't say anything about the effective predicate of the + // output of such an aggregation. + if (node.getGroupingKeys().isEmpty()) { + return TRUE_CONSTANT; + } + + RowExpression underlyingPredicate = node.getSource().accept(this, context); + + return pullExpressionThroughVariables(underlyingPredicate, node.getGroupingKeys()); + } + + @Override + public RowExpression visitFilter(FilterNode node, Void context) + { + RowExpression underlyingPredicate = node.getSource().accept(this, context); + + RowExpression predicate = node.getPredicate(); + + // Remove non-deterministic conjuncts + predicate = logicalRowExpressions.filterDeterministicConjuncts(predicate); + + return logicalRowExpressions.combineConjuncts(predicate, underlyingPredicate); + } + + @Override + public RowExpression visitExchange(ExchangeNode node, Void context) + { + return deriveCommonPredicates(node, source -> { + Map mappings = new HashMap<>(); + for (int i = 0; i < node.getInputs().get(source).size(); i++) { + mappings.put( + node.getOutputVariables().get(i), + node.getInputs().get(source).get(i)); + } + return mappings.entrySet(); + }); + } + + @Override + public RowExpression visitProject(ProjectNode node, Void context) + { + // TODO: add simple algebraic solver for projection translation (right now only considers identity projections) + + RowExpression underlyingPredicate = node.getSource().accept(this, context); + + List projectionEqualities = node.getAssignments().getMap().entrySet().stream() + .filter(this::notIdentityAssignment) + .filter(this::canCompareEquity) + .map(this::toEquality) + .collect(toImmutableList()); + + return pullExpressionThroughVariables(logicalRowExpressions.combineConjuncts( + ImmutableList.builder() + .addAll(projectionEqualities) + .add(underlyingPredicate) + .build()), + node.getOutputVariables()); + } + + @Override + public RowExpression visitTopN(TopNNode node, Void context) + { + return node.getSource().accept(this, context); + } + + @Override + public RowExpression visitLimit(LimitNode node, Void context) + { + return node.getSource().accept(this, context); + } + + @Override + public RowExpression visitAssignUniqueId(AssignUniqueId node, Void context) + { + return node.getSource().accept(this, context); + } + + @Override + public RowExpression visitDistinctLimit(DistinctLimitNode node, Void context) + { + return node.getSource().accept(this, context); + } + + @Override + public RowExpression visitTableScan(TableScanNode node, Void context) + { + Map assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse(); + return domainTranslator.toPredicate(node.getCurrentConstraint().simplify().transform(column -> assignments.containsKey(column) ? assignments.get(column) : null)); + } + + @Override + public RowExpression visitSort(SortNode node, Void context) + { + return node.getSource().accept(this, context); + } + + @Override + public RowExpression visitWindow(WindowNode node, Void context) + { + return node.getSource().accept(this, context); + } + + @Override + public RowExpression visitUnion(UnionNode node, Void context) + { + return deriveCommonPredicates(node, source -> outputMap(node, source).entries()); + } + + @Override + public RowExpression visitJoin(JoinNode node, Void context) + { + RowExpression leftPredicate = node.getLeft().accept(this, context); + RowExpression rightPredicate = node.getRight().accept(this, context); + + List joinConjuncts = node.getCriteria().stream() + .map(this::toRowExpression) + .collect(toImmutableList()); + + switch (node.getType()) { + case INNER: + return pullExpressionThroughVariables(logicalRowExpressions.combineConjuncts(ImmutableList.builder() + .add(leftPredicate) + .add(rightPredicate) + .add(logicalRowExpressions.combineConjuncts(joinConjuncts)) + .add(node.getFilter().orElse(TRUE_CONSTANT)) + .build()), node.getOutputVariables()); + case LEFT: + return logicalRowExpressions.combineConjuncts(ImmutableList.builder() + .add(pullExpressionThroughVariables(leftPredicate, node.getOutputVariables())) + .addAll(pullNullableConjunctsThroughOuterJoin(extractConjuncts(rightPredicate), node.getOutputVariables(), node.getRight().getOutputVariables()::contains)) + .addAll(pullNullableConjunctsThroughOuterJoin(joinConjuncts, node.getOutputVariables(), node.getRight().getOutputVariables()::contains)) + .build()); + case RIGHT: + return logicalRowExpressions.combineConjuncts(ImmutableList.builder() + .add(pullExpressionThroughVariables(rightPredicate, node.getOutputVariables())) + .addAll(pullNullableConjunctsThroughOuterJoin(extractConjuncts(leftPredicate), node.getOutputVariables(), node.getLeft().getOutputVariables()::contains)) + .addAll(pullNullableConjunctsThroughOuterJoin(joinConjuncts, node.getOutputVariables(), node.getLeft().getOutputVariables()::contains)) + .build()); + case FULL: + return logicalRowExpressions.combineConjuncts(ImmutableList.builder() + .addAll(pullNullableConjunctsThroughOuterJoin(extractConjuncts(leftPredicate), node.getOutputVariables(), node.getLeft().getOutputVariables()::contains)) + .addAll(pullNullableConjunctsThroughOuterJoin(extractConjuncts(rightPredicate), node.getOutputVariables(), node.getRight().getOutputVariables()::contains)) + .addAll(pullNullableConjunctsThroughOuterJoin(joinConjuncts, node.getOutputVariables(), node.getLeft().getOutputVariables()::contains, node.getRight().getOutputVariables()::contains)) + .build()); + default: + throw new UnsupportedOperationException("Unknown join type: " + node.getType()); + } + } + + private Iterable pullNullableConjunctsThroughOuterJoin(List conjuncts, Collection outputVariables, Predicate... nullVariableScopes) + { + // Conjuncts without any symbol dependencies cannot be applied to the effective predicate (e.g. FALSE literal) + return conjuncts.stream() + .map(expression -> pullExpressionThroughVariables(expression, outputVariables)) + .map(expression -> VariablesExtractor.extractAll(expression).isEmpty() ? TRUE_CONSTANT : expression) + .map(expressionOrNullVariables(nullVariableScopes)) + .collect(toImmutableList()); + } + + public Function expressionOrNullVariables(final Predicate... nullVariableScopes) + { + return expression -> { + ImmutableList.Builder resultDisjunct = ImmutableList.builder(); + resultDisjunct.add(expression); + + for (Predicate nullVariableScope : nullVariableScopes) { + List variables = VariablesExtractor.extractUnique(expression).stream() + .filter(nullVariableScope) + .collect(toImmutableList()); + + if (Iterables.isEmpty(variables)) { + continue; + } + + ImmutableList.Builder nullConjuncts = ImmutableList.builder(); + for (VariableReferenceExpression variable : variables) { + nullConjuncts.add(specialForm(IS_NULL, BOOLEAN, variable)); + } + + resultDisjunct.add(logicalRowExpressions.and(nullConjuncts.build())); + } + + return logicalRowExpressions.or(resultDisjunct.build()); + }; + } + + @Override + public RowExpression visitSemiJoin(SemiJoinNode node, Void context) + { + // Filtering source does not change the effective predicate over the output symbols + return node.getSource().accept(this, context); + } + + @Override + public RowExpression visitSpatialJoin(SpatialJoinNode node, Void context) + { + RowExpression leftPredicate = node.getLeft().accept(this, context); + RowExpression rightPredicate = node.getRight().accept(this, context); + + switch (node.getType()) { + case INNER: + return logicalRowExpressions.combineConjuncts(ImmutableList.builder() + .add(pullExpressionThroughVariables(leftPredicate, node.getOutputVariables())) + .add(pullExpressionThroughVariables(rightPredicate, node.getOutputVariables())) + .build()); + case LEFT: + return logicalRowExpressions.combineConjuncts(ImmutableList.builder() + .add(pullExpressionThroughVariables(leftPredicate, node.getOutputVariables())) + .addAll(pullNullableConjunctsThroughOuterJoin(extractConjuncts(rightPredicate), node.getOutputVariables(), node.getRight().getOutputVariables()::contains)) + .build()); + default: + throw new IllegalArgumentException("Unsupported spatial join type: " + node.getType()); + } + } + + private RowExpression toRowExpression(JoinNode.EquiJoinClause equiJoinClause) + { + return buildEqualsExpression(functionManger, equiJoinClause.getLeft(), equiJoinClause.getRight()); + } + + private RowExpression deriveCommonPredicates(PlanNode node, Function>> mapping) + { + // Find the predicates that can be pulled up from each source + List> sourceOutputConjuncts = new ArrayList<>(); + for (int i = 0; i < node.getSources().size(); i++) { + RowExpression underlyingPredicate = node.getSources().get(i).accept(this, null); + + List equalities = mapping.apply(i).stream() + .filter(this::notIdentityAssignment) + .filter(this::canCompareEquity) + .map(this::toEquality) + .collect(toImmutableList()); + + sourceOutputConjuncts.add(ImmutableSet.copyOf(extractConjuncts(pullExpressionThroughVariables(logicalRowExpressions.combineConjuncts( + ImmutableList.builder() + .addAll(equalities) + .add(underlyingPredicate) + .build()), + node.getOutputVariables())))); + } + + // Find the intersection of predicates across all sources + // TODO: use a more precise way to determine overlapping conjuncts (e.g. commutative predicates) + Iterator> iterator = sourceOutputConjuncts.iterator(); + Set potentialOutputConjuncts = iterator.next(); + while (iterator.hasNext()) { + potentialOutputConjuncts = Sets.intersection(potentialOutputConjuncts, iterator.next()); + } + + return logicalRowExpressions.combineConjuncts(potentialOutputConjuncts); + } + + private boolean notIdentityAssignment(Map.Entry entry) + { + return !entry.getKey().equals(entry.getValue()); + } + + private boolean canCompareEquity(Map.Entry entry) + { + try { + functionManger.resolveOperator(EQUAL, fromTypes(entry.getKey().getType(), entry.getValue().getType())); + return true; + } + catch (OperatorNotFoundException e) { + return false; + } + } + + private RowExpression toEquality(Map.Entry entry) + { + return buildEqualsExpression(functionManger, entry.getKey(), entry.getValue()); + } + + private static CallExpression buildEqualsExpression(FunctionManager functionManager, RowExpression left, RowExpression right) + { + return call( + EQUAL.getFunctionName().getFunctionName(), + functionManager.resolveOperator(EQUAL, fromTypes(left.getType(), right.getType())), + BOOLEAN, + left, + right); + } + + private RowExpression pullExpressionThroughVariables(RowExpression expression, Collection variables) + { + RowExpressionEqualityInference equalityInference = new RowExpressionEqualityInference.Builder(functionManger, typeManager) + .addEqualityInference(expression) + .build(); + + ImmutableList.Builder effectiveConjuncts = ImmutableList.builder(); + for (RowExpression conjunct : new RowExpressionEqualityInference.Builder(functionManger, typeManager).nonInferrableConjuncts(expression)) { + if (determinismEvaluator.isDeterministic(conjunct)) { + RowExpression rewritten = equalityInference.rewriteExpression(conjunct, in(variables)); + if (rewritten != null) { + effectiveConjuncts.add(rewritten); + } + } + } + + effectiveConjuncts.addAll(equalityInference.generateEqualitiesPartitionedBy(in(variables)).getScopeEqualities()); + + return logicalRowExpressions.combineConjuncts(effectiveConjuncts.build()); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionVariableInliner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionVariableInliner.java index 30867745b4463..d74e3b3100961 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionVariableInliner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/RowExpressionVariableInliner.java @@ -13,15 +13,16 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.expressions.RowExpressionRewriter; +import com.facebook.presto.expressions.RowExpressionTreeRewriter; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; -import com.facebook.presto.spi.relation.RowExpressionRewriter; -import com.facebook.presto.spi.relation.RowExpressionTreeRewriter; import com.facebook.presto.spi.relation.VariableReferenceExpression; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.function.Function; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -30,23 +31,28 @@ public final class RowExpressionVariableInliner extends RowExpressionRewriter { private final Set excludedNames = new HashSet<>(); - private final Map mapping; + private final Function mapping; - private RowExpressionVariableInliner(Map mapping) + private RowExpressionVariableInliner(Function mapping) { this.mapping = mapping; } - public static RowExpression inlineVariables(Map mapping, RowExpression expression) + public static RowExpression inlineVariables(Function mapping, RowExpression expression) { return RowExpressionTreeRewriter.rewriteWith(new RowExpressionVariableInliner(mapping), expression); } + public static RowExpression inlineVariables(Map mapping, RowExpression expression) + { + return inlineVariables(mapping::get, expression); + } + @Override public RowExpression rewriteVariableReference(VariableReferenceExpression node, Void context, RowExpressionTreeRewriter treeRewriter) { if (!excludedNames.contains(node.getName())) { - RowExpression result = mapping.get(node); + RowExpression result = mapping.apply(node); checkState(result != null, "Cannot resolve symbol %s", node.getName()); return result; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SortExpressionExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SortExpressionExtractor.java index df98fb3ea76f2..6a754359f0e79 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SortExpressionExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SortExpressionExtractor.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.OperatorType; @@ -21,7 +22,6 @@ import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.InputReferenceExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; -import com.facebook.presto.spi.relation.LogicalRowExpressions; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionVisitor; import com.facebook.presto.spi.relation.SpecialFormExpression; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java index c13c09e3845c9..275b04804cd5f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java @@ -13,18 +13,26 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; +import com.facebook.presto.execution.scheduler.TableWriteInfo; +import com.facebook.presto.execution.scheduler.TableWriteInfo.DeleteScanInfo; import com.facebook.presto.operator.StageExecutionDescriptor; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.split.SampledSplitSource; import com.facebook.presto.split.SplitSource; import com.facebook.presto.split.SplitSourceProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; @@ -35,11 +43,9 @@ import com.facebook.presto.sql.planner.plan.IndexJoinNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.MetadataDeleteNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; @@ -48,18 +54,17 @@ import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Supplier; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.GROUPED_SCHEDULING; @@ -79,11 +84,11 @@ public SplitSourceFactory(SplitSourceProvider splitSourceProvider) this.splitSourceProvider = requireNonNull(splitSourceProvider, "splitSourceProvider is null"); } - public Map createSplitSources(PlanFragment fragment, Session session) + public Map createSplitSources(PlanFragment fragment, Session session, TableWriteInfo tableWriteInfo) { ImmutableList.Builder splitSources = ImmutableList.builder(); try { - return fragment.getRoot().accept(new Visitor(session, fragment.getStageExecutionDescriptor(), splitSources), null); + return fragment.getRoot().accept(new Visitor(session, fragment.getStageExecutionDescriptor(), splitSources), new Context(tableWriteInfo)); } catch (Throwable t) { splitSources.build().forEach(SplitSourceFactory::closeSplitSource); @@ -113,7 +118,7 @@ private static SplitSchedulingStrategy getSplitSchedulingStrategy(StageExecution } private final class Visitor - extends InternalPlanVisitor, Void> + extends InternalPlanVisitor, Context> { private final Session session; private final StageExecutionDescriptor stageExecutionDescriptor; @@ -127,18 +132,26 @@ private Visitor(Session session, StageExecutionDescriptor stageExecutionDescript } @Override - public Map visitExplainAnalyze(ExplainAnalyzeNode node, Void context) + public Map visitExplainAnalyze(ExplainAnalyzeNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitTableScan(TableScanNode node, Void context) + public Map visitTableScan(TableScanNode node, Context context) { // get dataSource for table + TableHandle table; + Optional deleteScanInfo = context.getTableWriteInfo().getDeleteScanInfo(); + if (deleteScanInfo.isPresent() && deleteScanInfo.get().getId() == node.getId()) { + table = deleteScanInfo.get().getTableHandle(); + } + else { + table = node.getTable(); + } Supplier splitSourceSupplier = () -> splitSourceProvider.getSplits( session, - node.getTable(), + table, getSplitSchedulingStrategy(stageExecutionDescriptor, node.getId())); SplitSource splitSource = new LazySplitSource(splitSourceSupplier); @@ -149,7 +162,7 @@ public Map visitTableScan(TableScanNode node, Void cont } @Override - public Map visitJoin(JoinNode node, Void context) + public Map visitJoin(JoinNode node, Context context) { Map leftSplits = node.getLeft().accept(this, context); Map rightSplits = node.getRight().accept(this, context); @@ -160,7 +173,7 @@ public Map visitJoin(JoinNode node, Void context) } @Override - public Map visitSemiJoin(SemiJoinNode node, Void context) + public Map visitSemiJoin(SemiJoinNode node, Context context) { Map sourceSplits = node.getSource().accept(this, context); Map filteringSourceSplits = node.getFilteringSource().accept(this, context); @@ -171,7 +184,7 @@ public Map visitSemiJoin(SemiJoinNode node, Void contex } @Override - public Map visitSpatialJoin(SpatialJoinNode node, Void context) + public Map visitSpatialJoin(SpatialJoinNode node, Context context) { Map leftSplits = node.getLeft().accept(this, context); Map rightSplits = node.getRight().accept(this, context); @@ -182,33 +195,33 @@ public Map visitSpatialJoin(SpatialJoinNode node, Void } @Override - public Map visitIndexJoin(IndexJoinNode node, Void context) + public Map visitIndexJoin(IndexJoinNode node, Context context) { return node.getProbeSource().accept(this, context); } @Override - public Map visitRemoteSource(RemoteSourceNode node, Void context) + public Map visitRemoteSource(RemoteSourceNode node, Context context) { // remote source node does not have splits return ImmutableMap.of(); } @Override - public Map visitValues(ValuesNode node, Void context) + public Map visitValues(ValuesNode node, Context context) { // values node does not have splits return ImmutableMap.of(); } @Override - public Map visitFilter(FilterNode node, Void context) + public Map visitFilter(FilterNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitSample(SampleNode node, Void context) + public Map visitSample(SampleNode node, Context context) { switch (node.getSampleType()) { case BERNOULLI: @@ -230,139 +243,145 @@ public Map visitSample(SampleNode node, Void context) } @Override - public Map visitAggregation(AggregationNode node, Void context) + public Map visitAggregation(AggregationNode node, Context context) + { + return node.getSource().accept(this, context); + } + + @Override + public Map visitGroupId(GroupIdNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitGroupId(GroupIdNode node, Void context) + public Map visitMarkDistinct(MarkDistinctNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitMarkDistinct(MarkDistinctNode node, Void context) + public Map visitWindow(WindowNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitWindow(WindowNode node, Void context) + public Map visitRowNumber(RowNumberNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitRowNumber(RowNumberNode node, Void context) + public Map visitTopNRowNumber(TopNRowNumberNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitTopNRowNumber(TopNRowNumberNode node, Void context) + public Map visitProject(ProjectNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitProject(ProjectNode node, Void context) + public Map visitUnnest(UnnestNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitUnnest(UnnestNode node, Void context) + public Map visitTopN(TopNNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitTopN(TopNNode node, Void context) + public Map visitOutput(OutputNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitOutput(OutputNode node, Void context) + public Map visitEnforceSingleRow(EnforceSingleRowNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitEnforceSingleRow(EnforceSingleRowNode node, Void context) + public Map visitAssignUniqueId(AssignUniqueId node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitAssignUniqueId(AssignUniqueId node, Void context) + public Map visitLimit(LimitNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitLimit(LimitNode node, Void context) + public Map visitDistinctLimit(DistinctLimitNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitDistinctLimit(DistinctLimitNode node, Void context) + public Map visitSort(SortNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitSort(SortNode node, Void context) + public Map visitTableWriter(TableWriterNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitTableWriter(TableWriterNode node, Void context) + public Map visitTableWriteMerge(TableWriterMergeNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitTableFinish(TableFinishNode node, Void context) + public Map visitTableFinish(TableFinishNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitStatisticsWriterNode(StatisticsWriterNode node, Void context) + public Map visitStatisticsWriterNode(StatisticsWriterNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitDelete(DeleteNode node, Void context) + public Map visitDelete(DeleteNode node, Context context) { return node.getSource().accept(this, context); } @Override - public Map visitMetadataDelete(MetadataDeleteNode node, Void context) + public Map visitMetadataDelete(MetadataDeleteNode node, Context context) { // MetadataDelete node does not have splits return ImmutableMap.of(); } @Override - public Map visitUnion(UnionNode node, Void context) + public Map visitUnion(UnionNode node, Context context) { return processSources(node.getSources(), context); } @Override - public Map visitExchange(ExchangeNode node, Void context) + public Map visitExchange(ExchangeNode node, Context context) { return processSources(node.getSources(), context); } - private Map processSources(List sources, Void context) + private Map processSources(List sources, Context context) { ImmutableMap.Builder result = ImmutableMap.builder(); for (PlanNode child : sources) { @@ -373,9 +392,24 @@ private Map processSources(List sources, Void } @Override - public Map visitPlan(PlanNode node, Void context) + public Map visitPlan(PlanNode node, Context context) { throw new UnsupportedOperationException("not yet implemented: " + node.getClass().getName()); } } + + private static class Context + { + private final TableWriteInfo tableWriteInfo; + + public Context(TableWriteInfo tableWriteInfo) + { + this.tableWriteInfo = tableWriteInfo; + } + + public TableWriteInfo getTableWriteInfo() + { + return tableWriteInfo; + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java index 25f4c93159641..c4b1359a620fe 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.statistics.ColumnStatisticMetadata; @@ -28,7 +29,6 @@ import com.facebook.presto.spi.statistics.TableStatisticsMetadata; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.analyzer.TypeSignatureProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; import com.facebook.presto.sql.relational.FunctionResolution; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SubExpressionExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SubExpressionExtractor.java index 471245b9ebac1..f21b156a80a88 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SubExpressionExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SubExpressionExtractor.java @@ -22,6 +22,7 @@ /** * Extracts and returns the set of all expression subtrees within an Expression, including Expression itself */ +@Deprecated public final class SubExpressionExtractor { private SubExpressionExtractor() {} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SubqueryPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SubqueryPlanner.java index 872853f59a63f..708899681edaf 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SubqueryPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SubqueryPlanner.java @@ -15,20 +15,20 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.analyzer.Analysis; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignmentUtils; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.tree.BooleanLiteral; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/TypeProvider.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/TypeProvider.java index f3c5d6fc56cd9..d017494db8305 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/TypeProvider.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/TypeProvider.java @@ -72,4 +72,9 @@ public Set allVariables() .map(entry -> new VariableReferenceExpression(entry.getKey(), entry.getValue())) .collect(toImmutableSet()); } + + public Map allTypes() + { + return types; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/NoOpSymbolResolver.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/VariableResolver.java similarity index 70% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/NoOpSymbolResolver.java rename to presto-main/src/main/java/com/facebook/presto/sql/planner/VariableResolver.java index bbb9e7ceb1aea..e08fb43ce78a5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/NoOpSymbolResolver.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/VariableResolver.java @@ -13,14 +13,9 @@ */ package com.facebook.presto.sql.planner; -public class NoOpSymbolResolver - implements SymbolResolver -{ - public static final NoOpSymbolResolver INSTANCE = new NoOpSymbolResolver(); +import com.facebook.presto.spi.relation.VariableReferenceExpression; - @Override - public Object getValue(Symbol symbol) - { - return symbol.toSymbolReference(); - } +public interface VariableResolver +{ + Object getValue(VariableReferenceExpression variable); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/VariablesExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/VariablesExtractor.java index 9db8df0b5b643..2e65ceb2451e8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/VariablesExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/VariablesExtractor.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.expressions.DefaultRowExpressionTraversalVisitor; import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.relation.DefaultRowExpressionTraversalVisitor; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Lookup; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java index ce230d18aebed..aeda50f8de029 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java @@ -194,7 +194,7 @@ private boolean exploreChildren(int group, Context context, Matcher matcher) private Rule.Context ruleContext(Context context) { StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, Optional.of(context.memo), context.lookup, context.session, context.variableAllocator.getTypes()); - CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.of(context.memo), context.session, context.variableAllocator.getTypes()); + CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.of(context.memo), context.session); return new Rule.Context() { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/RuleStats.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/RuleStats.java index 60d5f6bb6a113..522ad1f2bf03b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/RuleStats.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/RuleStats.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.sql.planner.iterative; -import io.airlift.stats.TimeDistribution; +import com.facebook.airlift.stats.TimeDistribution; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/AddIntermediateAggregations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/AddIntermediateAggregations.java index e0669ef84e247..695a9652f0cdd 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/AddIntermediateAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/AddIntermediateAggregations.java @@ -16,17 +16,17 @@ import com.facebook.presto.Session; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Lookup; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.planner.plan.ExchangeNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -36,10 +36,10 @@ import static com.facebook.presto.SystemSessionProperties.getTaskConcurrency; import static com.facebook.presto.SystemSessionProperties.isEnableIntermediateAggregations; import static com.facebook.presto.matching.Pattern.empty; +import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.INTERMEDIATE; +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.sql.planner.optimizations.AggregationNodeUtils.extractAggregationUniqueVariables; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.FINAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.INTERMEDIATE; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.LOCAL; import static com.facebook.presto.sql.planner.plan.ExchangeNode.gatheringExchange; import static com.facebook.presto.sql.planner.plan.ExchangeNode.roundRobinExchange; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CreatePartialTopN.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CreatePartialTopN.java index add0c1d71d3af..03090dfec632c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CreatePartialTopN.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CreatePartialTopN.java @@ -15,14 +15,14 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.TopNNode; +import static com.facebook.presto.spi.plan.TopNNode.Step.FINAL; +import static com.facebook.presto.spi.plan.TopNNode.Step.PARTIAL; +import static com.facebook.presto.spi.plan.TopNNode.Step.SINGLE; import static com.facebook.presto.sql.planner.plan.Patterns.TopN.step; import static com.facebook.presto.sql.planner.plan.Patterns.topN; -import static com.facebook.presto.sql.planner.plan.TopNNode.Step.FINAL; -import static com.facebook.presto.sql.planner.plan.TopNNode.Step.PARTIAL; -import static com.facebook.presto.sql.planner.plan.TopNNode.Step.SINGLE; public class CreatePartialTopN implements Rule diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DesugarCurrentPath.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DesugarCurrentPath.java deleted file mode 100644 index 50c39f72e7d67..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DesugarCurrentPath.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sql.planner.iterative.rule; - -import com.facebook.presto.sql.tree.CurrentPath; -import com.facebook.presto.sql.tree.Expression; -import com.facebook.presto.sql.tree.ExpressionTreeRewriter; -import com.facebook.presto.sql.tree.FunctionCall; -import com.facebook.presto.sql.tree.QualifiedName; -import com.google.common.collect.ImmutableList; - -import static com.facebook.presto.sql.tree.ExpressionTreeRewriter.rewriteWith; - -public class DesugarCurrentPath - extends ExpressionRewriteRuleSet -{ - public DesugarCurrentPath() - { - super(createRewrite()); - } - - private static ExpressionRewriter createRewrite() - { - return (expression, context) -> rewriteWith(new com.facebook.presto.sql.tree.ExpressionRewriter() - { - @Override - public Expression rewriteCurrentPath(CurrentPath node, Void context, ExpressionTreeRewriter treeRewriter) - { - return DesugarCurrentPath.getCall(node); - } - }, expression); - } - - public static FunctionCall getCall(CurrentPath node) - { - return new FunctionCall(QualifiedName.of("$current_path"), ImmutableList.of()); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DetermineJoinDistributionType.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DetermineJoinDistributionType.java index a29538a65579f..09a877f66ffa4 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DetermineJoinDistributionType.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DetermineJoinDistributionType.java @@ -23,7 +23,6 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType; -import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.Ordering; @@ -142,7 +141,6 @@ private static boolean mustReplicate(JoinNode joinNode, Context context) private PlanNodeWithCost getJoinNodeWithCost(Context context, JoinNode possibleJoinNode) { - TypeProvider types = context.getVariableAllocator().getTypes(); StatsProvider stats = context.getStatsProvider(); boolean replicated = possibleJoinNode.getDistributionType().get().equals(REPLICATED); /* @@ -173,7 +171,6 @@ private PlanNodeWithCost getJoinNodeWithCost(Context context, JoinNode possibleJ possibleJoinNode.getLeft(), possibleJoinNode.getRight(), stats, - types, replicated, estimatedSourceDistributedTaskCount); return new PlanNodeWithCost(cost.toPlanCost(), possibleJoinNode); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DetermineSemiJoinDistributionType.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DetermineSemiJoinDistributionType.java index 3178049a9a8f1..ba9e496e37e78 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DetermineSemiJoinDistributionType.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/DetermineSemiJoinDistributionType.java @@ -36,7 +36,6 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType; -import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.google.common.collect.Ordering; @@ -129,7 +128,6 @@ private boolean canReplicate(SemiJoinNode node, Context context) private PlanNodeWithCost getSemiJoinNodeWithCost(SemiJoinNode possibleJoinNode, Context context) { - TypeProvider types = context.getVariableAllocator().getTypes(); StatsProvider stats = context.getStatsProvider(); boolean replicated = possibleJoinNode.getDistributionType().get().equals(REPLICATED); /* @@ -156,7 +154,6 @@ private PlanNodeWithCost getSemiJoinNodeWithCost(SemiJoinNode possibleJoinNode, possibleJoinNode.getSource(), possibleJoinNode.getFilteringSource(), stats, - types, replicated, estimatedSourceDistributedTaskCount); return new PlanNodeWithCost(cost.toPlanCost(), possibleJoinNode); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EliminateCrossJoins.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EliminateCrossJoins.java index 03a79ea484cef..7d134559a1a46 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EliminateCrossJoins.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EliminateCrossJoins.java @@ -16,17 +16,17 @@ import com.facebook.presto.Session; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.optimizations.joins.JoinGraph; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.tree.Expression; import com.google.common.collect.ImmutableList; @@ -210,6 +210,6 @@ public static PlanNode buildJoinTree(List expectedO // If needed, introduce a projection to constrain the outputs to what was originally expected // Some nodes are sensitive to what's produced (e.g., DistinctLimit node) - return restrictOutputs(idAllocator, result, ImmutableSet.copyOf(expectedOutputVariables)).orElse(result); + return restrictOutputs(idAllocator, result, ImmutableSet.copyOf(expectedOutputVariables), false).orElse(result); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroLimit.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroLimit.java index b5d7f7786403f..fb77f68fcb471 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroLimit.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroLimit.java @@ -15,9 +15,9 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.google.common.collect.ImmutableList; import static com.facebook.presto.sql.planner.plan.Patterns.Limit.count; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExpressionRewriteRuleSet.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExpressionRewriteRuleSet.java index 7734415c4063d..6ddc752d0b232 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExpressionRewriteRuleSet.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExpressionRewriteRuleSet.java @@ -15,19 +15,19 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignmentUtils; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.tree.Expression; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExtractSpatialJoins.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExtractSpatialJoins.java index 8412a2aaa2650..f44c524469a89 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExtractSpatialJoins.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ExtractSpatialJoins.java @@ -31,9 +31,11 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; @@ -47,9 +49,7 @@ import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.iterative.Rule.Context; import com.facebook.presto.sql.planner.iterative.Rule.Result; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.relational.OriginalExpressionUtils; @@ -76,6 +76,7 @@ import java.util.Optional; import java.util.Set; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.SystemSessionProperties.getSpatialPartitioningTableName; import static com.facebook.presto.SystemSessionProperties.isSpatialJoinEnabled; import static com.facebook.presto.matching.Capture.newCapture; @@ -101,7 +102,6 @@ import static com.facebook.presto.util.SpatialJoinUtils.extractSupportedSpatialFunctions; import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.Objects.requireNonNull; @@ -481,7 +481,7 @@ private static KdbTree loadKdbTree(String tableName, Session session, Metadata m List splits = splitBatch.getSplits(); for (Split split : splits) { - try (ConnectorPageSource pageSource = pageSourceManager.createPageSource(session, split, ImmutableList.of(kdbTreeColumn))) { + try (ConnectorPageSource pageSource = pageSourceManager.createPageSource(session, split, newTableHandle, ImmutableList.of(kdbTreeColumn))) { do { getFutureValue(pageSource.isBlocked()); Page page = pageSource.getNextPage(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/GatherAndMergeWindows.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/GatherAndMergeWindows.java index 55d970f154c0b..76d0d8872fecd 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/GatherAndMergeWindows.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/GatherAndMergeWindows.java @@ -17,15 +17,15 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; import com.facebook.presto.matching.PropertyPattern; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.VariablesExtractor; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.Assignments; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.google.common.collect.ImmutableList; @@ -211,7 +211,7 @@ protected Optional manipulateAdjacentWindowNodes(WindowNode parent, Wi parent.getPreSortedOrderPrefix()); return Optional.of( - restrictOutputs(context.getIdAllocator(), mergedWindowNode, ImmutableSet.copyOf(parent.getOutputVariables())) + restrictOutputs(context.getIdAllocator(), mergedWindowNode, ImmutableSet.copyOf(parent.getOutputVariables()), false) .orElse(mergedWindowNode)); } } @@ -230,7 +230,7 @@ protected Optional manipulateAdjacentWindowNodes(WindowNode parent, Wi if ((compare(parent, child) < 0) && (!dependsOn(parent, child, context.getVariableAllocator().getTypes()))) { PlanNode transposedWindows = transpose(parent, child); return Optional.of( - restrictOutputs(context.getIdAllocator(), transposedWindows, ImmutableSet.copyOf(parent.getOutputVariables())) + restrictOutputs(context.getIdAllocator(), transposedWindows, ImmutableSet.copyOf(parent.getOutputVariables()), false) .orElse(transposedWindows)); } else { @@ -292,8 +292,8 @@ else if (!o1.getOrderingScheme().isPresent() && o2.getOrderingScheme().isPresent OrderingScheme o1OrderingScheme = o1.getOrderingScheme().get(); OrderingScheme o2OrderingScheme = o2.getOrderingScheme().get(); - Iterator iterator1 = o1OrderingScheme.getOrderBy().iterator(); - Iterator iterator2 = o2OrderingScheme.getOrderBy().iterator(); + Iterator iterator1 = o1OrderingScheme.getOrderByVariables().iterator(); + Iterator iterator2 = o2OrderingScheme.getOrderByVariables().iterator(); while (iterator1.hasNext() && iterator2.hasNext()) { VariableReferenceExpression variable1 = iterator1.next(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ImplementFilteredAggregations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ImplementFilteredAggregations.java index 6a22e9994d3f0..48be05b01f456 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ImplementFilteredAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ImplementFilteredAggregations.java @@ -15,13 +15,13 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; -import com.facebook.presto.sql.planner.plan.Assignments; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.SymbolReference; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/InlineProjections.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/InlineProjections.java index deea53cf63ef3..452ed15d88661 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/InlineProjections.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/InlineProjections.java @@ -13,18 +13,24 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.expressions.DefaultRowExpressionTraversalVisitor; import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.Assignments.Builder; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.ExpressionVariableInliner; +import com.facebook.presto.sql.planner.RowExpressionVariableInliner; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.VariablesExtractor; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.Assignments; -import com.facebook.presto.sql.planner.plan.Assignments.Builder; -import com.facebook.presto.sql.planner.plan.ProjectNode; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.Literal; @@ -34,6 +40,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -46,6 +53,7 @@ import static com.facebook.presto.sql.planner.plan.Patterns.source; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.isExpression; import static java.util.stream.Collectors.toSet; /** @@ -62,6 +70,13 @@ public class InlineProjections private static final Pattern PATTERN = project() .with(source().matching(project().capturedAs(CHILD))); + private final FunctionResolution functionResolution; + + public InlineProjections(FunctionManager functionManager) + { + this.functionResolution = new FunctionResolution(functionManager); + } + @Override public Pattern getPattern() { @@ -84,7 +99,7 @@ public Result apply(ProjectNode parent, Captures captures, Context context) .entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, - entry -> castToRowExpression(inlineReferences(castToExpression(entry.getValue()), assignments, context.getVariableAllocator().getTypes())))); + entry -> inlineReferences(entry.getValue(), assignments, context.getVariableAllocator().getTypes()))); // Synthesize identity assignments for the inputs of expressions that were inlined // to place in the child projection. @@ -94,8 +109,7 @@ public Result apply(ProjectNode parent, Captures captures, Context context) .entrySet().stream() .filter(entry -> targets.contains(entry.getKey())) .map(Map.Entry::getValue) - .map(OriginalExpressionUtils::castToExpression) - .flatMap(entry -> VariablesExtractor.extractAll(entry, context.getVariableAllocator().getTypes()).stream()) + .flatMap(expression -> extractInputs(expression, context.getVariableAllocator().getTypes()).stream()) .collect(toSet()); Builder childAssignments = Assignments.builder(); @@ -104,8 +118,19 @@ public Result apply(ProjectNode parent, Captures captures, Context context) childAssignments.put(assignment); } } + + boolean allTranslated = child.getAssignments().entrySet() + .stream() + .map(Map.Entry::getValue) + .noneMatch(OriginalExpressionUtils::isExpression); + for (VariableReferenceExpression input : inputs) { - childAssignments.put(identityAsSymbolReference(input)); + if (allTranslated) { + childAssignments.put(input, input); + } + else { + childAssignments.put(identityAsSymbolReference(input)); + } } return Result.ofPlanNode( @@ -118,16 +143,18 @@ public Result apply(ProjectNode parent, Captures captures, Context context) Assignments.copyOf(parentAssignments))); } - private Expression inlineReferences(Expression expression, Assignments assignments, TypeProvider types) + private RowExpression inlineReferences(RowExpression expression, Assignments assignments, TypeProvider types) { - Function mapping = variable -> { - if (assignments.get(variable) == null) { - return new SymbolReference(variable.getName()); - } - return castToExpression(assignments.get(variable)); - }; - - return ExpressionVariableInliner.inlineVariables(mapping, expression, types); + if (isExpression(expression)) { + Function mapping = variable -> { + if (assignments.get(variable) == null) { + return new SymbolReference(variable.getName()); + } + return castToExpression(assignments.get(variable)); + }; + return castToRowExpression(ExpressionVariableInliner.inlineVariables(mapping, castToExpression(expression), types)); + } + return RowExpressionVariableInliner.inlineVariables(variable -> assignments.getMap().getOrDefault(variable, variable), expression); } private Sets.SetView extractInliningTargets(ProjectNode parent, ProjectNode child, Context context) @@ -141,25 +168,25 @@ private Sets.SetView extractInliningTargets(Project // which come from the child, as opposed to an enclosing scope. Set childOutputSet = ImmutableSet.copyOf(child.getOutputVariables()); + TypeProvider types = context.getVariableAllocator().getTypes(); Map dependencies = parent.getAssignments() .getExpressions() .stream() - .map(OriginalExpressionUtils::castToExpression) - .flatMap(expression -> VariablesExtractor.extractAll(expression, context.getVariableAllocator().getTypes()).stream()) + .flatMap(expression -> extractInputs(expression, context.getVariableAllocator().getTypes()).stream()) .filter(childOutputSet::contains) .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); // find references to simple constants Set constants = dependencies.keySet().stream() - .filter(input -> castToExpression(child.getAssignments().get(input)) instanceof Literal) + .filter(input -> isConstant(child.getAssignments().get(input))) .collect(toSet()); // exclude any complex inputs to TRY expressions. Inlining them would potentially // change the semantics of those expressions Set tryArguments = parent.getAssignments() .getExpressions().stream() - .flatMap(expression -> extractTryArguments(castToExpression(expression), context.getVariableAllocator().getTypes()).stream()) + .flatMap(expression -> extractTryArguments(expression, types).stream()) .collect(toSet()); Set singletons = dependencies.entrySet().stream() @@ -172,12 +199,43 @@ private Sets.SetView extractInliningTargets(Project return Sets.union(singletons, constants); } - private Set extractTryArguments(Expression expression, TypeProvider types) + private Set extractTryArguments(RowExpression expression, TypeProvider types) { - return AstUtils.preOrder(expression) - .filter(TryExpression.class::isInstance) - .map(TryExpression.class::cast) - .flatMap(tryExpression -> VariablesExtractor.extractAll(tryExpression, types).stream()) - .collect(toSet()); + if (isExpression(expression)) { + return AstUtils.preOrder(castToExpression(expression)) + .filter(TryExpression.class::isInstance) + .map(TryExpression.class::cast) + .flatMap(tryExpression -> VariablesExtractor.extractAll(tryExpression, types).stream()) + .collect(toSet()); + } + ImmutableSet.Builder builder = ImmutableSet.builder(); + expression.accept(new DefaultRowExpressionTraversalVisitor>() + { + @Override + public Void visitCall(CallExpression call, ImmutableSet.Builder context) + { + if (functionResolution.isTryFunction(call.getFunctionHandle())) { + context.addAll(VariablesExtractor.extractAll(call)); + } + return super.visitCall(call, context); + } + }, builder); + return builder.build(); + } + + private static List extractInputs(RowExpression expression, TypeProvider types) + { + if (isExpression(expression)) { + return VariablesExtractor.extractAll(castToExpression(expression), types); + } + return VariablesExtractor.extractAll(expression); + } + + private static boolean isConstant(RowExpression expression) + { + if (isExpression(expression)) { + return castToExpression(expression) instanceof Literal; + } + return expression instanceof ConstantExpression; } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimitWithDistinct.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimitWithDistinct.java index cf4be49a038e8..06894a7c72ea1 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimitWithDistinct.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimitWithDistinct.java @@ -16,10 +16,10 @@ import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import static com.facebook.presto.matching.Capture.newCapture; import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimitWithSort.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimitWithSort.java index bff515b96e7ea..bc0f4cba1a454 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimitWithSort.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimitWithSort.java @@ -16,10 +16,10 @@ import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.SortNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import static com.facebook.presto.matching.Capture.newCapture; import static com.facebook.presto.sql.planner.plan.Patterns.limit; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimitWithTopN.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimitWithTopN.java index 20f80d13dc53b..531742f0c78b8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimitWithTopN.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimitWithTopN.java @@ -16,9 +16,9 @@ import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.LimitNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import static com.facebook.presto.matching.Capture.newCapture; import static com.facebook.presto.sql.planner.plan.Patterns.limit; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimits.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimits.java index 926da823ef85a..a917cd4af4c1d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimits.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MergeLimits.java @@ -16,8 +16,8 @@ import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.LimitNode; import static com.facebook.presto.matching.Capture.newCapture; import static com.facebook.presto.sql.planner.plan.Patterns.limit; @@ -47,6 +47,6 @@ public Result apply(LimitNode parent, Captures captures, Context context) parent.getId(), child.getSource(), Math.min(parent.getCount(), child.getCount()), - parent.isPartial())); + parent.getStep())); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MultipleDistinctAggregationToMarkDistinct.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MultipleDistinctAggregationToMarkDistinct.java index 627da12125b0c..66a092147bdc0 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MultipleDistinctAggregationToMarkDistinct.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MultipleDistinctAggregationToMarkDistinct.java @@ -16,11 +16,11 @@ import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.google.common.base.Predicates; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickTableLayout.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickTableLayout.java index fcca0ee0d6da4..07094b42a4afa 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickTableLayout.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickTableLayout.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; @@ -33,6 +34,8 @@ import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.predicate.NullableValue; import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.Type; @@ -40,10 +43,15 @@ import com.facebook.presto.sql.planner.ExpressionDomainTranslator; import com.facebook.presto.sql.planner.ExpressionInterpreter; import com.facebook.presto.sql.planner.LiteralEncoder; -import com.facebook.presto.sql.planner.LookupSymbolResolver; +import com.facebook.presto.sql.planner.RowExpressionInterpreter; +import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.planner.VariableResolver; import com.facebook.presto.sql.planner.VariablesExtractor; import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.NodeRef; @@ -58,12 +66,15 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import static com.facebook.presto.SystemSessionProperties.isNewOptimizerEnabled; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression; import static com.facebook.presto.matching.Capture.newCapture; import static com.facebook.presto.metadata.TableLayoutResult.computeEnforced; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; -import static com.facebook.presto.spi.relation.RowExpressionNodeInliner.replaceExpression; +import static com.facebook.presto.spi.relation.DomainTranslator.BASIC_COLUMN_EXTRACTOR; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.sql.ExpressionUtils.combineConjuncts; import static com.facebook.presto.sql.ExpressionUtils.filterDeterministicConjuncts; import static com.facebook.presto.sql.ExpressionUtils.filterNonDeterministicConjuncts; @@ -74,7 +85,9 @@ import static com.facebook.presto.sql.planner.plan.Patterns.tableScan; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.isExpression; import static com.facebook.presto.sql.tree.BooleanLiteral.TRUE_LITERAL; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableBiMap.toImmutableBiMap; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; @@ -155,7 +168,7 @@ public Result apply(FilterNode filterNode, Captures captures, Context context) PlanNode rewritten = pushPredicateIntoTableScan( tableScan, - castToExpression(filterNode.getPredicate()), + filterNode.getPredicate(), false, context.getSession(), context.getVariableAllocator().getTypes(), @@ -267,9 +280,13 @@ public Result apply(TableScanNode tableScanNode, Captures captures, Context cont } } + /** + * @param predicate can be a RowExpression or an OriginalExpression. The method will handle both cases. + * Once Expression is migrated to RowExpression in PickTableLayout, the method should only support RowExpression. + */ public static PlanNode pushPredicateIntoTableScan( TableScanNode node, - Expression predicate, + RowExpression predicate, boolean pruneWithPredicateExpression, Session session, TypeProvider types, @@ -278,46 +295,82 @@ public static PlanNode pushPredicateIntoTableScan( SqlParser parser, ExpressionDomainTranslator domainTranslator) { - if (metadata.isPushdownFilterSupported(session, node.getTable())) { + DomainTranslator translator = new RowExpressionDomainTranslator(metadata); + + if (!metadata.isPushdownFilterSupported(session, node.getTable())) { + if (isExpression(predicate)) { + return pushPredicateIntoTableScan(node, castToExpression(predicate), pruneWithPredicateExpression, session, types, idAllocator, metadata, parser, domainTranslator); + } + return pushPredicateIntoTableScan(node, predicate, pruneWithPredicateExpression, session, idAllocator, metadata, translator); + } + + // filter pushdown; to be replaced by ConnectorPlanOptimizer + RowExpression expression; + if (isExpression(predicate)) { Map, Type> predicateTypes = getExpressionTypes( session, metadata, parser, types, - predicate, + castToExpression(predicate), emptyList(), WarningCollector.NOOP, false); - BiMap symbolToColumnMapping = node.getAssignments().entrySet().stream() - .collect(toImmutableBiMap( - Map.Entry::getKey, - entry -> new VariableReferenceExpression(getColumnName(session, metadata, node.getTable(), entry.getValue()), entry.getKey().getType()))); - RowExpression translatedPredicate = replaceExpression(SqlToRowExpressionTranslator.translate(predicate, predicateTypes, ImmutableMap.of(), metadata.getFunctionManager(), metadata.getTypeManager(), session, false), symbolToColumnMapping); + expression = SqlToRowExpressionTranslator.translate(castToExpression(predicate), predicateTypes, ImmutableMap.of(), metadata.getFunctionManager(), metadata.getTypeManager(), session); + } + else { + expression = predicate; + } - PushdownFilterResult pushdownFilterResult = metadata.pushdownFilter(session, node.getTable(), translatedPredicate); + // optimize rowExpression and return ValuesNode if false. e.g. 0=1 => false, 1>2 => false, (a = 1 and a = 2) => false + DomainTranslator.ExtractionResult translatedExpression = translator.fromPredicate(session.toConnectorSession(), expression, BASIC_COLUMN_EXTRACTOR); + if (translatedExpression.getTupleDomain().isNone()) { + return new ValuesNode(idAllocator.getNextId(), node.getOutputVariables(), ImmutableList.of()); + } - TableLayout layout = pushdownFilterResult.getLayout(); - if (layout.getPredicate().isNone()) { - return new ValuesNode(idAllocator.getNextId(), node.getOutputVariables(), ImmutableList.of()); - } + BiMap symbolToColumnMapping = node.getAssignments().entrySet().stream() + .collect(toImmutableBiMap( + Map.Entry::getKey, + entry -> new VariableReferenceExpression(getColumnName(session, metadata, node.getTable(), entry.getValue()), entry.getKey().getType()))); + PushdownFilterResult pushdownFilterResult = metadata.pushdownFilter(session, node.getTable(), replaceExpression(expression, symbolToColumnMapping)); - TableScanNode tableScan = new TableScanNode( - node.getId(), - layout.getNewTableHandle(), - node.getOutputVariables(), - node.getAssignments(), - layout.getPredicate(), - TupleDomain.all()); + TableLayout layout = pushdownFilterResult.getLayout(); + if (layout.getPredicate().isNone()) { + return new ValuesNode(idAllocator.getNextId(), node.getOutputVariables(), ImmutableList.of()); + } - RowExpression unenforcedFilter = pushdownFilterResult.getUnenforcedFilter(); - if (!TRUE_CONSTANT.equals(unenforcedFilter)) { - return new FilterNode(idAllocator.getNextId(), tableScan, replaceExpression(unenforcedFilter, symbolToColumnMapping.inverse())); - } + TableScanNode tableScan = new TableScanNode( + node.getId(), + layout.getNewTableHandle(), + node.getOutputVariables(), + node.getAssignments(), + layout.getPredicate(), + TupleDomain.all()); - return tableScan; + RowExpression unenforcedFilter = pushdownFilterResult.getUnenforcedFilter(); + if (!TRUE_CONSTANT.equals(unenforcedFilter)) { + return new FilterNode(idAllocator.getNextId(), tableScan, replaceExpression(unenforcedFilter, symbolToColumnMapping.inverse())); } + return tableScan; + } + + /** + * For Expression {@param predicate} + */ + @Deprecated + private static PlanNode pushPredicateIntoTableScan( + TableScanNode node, + Expression predicate, + boolean pruneWithPredicateExpression, + Session session, + TypeProvider types, + PlanNodeIdAllocator idAllocator, + Metadata metadata, + SqlParser parser, + ExpressionDomainTranslator domainTranslator) + { // don't include non-deterministic predicates Expression deterministicPredicate = filterDeterministicConjuncts(predicate); ExpressionDomainTranslator.ExtractionResult decomposedPredicate = ExpressionDomainTranslator.fromPredicate( @@ -334,7 +387,7 @@ public static PlanNode pushPredicateIntoTableScan( Constraint constraint; if (pruneWithPredicateExpression) { - LayoutConstraintEvaluator evaluator = new LayoutConstraintEvaluator( + LayoutConstraintEvaluatorForExpression evaluator = new LayoutConstraintEvaluatorForExpression( metadata, parser, session, @@ -396,18 +449,110 @@ public static PlanNode pushPredicateIntoTableScan( return tableScan; } + /** + * For RowExpression {@param predicate} + */ + private static PlanNode pushPredicateIntoTableScan( + TableScanNode node, + RowExpression predicate, + boolean pruneWithPredicateExpression, + Session session, + PlanNodeIdAllocator idAllocator, + Metadata metadata, + DomainTranslator domainTranslator) + { + // don't include non-deterministic predicates + LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions( + new RowExpressionDeterminismEvaluator(metadata.getFunctionManager()), + new FunctionResolution(metadata.getFunctionManager()), + metadata.getFunctionManager()); + RowExpression deterministicPredicate = logicalRowExpressions.filterDeterministicConjuncts(predicate); + DomainTranslator.ExtractionResult decomposedPredicate = domainTranslator.fromPredicate( + session.toConnectorSession(), + deterministicPredicate, + BASIC_COLUMN_EXTRACTOR); + + TupleDomain newDomain = decomposedPredicate.getTupleDomain() + .transform(variableName -> node.getAssignments().get(variableName)) + .intersect(node.getEnforcedConstraint()); + + Map assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse(); + + Constraint constraint; + if (pruneWithPredicateExpression) { + LayoutConstraintEvaluatorForRowExpression evaluator = new LayoutConstraintEvaluatorForRowExpression( + metadata, + session, + node.getAssignments(), + logicalRowExpressions.combineConjuncts( + deterministicPredicate, + // Simplify the tuple domain to avoid creating an expression with too many nodes, + // which would be expensive to evaluate in the call to isCandidate below. + domainTranslator.toPredicate(newDomain.simplify().transform(column -> assignments.getOrDefault(column, null))))); + constraint = new Constraint<>(newDomain, evaluator::isCandidate); + } + else { + // Currently, invoking the expression interpreter is very expensive. + // TODO invoke the interpreter unconditionally when the interpreter becomes cheap enough. + constraint = new Constraint<>(newDomain); + } + if (constraint.getSummary().isNone()) { + return new ValuesNode(idAllocator.getNextId(), node.getOutputVariables(), ImmutableList.of()); + } + + // Layouts will be returned in order of the connector's preference + TableLayoutResult layout = metadata.getLayout( + session, + node.getTable(), + constraint, + Optional.of(node.getOutputVariables().stream() + .map(variable -> node.getAssignments().get(variable)) + .collect(toImmutableSet()))); + + if (layout.getLayout().getPredicate().isNone()) { + return new ValuesNode(idAllocator.getNextId(), node.getOutputVariables(), ImmutableList.of()); + } + + TableScanNode tableScan = new TableScanNode( + node.getId(), + layout.getLayout().getNewTableHandle(), + node.getOutputVariables(), + node.getAssignments(), + layout.getLayout().getPredicate(), + computeEnforced(newDomain, layout.getUnenforcedConstraint())); + + // The order of the arguments to combineConjuncts matters: + // * Unenforced constraints go first because they can only be simple column references, + // which are not prone to logic errors such as out-of-bound access, div-by-zero, etc. + // * Conjuncts in non-deterministic expressions and non-TupleDomain-expressible expressions should + // retain their original (maybe intermixed) order from the input predicate. However, this is not implemented yet. + // * Short of implementing the previous bullet point, the current order of non-deterministic expressions + // and non-TupleDomain-expressible expressions should be retained. Changing the order can lead + // to failures of previously successful queries. + RowExpression resultingPredicate = logicalRowExpressions.combineConjuncts( + domainTranslator.toPredicate(layout.getUnenforcedConstraint().transform(assignments::get)), + logicalRowExpressions.filterNonDeterministicConjuncts(predicate), + decomposedPredicate.getRemainingExpression()); + + if (!TRUE_CONSTANT.equals(resultingPredicate)) { + return new FilterNode(idAllocator.getNextId(), tableScan, resultingPredicate); + } + return tableScan; + } + private static String getColumnName(Session session, Metadata metadata, TableHandle tableHandle, ColumnHandle columnHandle) { return metadata.getColumnMetadata(session, tableHandle, columnHandle).getName(); } - private static class LayoutConstraintEvaluator + @Deprecated + private static class LayoutConstraintEvaluatorForExpression { private final Map assignments; private final ExpressionInterpreter evaluator; private final Set arguments; - public LayoutConstraintEvaluator(Metadata metadata, SqlParser parser, Session session, TypeProvider types, Map assignments, Expression expression) + public LayoutConstraintEvaluatorForExpression(Metadata metadata, SqlParser parser, Session session, TypeProvider types, Map assignments, Expression expression) { this.assignments = assignments; @@ -424,18 +569,79 @@ private boolean isCandidate(Map bindings) if (intersection(bindings.keySet(), arguments).isEmpty()) { return true; } - LookupSymbolResolver inputs = new LookupSymbolResolver(assignments, bindings); + LookupVariableResolver inputs = new LookupVariableResolver(assignments, bindings, variable -> new Symbol(variable.getName()).toSymbolReference()); // Skip pruning if evaluation fails in a recoverable way. Failing here can cause // spurious query failures for partitions that would otherwise be filtered out. Object optimized = TryFunction.evaluate(() -> evaluator.optimize(inputs), true); // If any conjuncts evaluate to FALSE or null, then the whole predicate will never be true and so the partition should be pruned - if (Boolean.FALSE.equals(optimized) || optimized == null || optimized instanceof NullLiteral) { - return false; + return !Boolean.FALSE.equals(optimized) && optimized != null && !(optimized instanceof NullLiteral); + } + } + + private static class LayoutConstraintEvaluatorForRowExpression + { + private final Map assignments; + private final RowExpressionInterpreter evaluator; + private final Set arguments; + + public LayoutConstraintEvaluatorForRowExpression(Metadata metadata, Session session, Map assignments, RowExpression expression) + { + this.assignments = assignments; + + evaluator = new RowExpressionInterpreter(expression, metadata, session.toConnectorSession(), OPTIMIZED); + arguments = VariablesExtractor.extractUnique(expression).stream() + .map(assignments::get) + .collect(toImmutableSet()); + } + + private boolean isCandidate(Map bindings) + { + if (intersection(bindings.keySet(), arguments).isEmpty()) { + return true; + } + LookupVariableResolver inputs = new LookupVariableResolver(assignments, bindings, variable -> variable); + + // Skip pruning if evaluation fails in a recoverable way. Failing here can cause + // spurious query failures for partitions that would otherwise be filtered out. + Object optimized = TryFunction.evaluate(() -> evaluator.optimize(inputs), true); + + // If any conjuncts evaluate to FALSE or null, then the whole predicate will never be true and so the partition should be pruned + return !Boolean.FALSE.equals(optimized) && optimized != null && (!(optimized instanceof ConstantExpression) || !((ConstantExpression) optimized).isNull()); + } + } + + private static class LookupVariableResolver + implements VariableResolver + { + private final Map assignments; + private final Map bindings; + // Use Object type to let interpreters consume the result + // TODO: use RowExpression once the Expression-to-RowExpression is done + private final Function missingBindingSupplier; + + public LookupVariableResolver( + Map assignments, + Map bindings, + Function missingBindingSupplier) + { + this.assignments = requireNonNull(assignments, "assignments is null"); + this.bindings = ImmutableMap.copyOf(requireNonNull(bindings, "bindings is null")); + this.missingBindingSupplier = requireNonNull(missingBindingSupplier, "missingBindingSupplier is null"); + } + + @Override + public Object getValue(VariableReferenceExpression variable) + { + ColumnHandle column = assignments.get(variable); + checkArgument(column != null, "Missing column assignment for %s", variable); + + if (!bindings.containsKey(column)) { + return missingBindingSupplier.apply(variable); } - return true; + return bindings.get(column).getValue(); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ProjectOffPushDownRule.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ProjectOffPushDownRule.java index 8e158bbef2f6e..87d0ee4535438 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ProjectOffPushDownRule.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ProjectOffPushDownRule.java @@ -18,10 +18,10 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.google.common.collect.ImmutableList; import java.util.Optional; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneAggregationColumns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneAggregationColumns.java index d887884c8e94f..9747c7f8bbeae 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneAggregationColumns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneAggregationColumns.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.PlanVariableAllocator; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.google.common.collect.Maps; import java.util.Map; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneAggregationSourceColumns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneAggregationSourceColumns.java index 6a45310c3984e..2aa0d64f8bf4e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneAggregationSourceColumns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneAggregationSourceColumns.java @@ -15,11 +15,11 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.optimizations.AggregationNodeUtils; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.google.common.collect.Streams; import java.util.Set; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneCountAggregationOverScalar.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneCountAggregationOverScalar.java index 184cfe2bb7d3b..70c4ea6dacad9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneCountAggregationOverScalar.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneCountAggregationOverScalar.java @@ -17,10 +17,10 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneCrossJoinColumns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneCrossJoinColumns.java index e7a386783d6f2..8a8013119ec9d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneCrossJoinColumns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneCrossJoinColumns.java @@ -40,8 +40,8 @@ public PruneCrossJoinColumns() @Override protected Optional pushDownProjectOff(PlanNodeIdAllocator idAllocator, PlanVariableAllocator variableAllocator, JoinNode joinNode, Set referencedOutputs) { - Optional newLeft = restrictOutputs(idAllocator, joinNode.getLeft(), referencedOutputs); - Optional newRight = restrictOutputs(idAllocator, joinNode.getRight(), referencedOutputs); + Optional newLeft = restrictOutputs(idAllocator, joinNode.getLeft(), referencedOutputs, false); + Optional newRight = restrictOutputs(idAllocator, joinNode.getRight(), referencedOutputs, false); if (!newLeft.isPresent() && !newRight.isPresent()) { return Optional.empty(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneLimitColumns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneLimitColumns.java index ebb7405e1fd5a..d5db49acf38d9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneLimitColumns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneLimitColumns.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.PlanVariableAllocator; -import com.facebook.presto.sql.planner.plan.LimitNode; import java.util.Optional; import java.util.Set; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneOrderByInAggregation.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneOrderByInAggregation.java index 6c1a3d269499c..e7d1133177675 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneOrderByInAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneOrderByInAggregation.java @@ -16,15 +16,15 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.google.common.collect.ImmutableMap; import java.util.Map; import java.util.Optional; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; +import static com.facebook.presto.spi.plan.AggregationNode.Aggregation; import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneProjectColumns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneProjectColumns.java index 540bfcf1c786c..0dd1b6534604b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneProjectColumns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneProjectColumns.java @@ -15,9 +15,9 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.PlanVariableAllocator; -import com.facebook.presto.sql.planner.plan.ProjectNode; import java.util.Optional; import java.util.Set; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneSemiJoinColumns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneSemiJoinColumns.java index 057f6fd00c6ff..12cffb21beb42 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneSemiJoinColumns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneSemiJoinColumns.java @@ -51,7 +51,7 @@ protected Optional pushDownProjectOff(PlanNodeIdAllocator idAllocator, semiJoinNode.getSourceHashVariable().map(Stream::of).orElse(Stream.empty())) .collect(toImmutableSet()); - return restrictOutputs(idAllocator, semiJoinNode.getSource(), requiredSourceInputs) + return restrictOutputs(idAllocator, semiJoinNode.getSource(), requiredSourceInputs, false) .map(newSource -> semiJoinNode.replaceChildren(ImmutableList.of( newSource, semiJoinNode.getFilteringSource()))); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneSemiJoinFilteringSourceColumns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneSemiJoinFilteringSourceColumns.java index 77b17654dd22e..a319fde722c47 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneSemiJoinFilteringSourceColumns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneSemiJoinFilteringSourceColumns.java @@ -47,7 +47,7 @@ public Result apply(SemiJoinNode semiJoinNode, Captures captures, Context contex semiJoinNode.getFilteringSourceHashVariable().map(Stream::of).orElse(Stream.empty())) .collect(toImmutableSet()); - return restrictOutputs(context.getIdAllocator(), semiJoinNode.getFilteringSource(), requiredFilteringSourceInputs) + return restrictOutputs(context.getIdAllocator(), semiJoinNode.getFilteringSource(), requiredFilteringSourceInputs, false) .map(newFilteringSource -> semiJoinNode.replaceChildren(ImmutableList.of(semiJoinNode.getSource(), newFilteringSource))) .map(Result::ofPlanNode) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneTopNColumns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneTopNColumns.java index b0dbe2b90d2b8..c066ad18a34ea 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneTopNColumns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneTopNColumns.java @@ -15,9 +15,9 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.PlanVariableAllocator; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.google.common.collect.Streams; import java.util.Optional; @@ -40,7 +40,7 @@ protected Optional pushDownProjectOff(PlanNodeIdAllocator idAllocator, { Set prunedTopNInputs = Streams.concat( referencedOutputs.stream(), - topNNode.getOrderingScheme().getOrderBy().stream()) + topNNode.getOrderingScheme().getOrderByVariables().stream()) .collect(toImmutableSet()); return restrictChildOutputs(idAllocator, topNNode, prunedTopNInputs); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneWindowColumns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneWindowColumns.java index 75a049bc7e241..12cc3e6b003f7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneWindowColumns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneWindowColumns.java @@ -54,7 +54,7 @@ protected Optional pushDownProjectOff(PlanNodeIdAllocator idAllocator, windowNode.getOrderingScheme().ifPresent( orderingScheme -> orderingScheme - .getOrderBy() + .getOrderByVariables() .forEach(referencedInputs::add)); windowNode.getHashVariable().ifPresent(referencedInputs::add); @@ -66,7 +66,7 @@ protected Optional pushDownProjectOff(PlanNodeIdAllocator idAllocator, PlanNode prunedWindowNode = new WindowNode( windowNode.getId(), - restrictOutputs(idAllocator, windowNode.getSource(), referencedInputs.build()) + restrictOutputs(idAllocator, windowNode.getSource(), referencedInputs.build(), false) .orElse(windowNode.getSource()), windowNode.getSpecification(), referencedFunctions, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushAggregationThroughOuterJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushAggregationThroughOuterJoin.java index 2c2befbef35d3..cea7eb10b9701 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushAggregationThroughOuterJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushAggregationThroughOuterJoin.java @@ -19,21 +19,23 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Lookup; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.tree.CoalesceExpression; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.NullLiteral; @@ -49,11 +51,11 @@ import static com.facebook.presto.SystemSessionProperties.shouldPushAggregationThroughJoin; import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.spi.plan.AggregationNode.globalAggregation; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.ExpressionVariableInliner.inlineVariables; import static com.facebook.presto.sql.planner.PlannerUtils.toVariableReference; import static com.facebook.presto.sql.planner.optimizations.DistinctOutputQueryUtil.isDistinct; -import static com.facebook.presto.sql.planner.plan.AggregationNode.globalAggregation; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; import static com.facebook.presto.sql.planner.plan.Patterns.join; import static com.facebook.presto.sql.planner.plan.Patterns.source; @@ -329,8 +331,8 @@ private Optional createAggregationOverNull(AggregationNod aggregation.getOrderBy().map(orderBy -> inlineOrderByVariables(sourcesVariableMapping, orderBy)), aggregation.isDistinct(), aggregation.getMask().map(x -> new VariableReferenceExpression(sourcesVariableMapping.get(x).getName(), x.getType()))); - String functionName = functionManager.getFunctionMetadata(overNullAggregation.getFunctionHandle()).getName(); - VariableReferenceExpression overNull = variableAllocator.newVariable(functionName, aggregationVariable.getType()); + QualifiedFunctionName functionName = functionManager.getFunctionMetadata(overNullAggregation.getFunctionHandle()).getName(); + VariableReferenceExpression overNull = variableAllocator.newVariable(functionName.getFunctionName(), aggregationVariable.getType()); aggregationsOverNullBuilder.put(overNull, overNullAggregation); aggregationsVariableMappingBuilder.put(aggregationVariable, overNull); } @@ -355,12 +357,14 @@ private static OrderingScheme inlineOrderByVariables(Map orderBy = ImmutableList.builder(); ImmutableMap.Builder ordering = new ImmutableMap.Builder<>(); - for (VariableReferenceExpression variable : orderingScheme.getOrderBy()) { + for (VariableReferenceExpression variable : orderingScheme.getOrderByVariables()) { VariableReferenceExpression translated = new VariableReferenceExpression(variableMapping.get(variable).getName(), variable.getType()); orderBy.add(translated); ordering.put(translated, orderingScheme.getOrdering(variable)); } - return new OrderingScheme(orderBy.build(), ordering.build()); + + ImmutableMap orderingMap = ordering.build(); + return new OrderingScheme(orderBy.build().stream().map(variable -> new Ordering(variable, orderingMap.get(variable))).collect(toImmutableList())); } private static boolean isUsingVariables(AggregationNode.Aggregation aggregation, Set sourceVariables, TypeProvider types) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughMarkDistinct.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughMarkDistinct.java index d0152bbc39b46..05e3f454227ab 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughMarkDistinct.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughMarkDistinct.java @@ -16,8 +16,8 @@ import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import static com.facebook.presto.matching.Capture.newCapture; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughOuterJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughOuterJoin.java index a4a1d9be9a20c..3856ad6eaaafa 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughOuterJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughOuterJoin.java @@ -16,16 +16,17 @@ import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.sql.planner.iterative.Lookup; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.Range; import static com.facebook.presto.SystemSessionProperties.isPushLimitThroughOuterJoin; import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.spi.plan.LimitNode.Step.PARTIAL; import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.extractCardinality; import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; @@ -82,11 +83,11 @@ public Result apply(LimitNode parent, Captures captures, Context context) PlanNode right = joinNode.getRight(); if (joinNode.getType() == LEFT && !isLimited(left, context.getLookup(), parent.getCount())) { - left = new LimitNode(context.getIdAllocator().getNextId(), left, parent.getCount(), true); + left = new LimitNode(context.getIdAllocator().getNextId(), left, parent.getCount(), PARTIAL); } if (joinNode.getType() == RIGHT && !isLimited(right, context.getLookup(), parent.getCount())) { - right = new LimitNode(context.getIdAllocator().getNextId(), right, parent.getCount(), true); + right = new LimitNode(context.getIdAllocator().getNextId(), right, parent.getCount(), PARTIAL); } if (joinNode.getLeft() != left || joinNode.getRight() != right) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughProject.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughProject.java index 5ca4eff27e173..edebb79e6d14e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughProject.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughProject.java @@ -16,9 +16,9 @@ import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.LimitNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import static com.facebook.presto.matching.Capture.newCapture; import static com.facebook.presto.sql.planner.iterative.rule.Util.transpose; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughSemiJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughSemiJoin.java index 4a201096075c5..3bc94429d0abc 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughSemiJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushLimitThroughSemiJoin.java @@ -16,8 +16,8 @@ import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import static com.facebook.presto.matching.Capture.newCapture; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushPartialAggregationThroughExchange.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushPartialAggregationThroughExchange.java index f8a85acb9528a..f67335a5de42f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushPartialAggregationThroughExchange.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushPartialAggregationThroughExchange.java @@ -19,19 +19,18 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.operator.aggregation.InternalAggregationFunction; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.Partitioning; import com.facebook.presto.sql.planner.PartitioningScheme; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.optimizations.SymbolMapper; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.ExchangeNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.google.common.collect.ImmutableList; import java.util.ArrayList; @@ -43,9 +42,10 @@ import java.util.stream.Collectors; import static com.facebook.presto.SystemSessionProperties.preferPartialAggregation; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.FINAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.PARTIAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.operator.aggregation.AggregationUtils.isDecomposable; +import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; @@ -85,7 +85,7 @@ public Result apply(AggregationNode aggregationNode, Captures captures, Context { ExchangeNode exchangeNode = captures.get(EXCHANGE_NODE); - boolean decomposable = aggregationNode.isDecomposable(functionManager); + boolean decomposable = isDecomposable(aggregationNode, functionManager); if (aggregationNode.getStep().equals(SINGLE) && aggregationNode.hasEmptyGroupingSet() && @@ -118,8 +118,8 @@ public Result apply(AggregationNode aggregationNode, Captures captures, Context .getPartitioning() .getArguments() .stream() - .filter(Partitioning.ArgumentBinding::isVariable) - .map(Partitioning.ArgumentBinding::getVariableReference) + .filter(VariableReferenceExpression.class::isInstance) + .map(VariableReferenceExpression.class::cast) .collect(Collectors.toList()); if (!aggregationNode.getGroupingKeys().containsAll(partitioningColumns)) { @@ -191,6 +191,7 @@ private PlanNode pushPartial(AggregationNode aggregation, ExchangeNode exchange, partitioning, partials, ImmutableList.copyOf(Collections.nCopies(partials.size(), aggregationOutputs)), + exchange.isEnsureSourceOrdering(), Optional.empty()); } @@ -201,7 +202,7 @@ private PlanNode split(AggregationNode node, Context context) Map finalAggregation = new HashMap<>(); for (Map.Entry entry : node.getAggregations().entrySet()) { AggregationNode.Aggregation originalAggregation = entry.getValue(); - String functionName = functionManager.getFunctionMetadata(originalAggregation.getFunctionHandle()).getName(); + String functionName = functionManager.getFunctionMetadata(originalAggregation.getFunctionHandle()).getName().getFunctionName(); FunctionHandle functionHandle = originalAggregation.getFunctionHandle(); InternalAggregationFunction function = functionManager.getAggregateFunctionImplementation(functionHandle); VariableReferenceExpression intermediateVariable = context.getVariableAllocator().newVariable(functionName, function.getIntermediateType()); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushPartialAggregationThroughJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushPartialAggregationThroughJoin.java index bdac8202a8e39..880c52cf4cc96 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushPartialAggregationThroughJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushPartialAggregationThroughJoin.java @@ -17,12 +17,12 @@ import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.VariablesExtractor; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -35,10 +35,10 @@ import java.util.stream.Collectors; import static com.facebook.presto.SystemSessionProperties.isPushAggregationThroughJoin; +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.iterative.rule.Util.restrictOutputs; import static com.facebook.presto.sql.planner.optimizations.AggregationNodeUtils.extractAggregationUniqueVariables; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.PARTIAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; import static com.facebook.presto.sql.planner.plan.Patterns.join; import static com.facebook.presto.sql.planner.plan.Patterns.source; @@ -192,6 +192,6 @@ private PlanNode pushPartialToJoin( child.getLeftHashVariable(), child.getRightHashVariable(), child.getDistributionType()); - return restrictOutputs(context.getIdAllocator(), joinNode, ImmutableSet.copyOf(aggregation.getOutputVariables())).orElse(joinNode); + return restrictOutputs(context.getIdAllocator(), joinNode, ImmutableSet.copyOf(aggregation.getOutputVariables()), false).orElse(joinNode); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushProjectionThroughExchange.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushProjectionThroughExchange.java index 30af8e0e09e70..16093855df4d9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushProjectionThroughExchange.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushProjectionThroughExchange.java @@ -16,18 +16,15 @@ import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.planner.PartitioningScheme; -import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.planner.RowExpressionVariableInliner; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.ExchangeNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; -import com.facebook.presto.sql.relational.OriginalExpressionUtils; -import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -38,14 +35,13 @@ import java.util.Set; import static com.facebook.presto.matching.Capture.newCapture; -import static com.facebook.presto.sql.planner.ExpressionVariableInliner.inlineVariables; -import static com.facebook.presto.sql.planner.PlannerUtils.toVariableReference; import static com.facebook.presto.sql.planner.iterative.rule.Util.restrictOutputs; import static com.facebook.presto.sql.planner.plan.Patterns.exchange; import static com.facebook.presto.sql.planner.plan.Patterns.project; import static com.facebook.presto.sql.planner.plan.Patterns.source; -import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.isExpression; +import static com.google.common.base.Preconditions.checkArgument; /** * Transforms: @@ -92,9 +88,8 @@ public Result apply(ProjectNode project, Captures captures, Context context) ImmutableList.Builder newSourceBuilder = ImmutableList.builder(); ImmutableList.Builder> inputsBuilder = ImmutableList.builder(); - TypeProvider types = context.getVariableAllocator().getTypes(); for (int i = 0; i < exchange.getSources().size(); i++) { - Map outputToInputMap = extractExchangeOutputToInput(exchange, i); + Map outputToInputMap = extractExchangeOutputToInput(exchange, i); Assignments.Builder projections = Assignments.builder(); ImmutableList.Builder inputs = ImmutableList.builder(); @@ -102,9 +97,8 @@ public Result apply(ProjectNode project, Captures captures, Context context) // Need to retain the partition keys for the exchange partitioningColumns.stream() .map(outputToInputMap::get) - .forEach(nameReference -> { - VariableReferenceExpression variable = toVariableReference(nameReference, types); - projections.put(variable, castToRowExpression(nameReference)); + .forEach(variable -> { + projections.put(variable, variable); inputs.add(variable); }); @@ -117,22 +111,21 @@ public Result apply(ProjectNode project, Captures captures, Context context) if (exchange.getOrderingScheme().isPresent()) { // need to retain ordering columns for the exchange - exchange.getOrderingScheme().get().getOrderBy().stream() + exchange.getOrderingScheme().get().getOrderByVariables().stream() // do not project the same symbol twice as ExchangeNode verifies that source input symbols match partitioning scheme outputLayout .filter(variable -> !partitioningColumns.contains(variable)) .map(outputToInputMap::get) - .forEach(nameReference -> { - VariableReferenceExpression variable = toVariableReference(nameReference, types); - projections.put(variable, castToRowExpression(nameReference)); + .forEach(variable -> { + projections.put(variable, variable); inputs.add(variable); }); } for (Map.Entry projection : project.getAssignments().entrySet()) { - Expression translatedExpression = inlineVariables(outputToInputMap, castToExpression(projection.getValue()), types); - Type type = projection.getKey().getType(); - VariableReferenceExpression variable = context.getVariableAllocator().newVariable(translatedExpression, type); - projections.put(variable, castToRowExpression(translatedExpression)); + checkArgument(!isExpression(projection.getValue()), "Cannot contain RowExpression after AddExchange"); + RowExpression translatedExpression = RowExpressionVariableInliner.inlineVariables(outputToInputMap, projection.getValue()); + VariableReferenceExpression variable = context.getVariableAllocator().newVariable(translatedExpression); + projections.put(variable, translatedExpression); inputs.add(variable); } newSourceBuilder.add(new ProjectNode(context.getIdAllocator().getNextId(), exchange.getSources().get(i), projections.build())); @@ -144,7 +137,7 @@ public Result apply(ProjectNode project, Captures captures, Context context) partitioningColumns.forEach(outputBuilder::add); exchange.getPartitioningScheme().getHashColumn().ifPresent(outputBuilder::add); if (exchange.getOrderingScheme().isPresent()) { - exchange.getOrderingScheme().get().getOrderBy().stream() + exchange.getOrderingScheme().get().getOrderByVariables().stream() .filter(variable -> !partitioningColumns.contains(variable)) .forEach(outputBuilder::add); } @@ -167,22 +160,23 @@ public Result apply(ProjectNode project, Captures captures, Context context) partitioningScheme, newSourceBuilder.build(), inputsBuilder.build(), + exchange.isEnsureSourceOrdering(), exchange.getOrderingScheme()); // we need to strip unnecessary symbols (hash, partitioning columns). - return Result.ofPlanNode(restrictOutputs(context.getIdAllocator(), result, ImmutableSet.copyOf(project.getOutputVariables())).orElse(result)); + return Result.ofPlanNode(restrictOutputs(context.getIdAllocator(), result, ImmutableSet.copyOf(project.getOutputVariables()), true).orElse(result)); } private static boolean isSymbolToSymbolProjection(ProjectNode project) { - return project.getAssignments().getExpressions().stream().map(OriginalExpressionUtils::castToExpression).allMatch(e -> e instanceof SymbolReference); + return project.getAssignments().getExpressions().stream().allMatch(e -> e instanceof VariableReferenceExpression); } - private static Map extractExchangeOutputToInput(ExchangeNode exchange, int sourceIndex) + private static Map extractExchangeOutputToInput(ExchangeNode exchange, int sourceIndex) { - Map outputToInputMap = new HashMap<>(); + Map outputToInputMap = new HashMap<>(); for (int i = 0; i < exchange.getOutputVariables().size(); i++) { - outputToInputMap.put(exchange.getOutputVariables().get(i), new SymbolReference(exchange.getInputs().get(sourceIndex).get(i).getName())); + outputToInputMap.put(exchange.getOutputVariables().get(i), exchange.getInputs().get(sourceIndex).get(i)); } return outputToInputMap; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushProjectionThroughUnion.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushProjectionThroughUnion.java index dcbfdc4ea31fa..b3a0051de5651 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushProjectionThroughUnion.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushProjectionThroughUnion.java @@ -16,19 +16,21 @@ import com.facebook.presto.matching.Capture; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.planner.ExpressionVariableInliner; +import com.facebook.presto.sql.planner.RowExpressionVariableInliner; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.Assignments; -import com.facebook.presto.sql.planner.plan.ProjectNode; -import com.facebook.presto.sql.planner.plan.UnionNode; -import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ListMultimap; import com.google.common.collect.Maps; import java.util.HashMap; @@ -36,11 +38,13 @@ import java.util.Map; import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils.fromListMultimap; import static com.facebook.presto.sql.planner.plan.Patterns.project; import static com.facebook.presto.sql.planner.plan.Patterns.source; import static com.facebook.presto.sql.planner.plan.Patterns.union; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.isExpression; public class PushProjectionThroughUnion implements Rule @@ -71,7 +75,7 @@ public Result apply(ProjectNode parent, Captures captures, Context context) ImmutableList.Builder outputSources = ImmutableList.builder(); for (int i = 0; i < source.getSources().size(); i++) { - Map outputToInput = Maps.transformValues(source.sourceVariableMap(i), variable -> new SymbolReference(variable.getName())); // Map: output of union -> input of this source to the union + Map outputToInput = Maps.transformValues(source.sourceVariableMap(i), OriginalExpressionUtils::asSymbolReference); // Map: output of union -> input of this source to the union Assignments.Builder assignments = Assignments.builder(); // assignments for the new ProjectNode // mapping from current ProjectNode to new ProjectNode, used to identify the output layout @@ -79,16 +83,25 @@ public Result apply(ProjectNode parent, Captures captures, Context context) // Translate the assignments in the ProjectNode using symbols of the source of the UnionNode for (Map.Entry entry : parent.getAssignments().entrySet()) { - Expression translatedExpression = ExpressionVariableInliner.inlineVariables(outputToInput, castToExpression(entry.getValue()), context.getVariableAllocator().getTypes()); + RowExpression translatedExpression; Type type = entry.getKey().getType(); - VariableReferenceExpression variable = context.getVariableAllocator().newVariable(translatedExpression, type); - assignments.put(variable, castToRowExpression(translatedExpression)); - projectVariableMapping.put(new VariableReferenceExpression(entry.getKey().getName(), type), variable); + VariableReferenceExpression variable; + if (isExpression(entry.getValue())) { + translatedExpression = castToRowExpression(ExpressionVariableInliner.inlineVariables(outputToInput, castToExpression(entry.getValue()), context.getVariableAllocator().getTypes())); + variable = context.getVariableAllocator().newVariable(castToExpression(translatedExpression), type); + } + else { + translatedExpression = RowExpressionVariableInliner.inlineVariables(source.sourceVariableMap(i), entry.getValue()); + variable = context.getVariableAllocator().newVariable(translatedExpression); + } + assignments.put(variable, translatedExpression); + projectVariableMapping.put(entry.getKey(), variable); } outputSources.add(new ProjectNode(context.getIdAllocator().getNextId(), source.getSources().get(i), assignments.build())); outputLayout.forEach(variable -> mappings.put(variable, projectVariableMapping.get(variable))); } - return Result.ofPlanNode(new UnionNode(parent.getId(), outputSources.build(), mappings.build())); + ListMultimap outputsToInputs = mappings.build(); + return Result.ofPlanNode(new UnionNode(parent.getId(), outputSources.build(), ImmutableList.copyOf(outputsToInputs.keySet()), fromListMultimap(outputsToInputs))); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushRemoteExchangeThroughAssignUniqueId.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushRemoteExchangeThroughAssignUniqueId.java index 463ee1c4dae05..358408a8d8ddd 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushRemoteExchangeThroughAssignUniqueId.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushRemoteExchangeThroughAssignUniqueId.java @@ -82,6 +82,7 @@ public Result apply(ExchangeNode node, Captures captures, Context context) partitioningScheme.getBucketToPartition()), ImmutableList.of(assignUniqueId.getSource()), ImmutableList.of(removeVariable(getOnlyElement(node.getInputs()), assignUniqueId.getIdVariable())), + node.isEnsureSourceOrdering(), Optional.empty()), assignUniqueId.getIdVariable())); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushTableWriteThroughUnion.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushTableWriteThroughUnion.java index 7a2de288208b2..18718b7747e55 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushTableWriteThroughUnion.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushTableWriteThroughUnion.java @@ -18,14 +18,15 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.optimizations.SymbolMapper; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ListMultimap; import java.util.ArrayList; import java.util.List; @@ -33,6 +34,7 @@ import static com.facebook.presto.SystemSessionProperties.isPushTableWriteThroughUnion; import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils.fromListMultimap; import static com.facebook.presto.sql.planner.plan.Patterns.source; import static com.facebook.presto.sql.planner.plan.Patterns.tableWriterNode; import static com.facebook.presto.sql.planner.plan.Patterns.union; @@ -76,12 +78,14 @@ public Result apply(TableWriterNode writerNode, Captures captures, Context conte ImmutableListMultimap.Builder unionMappings = ImmutableListMultimap.builder(); sourceMappings.forEach(mappings -> mappings.forEach(unionMappings::put)); + ListMultimap mappings = unionMappings.build(); return Result.ofPlanNode( new UnionNode( context.getIdAllocator().getNextId(), rewrittenSources.build(), - unionMappings.build())); + ImmutableList.copyOf(mappings.keySet()), + fromListMultimap(mappings))); } private static TableWriterNode rewriteSource( @@ -112,9 +116,6 @@ private static TableWriterNode rewriteSource( private static Map getInputVariableMapping(UnionNode node, int source) { - return node.getVariableMapping() - .keySet() - .stream() - .collect(toImmutableMap(key -> key, key -> node.getVariableMapping().get(key).get(source))); + return node.getOutputVariables().stream().collect(toImmutableMap(key -> key, key -> node.getVariableMapping().get(key).get(source))); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushTopNThroughUnion.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushTopNThroughUnion.java index acff7bac35507..61975c3bed59d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushTopNThroughUnion.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushTopNThroughUnion.java @@ -17,22 +17,22 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.optimizations.SymbolMapper; -import com.facebook.presto.sql.planner.plan.TopNNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.Set; import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.spi.plan.TopNNode.Step.PARTIAL; import static com.facebook.presto.sql.planner.plan.Patterns.TopN.step; import static com.facebook.presto.sql.planner.plan.Patterns.source; import static com.facebook.presto.sql.planner.plan.Patterns.topN; import static com.facebook.presto.sql.planner.plan.Patterns.union; -import static com.facebook.presto.sql.planner.plan.TopNNode.Step.PARTIAL; import static com.google.common.collect.Iterables.getLast; import static com.google.common.collect.Sets.intersection; @@ -74,6 +74,7 @@ public Result apply(TopNNode topNNode, Captures captures, Context context) return Result.ofPlanNode(new UnionNode( unionNode.getId(), sources.build(), + unionNode.getOutputVariables(), unionNode.getVariableMapping())); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantIdentityProjections.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantIdentityProjections.java index c51781b9b6cbd..f6d857209a550 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantIdentityProjections.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantIdentityProjections.java @@ -15,8 +15,8 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.relational.ProjectNodeUtils; import com.google.common.collect.ImmutableSet; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ReorderJoins.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ReorderJoins.java index 30fd0f75fd373..b7e1767b0875d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ReorderJoins.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ReorderJoins.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.cost.CostComparator; import com.facebook.presto.cost.CostProvider; @@ -43,7 +44,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Ordering; -import io.airlift.log.Logger; import java.util.ArrayList; import java.util.HashMap; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java index 2f504910ac131..866ffbf3826ea 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java @@ -16,16 +16,17 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; -import com.facebook.presto.sql.planner.plan.Assignments; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.sql.tree.LongLiteral; import com.facebook.presto.sql.tree.QualifiedName; @@ -36,6 +37,7 @@ import java.util.Optional; import static com.facebook.presto.SystemSessionProperties.getHashPartitionCount; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; @@ -66,7 +68,7 @@ public class RewriteSpatialPartitioningAggregation implements Rule { private static final TypeSignature GEOMETRY_TYPE_SIGNATURE = parseTypeSignature("Geometry"); - private static final String NAME = "spatial_partitioning"; + private static final QualifiedFunctionName NAME = QualifiedFunctionName.of(DEFAULT_NAMESPACE, "spatial_partitioning"); private final Pattern pattern = aggregation().matching(this::hasSpatialPartitioningAggregation); private final Metadata metadata; @@ -97,7 +99,7 @@ public Result apply(AggregationNode node, Captures captures, Context context) ImmutableMap.Builder envelopeAssignments = ImmutableMap.builder(); for (Map.Entry entry : node.getAggregations().entrySet()) { Aggregation aggregation = entry.getValue(); - String name = metadata.getFunctionManager().getFunctionMetadata(aggregation.getFunctionHandle()).getName(); + QualifiedFunctionName name = metadata.getFunctionManager().getFunctionMetadata(aggregation.getFunctionHandle()).getName(); Type geometryType = metadata.getType(GEOMETRY_TYPE_SIGNATURE); if (name.equals(NAME) && aggregation.getArguments().size() == 1) { RowExpression geometry = getOnlyElement(aggregation.getArguments()); @@ -111,8 +113,8 @@ public Result apply(AggregationNode node, Captures captures, Context context) aggregations.put(entry.getKey(), new Aggregation( new CallExpression( - name, - metadata.getFunctionManager().lookupFunction(NAME, fromTypes(geometryType, INTEGER)), + name.getFunctionName(), + metadata.getFunctionManager().lookupFunction(NAME.getFunctionName(), fromTypes(geometryType, INTEGER)), entry.getKey().getType(), ImmutableList.of( castToRowExpression(asSymbolReference(envelopeVariable)), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TranslateExpressions.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RowExpressionRewriteRuleSet.java similarity index 53% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TranslateExpressions.java rename to presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RowExpressionRewriteRuleSet.java index 474853d696315..0e074e69fcb09 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TranslateExpressions.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RowExpressionRewriteRuleSet.java @@ -11,55 +11,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package com.facebook.presto.sql.planner.iterative.rule; -package com.facebook.presto.sql.planner.optimizations; - -import com.facebook.presto.Session; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; -import com.facebook.presto.metadata.Metadata; -import com.facebook.presto.operator.aggregation.InternalAggregationFunction; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.spi.type.FunctionType; -import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.WindowNode; -import com.facebook.presto.sql.planner.plan.WindowNode.Function; -import com.facebook.presto.sql.relational.OriginalExpressionUtils; -import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; -import com.facebook.presto.sql.tree.Expression; -import com.facebook.presto.sql.tree.LambdaArgumentDeclaration; -import com.facebook.presto.sql.tree.LambdaExpression; -import com.facebook.presto.sql.tree.NodeRef; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Optional; import java.util.Set; -import static com.facebook.presto.execution.warnings.WarningCollector.NOOP; -import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypes; import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; import static com.facebook.presto.sql.planner.plan.Patterns.applyNode; import static com.facebook.presto.sql.planner.plan.Patterns.filter; @@ -71,42 +51,120 @@ import static com.facebook.presto.sql.planner.plan.Patterns.values; import static com.facebook.presto.sql.planner.plan.Patterns.window; import static com.facebook.presto.sql.relational.Expressions.call; -import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; -import static com.facebook.presto.sql.relational.OriginalExpressionUtils.isExpression; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Verify.verify; -import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.builder; -import static java.util.Collections.emptyList; import static java.util.Objects.requireNonNull; -public class TranslateExpressions +public class RowExpressionRewriteRuleSet { - private final Metadata metadata; - private final SqlParser sqlParser; + public interface PlanRowExpressionRewriter + { + RowExpression rewrite(RowExpression expression, Rule.Context context); + } - public TranslateExpressions(Metadata metadata, SqlParser sqlParser) + protected final PlanRowExpressionRewriter rewriter; + + public RowExpressionRewriteRuleSet(PlanRowExpressionRewriter rewriter) { - this.metadata = requireNonNull(metadata, "metadata is null"); - this.sqlParser = requireNonNull(sqlParser, "sqlParseris null"); + this.rewriter = requireNonNull(rewriter, "rewriter is null"); } public Set> rules() { return ImmutableSet.of( - new ValuesExpressionTranslation(), - new FilterExpressionTranslation(), - new ProjectExpressionTranslation(), - new ApplyExpressionTranslation(), - new WindowExpressionTranslation(), - new JoinExpressionTranslation(), - new SpatialJoinExpressionTranslation(), - new AggregationExpressionTranslation(), - new TableFinishExpressionTranslation(), - new TableWriterExpressionTranslation()); + valueRowExpressionRewriteRule(), + filterRowExpressionRewriteRule(), + projectRowExpressionRewriteRule(), + applyNodeRowExpressionRewriteRule(), + windowRowExpressionRewriteRule(), + joinRowExpressionRewriteRule(), + spatialJoinRowExpressionRewriteRule(), + aggregationRowExpressionRewriteRule(), + tableFinishRowExpressionRewriteRule(), + tableWriterRowExpressionRewriteRule()); + } + + public Rule valueRowExpressionRewriteRule() + { + return new ValuesRowExpressionRewrite(); + } + + public Rule filterRowExpressionRewriteRule() + { + return new FilterRowExpressionRewrite(); + } + + public Rule projectRowExpressionRewriteRule() + { + return new ProjectRowExpressionRewrite(); + } + + public Rule applyNodeRowExpressionRewriteRule() + { + return new ApplyRowExpressionRewrite(); + } + + public Rule windowRowExpressionRewriteRule() + { + return new WindowRowExpressionRewrite(); } - private final class SpatialJoinExpressionTranslation + public Rule joinRowExpressionRewriteRule() + { + return new JoinRowExpressionRewrite(); + } + + public Rule spatialJoinRowExpressionRewriteRule() + { + return new SpatialJoinRowExpressionRewrite(); + } + + public Rule tableFinishRowExpressionRewriteRule() + { + return new TableFinishRowExpressionRewrite(); + } + + public Rule tableWriterRowExpressionRewriteRule() + { + return new TableWriterRowExpressionRewrite(); + } + + public Rule aggregationRowExpressionRewriteRule() + { + return new AggregationRowExpressionRewrite(); + } + + private final class ProjectRowExpressionRewrite + implements Rule + { + @Override + public Pattern getPattern() + { + return project(); + } + + @Override + public Result apply(ProjectNode projectNode, Captures captures, Context context) + { + Assignments.Builder builder = Assignments.builder(); + boolean anyRewritten = false; + for (Map.Entry entry : projectNode.getAssignments().getMap().entrySet()) { + RowExpression rewritten = rewriter.rewrite(entry.getValue(), context); + if (!rewritten.equals(entry.getValue())) { + anyRewritten = true; + } + builder.put(entry.getKey(), rewritten); + } + Assignments assignments = builder.build(); + if (anyRewritten) { + return Result.ofPlanNode(new ProjectNode(projectNode.getId(), projectNode.getSource(), assignments)); + } + return Result.empty(); + } + } + + private final class SpatialJoinRowExpressionRewrite implements Rule { @Override @@ -119,7 +177,7 @@ public Pattern getPattern() public Result apply(SpatialJoinNode spatialJoinNode, Captures captures, Context context) { RowExpression filter = spatialJoinNode.getFilter(); - RowExpression rewritten = removeOriginalExpression(filter, context); + RowExpression rewritten = rewriter.rewrite(filter, context); if (filter.equals(rewritten)) { return Result.empty(); @@ -137,7 +195,7 @@ public Result apply(SpatialJoinNode spatialJoinNode, Captures captures, Context } } - private final class JoinExpressionTranslation + private final class JoinRowExpressionRewrite implements Rule { @Override @@ -154,7 +212,7 @@ public Result apply(JoinNode joinNode, Captures captures, Context context) } RowExpression filter = joinNode.getFilter().get(); - RowExpression rewritten = removeOriginalExpression(filter, context); + RowExpression rewritten = rewriter.rewrite(filter, context); if (filter.equals(rewritten)) { return Result.empty(); @@ -173,7 +231,7 @@ public Result apply(JoinNode joinNode, Captures captures, Context context) } } - private final class WindowExpressionTranslation + private final class WindowRowExpressionRewrite implements Rule { @Override @@ -187,12 +245,12 @@ public Result apply(WindowNode windowNode, Captures captures, Context context) { checkState(windowNode.getSource() != null); boolean anyRewritten = false; - ImmutableMap.Builder functions = ImmutableMap.builder(); - for (Entry entry : windowNode.getWindowFunctions().entrySet()) { + ImmutableMap.Builder functions = ImmutableMap.builder(); + for (Map.Entry entry : windowNode.getWindowFunctions().entrySet()) { ImmutableList.Builder newArguments = ImmutableList.builder(); CallExpression callExpression = entry.getValue().getFunctionCall(); for (RowExpression argument : callExpression.getArguments()) { - RowExpression rewritten = removeOriginalExpression(argument, context); + RowExpression rewritten = rewriter.rewrite(argument, context); if (rewritten != argument) { anyRewritten = true; } @@ -200,13 +258,14 @@ public Result apply(WindowNode windowNode, Captures captures, Context context) } functions.put( entry.getKey(), - new Function( + new WindowNode.Function( call( callExpression.getDisplayName(), callExpression.getFunctionHandle(), callExpression.getType(), newArguments.build()), - entry.getValue().getFrame())); + entry.getValue().getFrame(), + entry.getValue().isIgnoreNulls())); } if (anyRewritten) { return Result.ofPlanNode(new WindowNode( @@ -222,29 +281,7 @@ public Result apply(WindowNode windowNode, Captures captures, Context context) } } - private final class ProjectExpressionTranslation - implements Rule - { - @Override - public Pattern getPattern() - { - return project(); - } - - @Override - public Result apply(ProjectNode projectNode, Captures captures, Context context) - { - Assignments assignments = projectNode.getAssignments(); - Optional rewrittenAssignments = translateAssignments(assignments, context); - - if (!rewrittenAssignments.isPresent()) { - return Result.empty(); - } - return Result.ofPlanNode(new ProjectNode(projectNode.getId(), projectNode.getSource(), rewrittenAssignments.get())); - } - } - - private final class ApplyExpressionTranslation + private final class ApplyRowExpressionRewrite implements Rule { @Override @@ -272,7 +309,21 @@ public Result apply(ApplyNode applyNode, Captures captures, Context context) } } - private final class FilterExpressionTranslation + private Optional translateAssignments(Assignments assignments, Rule.Context context) + { + Assignments.Builder builder = Assignments.builder(); + assignments.getMap() + .entrySet() + .stream() + .forEach(entry -> builder.put(entry.getKey(), rewriter.rewrite(entry.getValue(), context))); + Assignments rewritten = builder.build(); + if (rewritten.equals(assignments)) { + return Optional.empty(); + } + return Optional.of(rewritten); + } + + private final class FilterRowExpressionRewrite implements Rule { @Override @@ -285,7 +336,7 @@ public Pattern getPattern() public Result apply(FilterNode filterNode, Captures captures, Context context) { checkState(filterNode.getSource() != null); - RowExpression rewritten = removeOriginalExpression(filterNode.getPredicate(), context); + RowExpression rewritten = rewriter.rewrite(filterNode.getPredicate(), context); if (filterNode.getPredicate().equals(rewritten)) { return Result.empty(); @@ -294,7 +345,7 @@ public Result apply(FilterNode filterNode, Captures captures, Context context) } } - private final class ValuesExpressionTranslation + private final class ValuesRowExpressionRewrite implements Rule { @Override @@ -311,8 +362,8 @@ public Result apply(ValuesNode valuesNode, Captures captures, Context context) for (List row : valuesNode.getRows()) { ImmutableList.Builder newRow = ImmutableList.builder(); for (RowExpression rowExpression : row) { - RowExpression rewritten = removeOriginalExpression(rowExpression, context); - if (rowExpression != rewritten) { + RowExpression rewritten = rewriter.rewrite(rowExpression, context); + if (!rewritten.equals(rowExpression)) { anyRewritten = true; } newRow.add(rewritten); @@ -326,7 +377,7 @@ public Result apply(ValuesNode valuesNode, Captures captures, Context context) } } - private final class AggregationExpressionTranslation + private final class AggregationRowExpressionRewrite implements Rule { @Override @@ -343,7 +394,7 @@ public Result apply(AggregationNode node, Captures captures, Context context) boolean changed = false; ImmutableMap.Builder rewrittenAggregation = builder(); for (Map.Entry entry : node.getAggregations().entrySet()) { - AggregationNode.Aggregation rewritten = translateAggregation(entry.getValue(), context.getSession(), context.getVariableAllocator().getTypes()); + AggregationNode.Aggregation rewritten = rewriteAggregation(entry.getValue(), context); rewrittenAggregation.put(entry.getKey(), rewritten); if (!rewritten.equals(entry.getValue())) { changed = true; @@ -366,7 +417,7 @@ public Result apply(AggregationNode node, Captures captures, Context context) } } - private final class TableFinishExpressionTranslation + private final class TableFinishRowExpressionRewrite implements Rule { @Override @@ -399,7 +450,24 @@ public Result apply(TableFinishNode node, Captures captures, Context context) } } - private final class TableWriterExpressionTranslation + private Optional translateStatisticAggregation(StatisticAggregations statisticAggregations, Rule.Context context) + { + ImmutableMap.Builder rewrittenAggregation = builder(); + boolean changed = false; + for (Map.Entry entry : statisticAggregations.getAggregations().entrySet()) { + AggregationNode.Aggregation rewritten = rewriteAggregation(entry.getValue(), context); + rewrittenAggregation.put(entry.getKey(), rewritten); + if (!rewritten.equals(entry.getValue())) { + changed = true; + } + } + if (changed) { + return Optional.of(new StatisticAggregations(rewrittenAggregation.build(), statisticAggregations.getGroupingVariables())); + } + return Optional.empty(); + } + + private final class TableWriterRowExpressionRewrite implements Rule { @Override @@ -430,180 +498,21 @@ public Result apply(TableWriterNode node, Captures captures, Context context) node.getColumns(), node.getColumnNames(), node.getPartitioningScheme(), - rewrittenStatisticsAggregation, - node.getStatisticsAggregationDescriptor())); + rewrittenStatisticsAggregation)); } return Result.empty(); } } - private Optional translateStatisticAggregation(StatisticAggregations statisticAggregations, Rule.Context context) - { - ImmutableMap.Builder rewrittenAggregation = builder(); - boolean changed = false; - for (Map.Entry entry : statisticAggregations.getAggregations().entrySet()) { - AggregationNode.Aggregation rewritten = translateAggregation(entry.getValue(), context.getSession(), context.getVariableAllocator().getTypes()); - rewrittenAggregation.put(entry.getKey(), rewritten); - if (!rewritten.equals(entry.getValue())) { - changed = true; - } - } - if (changed) { - return Optional.of(new StatisticAggregations(rewrittenAggregation.build(), statisticAggregations.getGroupingVariables())); - } - return Optional.empty(); - } - - @VisibleForTesting - public AggregationNode.Aggregation translateAggregation(AggregationNode.Aggregation aggregation, Session session, TypeProvider typeProvider) + private AggregationNode.Aggregation rewriteAggregation(AggregationNode.Aggregation aggregation, Rule.Context context) { - Map, Type> types = analyzeAggregationExpressionTypes(aggregation, session, typeProvider); - + RowExpression rewrittenCall = rewriter.rewrite(aggregation.getCall(), context); + checkArgument(rewrittenCall instanceof CallExpression, "Aggregation CallExpression must be rewritten to CallExpression"); return new AggregationNode.Aggregation( - new CallExpression( - aggregation.getCall().getDisplayName(), - aggregation.getCall().getFunctionHandle(), - aggregation.getCall().getType(), - aggregation.getArguments().stream().map(argument -> removeOriginalExpression(argument, session, types)).collect(toImmutableList())), - aggregation.getFilter().map(filter -> removeOriginalExpression(filter, session, types)), + (CallExpression) rewrittenCall, + aggregation.getFilter().map(filter -> rewriter.rewrite(filter, context)), aggregation.getOrderBy(), aggregation.isDistinct(), aggregation.getMask()); } - - private Map, Type> analyzeAggregationExpressionTypes(AggregationNode.Aggregation aggregation, Session session, TypeProvider typeProvider) - { - List lambdaExpressions = aggregation.getArguments().stream() - .filter(OriginalExpressionUtils::isExpression) - .map(OriginalExpressionUtils::castToExpression) - .filter(LambdaExpression.class::isInstance) - .map(LambdaExpression.class::cast) - .collect(toImmutableList()); - ImmutableMap.Builder, Type> builder = ImmutableMap., Type>builder(); - if (!lambdaExpressions.isEmpty()) { - List functionTypes = metadata.getFunctionManager().getFunctionMetadata(aggregation.getFunctionHandle()).getArgumentTypes().stream() - .filter(typeSignature -> typeSignature.getBase().equals(FunctionType.NAME)) - .map(typeSignature -> (FunctionType) (metadata.getTypeManager().getType(typeSignature))) - .collect(toImmutableList()); - InternalAggregationFunction internalAggregationFunction = metadata.getFunctionManager().getAggregateFunctionImplementation(aggregation.getFunctionHandle()); - List lambdaInterfaces = internalAggregationFunction.getLambdaInterfaces(); - verify(lambdaExpressions.size() == functionTypes.size()); - verify(lambdaExpressions.size() == lambdaInterfaces.size()); - - for (int i = 0; i < lambdaExpressions.size(); i++) { - LambdaExpression lambdaExpression = lambdaExpressions.get(i); - FunctionType functionType = functionTypes.get(i); - - // To compile lambda, LambdaDefinitionExpression needs to be generated from LambdaExpression, - // which requires the types of all sub-expressions. - // - // In project and filter expression compilation, ExpressionAnalyzer.getExpressionTypesFromInput - // is used to generate the types of all sub-expressions. (see visitScanFilterAndProject and visitFilter) - // - // This does not work here since the function call representation in final aggregation node - // is currently a hack: it takes intermediate type as input, and may not be a valid - // function call in Presto. - // - // TODO: Once the final aggregation function call representation is fixed, - // the same mechanism in project and filter expression should be used here. - verify(lambdaExpression.getArguments().size() == functionType.getArgumentTypes().size()); - Map, Type> lambdaArgumentExpressionTypes = new HashMap<>(); - Map lambdaArgumentSymbolTypes = new HashMap<>(); - for (int j = 0; j < lambdaExpression.getArguments().size(); j++) { - LambdaArgumentDeclaration argument = lambdaExpression.getArguments().get(j); - Type type = functionType.getArgumentTypes().get(j); - lambdaArgumentExpressionTypes.put(NodeRef.of(argument), type); - lambdaArgumentSymbolTypes.put(argument.getName().getValue(), type); - } - // the lambda expression itself - builder.put(NodeRef.of(lambdaExpression), functionType) - // expressions from lambda arguments - .putAll(lambdaArgumentExpressionTypes) - // expressions from lambda body - .putAll(getExpressionTypes( - session, - metadata, - sqlParser, - TypeProvider.copyOf(lambdaArgumentSymbolTypes), - lambdaExpression.getBody(), - emptyList(), - NOOP)); - } - } - for (RowExpression argument : aggregation.getArguments()) { - if (!isExpression(argument) || castToExpression(argument) instanceof LambdaExpression) { - continue; - } - builder.putAll(analyze(castToExpression(argument), session, typeProvider)); - } - if (aggregation.getFilter().isPresent() && isExpression(aggregation.getFilter().get())) { - builder.putAll(analyze(castToExpression(aggregation.getFilter().get()), session, typeProvider)); - } - return builder.build(); - } - - private Map, Type> analyze(Expression expression, Session session, TypeProvider typeProvider) - { - return getExpressionTypes( - session, - metadata, - sqlParser, - typeProvider, - expression, - emptyList(), - NOOP); - } - - private RowExpression toRowExpression(Expression expression, Session session, Map, Type> types) - { - return SqlToRowExpressionTranslator.translate(expression, types, ImmutableMap.of(), metadata.getFunctionManager(), metadata.getTypeManager(), session, false); - } - - private RowExpression removeOriginalExpression(RowExpression expression, Rule.Context context) - { - if (isExpression(expression)) { - return toRowExpression( - castToExpression(expression), - context.getSession(), - analyze(castToExpression(expression), context.getSession(), context.getVariableAllocator().getTypes())); - } - return expression; - } - - private RowExpression removeOriginalExpression(RowExpression rowExpression, Session session, Map, Type> types) - { - if (isExpression(rowExpression)) { - Expression expression = castToExpression(rowExpression); - return toRowExpression(expression, session, types); - } - return rowExpression; - } - - /** - * Return Optional.empty() to denote unchanged assignments - */ - private Optional translateAssignments(Assignments assignments, Rule.Context context) - { - Assignments.Builder builder = Assignments.builder(); - boolean anyRewritten = false; - for (Map.Entry entry : assignments.entrySet()) { - RowExpression expression = entry.getValue(); - RowExpression rewritten; - if (isExpression(expression)) { - rewritten = toRowExpression( - castToExpression(expression), - context.getSession(), - analyze(castToExpression(expression), context.getSession(), context.getVariableAllocator().getTypes())); - anyRewritten = true; - } - else { - rewritten = expression; - } - builder.put(entry.getKey(), rewritten); - } - if (!anyRewritten) { - return Optional.empty(); - } - return Optional.of(builder.build()); - } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyArrayOperations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyArrayOperations.java new file mode 100644 index 0000000000000..f75c29d1f7fa7 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyArrayOperations.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import static com.facebook.presto.sql.planner.iterative.rule.SimplifyArrayOperationsRewriter.simplifyArrayOperations; + +public class SimplifyArrayOperations + extends ExpressionRewriteRuleSet +{ + public SimplifyArrayOperations() + { + super((expression, context) -> simplifyArrayOperations(expression, context.getSession())); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyArrayOperationsRewriter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyArrayOperationsRewriter.java new file mode 100644 index 0000000000000..80278ca9785bd --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyArrayOperationsRewriter.java @@ -0,0 +1,222 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.Session; +import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; +import com.facebook.presto.sql.tree.Cast; +import com.facebook.presto.sql.tree.ComparisonExpression; +import com.facebook.presto.sql.tree.ComparisonExpression.Operator; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.ExpressionRewriter; +import com.facebook.presto.sql.tree.ExpressionTreeRewriter; +import com.facebook.presto.sql.tree.FunctionCall; +import com.facebook.presto.sql.tree.Identifier; +import com.facebook.presto.sql.tree.LambdaArgumentDeclaration; +import com.facebook.presto.sql.tree.LambdaExpression; +import com.facebook.presto.sql.tree.Literal; +import com.facebook.presto.sql.tree.LongLiteral; +import com.facebook.presto.sql.tree.NodeLocation; +import com.facebook.presto.sql.tree.QualifiedName; +import com.facebook.presto.sql.tree.SearchedCaseExpression; +import com.facebook.presto.sql.tree.WhenClause; +import com.google.common.collect.ImmutableList; + +import java.util.Optional; + +import static com.facebook.presto.sql.tree.ComparisonExpression.Operator.EQUAL; + +public class SimplifyArrayOperationsRewriter +{ + private SimplifyArrayOperationsRewriter() {} + + public static Expression simplifyArrayOperations(Expression expression, Session session) + { + if (SystemSessionProperties.isSimplifyArrayOperations(session)) { + return ExpressionTreeRewriter.rewriteWith(new Visitor(), expression); + } + + return expression; + } + + private static class Visitor + extends ExpressionRewriter + { + private static boolean isConstant(Expression expression) + { + if (expression instanceof Cast) { + return isConstant(((Cast) expression).getExpression()); + } + + return expression instanceof Literal; + } + + private static boolean isZero(Expression expression) + { + if (expression instanceof Cast) { + return isZero(((Cast) expression).getExpression()); + } + + return expression instanceof LongLiteral && ((LongLiteral) expression).getValue() == 0; + } + + private static boolean isCardinalityOfFilter(Expression expression) + { + if (expression instanceof FunctionCall) { + FunctionCall functionCall = (FunctionCall) expression; + return functionCall.getName().toString().equals("cardinality") && + functionCall.getArguments().size() == 1 && + functionCall.getArguments().get(0) instanceof FunctionCall && + ((FunctionCall) (functionCall.getArguments().get(0))).getName().toString().equals("filter"); + } + + return false; + } + + private static boolean isSimpleComaparison(ComparisonExpression comparisonExpression) + { + switch (comparisonExpression.getOperator()) { + case EQUAL: + case GREATER_THAN: + case LESS_THAN: + case NOT_EQUAL: + case GREATER_THAN_OR_EQUAL: + case LESS_THAN_OR_EQUAL: + return true; + } + + return false; + } + + private Expression simplifyCardinalityOfFilterComparedToZero( + NodeLocation location, + Expression array, + Operator operator, + LambdaExpression origLambda) + { + // Comparing to zero can be rewrtitten to none_match(= 0) or any_match(> 0) + return new FunctionCall( + location, + QualifiedName.of(operator == EQUAL ? "none_match" : "any_match"), + ImmutableList.of(array, origLambda)); + } + + private Expression simplifyCardinalityOfFilter( + FunctionCall cardinalityOfFilter, + Optional operator, + Optional rhs, + Void context, + ExpressionTreeRewriter treeRewriter) + { + FunctionCall filter = (FunctionCall) cardinalityOfFilter.getArguments().get(0); + Expression array = treeRewriter.defaultRewrite(filter.getArguments().get(0), context); + LambdaExpression origLambda = (LambdaExpression) filter.getArguments().get(1); + boolean isComparison = operator.isPresent() && rhs.isPresent(); + + if (isComparison && (operator.get() == Operator.EQUAL || operator.get() == Operator.GREATER_THAN) && isZero(rhs.get())) { + return simplifyCardinalityOfFilterComparedToZero(cardinalityOfFilter.getLocation().orElse(new NodeLocation(0, 0)), array, operator.get(), origLambda); + } + + // Rewrite CARDINALITY(FILTER(arr, x -> f(x))) as REDUCE(arr, cast(0 as bigint), (s, x) -> case when f(x) then s + 1 else s, s -> s) + LambdaArgumentDeclaration origLambdaArgument = origLambda.getArguments().get(0); + Expression origLambdaBody = origLambda.getBody(); + NodeLocation location = filter.getLocation().orElse(new NodeLocation(0, 0)); + + // New lambda arguments + // TODO(viswanadha): Fix it to get a unique name that doesn't clash with existing ones. + String combineFunctionArgumentName = origLambdaArgument.getName().getValue() + "__1__"; + LambdaArgumentDeclaration combineFunctionArgument = new LambdaArgumentDeclaration(new Identifier(location, combineFunctionArgumentName, false)); + + // Final lambda arguments + String outputFunctionArgumentName = origLambdaArgument.getName().getValue() + "__2__"; + LambdaArgumentDeclaration outputFunctionArgument = new LambdaArgumentDeclaration(new Identifier(location, outputFunctionArgumentName, false)); + + ImmutableList.Builder builder = new ImmutableList.Builder(); + Expression elsePart = new Identifier(location, combineFunctionArgumentName, false); + + // New lambda body + ArithmeticBinaryExpression plus1 = + new ArithmeticBinaryExpression(location, + ArithmeticBinaryExpression.Operator.ADD, + new Identifier(location, combineFunctionArgumentName, false), + new LongLiteral(location, "1")); + + if (isComparison) { + // If it's a comparison, stop when the condition is true. So cardinality(filter(a, x -> f(x))) > 0 becomes reduce(a, 0, (s, x) ->tudligithfnithffeekfhuhrevjltijucase when s > 0 then s when f(x) then s + 1 else s, s->s > 0) + builder.add( + new WhenClause( + new ComparisonExpression( + location, + operator.get(), + new Identifier(location, combineFunctionArgumentName, false), + rhs.get()), + new Identifier(location, combineFunctionArgumentName, false))); + } + + builder.add(new WhenClause(origLambdaBody, plus1)); + LambdaExpression combineFunction = new LambdaExpression( + location, + ImmutableList.of(combineFunctionArgument, origLambdaArgument), + new SearchedCaseExpression(location, builder.build(), Optional.of(elsePart))); + + // Final argument to reduce. s -> s + Expression outputFunctionBody = new Identifier(location, outputFunctionArgumentName, false); + if (isComparison) { + // If it's a comparison, simply return the comparison + outputFunctionBody = new ComparisonExpression(cardinalityOfFilter.getLocation().orElse(new NodeLocation(0, 0)), operator.get(), outputFunctionBody, rhs.get()); + } + + LambdaExpression outputFunction = new LambdaExpression(location, ImmutableList.of(outputFunctionArgument), outputFunctionBody); + // Now make the reduce + return new FunctionCall( + location, + QualifiedName.of("reduce"), + ImmutableList.of( + array, + new Cast(location, new LongLiteral(location, "0"), "BIGINT"), + combineFunction, + outputFunction)); + } + + @Override + public Expression rewriteFunctionCall(FunctionCall functionCall, Void context, ExpressionTreeRewriter treeRewriter) + { + if (isCardinalityOfFilter(functionCall)) { + return simplifyCardinalityOfFilter(functionCall, Optional.empty(), Optional.empty(), context, treeRewriter); + } + + return treeRewriter.defaultRewrite(functionCall, context); + } + + @Override + public Expression rewriteComparisonExpression(ComparisonExpression comparisonExpression, Void context, ExpressionTreeRewriter treeRewriter) + { + Expression left = comparisonExpression.getLeft(); + Expression right = comparisonExpression.getRight(); + Operator operator = comparisonExpression.getOperator(); + + if (isSimpleComaparison(comparisonExpression) && isCardinalityOfFilter(left) && isConstant(right)) { + return simplifyCardinalityOfFilter((FunctionCall) left, Optional.of(operator), Optional.of(right), context, treeRewriter); + } + + // If the left is a literal and right is cardinality, we simply normalize it to reverse the operation. + if (isSimpleComaparison(comparisonExpression) && isCardinalityOfFilter(right) && isConstant(left)) { + return simplifyCardinalityOfFilter((FunctionCall) left, Optional.of(operator.negate()), Optional.of(right), context, treeRewriter); + } + + return treeRewriter.defaultRewrite(comparisonExpression, context); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCountOverConstant.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCountOverConstant.java index 71c9267a3eef5..e7254fe071d83 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCountOverConstant.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCountOverConstant.java @@ -18,14 +18,14 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.Assignments; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.Literal; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyExpressions.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyExpressions.java index 11ccf1023bef5..050c21796baba 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyExpressions.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyExpressions.java @@ -20,7 +20,7 @@ import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.ExpressionInterpreter; import com.facebook.presto.sql.planner.LiteralEncoder; -import com.facebook.presto.sql.planner.NoOpSymbolResolver; +import com.facebook.presto.sql.planner.NoOpVariableResolver; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.tree.Expression; @@ -53,7 +53,7 @@ static Expression rewrite(Expression expression, Session session, PlanVariableAl expression = extractCommonPredicates(expression); Map, Type> expressionTypes = getExpressionTypes(session, metadata, sqlParser, variableAllocator.getTypes(), expression, emptyList(), WarningCollector.NOOP); ExpressionInterpreter interpreter = ExpressionInterpreter.expressionOptimizer(expression, metadata, session, expressionTypes); - return literalEncoder.toExpression(interpreter.optimize(NoOpSymbolResolver.INSTANCE), expressionTypes.get(NodeRef.of(expression))); + return literalEncoder.toExpression(interpreter.optimize(NoOpVariableResolver.INSTANCE), expressionTypes.get(NodeRef.of(expression))); } public SimplifyExpressions(Metadata metadata, SqlParser sqlParser) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java new file mode 100644 index 0000000000000..7ddc65b1cd384 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.expressions.LogicalRowExpressions; +import com.facebook.presto.expressions.RowExpressionRewriter; +import com.facebook.presto.expressions.RowExpressionTreeRewriter; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.type.BooleanType; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; +import com.google.common.annotations.VisibleForTesting; + +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.SERIALIZABLE; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.OR; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public class SimplifyRowExpressions + extends RowExpressionRewriteRuleSet +{ + public SimplifyRowExpressions(Metadata metadata) + { + super(new Rewriter(metadata)); + } + + private static class Rewriter + implements PlanRowExpressionRewriter + { + private final RowExpressionOptimizer optimizer; + private final LogicalExpressionRewriter logicalExpressionRewriter; + + public Rewriter(Metadata metadata) + { + requireNonNull(metadata, "metadata is null"); + this.optimizer = new RowExpressionOptimizer(metadata); + this.logicalExpressionRewriter = new LogicalExpressionRewriter(metadata.getFunctionManager()); + } + + @Override + public RowExpression rewrite(RowExpression expression, Rule.Context context) + { + return rewrite(expression, context.getSession().toConnectorSession()); + } + + private RowExpression rewrite(RowExpression expression, ConnectorSession session) + { + RowExpression optimizedRowExpression = optimizer.optimize(expression, SERIALIZABLE, session); + if (optimizedRowExpression instanceof ConstantExpression || !BooleanType.BOOLEAN.equals(optimizedRowExpression.getType())) { + return optimizedRowExpression; + } + return RowExpressionTreeRewriter.rewriteWith(logicalExpressionRewriter, optimizedRowExpression, true); + } + } + + @VisibleForTesting + public static RowExpression rewrite(RowExpression expression, Metadata metadata, ConnectorSession session) + { + return new Rewriter(metadata).rewrite(expression, session); + } + + private static class LogicalExpressionRewriter + extends RowExpressionRewriter + { + private final FunctionResolution functionResolution; + private final LogicalRowExpressions logicalRowExpressions; + + public LogicalExpressionRewriter(FunctionManager functionManager) + { + requireNonNull(functionManager, "functionManager is null"); + this.functionResolution = new FunctionResolution(functionManager); + this.logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(functionManager), new FunctionResolution(functionManager), functionManager); + } + + @Override + public RowExpression rewriteCall(CallExpression node, Boolean isRoot, RowExpressionTreeRewriter treeRewriter) + { + if (functionResolution.isNotFunction(node.getFunctionHandle())) { + checkState(BooleanType.BOOLEAN.equals(node.getType()), "NOT must be boolean function"); + return rewriteBooleanExpression(node, isRoot); + } + if (isRoot) { + return treeRewriter.rewrite(node, false); + } + return null; + } + + @Override + public RowExpression rewriteSpecialForm(SpecialFormExpression node, Boolean isRoot, RowExpressionTreeRewriter treeRewriter) + { + if (isConjunctiveDisjunctive(node.getForm())) { + checkState(BooleanType.BOOLEAN.equals(node.getType()), "AND/OR must be boolean function"); + return rewriteBooleanExpression(node, isRoot); + } + if (isRoot) { + return treeRewriter.rewrite(node, false); + } + return null; + } + + private boolean isConjunctiveDisjunctive(Form form) + { + return form == AND || form == OR; + } + + private RowExpression rewriteBooleanExpression(RowExpression expression, boolean isRoot) + { + if (isRoot) { + return logicalRowExpressions.convertToConjunctiveNormalForm(expression); + } + return logicalRowExpressions.minimalNormalForm(expression); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SingleDistinctAggregationToGroupBy.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SingleDistinctAggregationToGroupBy.java index e5e35b50d4ac9..9ffb5b6003f53 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SingleDistinctAggregationToGroupBy.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SingleDistinctAggregationToGroupBy.java @@ -15,11 +15,11 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -33,8 +33,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Collections.emptyList; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java index 91afa36020a26..86ceb5055a2ba 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedInPredicateToJoin.java @@ -17,9 +17,12 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.PlanVariableAllocator; @@ -27,14 +30,11 @@ import com.facebook.presto.sql.planner.VariablesExtractor; import com.facebook.presto.sql.planner.iterative.Lookup; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.AssignmentUtils; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.BooleanLiteral; import com.facebook.presto.sql.tree.Cast; @@ -61,11 +61,11 @@ import java.util.Set; import static com.facebook.presto.matching.Pattern.nonEmpty; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.sql.ExpressionUtils.and; import static com.facebook.presto.sql.ExpressionUtils.or; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identitiesAsSymbolReferences; import static com.facebook.presto.sql.planner.plan.Patterns.Apply.correlation; import static com.facebook.presto.sql.planner.plan.Patterns.applyNode; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedScalarAggregationToJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedScalarAggregationToJoin.java index ed63493991824..025d7645355de 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedScalarAggregationToJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedScalarAggregationToJoin.java @@ -16,14 +16,14 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.sql.planner.iterative.Lookup; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.optimizations.ScalarAggregationToJoinRewriter; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import java.util.Optional; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedScalarSubquery.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedScalarSubquery.java index 532aaa0f91040..9eb40b52a790d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedScalarSubquery.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedScalarSubquery.java @@ -17,6 +17,7 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.BooleanType; import com.facebook.presto.sql.planner.iterative.Rule; @@ -24,7 +25,6 @@ import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.tree.Cast; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.sql.tree.LongLiteral; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedSingleRowSubqueryToProject.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedSingleRowSubqueryToProject.java index 71a911d304a60..ea84ed1ef01be 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedSingleRowSubqueryToProject.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformCorrelatedSingleRowSubqueryToProject.java @@ -15,12 +15,12 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformExistsApplyToLateralNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformExistsApplyToLateralNode.java index 2be2eb6d3e19c..1a475c17d2c46 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformExistsApplyToLateralNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TransformExistsApplyToLateralNode.java @@ -17,18 +17,18 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.optimizations.PlanNodeDecorrelator; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.planner.plan.ApplyNode; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.BooleanLiteral; import com.facebook.presto.sql.tree.Cast; @@ -43,9 +43,10 @@ import java.util.Optional; +import static com.facebook.presto.spi.plan.AggregationNode.globalAggregation; +import static com.facebook.presto.spi.plan.LimitNode.Step.FINAL; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; -import static com.facebook.presto.sql.planner.plan.AggregationNode.globalAggregation; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identitiesAsSymbolReferences; import static com.facebook.presto.sql.planner.plan.LateralJoinNode.Type.INNER; import static com.facebook.presto.sql.planner.plan.LateralJoinNode.Type.LEFT; @@ -133,7 +134,7 @@ private Optional rewriteToNonDefaultAggregation(ApplyNode applyNode, C context.getIdAllocator().getNextId(), applyNode.getSubquery(), 1L, - false), + FINAL), Assignments.of(subqueryTrue, castToRowExpression(TRUE_LITERAL))); PlanNodeDecorrelator decorrelator = new PlanNodeDecorrelator(context.getIdAllocator(), context.getVariableAllocator(), context.getLookup()); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TranslateExpressions.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TranslateExpressions.java new file mode 100644 index 0000000000000..183b288354b68 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/TranslateExpressions.java @@ -0,0 +1,187 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.Session; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.operator.aggregation.InternalAggregationFunction; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.type.FunctionType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.planner.PlanVariableAllocator; +import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.relational.OriginalExpressionUtils; +import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.LambdaArgumentDeclaration; +import com.facebook.presto.sql.tree.LambdaExpression; +import com.facebook.presto.sql.tree.NodeRef; +import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.facebook.presto.execution.warnings.WarningCollector.NOOP; +import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypes; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.isExpression; +import static com.google.common.base.Verify.verify; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Collections.emptyList; + +public class TranslateExpressions + extends RowExpressionRewriteRuleSet +{ + public TranslateExpressions(Metadata metadata, SqlParser sqlParser) + { + super(createRewriter(metadata, sqlParser)); + } + + private static PlanRowExpressionRewriter createRewriter(Metadata metadata, SqlParser sqlParser) + { + return new PlanRowExpressionRewriter() + { + @Override + public RowExpression rewrite(RowExpression expression, Rule.Context context) + { + // special treatment of the CallExpression in Aggregation + if (expression instanceof CallExpression && ((CallExpression) expression).getArguments().stream().anyMatch(OriginalExpressionUtils::isExpression)) { + return removeOriginalExpressionArguments((CallExpression) expression, context.getSession(), context.getVariableAllocator()); + } + return removeOriginalExpression(expression, context); + } + + private RowExpression removeOriginalExpressionArguments(CallExpression callExpression, Session session, PlanVariableAllocator variableAllocator) + { + Map, Type> types = analyzeCallExpressionTypes(callExpression, session, variableAllocator.getTypes()); + return new CallExpression( + callExpression.getDisplayName(), + callExpression.getFunctionHandle(), + callExpression.getType(), + callExpression.getArguments().stream() + .map(expression -> removeOriginalExpression(expression, session, types)) + .collect(toImmutableList())); + } + + private Map, Type> analyzeCallExpressionTypes(CallExpression callExpression, Session session, TypeProvider typeProvider) + { + List lambdaExpressions = callExpression.getArguments().stream() + .filter(OriginalExpressionUtils::isExpression) + .map(OriginalExpressionUtils::castToExpression) + .filter(LambdaExpression.class::isInstance) + .map(LambdaExpression.class::cast) + .collect(toImmutableList()); + ImmutableMap.Builder, Type> builder = ImmutableMap., Type>builder(); + if (!lambdaExpressions.isEmpty()) { + List functionTypes = metadata.getFunctionManager().getFunctionMetadata(callExpression.getFunctionHandle()).getArgumentTypes().stream() + .filter(typeSignature -> typeSignature.getBase().equals(FunctionType.NAME)) + .map(typeSignature -> (FunctionType) (metadata.getTypeManager().getType(typeSignature))) + .collect(toImmutableList()); + InternalAggregationFunction internalAggregationFunction = metadata.getFunctionManager().getAggregateFunctionImplementation(callExpression.getFunctionHandle()); + List lambdaInterfaces = internalAggregationFunction.getLambdaInterfaces(); + verify(lambdaExpressions.size() == functionTypes.size()); + verify(lambdaExpressions.size() == lambdaInterfaces.size()); + + for (int i = 0; i < lambdaExpressions.size(); i++) { + LambdaExpression lambdaExpression = lambdaExpressions.get(i); + FunctionType functionType = functionTypes.get(i); + + // To compile lambda, LambdaDefinitionExpression needs to be generated from LambdaExpression, + // which requires the types of all sub-expressions. + // + // In project and filter expression compilation, ExpressionAnalyzer.getExpressionTypesFromInput + // is used to generate the types of all sub-expressions. (see visitScanFilterAndProject and visitFilter) + // + // This does not work here since the function call representation in final aggregation node + // is currently a hack: it takes intermediate type as input, and may not be a valid + // function call in Presto. + // + // TODO: Once the final aggregation function call representation is fixed, + // the same mechanism in project and filter expression should be used here. + verify(lambdaExpression.getArguments().size() == functionType.getArgumentTypes().size()); + Map, Type> lambdaArgumentExpressionTypes = new HashMap<>(); + Map lambdaArgumentSymbolTypes = new HashMap<>(); + for (int j = 0; j < lambdaExpression.getArguments().size(); j++) { + LambdaArgumentDeclaration argument = lambdaExpression.getArguments().get(j); + Type type = functionType.getArgumentTypes().get(j); + lambdaArgumentExpressionTypes.put(NodeRef.of(argument), type); + lambdaArgumentSymbolTypes.put(argument.getName().getValue(), type); + } + // the lambda expression itself + builder.put(NodeRef.of(lambdaExpression), functionType) + // expressions from lambda arguments + .putAll(lambdaArgumentExpressionTypes) + // expressions from lambda body + .putAll(getExpressionTypes( + session, + metadata, + sqlParser, + TypeProvider.copyOf(lambdaArgumentSymbolTypes), + lambdaExpression.getBody(), + emptyList(), + NOOP)); + } + } + for (RowExpression argument : callExpression.getArguments()) { + if (!isExpression(argument) || castToExpression(argument) instanceof LambdaExpression) { + continue; + } + builder.putAll(analyze(castToExpression(argument), session, typeProvider)); + } + return builder.build(); + } + + private Map, Type> analyze(Expression expression, Session session, TypeProvider typeProvider) + { + return getExpressionTypes( + session, + metadata, + sqlParser, + typeProvider, + expression, + emptyList(), + NOOP); + } + + private RowExpression toRowExpression(Expression expression, Session session, Map, Type> types) + { + return SqlToRowExpressionTranslator.translate(expression, types, ImmutableMap.of(), metadata.getFunctionManager(), metadata.getTypeManager(), session); + } + + private RowExpression removeOriginalExpression(RowExpression expression, Rule.Context context) + { + if (isExpression(expression)) { + return toRowExpression( + castToExpression(expression), + context.getSession(), + analyze(castToExpression(expression), context.getSession(), context.getVariableAllocator().getTypes())); + } + return expression; + } + + private RowExpression removeOriginalExpression(RowExpression rowExpression, Session session, Map, Type> types) + { + if (isExpression(rowExpression)) { + Expression expression = castToExpression(rowExpression); + return toRowExpression(expression, session, types); + } + return rowExpression; + } + }; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/Util.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/Util.java index 4053a1508b0f7..ad26810507fdb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/Util.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/Util.java @@ -15,11 +15,11 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.VariablesExtractor; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -30,6 +30,7 @@ import java.util.Optional; import java.util.Set; +import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignments; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignmentsAsSymbolReferences; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -88,7 +89,7 @@ public static PlanNode transpose(PlanNode parent, PlanNode child) /** * @return If the node has outputs not in permittedOutputs, returns an identity projection containing only those node outputs also in permittedOutputs. */ - public static Optional restrictOutputs(PlanNodeIdAllocator idAllocator, PlanNode node, Set permittedOutputs) + public static Optional restrictOutputs(PlanNodeIdAllocator idAllocator, PlanNode node, Set permittedOutputs, boolean useRowExpression) { List restrictedOutputs = node.getOutputVariables().stream() .filter(permittedOutputs::contains) @@ -102,7 +103,7 @@ public static Optional restrictOutputs(PlanNodeIdAllocator idAllocator new ProjectNode( idAllocator.getNextId(), node, - identityAssignmentsAsSymbolReferences(restrictedOutputs))); + useRowExpression ? identityAssignments(restrictedOutputs) : identityAssignmentsAsSymbolReferences(restrictedOutputs))); } /** @@ -125,7 +126,7 @@ public static Optional restrictChildOutputs(PlanNodeIdAllocator idAllo for (int i = 0; i < node.getSources().size(); ++i) { PlanNode oldChild = node.getSources().get(i); - Optional newChild = restrictOutputs(idAllocator, oldChild, permittedChildOutputs.get(i)); + Optional newChild = restrictOutputs(idAllocator, oldChild, permittedChildOutputs.get(i), false); rewroteChildren |= newChild.isPresent(); newChildrenBuilder.add(newChild.orElse(oldChild)); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ActualProperties.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ActualProperties.java index f419eeb8ede87..7e807c80c7f0e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ActualProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ActualProperties.java @@ -18,9 +18,12 @@ import com.facebook.presto.spi.ConstantProperty; import com.facebook.presto.spi.LocalProperty; import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.Partitioning; import com.facebook.presto.sql.planner.PartitioningHandle; +import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -36,9 +39,12 @@ import java.util.Set; import java.util.function.Function; +import static com.facebook.presto.sql.planner.PlannerUtils.toVariableReference; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.COORDINATOR_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.isExpression; import static com.facebook.presto.util.MoreLists.filteredCopy; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; @@ -169,7 +175,7 @@ public boolean isStreamRepartitionEffective(Collection> translator) + public ActualProperties translateVariable(Function> translator) { Map translatedConstants = new HashMap<>(); for (Map.Entry entry : constants.entrySet()) { @@ -179,12 +185,54 @@ public ActualProperties translate(Function Optional.ofNullable(constants.get(symbol)))) + .global(global.translateVariableToRowExpression(variable -> { + Optional translated = translator.apply(variable).map(RowExpression.class::cast); + if (!translated.isPresent()) { + translated = Optional.ofNullable(constants.get(variable)); + } + return translated; + })) .local(LocalProperties.translate(localProperties, translator)) .constants(translatedConstants) .build(); } + public ActualProperties translateRowExpression(Map assignments, TypeProvider types) + { + Map inputToOutputVariables = new HashMap<>(); + for (Map.Entry assignment : assignments.entrySet()) { + RowExpression expression = assignment.getValue(); + if (isExpression(expression)) { + if (castToExpression(expression) instanceof SymbolReference) { + inputToOutputVariables.put(toVariableReference(castToExpression(expression), types), assignment.getKey()); + } + } + else { + if (expression instanceof VariableReferenceExpression) { + inputToOutputVariables.put((VariableReferenceExpression) expression, assignment.getKey()); + } + } + } + + Map translatedConstants = new HashMap<>(); + for (Map.Entry entry : constants.entrySet()) { + if (inputToOutputVariables.containsKey(entry.getKey())) { + translatedConstants.put(inputToOutputVariables.get(entry.getKey()), entry.getValue()); + } + } + + ImmutableMap.Builder inputToOutputMappings = ImmutableMap.builder(); + inputToOutputMappings.putAll(inputToOutputVariables); + constants.entrySet().stream() + .filter(entry -> !inputToOutputVariables.containsKey(entry.getKey())) + .forEach(inputToOutputMappings::put); + return builder() + .global(global.translateRowExpression(inputToOutputMappings.build(), assignments, types)) + .local(LocalProperties.translate(localProperties, variable -> Optional.ofNullable(inputToOutputVariables.get(variable)))) + .constants(translatedConstants) + .build(); + } + public Optional getNodePartitioning() { return global.getNodePartitioning(); @@ -355,10 +403,10 @@ public static Global arbitraryPartition() return new Global(Optional.empty(), Optional.empty(), false); } - public static Global partitionedOn( + public static Global partitionedOn( PartitioningHandle nodePartitioningHandle, - List nodePartitioning, - Optional> streamPartitioning) + List nodePartitioning, + Optional> streamPartitioning) { return new Global( Optional.of(Partitioning.create(nodePartitioningHandle, nodePartitioning)), @@ -382,6 +430,11 @@ public static Global streamPartitionedOn(List strea false); } + public static Global partitionedOnCoalesce(Partitioning one, Partitioning other) + { + return new Global(one.translateToCoalesce(other), Optional.empty(), false); + } + public Global withReplicatedNulls(boolean replicatedNulls) { return new Global(nodePartitioning, streamPartitioning, replicatedNulls); @@ -494,13 +547,20 @@ private boolean isStreamRepartitionEffective(Collection> translator, - Function> constants) + private Global translateVariableToRowExpression( + Function> translator) + { + return new Global( + nodePartitioning.flatMap(partitioning -> partitioning.translateVariableToRowExpression(translator)), + streamPartitioning.flatMap(partitioning -> partitioning.translateVariableToRowExpression(translator)), + nullsAndAnyReplicated); + } + + private Global translateRowExpression(Map inputToOutputMappings, Map assignments, TypeProvider types) { return new Global( - nodePartitioning.flatMap(partitioning -> partitioning.translate(translator, constants)), - streamPartitioning.flatMap(partitioning -> partitioning.translate(translator, constants)), + nodePartitioning.flatMap(partitioning -> partitioning.translateRowExpression(inputToOutputMappings, assignments, types)), + streamPartitioning.flatMap(partitioning -> partitioning.translateRowExpression(inputToOutputMappings, assignments, types)), nullsAndAnyReplicated); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java index 55d118ee1e5aa..7019ff58dfa92 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java @@ -22,10 +22,16 @@ import com.facebook.presto.spi.LocalProperty; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SortingProperty; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -40,9 +46,7 @@ import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.optimizations.PreferredProperties.PartitioningProperties; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.ChildReplacer; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; @@ -55,10 +59,8 @@ import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SortNode; @@ -66,13 +68,9 @@ import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; -import com.facebook.presto.sql.tree.Expression; -import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -82,6 +80,7 @@ import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.ListMultimap; import com.google.common.collect.SetMultimap; import java.util.ArrayList; @@ -105,10 +104,12 @@ import static com.facebook.presto.SystemSessionProperties.isRedistributeWrites; import static com.facebook.presto.SystemSessionProperties.isScaleWriters; import static com.facebook.presto.SystemSessionProperties.preferStreamingOperators; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.operator.aggregation.AggregationUtils.hasSingleNodeExecutionPreference; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.spi.plan.LimitNode.Step.PARTIAL; import static com.facebook.presto.sql.planner.FragmentTableScanCounter.getNumberOfTableScans; import static com.facebook.presto.sql.planner.FragmentTableScanCounter.hasMultipleTableScans; -import static com.facebook.presto.sql.planner.PlannerUtils.toVariableReference; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_ARBITRARY_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SCALED_WRITER_DISTRIBUTION; @@ -118,17 +119,17 @@ import static com.facebook.presto.sql.planner.optimizations.ActualProperties.Global.partitionedOn; import static com.facebook.presto.sql.planner.optimizations.ActualProperties.Global.singleStreamPartition; import static com.facebook.presto.sql.planner.optimizations.LocalProperties.grouped; +import static com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils.fromListMultimap; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_MATERIALIZED; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; +import static com.facebook.presto.sql.planner.plan.ExchangeNode.ensureSourceOrderingGatheringExchange; import static com.facebook.presto.sql.planner.plan.ExchangeNode.gatheringExchange; import static com.facebook.presto.sql.planner.plan.ExchangeNode.mergingExchange; import static com.facebook.presto.sql.planner.plan.ExchangeNode.partitionedExchange; import static com.facebook.presto.sql.planner.plan.ExchangeNode.replicatedExchange; import static com.facebook.presto.sql.planner.plan.ExchangeNode.roundRobinExchange; -import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; -import static com.facebook.presto.sql.tree.BooleanLiteral.TRUE_LITERAL; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; @@ -199,7 +200,7 @@ public PlanWithProperties visitPlan(PlanNode node, PreferredProperties preferred @Override public PlanWithProperties visitProject(ProjectNode node, PreferredProperties preferredProperties) { - Map identities = computeIdentityTranslations(node.getAssignments(), types); + Map identities = computeIdentityTranslations(node.getAssignments()); PreferredProperties translatedPreferred = preferredProperties.translate(symbol -> Optional.ofNullable(identities.get(symbol))); return rebaseAndDeriveProperties(node, planChild(node, translatedPreferred)); @@ -238,7 +239,7 @@ public PlanWithProperties visitAggregation(AggregationNode node, PreferredProper { Set partitioningRequirement = ImmutableSet.copyOf(node.getGroupingKeys()); - boolean preferSingleNode = node.hasSingleNodeExecutionPreference(metadata.getFunctionManager()); + boolean preferSingleNode = hasSingleNodeExecutionPreference(node, metadata.getFunctionManager()); PreferredProperties preferredProperties = preferSingleNode ? PreferredProperties.undistributed() : PreferredProperties.any(); if (!node.getGroupingKeys().isEmpty()) { @@ -324,7 +325,7 @@ public PlanWithProperties visitWindow(WindowNode node, PreferredProperties prefe desiredProperties.add(new GroupingProperty<>(node.getPartitionBy())); } node.getOrderingScheme().ifPresent(orderingScheme -> - orderingScheme.getOrderBy().stream() + orderingScheme.getOrderByVariables().stream() .map(variable -> new SortingProperty<>(variable, orderingScheme.getOrdering(variable))) .forEach(desiredProperties::add)); @@ -468,7 +469,7 @@ public PlanWithProperties visitSort(SortNode node, PreferredProperties preferred // skip the SortNode if the local properties guarantee ordering on Sort keys // TODO: This should be extracted as a separate optimizer once the planner is able to reason about the ordering of each operator List> desiredProperties = new ArrayList<>(); - for (VariableReferenceExpression variable : node.getOrderingScheme().getOrderBy()) { + for (VariableReferenceExpression variable : node.getOrderingScheme().getOrderByVariables()) { desiredProperties.add(new SortingProperty<>(variable, node.getOrderingScheme().getOrdering(variable))); } @@ -489,7 +490,8 @@ public PlanWithProperties visitSort(SortNode node, PreferredProperties preferred new SortNode( idAllocator.getNextId(), source, - node.getOrderingScheme()), + node.getOrderingScheme(), + true), node.getOrderingScheme()), child.getProperties()); } @@ -510,7 +512,7 @@ public PlanWithProperties visitLimit(LimitNode node, PreferredProperties preferr if (!child.getProperties().isSingleNode()) { child = withDerivedProperties( - new LimitNode(idAllocator.getNextId(), child.getNode(), node.getCount(), true), + new LimitNode(idAllocator.getNextId(), child.getNode(), node.getCount(), PARTIAL), child.getProperties()); child = withDerivedProperties( @@ -542,7 +544,7 @@ public PlanWithProperties visitDistinctLimit(DistinctLimitNode node, PreferredPr public PlanWithProperties visitFilter(FilterNode node, PreferredProperties preferredProperties) { if (node.getSource() instanceof TableScanNode) { - return planTableScan((TableScanNode) node.getSource(), castToExpression(node.getPredicate())); + return planTableScan((TableScanNode) node.getSource(), node.getPredicate()); } return rebaseAndDeriveProperties(node, planChild(node, preferredProperties)); @@ -551,7 +553,7 @@ public PlanWithProperties visitFilter(FilterNode node, PreferredProperties prefe @Override public PlanWithProperties visitTableScan(TableScanNode node, PreferredProperties preferredProperties) { - return planTableScan(node, TRUE_LITERAL); + return planTableScan(node, TRUE_CONSTANT); } @Override @@ -585,7 +587,7 @@ else if (redistributeWrites) { return rebaseAndDeriveProperties(node, source); } - private PlanWithProperties planTableScan(TableScanNode node, Expression predicate) + private PlanWithProperties planTableScan(TableScanNode node, RowExpression predicate) { PlanNode plan = pushPredicateIntoTableScan(node, predicate, true, session, types, idAllocator, metadata, parser, domainTranslator); // TODO: Support selecting layout with best local property once connector can participate in query optimization. @@ -642,20 +644,29 @@ public PlanWithProperties visitStatisticsWriterNode(StatisticsWriterNode node, P @Override public PlanWithProperties visitTableFinish(TableFinishNode node, PreferredProperties preferredProperties) { - PlanWithProperties child = planChild(node, PreferredProperties.any()); + PlanNode child = planChild(node, PreferredProperties.any()).getNode(); - // if the child is already a gathering exchange, don't add another - if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType().equals(GATHER)) { - return rebaseAndDeriveProperties(node, child); + ExchangeNode gather; + // in case the input is a union (see PushTableWriteThroughUnion), don't add another exchange + if (child instanceof ExchangeNode) { + ExchangeNode exchangeNode = (ExchangeNode) child; + gather = new ExchangeNode( + idAllocator.getNextId(), + GATHER, + REMOTE_STREAMING, + new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), exchangeNode.getOutputVariables()), + exchangeNode.getSources(), + exchangeNode.getInputs(), + true, + Optional.empty()); } - - if (!child.getProperties().isCoordinatorOnly()) { - child = withDerivedProperties( - gatheringExchange(idAllocator.getNextId(), REMOTE_STREAMING, child.getNode()), - child.getProperties()); + else { + gather = ensureSourceOrderingGatheringExchange(idAllocator.getNextId(), REMOTE_STREAMING, child); } - return rebaseAndDeriveProperties(node, child); + return withDerivedProperties( + ChildReplacer.replaceChildren(node, ImmutableList.of(gather)), + ImmutableList.of()); } private SetMultimap createMapping(List keys, List values) @@ -719,7 +730,7 @@ private PlanWithProperties planPartitionedJoin(JoinNode node, List> outputToInputTranslator(UnionNode node, int sourceIndex, TypeProvider types) + private Function> outputToInputTranslator(UnionNode node, int sourceIndex) { return variable -> Optional.of(node.getVariableMapping().get(variable).get(sourceIndex)); } @@ -1054,7 +1065,7 @@ private Partitioning selectUnionPartitioning(UnionNode node, PartitioningPropert // Try planning the children to see if any of them naturally produce a partitioning (for now, just select the first) boolean nullsAndAnyReplicated = parentPreference.isNullsAndAnyReplicated(); for (int sourceIndex = 0; sourceIndex < node.getSources().size(); sourceIndex++) { - PreferredProperties.PartitioningProperties childPartitioning = parentPreference.translate(outputToInputTranslator(node, sourceIndex, types)).get(); + PreferredProperties.PartitioningProperties childPartitioning = parentPreference.translateVariable(outputToInputTranslator(node, sourceIndex)).get(); PreferredProperties childPreferred = PreferredProperties.builder() .global(PreferredProperties.Global.distributed(childPartitioning.withNullsAndAnyReplicated(nullsAndAnyReplicated))) .build(); @@ -1066,7 +1077,7 @@ private Partitioning selectUnionPartitioning(UnionNode node, PartitioningPropert Function> childToParent = createTranslator(createMapping( node.sourceOutputLayout(sourceIndex), node.getOutputVariables())); - return child.getProperties().translate(childToParent).getNodePartitioning().get(); + return child.getProperties().translateVariable(childToParent).getNodePartitioning().get(); } } @@ -1089,7 +1100,7 @@ public PlanWithProperties visitUnion(UnionNode node, PreferredProperties parentP ImmutableListMultimap.Builder outputToSourcesMapping = ImmutableListMultimap.builder(); for (int sourceIndex = 0; sourceIndex < node.getSources().size(); sourceIndex++) { - Partitioning childPartitioning = desiredParentPartitioning.translate(createDirectTranslator(createMapping( + Partitioning childPartitioning = desiredParentPartitioning.translateVariable(createDirectTranslator(createMapping( node.getOutputVariables(), node.sourceOutputLayout(sourceIndex)))); @@ -1121,10 +1132,13 @@ public PlanWithProperties visitUnion(UnionNode node, PreferredProperties parentP outputToSourcesMapping.put(node.getOutputVariables().get(column), node.sourceOutputLayout(sourceIndex).get(column)); } } + + ListMultimap outputsToInputs = outputToSourcesMapping.build(); UnionNode newNode = new UnionNode( node.getId(), partitionedSources.build(), - outputToSourcesMapping.build()); + ImmutableList.copyOf(outputsToInputs.keySet()), + fromListMultimap(outputsToInputs)); return new PlanWithProperties( newNode, @@ -1176,6 +1190,7 @@ public PlanWithProperties visitUnion(UnionNode node, PreferredProperties parentP new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), node.getOutputVariables()), distributedChildren, distributedOutputLayouts, + false, Optional.empty()); } else if (!singleNodeChildren.isEmpty()) { @@ -1194,6 +1209,7 @@ else if (!singleNodeChildren.isEmpty()) { new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), exchangeOutputLayout), distributedChildren, distributedOutputLayouts, + false, Optional.empty()); singleNodeChildren.add(result); @@ -1209,7 +1225,8 @@ else if (!singleNodeChildren.isEmpty()) { } // add local union for all unpartitioned inputs - result = new UnionNode(node.getId(), singleNodeChildren, mappings.build()); + ListMultimap outputsToInputs = mappings.build(); + result = new UnionNode(node.getId(), singleNodeChildren, ImmutableList.copyOf(outputsToInputs.keySet()), fromListMultimap(outputsToInputs)); } else { throw new IllegalStateException("both singleNodeChildren distributedChildren are empty"); @@ -1245,6 +1262,7 @@ private PlanWithProperties arbitraryDistributeUnion( new PartitioningScheme(Partitioning.create(FIXED_ARBITRARY_DISTRIBUTION, ImmutableList.of()), node.getOutputVariables()), distributedChildren, distributedOutputLayouts, + false, Optional.empty())); } } @@ -1282,6 +1300,11 @@ private PlanWithProperties rebaseAndDeriveProperties(PlanNode node, List inputProperties) + { + return new PlanWithProperties(node, deriveProperties(node, inputProperties)); + } + private PlanWithProperties withDerivedProperties(PlanNode node, ActualProperties inputProperties) { return new PlanWithProperties(node, deriveProperties(node, inputProperties)); @@ -1324,7 +1347,10 @@ private Partitioning createPartitioning(List partit if (e.getErrorCode().equals(NOT_SUPPORTED.toErrorCode())) { throw new PrestoException( NOT_SUPPORTED, - format("Catalog \"%s\" does not support custom partitioning and cannot be used as a partitioning provider", partitioningProviderCatalog), + format( + "Catalog \"%s\" cannot be used as a partitioning provider: %s", + partitioningProviderCatalog, + e.getMessage()), e); } throw e; @@ -1386,12 +1412,12 @@ private boolean canPushdownPartialMergeThroughLowMemoryOperators(PlanNode node) .allMatch(this::canPushdownPartialMergeThroughLowMemoryOperators); } - public static Map computeIdentityTranslations(Assignments assignments, TypeProvider types) + public static Map computeIdentityTranslations(Assignments assignments) { Map outputToInput = new HashMap<>(); for (Map.Entry assignment : assignments.getMap().entrySet()) { - if (castToExpression(assignment.getValue()) instanceof SymbolReference) { - outputToInput.put(assignment.getKey(), toVariableReference(castToExpression(assignment.getValue()), types)); + if (assignment.getValue() instanceof VariableReferenceExpression) { + outputToInput.put(assignment.getKey(), (VariableReferenceExpression) assignment.getValue()); } } return outputToInput; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java index 5e0b0b1e7aa91..d276ff0409469 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java @@ -20,8 +20,12 @@ import com.facebook.presto.spi.GroupingProperty; import com.facebook.presto.spi.LocalProperty; import com.facebook.presto.spi.SortingProperty; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.Partitioning; @@ -29,7 +33,6 @@ import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.optimizations.StreamPropertyDerivations.StreamProperties; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; @@ -39,19 +42,18 @@ import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.OutputNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; +import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -63,9 +65,15 @@ import java.util.Set; import static com.facebook.presto.SystemSessionProperties.getTaskConcurrency; +import static com.facebook.presto.SystemSessionProperties.getTaskPartitionedWriterCount; import static com.facebook.presto.SystemSessionProperties.getTaskWriterCount; import static com.facebook.presto.SystemSessionProperties.isDistributedSortEnabled; import static com.facebook.presto.SystemSessionProperties.isSpillEnabled; +import static com.facebook.presto.SystemSessionProperties.isTableWriterMergeOperatorEnabled; +import static com.facebook.presto.operator.aggregation.AggregationUtils.hasSingleNodeExecutionPreference; +import static com.facebook.presto.operator.aggregation.AggregationUtils.isDecomposable; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_ARBITRARY_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; @@ -82,6 +90,7 @@ import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; import static com.facebook.presto.sql.planner.plan.ExchangeNode.gatheringExchange; import static com.facebook.presto.sql.planner.plan.ExchangeNode.mergingExchange; +import static com.facebook.presto.sql.planner.plan.ExchangeNode.partitionedExchange; import static com.facebook.presto.sql.planner.plan.ExchangeNode.roundRobinExchange; import static com.facebook.presto.sql.planner.plan.ExchangeNode.systemPartitionedExchange; import static com.google.common.base.Preconditions.checkArgument; @@ -113,12 +122,14 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Pla private class Rewriter extends InternalPlanVisitor { + private final PlanVariableAllocator variableAllocator; private final PlanNodeIdAllocator idAllocator; private final Session session; private final TypeProvider types; public Rewriter(PlanVariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, Session session) { + this.variableAllocator = variableAllocator; this.types = variableAllocator.getTypes(); this.idAllocator = idAllocator; this.session = session; @@ -278,13 +289,13 @@ public PlanWithProperties visitAggregation(AggregationNode node, StreamPreferred { checkState(node.getStep() == AggregationNode.Step.SINGLE, "step of aggregation is expected to be SINGLE, but it is %s", node.getStep()); - if (node.hasSingleNodeExecutionPreference(metadata.getFunctionManager())) { + if (hasSingleNodeExecutionPreference(node, metadata.getFunctionManager())) { return planAndEnforceChildren(node, singleStream(), defaultParallelism(session)); } List groupingKeys = node.getGroupingKeys(); if (node.hasDefaultOutput()) { - checkState(node.isDecomposable(metadata.getFunctionManager())); + checkState(isDecomposable(node, metadata.getFunctionManager())); // Put fixed local exchange directly below final aggregation to ensure that final and partial aggregations are separated by exchange (in a local runner mode) // This is required so that default outputs from multiple instances of partial aggregations are passed to a single final aggregation. @@ -341,7 +352,7 @@ public PlanWithProperties visitWindow(WindowNode node, StreamPreferredProperties desiredProperties.add(new GroupingProperty<>(node.getPartitionBy())); } node.getOrderingScheme().ifPresent(orderingScheme -> - orderingScheme.getOrderBy().stream() + orderingScheme.getOrderByVariables().stream() .map(variable -> new SortingProperty<>(variable, orderingScheme.getOrdering(variable))) .forEach(desiredProperties::add)); Iterator>> matchIterator = LocalProperties.match(child.getProperties().getLocalProperties(), desiredProperties).iterator(); @@ -473,19 +484,90 @@ public PlanWithProperties visitTopNRowNumber(TopNRowNumberNode node, StreamPrefe // @Override - public PlanWithProperties visitTableWriter(TableWriterNode node, StreamPreferredProperties parentPreferences) + public PlanWithProperties visitTableWriter(TableWriterNode originalTableWriterNode, StreamPreferredProperties parentPreferences) { - StreamPreferredProperties requiredProperties; - StreamPreferredProperties preferredProperties; - if (getTaskWriterCount(session) > 1) { - requiredProperties = fixedParallelism(); - preferredProperties = fixedParallelism(); + if (originalTableWriterNode.getPartitioningScheme().isPresent() && getTaskPartitionedWriterCount(session) == 1) { + return planAndEnforceChildren(originalTableWriterNode, singleStream(), defaultParallelism(session)); + } + + if (!originalTableWriterNode.getPartitioningScheme().isPresent() && getTaskWriterCount(session) == 1) { + return planAndEnforceChildren(originalTableWriterNode, singleStream(), defaultParallelism(session)); + } + + if (!isTableWriterMergeOperatorEnabled(session)) { + return planAndEnforceChildren(originalTableWriterNode, fixedParallelism(), fixedParallelism()); + } + + Optional statisticAggregations = originalTableWriterNode + .getStatisticsAggregation() + .map(aggregations -> aggregations.splitIntoPartialAndIntermediate( + variableAllocator, + metadata.getFunctionManager())); + + PlanWithProperties tableWriter; + + if (!originalTableWriterNode.getPartitioningScheme().isPresent()) { + tableWriter = planAndEnforceChildren( + new TableWriterNode( + originalTableWriterNode.getId(), + originalTableWriterNode.getSource(), + originalTableWriterNode.getTarget(), + variableAllocator.newVariable("partialrowcount", BIGINT), + variableAllocator.newVariable("partialfragments", VARBINARY), + variableAllocator.newVariable("partialcontext", VARBINARY), + originalTableWriterNode.getColumns(), + originalTableWriterNode.getColumnNames(), + originalTableWriterNode.getPartitioningScheme(), + statisticAggregations.map(StatisticAggregations.Parts::getPartialAggregation)), + fixedParallelism(), + fixedParallelism()); } else { - requiredProperties = singleStream(); - preferredProperties = defaultParallelism(session); + PlanWithProperties source = originalTableWriterNode.getSource().accept(this, fixedParallelism()); + PlanWithProperties exchange = deriveProperties( + partitionedExchange( + idAllocator.getNextId(), + LOCAL, + source.getNode(), + originalTableWriterNode.getPartitioningScheme().get()), + source.getProperties()); + tableWriter = deriveProperties( + new TableWriterNode( + originalTableWriterNode.getId(), + exchange.getNode(), + originalTableWriterNode.getTarget(), + variableAllocator.newVariable("partialrowcount", BIGINT), + variableAllocator.newVariable("partialfragments", VARBINARY), + variableAllocator.newVariable("partialcontext", VARBINARY), + originalTableWriterNode.getColumns(), + originalTableWriterNode.getColumnNames(), + originalTableWriterNode.getPartitioningScheme(), + statisticAggregations.map(StatisticAggregations.Parts::getPartialAggregation)), + exchange.getProperties()); } - return planAndEnforceChildren(node, requiredProperties, preferredProperties); + + PlanWithProperties gatheringExchange = deriveProperties( + gatheringExchange( + idAllocator.getNextId(), + LOCAL, + tableWriter.getNode()), + tableWriter.getProperties()); + + return deriveProperties( + new TableWriterMergeNode( + idAllocator.getNextId(), + gatheringExchange.getNode(), + originalTableWriterNode.getRowCountVariable(), + originalTableWriterNode.getFragmentVariable(), + originalTableWriterNode.getTableCommitContextVariable(), + statisticAggregations.map(StatisticAggregations.Parts::getIntermediateAggregation)), + gatheringExchange.getProperties()); + } + + @Override + public PlanWithProperties visitTableWriteMerge(TableWriterMergeNode node, StreamPreferredProperties context) + { + throw new IllegalArgumentException("Unexpected TableWriterMergeNode"); } // @@ -535,6 +617,7 @@ public PlanWithProperties visitUnion(UnionNode node, StreamPreferredProperties p new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), node.getOutputVariables()), sources, inputLayouts, + false, Optional.empty()); return deriveProperties(exchangeNode, inputProperties); } @@ -550,6 +633,7 @@ public PlanWithProperties visitUnion(UnionNode node, StreamPreferredProperties p node.getOutputVariables()), sources, inputLayouts, + false, Optional.empty()); return deriveProperties(exchangeNode, inputProperties); } @@ -562,6 +646,7 @@ public PlanWithProperties visitUnion(UnionNode node, StreamPreferredProperties p new PartitioningScheme(Partitioning.create(FIXED_ARBITRARY_DISTRIBUTION, ImmutableList.of()), node.getOutputVariables()), sources, inputLayouts, + false, Optional.empty()); ExchangeNode exchangeNode = result; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AggregationNodeUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AggregationNodeUtils.java index ff6f42ce33d63..aa2cf51795aac 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AggregationNodeUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AggregationNodeUtils.java @@ -14,12 +14,12 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.VariablesExtractor; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -56,7 +56,7 @@ public static Set extractAggregationUniqueVariables ImmutableSet.Builder builder = ImmutableSet.builder(); aggregation.getArguments().forEach(argument -> builder.addAll(extractAll(argument, types))); aggregation.getFilter().ifPresent(filter -> builder.addAll(extractAll(filter, types))); - aggregation.getOrderBy().ifPresent(orderingScheme -> builder.addAll(orderingScheme.getOrderBy())); + aggregation.getOrderBy().ifPresent(orderingScheme -> builder.addAll(orderingScheme.getOrderByVariables())); return builder.build(); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java index 03dfab3e56e4e..e4f23a0678245 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java @@ -17,10 +17,17 @@ import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorPlanOptimizer; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; @@ -48,7 +55,14 @@ public class ApplyConnectorOptimization static final Set> CONNECTOR_ACCESSIBLE_PLAN_NODES = ImmutableSet.of( FilterNode.class, TableScanNode.class, - ValuesNode.class); + LimitNode.class, + TopNNode.class, + ValuesNode.class, + ProjectNode.class, + AggregationNode.class, + UnionNode.class, + IntersectNode.class, + ExceptNode.class); // for a leaf node that does not belong to any connector (e.g., ValuesNode) private static final ConnectorId EMPTY_CONNECTOR_ID = new ConnectorId("$internal$" + ApplyConnectorOptimization.class + "_CONNECTOR"); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyNodeUtil.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyNodeUtil.java index 7f914e0b7b46c..631ccc5bb3c6b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyNodeUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyNodeUtil.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.sql.planner.optimizations; -import com.facebook.presto.sql.planner.plan.Assignments; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.tree.ExistsPredicate; import com.facebook.presto.sql.tree.Expression; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/BeginTableWrite.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/BeginTableWrite.java deleted file mode 100644 index 8eb93a9a9d413..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/BeginTableWrite.java +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sql.planner.optimizations; - -import com.facebook.presto.Session; -import com.facebook.presto.execution.warnings.WarningCollector; -import com.facebook.presto.metadata.Metadata; -import com.facebook.presto.metadata.TableLayoutResult; -import com.facebook.presto.spi.ColumnHandle; -import com.facebook.presto.spi.Constraint; -import com.facebook.presto.spi.TableHandle; -import com.facebook.presto.spi.plan.FilterNode; -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeIdAllocator; -import com.facebook.presto.spi.plan.TableScanNode; -import com.facebook.presto.spi.predicate.TupleDomain; -import com.facebook.presto.sql.planner.PlanVariableAllocator; -import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.DeleteNode; -import com.facebook.presto.sql.planner.plan.ExchangeNode; -import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; -import com.facebook.presto.sql.planner.plan.SemiJoinNode; -import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; -import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.UnionNode; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; - -import java.util.Optional; -import java.util.Set; - -import static com.facebook.presto.metadata.TableLayoutResult.computeEnforced; -import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMostScalar; -import static com.facebook.presto.sql.planner.plan.ChildReplacer.replaceChildren; -import static com.google.common.base.Preconditions.checkState; -import static java.util.stream.Collectors.toSet; - -/* - * Major HACK alert!!! - * - * This logic should be invoked on query start, not during planning. At that point, the token - * returned by beginCreate/beginInsert should be handed down to tasks in a mapping separate - * from the plan that links plan nodes to the corresponding token. - */ -public class BeginTableWrite - implements PlanOptimizer -{ - private final Metadata metadata; - - public BeginTableWrite(Metadata metadata) - { - this.metadata = metadata; - } - - @Override - public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, PlanVariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) - { - return SimplePlanRewriter.rewriteWith(new Rewriter(session), plan, new Context()); - } - - private class Rewriter - extends SimplePlanRewriter - { - private final Session session; - - public Rewriter(Session session) - { - this.session = session; - } - - @Override - public PlanNode visitTableWriter(TableWriterNode node, RewriteContext context) - { - // Part of the plan should be an Optional> and this - // callback can create the table and abort the table creation if the query fails. - - TableWriterNode.WriterTarget writerTarget = context.get().getMaterializedHandle(node.getTarget()).get(); - return new TableWriterNode( - node.getId(), - node.getSource().accept(this, context), - writerTarget, - node.getRowCountVariable(), - node.getFragmentVariable(), - node.getTableCommitContextVariable(), - node.getColumns(), - node.getColumnNames(), - node.getPartitioningScheme(), - node.getStatisticsAggregation(), - node.getStatisticsAggregationDescriptor()); - } - - @Override - public PlanNode visitDelete(DeleteNode node, RewriteContext context) - { - TableWriterNode.DeleteHandle deleteHandle = (TableWriterNode.DeleteHandle) context.get().getMaterializedHandle(node.getTarget()).get(); - return new DeleteNode( - node.getId(), - rewriteDeleteTableScan(node.getSource(), deleteHandle.getHandle()), - deleteHandle, - node.getRowId(), - node.getOutputVariables()); - } - - @Override - public PlanNode visitStatisticsWriterNode(StatisticsWriterNode node, RewriteContext context) - { - PlanNode child = node.getSource(); - child = child.accept(this, context); - - StatisticsWriterNode.WriteStatisticsHandle analyzeHandle = - new StatisticsWriterNode.WriteStatisticsHandle(metadata.beginStatisticsCollection(session, ((StatisticsWriterNode.WriteStatisticsReference) node.getTarget()).getHandle())); - - return new StatisticsWriterNode( - node.getId(), - child, - analyzeHandle, - node.getRowCountVariable(), - node.isRowCountEnabled(), - node.getDescriptor()); - } - - @Override - public PlanNode visitTableFinish(TableFinishNode node, RewriteContext context) - { - PlanNode child = node.getSource(); - - TableWriterNode.WriterTarget originalTarget = getTarget(child); - TableWriterNode.WriterTarget newTarget = createWriterTarget(originalTarget); - - context.get().addMaterializedHandle(originalTarget, newTarget); - child = child.accept(this, context); - - return new TableFinishNode( - node.getId(), - child, - newTarget, - node.getRowCountVariable(), - node.getStatisticsAggregation(), - node.getStatisticsAggregationDescriptor()); - } - - public TableWriterNode.WriterTarget getTarget(PlanNode node) - { - if (node instanceof TableWriterNode) { - return ((TableWriterNode) node).getTarget(); - } - if (node instanceof DeleteNode) { - return ((DeleteNode) node).getTarget(); - } - if (node instanceof ExchangeNode || node instanceof UnionNode) { - Set writerTargets = node.getSources().stream() - .map(this::getTarget) - .collect(toSet()); - return Iterables.getOnlyElement(writerTargets); - } - throw new IllegalArgumentException("Invalid child for TableCommitNode: " + node.getClass().getSimpleName()); - } - - private TableWriterNode.WriterTarget createWriterTarget(TableWriterNode.WriterTarget target) - { - // TODO: begin these operations in pre-execution step, not here - // TODO: we shouldn't need to store the schemaTableName in the handles, but there isn't a good way to pass this around with the current architecture - if (target instanceof TableWriterNode.CreateName) { - TableWriterNode.CreateName create = (TableWriterNode.CreateName) target; - return new TableWriterNode.CreateHandle(metadata.beginCreateTable(session, create.getCatalog(), create.getTableMetadata(), create.getLayout()), create.getTableMetadata().getTable()); - } - if (target instanceof TableWriterNode.InsertReference) { - TableWriterNode.InsertReference insert = (TableWriterNode.InsertReference) target; - return new TableWriterNode.InsertHandle(metadata.beginInsert(session, insert.getHandle()), metadata.getTableMetadata(session, insert.getHandle()).getTable()); - } - if (target instanceof TableWriterNode.DeleteHandle) { - TableWriterNode.DeleteHandle delete = (TableWriterNode.DeleteHandle) target; - return new TableWriterNode.DeleteHandle(metadata.beginDelete(session, delete.getHandle()), delete.getSchemaTableName()); - } - throw new IllegalArgumentException("Unhandled target type: " + target.getClass().getSimpleName()); - } - - private PlanNode rewriteDeleteTableScan(PlanNode node, TableHandle handle) - { - if (node instanceof TableScanNode) { - TableScanNode scan = (TableScanNode) node; - TupleDomain originalEnforcedConstraint = scan.getEnforcedConstraint(); - - TableLayoutResult layoutResult = metadata.getLayout( - session, - handle, - new Constraint<>(originalEnforcedConstraint), - Optional.of(ImmutableSet.copyOf(scan.getAssignments().values()))); - - return new TableScanNode( - scan.getId(), - layoutResult.getLayout().getNewTableHandle(), - scan.getOutputVariables(), - scan.getAssignments(), - layoutResult.getLayout().getPredicate(), - computeEnforced(originalEnforcedConstraint, layoutResult.getUnenforcedConstraint())); - } - - if (node instanceof FilterNode) { - PlanNode source = rewriteDeleteTableScan(((FilterNode) node).getSource(), handle); - return replaceChildren(node, ImmutableList.of(source)); - } - if (node instanceof ProjectNode) { - PlanNode source = rewriteDeleteTableScan(((ProjectNode) node).getSource(), handle); - return replaceChildren(node, ImmutableList.of(source)); - } - if (node instanceof SemiJoinNode) { - PlanNode source = rewriteDeleteTableScan(((SemiJoinNode) node).getSource(), handle); - return replaceChildren(node, ImmutableList.of(source, ((SemiJoinNode) node).getFilteringSource())); - } - if (node instanceof JoinNode) { - JoinNode joinNode = (JoinNode) node; - if (joinNode.getType() == JoinNode.Type.INNER && isAtMostScalar(joinNode.getRight())) { - PlanNode source = rewriteDeleteTableScan(joinNode.getLeft(), handle); - return replaceChildren(node, ImmutableList.of(source, joinNode.getRight())); - } - } - throw new IllegalArgumentException("Invalid descendant for DeleteNode: " + node.getClass().getName()); - } - } - - public static class Context - { - private Optional handle = Optional.empty(); - private Optional materializedHandle = Optional.empty(); - - public void addMaterializedHandle(TableWriterNode.WriterTarget handle, TableWriterNode.WriterTarget materializedHandle) - { - checkState(!this.handle.isPresent(), "can only have one WriterTarget in a subtree"); - this.handle = Optional.of(handle); - this.materializedHandle = Optional.of(materializedHandle); - } - - public Optional getMaterializedHandle(TableWriterNode.WriterTarget handle) - { - checkState(this.handle.get().equals(handle), "can't find materialized handle for WriterTarget"); - return materializedHandle; - } - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CheckSubqueryNodesAreRewritten.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CheckSubqueryNodesAreRewritten.java index fad7eebf3dfc6..db23bf712e596 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CheckSubqueryNodesAreRewritten.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CheckSubqueryNodesAreRewritten.java @@ -16,10 +16,10 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.analyzer.SemanticException; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.plan.ApplyNode; @@ -27,6 +27,7 @@ import java.util.List; +import static com.facebook.presto.spi.StandardErrorCode.UNSUPPORTED_SUBQUERY; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; import static com.google.common.base.Preconditions.checkState; import static java.lang.String.format; @@ -41,22 +42,22 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Pla .findFirst() .ifPresent(node -> { ApplyNode applyNode = (ApplyNode) node; - throw error(applyNode.getCorrelation(), applyNode.getOriginSubqueryError()); + error(applyNode.getCorrelation(), applyNode.getOriginSubqueryError()); }); searchFrom(plan).where(LateralJoinNode.class::isInstance) .findFirst() .ifPresent(node -> { LateralJoinNode lateralJoinNode = (LateralJoinNode) node; - throw error(lateralJoinNode.getCorrelation(), lateralJoinNode.getOriginSubqueryError()); + error(lateralJoinNode.getCorrelation(), lateralJoinNode.getOriginSubqueryError()); }); return plan; } - private SemanticException error(List correlation, String originSubqueryError) + private void error(List correlation, String originSubqueryError) { checkState(!correlation.isEmpty(), "All the non correlated subqueries should be rewritten at this point"); - throw new RuntimeException(format(originSubqueryError, "Given correlated subquery is not supported")); + throw new PrestoException(UNSUPPORTED_SUBQUERY, format(originSubqueryError, "Given correlated subquery is not supported")); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/DistinctOutputQueryUtil.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/DistinctOutputQueryUtil.java index 3f861548ea694..b0daa8130af99 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/DistinctOutputQueryUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/DistinctOutputQueryUtil.java @@ -13,18 +13,18 @@ */ package com.facebook.presto.sql.planner.optimizations; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.ValuesNode; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; -import com.facebook.presto.sql.planner.plan.ExceptNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; -import com.facebook.presto.sql.planner.plan.IntersectNode; -import com.facebook.presto.sql.planner.plan.LimitNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import java.util.function.Function; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ExpressionEquivalence.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ExpressionEquivalence.java index d8e89accaf1f9..a3408698a3af3 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ExpressionEquivalence.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ExpressionEquivalence.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.InputReferenceExpression; @@ -45,7 +46,6 @@ import java.util.Map; import java.util.Set; -import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; import static com.facebook.presto.spi.function.OperatorType.EQUAL; import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN_OR_EQUAL; @@ -96,6 +96,14 @@ public boolean areExpressionsEquivalent(Session session, Expression leftExpressi return canonicalizedLeft.equals(canonicalizedRight); } + public boolean areExpressionsEquivalent(RowExpression leftExpression, RowExpression rightExpression) + { + RowExpression canonicalizedLeft = leftExpression.accept(canonicalizationVisitor, null); + RowExpression canonicalizedRight = rightExpression.accept(canonicalizationVisitor, null); + + return canonicalizedLeft.equals(canonicalizedRight); + } + private RowExpression toRowExpression(Session session, Expression expression, Map variableInput, TypeProvider types) { // replace qualified names with input references since row expressions do not support these @@ -111,7 +119,7 @@ private RowExpression toRowExpression(Session session, Expression expression, Ma WarningCollector.NOOP); // convert to row expression - return translate(expression, expressionTypes, variableInput, metadata.getFunctionManager(), metadata.getTypeManager(), session, false); + return translate(expression, expressionTypes, variableInput, metadata.getFunctionManager(), metadata.getTypeManager(), session); } private static class CanonicalizationVisitor @@ -135,9 +143,9 @@ public RowExpression visitCall(CallExpression call, Void context) .map(expression -> expression.accept(this, context)) .collect(toImmutableList())); - String callName = functionManager.getFunctionMetadata(call.getFunctionHandle()).getName(); + QualifiedFunctionName callName = functionManager.getFunctionMetadata(call.getFunctionHandle()).getName(); - if (callName.equals(mangleOperatorName(EQUAL)) || callName.equals(mangleOperatorName(NOT_EQUAL)) || callName.equals(mangleOperatorName(IS_DISTINCT_FROM))) { + if (callName.equals(EQUAL.getFunctionName()) || callName.equals(NOT_EQUAL.getFunctionName()) || callName.equals(IS_DISTINCT_FROM.getFunctionName())) { // sort arguments return new CallExpression( call.getDisplayName(), @@ -146,11 +154,11 @@ public RowExpression visitCall(CallExpression call, Void context) ROW_EXPRESSION_ORDERING.sortedCopy(call.getArguments())); } - if (callName.equals(mangleOperatorName(GREATER_THAN)) || callName.equals(mangleOperatorName(GREATER_THAN_OR_EQUAL))) { + if (callName.equals(GREATER_THAN.getFunctionName()) || callName.equals(GREATER_THAN_OR_EQUAL.getFunctionName())) { // convert greater than to less than FunctionHandle functionHandle = functionManager.resolveOperator( - callName.equals(mangleOperatorName(GREATER_THAN)) ? LESS_THAN : LESS_THAN_OR_EQUAL, + callName.equals(GREATER_THAN.getFunctionName()) ? LESS_THAN : LESS_THAN_OR_EQUAL, swapPair(fromTypes(call.getArguments().stream().map(RowExpression::getType).collect(toImmutableList())))); return new CallExpression( call.getDisplayName(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java index 8ec677c41b0c1..262439dc7b0b2 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java @@ -17,19 +17,21 @@ import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.Partitioning.ArgumentBinding; import com.facebook.presto.sql.planner.PartitioningScheme; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; @@ -39,12 +41,10 @@ import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.google.common.collect.BiMap; @@ -54,6 +54,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; import java.util.HashMap; @@ -65,9 +66,9 @@ import java.util.Set; import java.util.function.Function; -import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION; +import static com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils.fromListMultimap; import static com.facebook.presto.sql.planner.plan.ChildReplacer.replaceChildren; import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; @@ -90,7 +91,7 @@ public class HashGenerationOptimizer implements PlanOptimizer { public static final long INITIAL_HASH_VALUE = 0; - private static final String HASH_CODE = mangleOperatorName("HASH_CODE"); + private static final String HASH_CODE = OperatorType.HASH_CODE.getFunctionName().getFunctionName(); private final FunctionManager functionManager; @@ -490,11 +491,11 @@ public PlanWithProperties visitExchange(ExchangeNode node, HashComputationSet pa Optional partitionVariables = Optional.empty(); PartitioningScheme partitioningScheme = node.getPartitioningScheme(); if (partitioningScheme.getPartitioning().getHandle().equals(FIXED_HASH_DISTRIBUTION) && - partitioningScheme.getPartitioning().getArguments().stream().allMatch(ArgumentBinding::isVariable)) { + partitioningScheme.getPartitioning().getArguments().stream().allMatch(VariableReferenceExpression.class::isInstance)) { // add precomputed hash for exchange partitionVariables = computeHash( partitioningScheme.getPartitioning().getArguments().stream() - .map(ArgumentBinding::getVariableReference) + .map(VariableReferenceExpression.class::cast) .collect(toImmutableList()), functionManager); preference = preference.withHashComputation(partitionVariables); @@ -556,6 +557,7 @@ public PlanWithProperties visitExchange(ExchangeNode node, HashComputationSet pa partitioningScheme, newSources.build(), newInputs.build(), + node.isEnsureSourceOrdering(), node.getOrderingScheme()), newHashVariables); } @@ -574,7 +576,7 @@ public PlanWithProperties visitUnion(UnionNode node, HashComputationSet parentPr // add hash variables to sources ImmutableListMultimap.Builder newVariableMapping = ImmutableListMultimap.builder(); - newVariableMapping.putAll(node.getVariableMapping()); + node.getVariableMapping().forEach(newVariableMapping::putAll); ImmutableList.Builder newSources = ImmutableList.builder(); for (int sourceId = 0; sourceId < node.getSources().size(); sourceId++) { // translate preference to input variables @@ -595,11 +597,13 @@ public PlanWithProperties visitUnion(UnionNode node, HashComputationSet parentPr } } + ListMultimap outputsToInputs = newVariableMapping.build(); return new PlanWithProperties( new UnionNode( node.getId(), newSources.build(), - newVariableMapping.build()), + ImmutableList.copyOf(outputsToInputs.keySet()), + fromListMultimap(outputsToInputs)), newHashVariables); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ImplementIntersectAndExceptAsUnion.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ImplementIntersectAndExceptAsUnion.java index a986d8ebf0e4d..3f26f32fc9346 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ImplementIntersectAndExceptAsUnion.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ImplementIntersectAndExceptAsUnion.java @@ -17,9 +17,17 @@ import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.IntersectNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.SetOperationNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.StandardTypes; @@ -27,15 +35,7 @@ import com.facebook.presto.sql.ExpressionUtils; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; -import com.facebook.presto.sql.planner.plan.Assignments; -import com.facebook.presto.sql.planner.plan.ExceptNode; -import com.facebook.presto.sql.planner.plan.IntersectNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; -import com.facebook.presto.sql.planner.plan.SetOperationNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.Cast; import com.facebook.presto.sql.tree.ComparisonExpression; @@ -46,16 +46,18 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ListMultimap; import com.google.common.collect.Maps; import java.util.List; import java.util.Map; import java.util.Optional; +import static com.facebook.presto.spi.plan.AggregationNode.Step; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; +import static com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils.fromListMultimap; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignmentsAsSymbolReferences; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.asSymbolReference; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; @@ -241,7 +243,8 @@ private UnionNode union(List nodes, List } } - return new UnionNode(idAllocator.getNextId(), nodes, outputsToInputs.build()); + ListMultimap mapping = outputsToInputs.build(); + return new UnionNode(idAllocator.getNextId(), nodes, ImmutableList.copyOf(mapping.keySet()), fromListMultimap(mapping)); } private AggregationNode computeCounts(UnionNode sourceNode, List originalColumns, List markers, List aggregationOutputs) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/IndexJoinOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/IndexJoinOptimizer.java index 5dfb57c623434..1e94bd187f258 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/IndexJoinOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/IndexJoinOptimizer.java @@ -18,9 +18,11 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.ResolvedIndex; import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.RowExpression; @@ -29,12 +31,10 @@ import com.facebook.presto.sql.planner.LiteralEncoder; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.IndexJoinNode; import com.facebook.presto.sql.planner.plan.IndexSourceNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.WindowNode; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/LimitPushDown.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/LimitPushDown.java index 065a72b0380dc..dc49368b22cbf 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/LimitPushDown.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/LimitPushDown.java @@ -15,27 +15,29 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.planner.plan.SortNode; -import com.facebook.presto.sql.planner.plan.TopNNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import static com.facebook.presto.spi.plan.LimitNode.Step.FINAL; +import static com.facebook.presto.spi.plan.LimitNode.Step.PARTIAL; import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; @@ -57,12 +59,12 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Pla private static class LimitContext { private final long count; - private final boolean partial; + private final LimitNode.Step step; - public LimitContext(long count, boolean partial) + public LimitContext(long count, LimitNode.Step step) { this.count = count; - this.partial = partial; + this.step = step; } public long getCount() @@ -70,9 +72,9 @@ public long getCount() return count; } - public boolean isPartial() + public LimitNode.Step getStep() { - return partial; + return step; } @Override @@ -80,7 +82,7 @@ public String toString() { return toStringHelper(this) .add("count", count) - .add("partial", partial) + .add("step", step) .toString(); } } @@ -103,7 +105,7 @@ public PlanNode visitPlan(PlanNode node, RewriteContext context) LimitContext limit = context.get(); if (limit != null) { // Drop in a LimitNode b/c we cannot push our limit down any further - rewrittenNode = new LimitNode(idAllocator.getNextId(), rewrittenNode, limit.getCount(), limit.isPartial()); + rewrittenNode = new LimitNode(idAllocator.getNextId(), rewrittenNode, limit.getCount(), limit.getStep()); } return rewrittenNode; } @@ -124,7 +126,7 @@ public PlanNode visitLimit(LimitNode node, RewriteContext context) } // default visitPlan logic will insert the limit node - return context.rewrite(node.getSource(), new LimitContext(count, false)); + return context.rewrite(node.getSource(), new LimitContext(count, FINAL)); } @Override @@ -143,7 +145,7 @@ public PlanNode visitAggregation(AggregationNode node, RewriteContext context) return new TopNNode(node.getId(), rewrittenSource, limit.getCount(), node.getOrderingScheme(), TopNNode.Step.SINGLE); } else if (rewrittenSource != node.getSource()) { - return new SortNode(node.getId(), rewrittenSource, node.getOrderingScheme()); + return new SortNode(node.getId(), rewrittenSource, node.getOrderingScheme(), node.isPartial()); } return node; } @@ -204,7 +206,7 @@ public PlanNode visitUnion(UnionNode node, RewriteContext context) LimitContext childLimit = null; if (limit != null) { - childLimit = new LimitContext(limit.getCount(), true); + childLimit = new LimitContext(limit.getCount(), PARTIAL); } List sources = new ArrayList<>(); @@ -212,9 +214,9 @@ public PlanNode visitUnion(UnionNode node, RewriteContext context) sources.add(context.rewrite(node.getSources().get(i), childLimit)); } - PlanNode output = new UnionNode(node.getId(), sources, node.getVariableMapping()); + PlanNode output = new UnionNode(node.getId(), sources, node.getOutputVariables(), node.getVariableMapping()); if (limit != null) { - output = new LimitNode(idAllocator.getNextId(), output, limit.getCount(), limit.isPartial()); + output = new LimitNode(idAllocator.getNextId(), output, limit.getCount(), limit.getStep()); } return output; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataDeleteOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataDeleteOptimizer.java index f154735ff6ed8..e7bd8c8df91d8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataDeleteOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataDeleteOptimizer.java @@ -31,7 +31,6 @@ import java.util.List; import java.util.Optional; -import static com.facebook.presto.sql.planner.plan.TableWriterNode.DeleteHandle; import static java.util.Objects.requireNonNull; /** @@ -96,7 +95,7 @@ public PlanNode visitTableFinish(TableFinishNode node, RewriteContext cont return new MetadataDeleteNode( idAllocator.getNextId(), - new DeleteHandle(tableScanNode.getTable(), delete.get().getTarget().getSchemaTableName()), + tableScanNode.getTable(), Iterables.getOnlyElement(node.getOutputVariables())); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataQueryOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataQueryOptimizer.java index db8ffb645fce0..74d34a3743dea 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataQueryOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataQueryOptimizer.java @@ -21,10 +21,16 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.Constraint; import com.facebook.presto.spi.DiscretePredicates; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.predicate.NullableValue; import com.facebook.presto.spi.predicate.TupleDomain; @@ -34,14 +40,9 @@ import com.facebook.presto.sql.planner.LiteralEncoder; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.planner.plan.SortNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -53,6 +54,7 @@ import java.util.Optional; import java.util.Set; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.sql.relational.Expressions.constant; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; @@ -64,7 +66,10 @@ public class MetadataQueryOptimizer implements PlanOptimizer { - private static final Set ALLOWED_FUNCTIONS = ImmutableSet.of("max", "min", "approx_distinct"); + private static final Set ALLOWED_FUNCTIONS = ImmutableSet.of( + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "max"), + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "min"), + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "approx_distinct")); private final Metadata metadata; private final LiteralEncoder literalEncoder; @@ -107,7 +112,7 @@ public PlanNode visitAggregation(AggregationNode node, RewriteContext cont { // supported functions are only MIN/MAX/APPROX_DISTINCT or distinct aggregates for (Aggregation aggregation : node.getAggregations().values()) { - String functionName = metadata.getFunctionManager().getFunctionMetadata(aggregation.getFunctionHandle()).getName(); + QualifiedFunctionName functionName = metadata.getFunctionManager().getFunctionMetadata(aggregation.getFunctionHandle()).getName(); if (!ALLOWED_FUNCTIONS.contains(functionName) && !aggregation.isDistinct()) { return context.defaultRewrite(node); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java index 98d35ba561418..69e59df27b5a4 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java @@ -16,8 +16,13 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -25,12 +30,8 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.GroupIdNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.tree.Cast; @@ -55,10 +56,11 @@ import java.util.stream.Collectors; import static com.facebook.presto.SystemSessionProperties.isOptimizeDistinctAggregationEnabled; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.planner.PlannerUtils.toVariableReference; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAsSymbolReference; import static com.facebook.presto.sql.relational.Expressions.variable; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.asSymbolReference; @@ -191,8 +193,10 @@ public PlanNode visitAggregation(AggregationNode node, RewriteContext cont node.getPartitioningScheme(), builder.build(), node.getInputs(), + node.isEnsureSourceOrdering(), node.getOrderingScheme()); } @@ -359,7 +361,7 @@ public PlanNode visitUnion(UnionNode node, RewriteContext context) } if (modified) { - return new UnionNode(node.getId(), builder.build(), node.getVariableMapping()); + return new UnionNode(node.getId(), builder.build(), node.getOutputVariables(), node.getVariableMapping()); } return node; @@ -975,7 +977,7 @@ private Expression simplifyExpression(Expression expression) emptyList(), /* parameters have already been replaced */ WarningCollector.NOOP); ExpressionInterpreter optimizer = ExpressionInterpreter.expressionOptimizer(expression, metadata, session, expressionTypes); - return literalEncoder.toExpression(optimizer.optimize(NoOpSymbolResolver.INSTANCE), expressionTypes.get(NodeRef.of(expression))); + return literalEncoder.toExpression(optimizer.optimize(NoOpVariableResolver.INSTANCE), expressionTypes.get(NodeRef.of(expression))); } private boolean areExpressionsEquivalent(Expression leftExpression, Expression rightExpression) @@ -1000,7 +1002,7 @@ private Object nullInputEvaluator(final Collection emptyList(), /* parameters have already been replaced */ WarningCollector.NOOP); return ExpressionInterpreter.expressionOptimizer(expression, metadata, session, expressionTypes) - .optimize(symbol -> nullVariableNames.contains(symbol.getName()) ? null : symbol.toSymbolReference()); + .optimize(variable -> nullVariableNames.contains(variable.getName()) ? null : new Symbol(variable.getName()).toSymbolReference()); } private Predicate joinEqualityExpression(final Collection leftVariables) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PreferredProperties.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PreferredProperties.java index a8e96d3508eb3..abfea1e5f83a1 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PreferredProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PreferredProperties.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.spi.LocalProperty; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.Partitioning; import com.google.common.collect.ImmutableList; @@ -267,7 +268,7 @@ public Global translate(Function properties.translate(translator))); + return distributed(partitioningProperties.flatMap(properties -> properties.translateVariable(translator))); } @Override @@ -374,7 +375,7 @@ public PartitioningProperties mergeWithParent(PartitioningProperties parent) return common.isEmpty() ? this : partitioned(common).withNullsAndAnyReplicated(nullsAndAnyReplicated); } - public Optional translate(Function> translator) + public Optional translateVariable(Function> translator) { Set newPartitioningColumns = partitioningColumns.stream() .map(translator) @@ -391,7 +392,7 @@ public Optional translate(Function newPartitioning = partitioning.get().translate(translator, symbol -> Optional.empty()); + Optional newPartitioning = partitioning.get().translateVariableToRowExpression(variable -> translator.apply(variable).map(RowExpression.class::cast)); if (!newPartitioning.isPresent()) { return Optional.empty(); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java index c07a1d8076ba9..014067a379071 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java @@ -24,9 +24,14 @@ import com.facebook.presto.spi.GroupingProperty; import com.facebook.presto.spi.LocalProperty; import com.facebook.presto.spi.SortingProperty; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.ConstantExpression; @@ -36,12 +41,10 @@ import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.ExpressionDomainTranslator; import com.facebook.presto.sql.planner.ExpressionInterpreter; -import com.facebook.presto.sql.planner.NoOpSymbolResolver; -import com.facebook.presto.sql.planner.OrderingScheme; +import com.facebook.presto.sql.planner.NoOpVariableResolver; import com.facebook.presto.sql.planner.RowExpressionInterpreter; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.optimizations.ActualProperties.Global; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.DeleteNode; @@ -55,10 +58,8 @@ import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; @@ -66,8 +67,8 @@ import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; @@ -90,15 +91,18 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.facebook.presto.SystemSessionProperties.isOptimizeFullOuterJoinWithCoalesce; import static com.facebook.presto.SystemSessionProperties.planWithTableNodePartitioning; import static com.facebook.presto.spi.predicate.TupleDomain.extractFixedValuesToConstantExpressions; import static com.facebook.presto.spi.relation.DomainTranslator.BASIC_COLUMN_EXTRACTOR; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypes; import static com.facebook.presto.sql.planner.PlannerUtils.toVariableReference; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.ARBITRARY_DISTRIBUTION; import static com.facebook.presto.sql.planner.optimizations.ActualProperties.Global.arbitraryPartition; import static com.facebook.presto.sql.planner.optimizations.ActualProperties.Global.coordinatorSingleStreamPartition; import static com.facebook.presto.sql.planner.optimizations.ActualProperties.Global.partitionedOn; +import static com.facebook.presto.sql.planner.optimizations.ActualProperties.Global.partitionedOnCoalesce; import static com.facebook.presto.sql.planner.optimizations.ActualProperties.Global.singleStreamPartition; import static com.facebook.presto.sql.planner.optimizations.ActualProperties.Global.streamPartitionedOn; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; @@ -181,7 +185,7 @@ public ActualProperties visitExplainAnalyze(ExplainAnalyzeNode node, List inputProperties) { return Iterables.getOnlyElement(inputProperties) - .translate(column -> PropertyDerivations.filterIfMissing(node.getOutputVariables(), column)); + .translateVariable(column -> PropertyDerivations.filterIfMissing(node.getOutputVariables(), column)); } @Override @@ -240,7 +244,7 @@ public ActualProperties visitWindow(WindowNode node, List inpu // If the input is completely pre-partitioned and sorted, then the original input properties will be respected Optional orderingScheme = node.getOrderingScheme(); if (ImmutableSet.copyOf(node.getPartitionBy()).equals(node.getPrePartitionedInputs()) - && (!orderingScheme.isPresent() || node.getPreSortedOrderPrefix() == orderingScheme.get().getOrderBy().size())) { + && (!orderingScheme.isPresent() || node.getPreSortedOrderPrefix() == orderingScheme.get().getOrderByVariables().size())) { return properties; } @@ -264,7 +268,7 @@ public ActualProperties visitWindow(WindowNode node, List inpu } orderingScheme.ifPresent(scheme -> - scheme.getOrderBy().stream() + scheme.getOrderByVariables().stream() .map(column -> new SortingProperty<>(column, scheme.getOrdering(column))) .forEach(localProperties::add)); @@ -291,7 +295,7 @@ public ActualProperties visitGroupId(GroupIdNode node, List in inputToOutputMappings.putIfAbsent(argument, argument); } - return Iterables.getOnlyElement(inputProperties).translate(column -> Optional.ofNullable(inputToOutputMappings.get(column))); + return Iterables.getOnlyElement(inputProperties).translateVariable(column -> Optional.ofNullable(inputToOutputMappings.get(column))); } @Override @@ -299,7 +303,7 @@ public ActualProperties visitAggregation(AggregationNode node, List node.getGroupingKeys().contains(variable) ? Optional.of(variable) : Optional.empty()); + ActualProperties translated = properties.translateVariable(variable -> node.getGroupingKeys().contains(variable) ? Optional.of(variable) : Optional.empty()); return ActualProperties.builderFrom(translated) .local(LocalProperties.grouped(node.getGroupingKeys())) @@ -319,7 +323,7 @@ public ActualProperties visitTopNRowNumber(TopNRowNumberNode node, List> localProperties = ImmutableList.builder(); localProperties.add(new GroupingProperty<>(node.getPartitionBy())); - for (VariableReferenceExpression column : node.getOrderingScheme().getOrderBy()) { + for (VariableReferenceExpression column : node.getOrderingScheme().getOrderByVariables()) { localProperties.add(new SortingProperty<>(column, node.getOrderingScheme().getOrdering(column))); } @@ -333,7 +337,7 @@ public ActualProperties visitTopN(TopNNode node, List inputPro { ActualProperties properties = Iterables.getOnlyElement(inputProperties); - List> localProperties = node.getOrderingScheme().getOrderBy().stream() + List> localProperties = node.getOrderingScheme().getOrderByVariables().stream() .map(column -> new SortingProperty<>(column, node.getOrderingScheme().getOrdering(column))) .collect(toImmutableList()); @@ -347,7 +351,7 @@ public ActualProperties visitSort(SortNode node, List inputPro { ActualProperties properties = Iterables.getOnlyElement(inputProperties); - List> localProperties = node.getOrderingScheme().getOrderBy().stream() + List> localProperties = node.getOrderingScheme().getOrderByVariables().stream() .map(column -> new SortingProperty<>(column, node.getOrderingScheme().getOrdering(column))) .collect(toImmutableList()); @@ -392,7 +396,7 @@ public ActualProperties visitTableFinish(TableFinishNode node, List inputProperties) { // drop all symbols in property because delete doesn't pass on any of the columns - return Iterables.getOnlyElement(inputProperties).translate(symbol -> Optional.empty()); + return Iterables.getOnlyElement(inputProperties).translateVariable(symbol -> Optional.empty()); } @Override @@ -406,8 +410,8 @@ public ActualProperties visitJoin(JoinNode node, List inputPro switch (node.getType()) { case INNER: - probeProperties = probeProperties.translate(column -> filterOrRewrite(outputVariableReferences, node.getCriteria(), column)); - buildProperties = buildProperties.translate(column -> filterOrRewrite(outputVariableReferences, node.getCriteria(), column)); + probeProperties = probeProperties.translateVariable(column -> filterOrRewrite(outputVariableReferences, node.getCriteria(), column)); + buildProperties = buildProperties.translateVariable(column -> filterOrRewrite(outputVariableReferences, node.getCriteria(), column)); Map constants = new HashMap<>(); constants.putAll(probeProperties.getConstants()); @@ -428,21 +432,31 @@ public ActualProperties visitJoin(JoinNode node, List inputPro .unordered(unordered) .build(); case LEFT: - return ActualProperties.builderFrom(probeProperties.translate(column -> filterIfMissing(outputVariableReferences, column))) + return ActualProperties.builderFrom(probeProperties.translateVariable(column -> filterIfMissing(outputVariableReferences, column))) .unordered(unordered) .build(); case RIGHT: - buildProperties = buildProperties.translate(column -> filterIfMissing(node.getOutputVariables(), column)); + buildProperties = buildProperties.translateVariable(column -> filterIfMissing(node.getOutputVariables(), column)); - return ActualProperties.builderFrom(buildProperties.translate(column -> filterIfMissing(outputVariableReferences, column))) + return ActualProperties.builderFrom(buildProperties.translateVariable(column -> filterIfMissing(outputVariableReferences, column))) .local(ImmutableList.of()) .unordered(true) .build(); case FULL: - // We can't say anything about the partitioning scheme because any partition of - // a hash-partitioned join can produce nulls in case of a lack of matches + if (probeProperties.isSingleNode()) { + return ActualProperties.builder() + .global(singleStreamPartition()) + .build(); + } + + if (isOptimizeFullOuterJoinWithCoalesce(session) && probeProperties.getNodePartitioning().isPresent() && buildProperties.getNodePartitioning().isPresent()) { + return ActualProperties.builder() + .global(partitionedOnCoalesce(probeProperties.getNodePartitioning().get(), buildProperties.getNodePartitioning().get())) + .build(); + } + return ActualProperties.builder() - .global(probeProperties.isSingleNode() ? singleStreamPartition() : arbitraryPartition()) + .global(arbitraryPartition()) .build(); default: throw new UnsupportedOperationException("Unsupported join type: " + node.getType()); @@ -464,8 +478,8 @@ public ActualProperties visitSpatialJoin(SpatialJoinNode node, List filterIfMissing(outputs, column)); - buildProperties = buildProperties.translate(column -> filterIfMissing(outputs, column)); + probeProperties = probeProperties.translateVariable(column -> filterIfMissing(outputs, column)); + buildProperties = buildProperties.translateVariable(column -> filterIfMissing(outputs, column)); Map constants = new HashMap<>(); constants.putAll(probeProperties.getConstants()); @@ -475,7 +489,7 @@ public ActualProperties visitSpatialJoin(SpatialJoinNode node, List filterIfMissing(outputs, column))) + return ActualProperties.builderFrom(probeProperties.translateVariable(column -> filterIfMissing(outputs, column))) .build(); default: throw new IllegalArgumentException("Unsupported spatial join type: " + node.getType()); @@ -527,7 +541,7 @@ public ActualProperties visitExchange(ExchangeNode node, List inputToOutput.put(inputVariables.get(i), node.getOutputVariables().get(i)); } - ActualProperties translated = inputProperties.get(sourceIndex).translate(variable -> Optional.ofNullable(inputToOutput.get(variable))); + ActualProperties translated = inputProperties.get(sourceIndex).translateVariable(variable -> Optional.ofNullable(inputToOutput.get(variable))); entries = (entries == null) ? translated.getConstants().entrySet() : Sets.intersection(entries, translated.getConstants().entrySet()); } @@ -538,7 +552,7 @@ public ActualProperties visitExchange(ExchangeNode node, List ImmutableList.Builder> localProperties = ImmutableList.builder(); if (node.getOrderingScheme().isPresent()) { - node.getOrderingScheme().get().getOrderBy().stream() + node.getOrderingScheme().get().getOrderByVariables().stream() .map(column -> new SortingProperty<>(column, node.getOrderingScheme().get().getOrdering(column))) .forEach(localProperties::add); } @@ -621,9 +635,7 @@ public ActualProperties visitProject(ProjectNode node, List in { ActualProperties properties = Iterables.getOnlyElement(inputProperties); - Map identities = computeIdentityTranslations(node.getAssignments().getMap(), types); - - ActualProperties translatedProperties = properties.translate(column -> Optional.ofNullable(identities.get(column))); + ActualProperties translatedProperties = properties.translateRowExpression(node.getAssignments().getMap(), types); // Extract additional constants Map constants = new HashMap<>(); @@ -640,7 +652,7 @@ public ActualProperties visitProject(ProjectNode node, List in Map, Type> expressionTypes = getExpressionTypes(session, metadata, parser, types, castToExpression(expression), emptyList(), WarningCollector.NOOP); Type type = requireNonNull(expressionTypes.get(NodeRef.of(castToExpression(expression)))); ExpressionInterpreter optimizer = ExpressionInterpreter.expressionOptimizer(castToExpression(expression), metadata, session, expressionTypes); - Object value = optimizer.optimize(NoOpSymbolResolver.INSTANCE); + Object value = optimizer.optimize(NoOpVariableResolver.INSTANCE); if (value instanceof SymbolReference) { VariableReferenceExpression variable = toVariableReference((SymbolReference) value, types); @@ -654,7 +666,7 @@ else if (!(value instanceof Expression)) { } } else { - Object value = new RowExpressionInterpreter(expression, metadata, session.toConnectorSession(), true).optimize(); + Object value = new RowExpressionInterpreter(expression, metadata, session.toConnectorSession(), OPTIMIZED).optimize(); if (value instanceof VariableReferenceExpression) { ConstantExpression existingConstantValue = constants.get(value); @@ -689,6 +701,12 @@ public ActualProperties visitTableWriter(TableWriterNode node, List inputProperties) + { + return Iterables.getOnlyElement(inputProperties); + } + @Override public ActualProperties visitSample(SampleNode node, List inputProperties) { @@ -700,7 +718,7 @@ public ActualProperties visitUnnest(UnnestNode node, List inpu { Set passThroughInputs = ImmutableSet.copyOf(node.getReplicateVariables()); - return Iterables.getOnlyElement(inputProperties).translate(column -> { + return Iterables.getOnlyElement(inputProperties).translateVariable(column -> { if (passThroughInputs.contains(column)) { return Optional.of(column); } @@ -756,9 +774,14 @@ private Global deriveGlobalProperties(TableLayout layout, Map arguments = tablePartitioning.getPartitioningColumns().stream() - .map(assignments::get) + + Set assignmentsAndConstants = ImmutableSet.builder() + .addAll(assignments.keySet()) + .addAll(constants.keySet()) + .build(); + if (assignmentsAndConstants.containsAll(tablePartitioning.getPartitioningColumns())) { + List arguments = tablePartitioning.getPartitioningColumns().stream() + .map(column -> assignments.containsKey(column) ? assignments.get(column) : constants.get(column)) .collect(toImmutableList()); return partitionedOn(tablePartitioning.getPartitioningHandle(), arguments, streamPartitioning); @@ -792,25 +815,6 @@ private static Optional> translateToNonConstan return Optional.of(ImmutableList.copyOf(builder.build())); } - - private static Map computeIdentityTranslations(Map assignments, TypeProvider types) - { - Map inputToOutput = new HashMap<>(); - for (Map.Entry assignment : assignments.entrySet()) { - RowExpression expression = assignment.getValue(); - if (isExpression(expression)) { - if (castToExpression(expression) instanceof SymbolReference) { - inputToOutput.put(toVariableReference(castToExpression(expression), types), assignment.getKey()); - } - } - else { - if (expression instanceof VariableReferenceExpression) { - inputToOutput.put((VariableReferenceExpression) expression, assignment.getKey()); - } - } - } - return inputToOutput; - } } static boolean spillPossible(Session session, JoinNode.Type joinType) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java index f58424bf09f07..8cd4be7ec6aac 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java @@ -16,10 +16,20 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.SetOperationNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -27,42 +37,32 @@ import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.VariablesExtractor; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; -import com.facebook.presto.sql.planner.plan.ExceptNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; import com.facebook.presto.sql.planner.plan.GroupIdNode; import com.facebook.presto.sql.planner.plan.IndexJoinNode; import com.facebook.presto.sql.planner.plan.IndexSourceNode; -import com.facebook.presto.sql.planner.plan.IntersectNode; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; -import com.facebook.presto.sql.planner.plan.SetOperationNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; -import com.facebook.presto.sql.tree.Expression; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; @@ -84,8 +84,8 @@ import static com.facebook.presto.sql.planner.optimizations.AggregationNodeUtils.extractAggregationUniqueVariables; import static com.facebook.presto.sql.planner.optimizations.ApplyNodeUtil.verifySubquerySupported; import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isScalar; +import static com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils.fromListMultimap; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; -import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.isExpression; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; @@ -144,7 +144,7 @@ public PlanNode visitExchange(ExchangeNode node, RewriteContext expectedOutputVariables.addAll(orderingScheme.getOrderBy())); + node.getOrderingScheme().ifPresent(orderingScheme -> expectedOutputVariables.addAll(orderingScheme.getOrderByVariables())); List> inputsBySource = new ArrayList<>(node.getInputs().size()); for (int i = 0; i < node.getInputs().size(); i++) { @@ -187,6 +187,7 @@ public PlanNode visitExchange(ExchangeNode node, RewriteContext expectedFilterInputs = new HashSet<>(); if (node.getFilter().isPresent()) { - expectedFilterInputs = ImmutableSet.builder() - .addAll(VariablesExtractor.extractUnique(castToExpression(node.getFilter().get()), variableAllocator.getTypes())) - .addAll(context.get()) - .build(); + if (isExpression(node.getFilter().get())) { + expectedFilterInputs = ImmutableSet.builder() + .addAll(VariablesExtractor.extractUnique(castToExpression(node.getFilter().get()), variableAllocator.getTypes())) + .addAll(context.get()) + .build(); + } + else { + expectedFilterInputs = ImmutableSet.builder() + .addAll(VariablesExtractor.extractUnique(node.getFilter().get())) + .addAll(context.get()) + .build(); + } } ImmutableSet.Builder leftInputsBuilder = ImmutableSet.builder(); @@ -385,7 +394,7 @@ public PlanNode visitWindow(WindowNode node, RewriteContext - orderingScheme.getOrderBy() + orderingScheme.getOrderByVariables() .forEach(expectedInputs::add)); for (WindowNode.Frame frame : node.getFrames()) { @@ -451,10 +460,19 @@ public PlanNode visitTableScan(TableScanNode node, RewriteContext> context) { - Set expectedInputs = ImmutableSet.builder() - .addAll(VariablesExtractor.extractUnique(castToExpression(node.getPredicate()), variableAllocator.getTypes())) - .addAll(context.get()) - .build(); + Set expectedInputs; + if (isExpression(node.getPredicate())) { + expectedInputs = ImmutableSet.builder() + .addAll(VariablesExtractor.extractUnique(castToExpression(node.getPredicate()), variableAllocator.getTypes())) + .addAll(context.get()) + .build(); + } + else { + expectedInputs = ImmutableSet.builder() + .addAll(VariablesExtractor.extractUnique(node.getPredicate())) + .addAll(context.get()) + .build(); + } PlanNode source = context.rewrite(node.getSource(), expectedInputs); @@ -540,7 +558,12 @@ public PlanNode visitProject(ProjectNode node, RewriteContext { if (context.get().contains(variable)) { - expectedInputs.addAll(VariablesExtractor.extractUnique(castToExpression(expression), variableAllocator.getTypes())); + if (isExpression(expression)) { + expectedInputs.addAll(VariablesExtractor.extractUnique(castToExpression(expression), variableAllocator.getTypes())); + } + else { + expectedInputs.addAll(VariablesExtractor.extractUnique(expression)); + } builder.put(variable, expression); } }); @@ -564,7 +587,7 @@ public PlanNode visitLimit(LimitNode node, RewriteContext expectedInputs = ImmutableSet.builder() .addAll(context.get()); PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); - return new LimitNode(node.getId(), source, node.getCount(), node.isPartial()); + return new LimitNode(node.getId(), source, node.getCount(), node.getStep()); } @Override @@ -586,7 +609,7 @@ public PlanNode visitTopN(TopNNode node, RewriteContext expectedInputs = ImmutableSet.builder() .addAll(context.get()) - .addAll(node.getOrderingScheme().getOrderBy()); + .addAll(node.getOrderingScheme().getOrderByVariables()); PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); @@ -615,7 +638,7 @@ public PlanNode visitTopNRowNumber(TopNRowNumberNode node, RewriteContext expectedInputs = ImmutableSet.builder() .addAll(context.get()) .addAll(node.getPartitionBy()) - .addAll(node.getOrderingScheme().getOrderBy()); + .addAll(node.getOrderingScheme().getOrderByVariables()); if (node.getHashVariable().isPresent()) { expectedInputs.add(node.getHashVariable().get()); @@ -634,11 +657,11 @@ public PlanNode visitTopNRowNumber(TopNRowNumberNode node, RewriteContext> context) { - Set expectedInputs = ImmutableSet.copyOf(concat(context.get(), node.getOrderingScheme().getOrderBy())); + Set expectedInputs = ImmutableSet.copyOf(concat(context.get(), node.getOrderingScheme().getOrderByVariables())); PlanNode source = context.rewrite(node.getSource(), expectedInputs); - return new SortNode(node.getId(), source, node.getOrderingScheme()); + return new SortNode(node.getId(), source, node.getOrderingScheme(), node.isPartial()); } @Override @@ -669,8 +692,20 @@ public PlanNode visitTableWriter(TableWriterNode node, RewriteContext> context) + { + PlanNode source = context.rewrite(node.getSource(), ImmutableSet.copyOf(node.getSource().getOutputVariables())); + return new TableWriterMergeNode( + node.getId(), + source, + node.getRowCountVariable(), + node.getFragmentVariable(), + node.getTableCommitContextVariable(), + node.getStatisticsAggregation()); } @Override @@ -680,7 +715,7 @@ public PlanNode visitStatisticsWriterNode(StatisticsWriterNode node, RewriteCont return new StatisticsWriterNode( node.getId(), source, - node.getTarget(), + node.getTableHandle(), node.getRowCountVariable(), node.isRowCountEnabled(), node.getDescriptor()); @@ -703,7 +738,7 @@ public PlanNode visitTableFinish(TableFinishNode node, RewriteContext> context) { PlanNode source = context.rewrite(node.getSource(), ImmutableSet.of(node.getRowId())); - return new DeleteNode(node.getId(), source, node.getTarget(), node.getRowId(), node.getOutputVariables()); + return new DeleteNode(node.getId(), source, node.getRowId(), node.getOutputVariables()); } @Override @@ -711,7 +746,7 @@ public PlanNode visitUnion(UnionNode node, RewriteContext rewrittenVariableMapping = rewriteSetOperationVariableMapping(node, context); ImmutableList rewrittenSubPlans = rewriteSetOperationSubPlans(node, context, rewrittenVariableMapping); - return new UnionNode(node.getId(), rewrittenSubPlans, rewrittenVariableMapping); + return new UnionNode(node.getId(), rewrittenSubPlans, ImmutableList.copyOf(rewrittenVariableMapping.keySet()), fromListMultimap(rewrittenVariableMapping)); } @Override @@ -719,7 +754,7 @@ public PlanNode visitIntersect(IntersectNode node, RewriteContext rewrittenVariableMapping = rewriteSetOperationVariableMapping(node, context); ImmutableList rewrittenSubPlans = rewriteSetOperationSubPlans(node, context, rewrittenVariableMapping); - return new IntersectNode(node.getId(), rewrittenSubPlans, rewrittenVariableMapping); + return new IntersectNode(node.getId(), rewrittenSubPlans, ImmutableList.copyOf(rewrittenVariableMapping.keySet()), fromListMultimap(rewrittenVariableMapping)); } @Override @@ -727,7 +762,7 @@ public PlanNode visitExcept(ExceptNode node, RewriteContext rewrittenVariableMapping = rewriteSetOperationVariableMapping(node, context); ImmutableList rewrittenSubPlans = rewriteSetOperationSubPlans(node, context, rewrittenVariableMapping); - return new ExceptNode(node.getId(), rewrittenSubPlans, rewrittenVariableMapping); + return new ExceptNode(node.getId(), rewrittenSubPlans, ImmutableList.copyOf(rewrittenVariableMapping.keySet()), fromListMultimap(rewrittenVariableMapping)); } private ListMultimap rewriteSetOperationVariableMapping(SetOperationNode node, RewriteContext> context) @@ -798,10 +833,15 @@ public PlanNode visitApply(ApplyNode node, RewriteContext entry : node.getSubqueryAssignments().getMap().entrySet()) { VariableReferenceExpression output = entry.getKey(); - Expression expression = castToExpression(entry.getValue()); + RowExpression expression = entry.getValue(); if (context.get().contains(output)) { - subqueryAssignmentsVariablesBuilder.addAll(VariablesExtractor.extractUnique(expression, variableAllocator.getTypes())); - subqueryAssignments.put(output, castToRowExpression(expression)); + if (isExpression(expression)) { + subqueryAssignmentsVariablesBuilder.addAll(VariablesExtractor.extractUnique(castToExpression(expression), variableAllocator.getTypes())); + } + else { + subqueryAssignmentsVariablesBuilder.addAll(VariablesExtractor.extractUnique(expression)); + } + subqueryAssignments.put(output, expression); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java index dc240dbb6c76e..bc88c3b329c72 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java @@ -20,20 +20,24 @@ import com.facebook.presto.spi.Subfield; import com.facebook.presto.spi.Subfield.NestedField; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.TypeSignature; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; @@ -42,16 +46,13 @@ import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.relational.OriginalExpressionUtils; @@ -71,11 +72,14 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; +import static com.facebook.presto.SystemSessionProperties.isLegacyUnnest; import static com.facebook.presto.SystemSessionProperties.isPushdownSubfieldsEnabled; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.spi.Subfield.allSubscripts; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; @@ -114,6 +118,7 @@ private static class Rewriter private final Metadata metadata; private final TypeProvider types; private final SubfieldExtractor subfieldExtractor; + private static final QualifiedFunctionName ARBITRARY_AGGREGATE_FUNCTION = QualifiedFunctionName.of(DEFAULT_NAMESPACE, "arbitrary"); public Rewriter(Session session, Metadata metadata, TypeProvider types) { @@ -128,13 +133,24 @@ public PlanNode visitAggregation(AggregationNode node, RewriteContext c { context.get().variables.addAll(node.getGroupingKeys()); - for (AggregationNode.Aggregation aggregation : node.getAggregations().values()) { - aggregation.getArguments().forEach(expression -> subfieldExtractor.process(castToExpression(expression), context.get())); + for (Map.Entry entry : node.getAggregations().entrySet()) { + VariableReferenceExpression variable = entry.getKey(); + AggregationNode.Aggregation aggregation = entry.getValue(); + + // Allow sub-field pruning to pass through the arbitrary() aggregation + QualifiedFunctionName aggregateName = metadata.getFunctionManager().getFunctionMetadata(aggregation.getCall().getFunctionHandle()).getName(); + if (ARBITRARY_AGGREGATE_FUNCTION.equals(aggregateName)) { + SymbolReference argument = (SymbolReference) castToExpression(aggregation.getArguments().get(0)); + context.get().addAssignment(variable, new VariableReferenceExpression(argument.getName(), types.get(argument))); + } + else { + aggregation.getArguments().forEach(expression -> subfieldExtractor.process(castToExpression(expression), context.get())); + } aggregation.getFilter().ifPresent(expression -> subfieldExtractor.process(castToExpression(expression), context.get())); aggregation.getOrderBy() - .map(OrderingScheme::getOrderBy) + .map(OrderingScheme::getOrderByVariables) .ifPresent(context.get().variables::addAll); aggregation.getMask().ifPresent(context.get().variables::add); @@ -267,7 +283,7 @@ public PlanNode visitSemiJoin(SemiJoinNode node, RewriteContext context @Override public PlanNode visitSort(SortNode node, RewriteContext context) { - context.get().variables.addAll(node.getOrderingScheme().getOrderBy()); + context.get().variables.addAll(node.getOrderingScheme().getOrderByVariables()); return context.defaultRewrite(node, context.get()); } @@ -330,7 +346,7 @@ public PlanNode visitTableWriter(TableWriterNode node, RewriteContext c @Override public PlanNode visitTopN(TopNNode node, RewriteContext context) { - context.get().variables.addAll(node.getOrderingScheme().getOrderBy()); + context.get().variables.addAll(node.getOrderingScheme().getOrderByVariables()); return context.defaultRewrite(node, context.get()); } @@ -339,14 +355,14 @@ public PlanNode visitTopNRowNumber(TopNRowNumberNode node, RewriteContext context) { - for (Map.Entry> entry : node.getVariableMapping().asMap().entrySet()) { + for (Map.Entry> entry : node.getVariableMapping().entrySet()) { entry.getValue().forEach(variable -> context.get().addAssignment(entry.getKey(), variable)); } @@ -361,11 +377,11 @@ public PlanNode visitUnnest(UnnestNode node, RewriteContext context) VariableReferenceExpression container = entry.getKey(); boolean found = false; - if (isRowType(container)) { + if (isRowType(container) && !isLegacyUnnest(session)) { for (VariableReferenceExpression field : entry.getValue()) { if (context.get().variables.contains(field)) { found = true; - newSubfields.add(new Subfield(container.getName(), ImmutableList.of(allSubscripts(), new NestedField(field.getName())))); + newSubfields.add(new Subfield(container.getName(), ImmutableList.of(allSubscripts(), nestedField(field.getName())))); } else { List matchingSubfields = context.get().findSubfields(field.getName()); @@ -375,7 +391,7 @@ public PlanNode visitUnnest(UnnestNode node, RewriteContext context) .map(Subfield::getPath) .map(path -> new Subfield(container.getName(), ImmutableList.builder() .add(allSubscripts()) - .add(new NestedField(field.getName())) + .add(nestedField(field.getName())) .addAll(path) .build())) .forEach(newSubfields::add); @@ -420,7 +436,7 @@ public PlanNode visitWindow(WindowNode node, RewriteContext context) context.get().variables.addAll(node.getSpecification().getPartitionBy()); node.getSpecification().getOrderingScheme() - .map(OrderingScheme::getOrderBy) + .map(OrderingScheme::getOrderByVariables) .ifPresent(context.get().variables::addAll); node.getWindowFunctions().values().stream() @@ -472,7 +488,7 @@ private static Optional toSubfield(Node expression) if (expression instanceof DereferenceExpression) { DereferenceExpression dereference = (DereferenceExpression) expression; - elements.add(new NestedField(dereference.getField().getValue())); + elements.add(nestedField(dereference.getField().getValue())); expression = dereference.getBase(); } else if (expression instanceof SubscriptExpression) { @@ -507,6 +523,11 @@ else if (index instanceof GenericLiteral) { } } + private static NestedField nestedField(String name) + { + return new NestedField(name.toLowerCase(Locale.ENGLISH)); + } + private static final class SubfieldExtractor extends DefaultExpressionTraversalVisitor { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/QueryCardinalityUtil.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/QueryCardinalityUtil.java index 6ba4963b5359f..c1d0984ee9041 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/QueryCardinalityUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/QueryCardinalityUtil.java @@ -13,17 +13,17 @@ */ package com.facebook.presto.sql.planner.optimizations; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.iterative.GroupReference; import com.facebook.presto.sql.planner.iterative.Lookup; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; -import com.facebook.presto.sql.planner.plan.LimitNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.google.common.collect.Range; import static com.facebook.presto.sql.planner.iterative.Lookup.noLookup; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ReplicateSemiJoinInDelete.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ReplicateSemiJoinInDelete.java index 4f30810245670..128354d666eb7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ReplicateSemiJoinInDelete.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ReplicateSemiJoinInDelete.java @@ -75,7 +75,6 @@ public PlanNode visitDelete(DeleteNode node, RewriteContext context) return new DeleteNode( node.getId(), rewrittenSource, - node.getTarget(), node.getRowId(), node.getOutputVariables()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RowExpressionPredicatePushDown.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RowExpressionPredicatePushDown.java new file mode 100644 index 0000000000000..f6209736b9a9b --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/RowExpressionPredicatePushDown.java @@ -0,0 +1,1328 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.optimizations; + +import com.facebook.presto.Session; +import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.expressions.LogicalRowExpressions; +import com.facebook.presto.expressions.RowExpressionNodeInliner; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.UnionNode; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.planner.PlanVariableAllocator; +import com.facebook.presto.sql.planner.RowExpressionEqualityInference; +import com.facebook.presto.sql.planner.RowExpressionPredicateExtractor; +import com.facebook.presto.sql.planner.RowExpressionVariableInliner; +import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.planner.VariablesExtractor; +import com.facebook.presto.sql.planner.plan.AssignUniqueId; +import com.facebook.presto.sql.planner.plan.ExchangeNode; +import com.facebook.presto.sql.planner.plan.GroupIdNode; +import com.facebook.presto.sql.planner.plan.JoinNode; +import com.facebook.presto.sql.planner.plan.MarkDistinctNode; +import com.facebook.presto.sql.planner.plan.SampleNode; +import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; +import com.facebook.presto.sql.planner.plan.SortNode; +import com.facebook.presto.sql.planner.plan.SpatialJoinNode; +import com.facebook.presto.sql.planner.plan.UnnestNode; +import com.facebook.presto.sql.planner.plan.WindowNode; +import com.facebook.presto.sql.relational.Expressions; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static com.facebook.presto.expressions.LogicalRowExpressions.FALSE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.extractConjuncts; +import static com.facebook.presto.spi.function.OperatorType.EQUAL; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; +import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignments; +import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED; +import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; +import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; +import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; +import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; +import static com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.sql.relational.Expressions.constant; +import static com.facebook.presto.sql.relational.Expressions.constantNull; +import static com.facebook.presto.sql.relational.Expressions.uniqueSubExpressions; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Predicates.in; +import static com.google.common.base.Predicates.not; +import static com.google.common.base.Verify.verify; +import static com.google.common.collect.Iterables.filter; +import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; + +public class RowExpressionPredicatePushDown + implements PlanOptimizer +{ + private final Metadata metadata; + private final RowExpressionPredicateExtractor effectivePredicateExtractor; + private final SqlParser sqlParser; + + public RowExpressionPredicatePushDown(Metadata metadata, SqlParser sqlParser) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + this.effectivePredicateExtractor = new RowExpressionPredicateExtractor(new RowExpressionDomainTranslator(metadata), metadata.getFunctionManager(), metadata.getTypeManager()); + this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + } + + @Override + public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, PlanVariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) + { + requireNonNull(plan, "plan is null"); + requireNonNull(session, "session is null"); + requireNonNull(types, "types is null"); + requireNonNull(idAllocator, "idAllocator is null"); + + return SimplePlanRewriter.rewriteWith( + new Rewriter(variableAllocator, idAllocator, metadata, effectivePredicateExtractor, sqlParser, session), + plan, + TRUE_CONSTANT); + } + + private static class Rewriter + extends SimplePlanRewriter + { + private final PlanVariableAllocator variableAllocator; + private final PlanNodeIdAllocator idAllocator; + private final Metadata metadata; + private final RowExpressionPredicateExtractor effectivePredicateExtractor; + private final Session session; + private final ExpressionEquivalence expressionEquivalence; + private final RowExpressionDeterminismEvaluator determinismEvaluator; + private final LogicalRowExpressions logicalRowExpressions; + private final TypeManager typeManager; + private final FunctionManager functionManager; + + private Rewriter( + PlanVariableAllocator variableAllocator, + PlanNodeIdAllocator idAllocator, + Metadata metadata, + RowExpressionPredicateExtractor effectivePredicateExtractor, + SqlParser sqlParser, + Session session) + { + this.variableAllocator = requireNonNull(variableAllocator, "variableAllocator is null"); + this.idAllocator = requireNonNull(idAllocator, "idAllocator is null"); + this.metadata = requireNonNull(metadata, "metadata is null"); + this.effectivePredicateExtractor = requireNonNull(effectivePredicateExtractor, "effectivePredicateExtractor is null"); + this.session = requireNonNull(session, "session is null"); + this.expressionEquivalence = new ExpressionEquivalence(metadata, sqlParser); + this.determinismEvaluator = new RowExpressionDeterminismEvaluator(metadata); + this.logicalRowExpressions = new LogicalRowExpressions(determinismEvaluator, new FunctionResolution(metadata.getFunctionManager()), metadata.getFunctionManager()); + this.typeManager = metadata.getTypeManager(); + this.functionManager = metadata.getFunctionManager(); + } + + @Override + public PlanNode visitPlan(PlanNode node, RewriteContext context) + { + PlanNode rewrittenNode = context.defaultRewrite(node, TRUE_CONSTANT); + if (!context.get().equals(TRUE_CONSTANT)) { + // Drop in a FilterNode b/c we cannot push our predicate down any further + rewrittenNode = new FilterNode(idAllocator.getNextId(), rewrittenNode, context.get()); + } + return rewrittenNode; + } + + @Override + public PlanNode visitExchange(ExchangeNode node, RewriteContext context) + { + boolean modified = false; + ImmutableList.Builder builder = ImmutableList.builder(); + for (int i = 0; i < node.getSources().size(); i++) { + Map outputsToInputs = new HashMap<>(); + for (int index = 0; index < node.getInputs().get(i).size(); index++) { + outputsToInputs.put( + node.getOutputVariables().get(index), + node.getInputs().get(i).get(index)); + } + + RowExpression sourcePredicate = RowExpressionVariableInliner.inlineVariables(outputsToInputs, context.get()); + PlanNode source = node.getSources().get(i); + PlanNode rewrittenSource = context.rewrite(source, sourcePredicate); + if (rewrittenSource != source) { + modified = true; + } + builder.add(rewrittenSource); + } + + if (modified) { + return new ExchangeNode( + node.getId(), + node.getType(), + node.getScope(), + node.getPartitioningScheme(), + builder.build(), + node.getInputs(), + node.isEnsureSourceOrdering(), + node.getOrderingScheme()); + } + + return node; + } + + @Override + public PlanNode visitWindow(WindowNode node, RewriteContext context) + { + // TODO: This could be broader. We can push down conjucts if they are constant for all rows in a window partition. + // The simplest way to guarantee this is if the conjucts are deterministic functions of the partitioning variables. + // This can leave out cases where they're both functions of some set of common expressions and the partitioning + // function is injective, but that's a rare case. The majority of window nodes are expected to be partitioned by + // pre-projected variables. + Predicate isSupported = conjunct -> + determinismEvaluator.isDeterministic(conjunct) && + VariablesExtractor.extractUnique(conjunct).stream().allMatch(node.getPartitionBy()::contains); + + Map> conjuncts = extractConjuncts(context.get()).stream().collect(Collectors.partitioningBy(isSupported)); + + PlanNode rewrittenNode = context.defaultRewrite(node, logicalRowExpressions.combineConjuncts(conjuncts.get(true))); + + if (!conjuncts.get(false).isEmpty()) { + rewrittenNode = new FilterNode(idAllocator.getNextId(), rewrittenNode, logicalRowExpressions.combineConjuncts(conjuncts.get(false))); + } + + return rewrittenNode; + } + + @Override + public PlanNode visitProject(ProjectNode node, RewriteContext context) + { + Set deterministicVariables = node.getAssignments().entrySet().stream() + .filter(entry -> determinismEvaluator.isDeterministic(entry.getValue())) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + + Predicate deterministic = conjunct -> deterministicVariables.containsAll(VariablesExtractor.extractUnique(conjunct)); + + Map> conjuncts = extractConjuncts(context.get()).stream().collect(Collectors.partitioningBy(deterministic)); + + // Push down conjuncts from the inherited predicate that only depend on deterministic assignments with + // certain limitations. + List deterministicConjuncts = conjuncts.get(true); + + // We partition the expressions in the deterministicConjuncts into two lists, and only inline the + // expressions that are in the inlining targets list. + Map> inlineConjuncts = deterministicConjuncts.stream() + .collect(Collectors.partitioningBy(expression -> isInliningCandidate(expression, node))); + + List inlinedDeterministicConjuncts = inlineConjuncts.get(true).stream() + .map(entry -> RowExpressionVariableInliner.inlineVariables(node.getAssignments().getMap(), entry)) + .collect(Collectors.toList()); + + PlanNode rewrittenNode = context.defaultRewrite(node, logicalRowExpressions.combineConjuncts(inlinedDeterministicConjuncts)); + + // All deterministic conjuncts that contains non-inlining targets, and non-deterministic conjuncts, + // if any, will be in the filter node. + List nonInliningConjuncts = inlineConjuncts.get(false); + nonInliningConjuncts.addAll(conjuncts.get(false)); + + if (!nonInliningConjuncts.isEmpty()) { + rewrittenNode = new FilterNode(idAllocator.getNextId(), rewrittenNode, logicalRowExpressions.combineConjuncts(nonInliningConjuncts)); + } + + return rewrittenNode; + } + + private boolean isInliningCandidate(RowExpression expression, ProjectNode node) + { + // TryExpressions should not be pushed down. However they are now being handled as lambda + // passed to a FunctionCall now and should not affect predicate push down. So we want to make + // sure the conjuncts are not TryExpressions. + FunctionResolution functionResolution = new FunctionResolution(functionManager); + verify(uniqueSubExpressions(expression) + .stream() + .noneMatch(subExpression -> subExpression instanceof CallExpression && + functionResolution.isTryFunction(((CallExpression) subExpression).getFunctionHandle()))); + + // candidate symbols for inlining are + // 1. references to simple constants + // 2. references to complex expressions that appear only once + // which come from the node, as opposed to an enclosing scope. + Set childOutputSet = ImmutableSet.copyOf(node.getOutputVariables()); + Map dependencies = VariablesExtractor.extractAll(expression).stream() + .filter(childOutputSet::contains) + .collect(Collectors.groupingBy(identity(), Collectors.counting())); + + return dependencies.entrySet().stream() + .allMatch(entry -> entry.getValue() == 1 || node.getAssignments().get(entry.getKey()) instanceof ConstantExpression); + } + + @Override + public PlanNode visitGroupId(GroupIdNode node, RewriteContext context) + { + Map commonGroupingVariableMapping = node.getGroupingColumns().entrySet().stream() + .filter(entry -> node.getCommonGroupingColumns().contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Predicate pushdownEligiblePredicate = conjunct -> VariablesExtractor.extractUnique(conjunct).stream() + .allMatch(commonGroupingVariableMapping.keySet()::contains); + + Map> conjuncts = extractConjuncts(context.get()).stream().collect(Collectors.partitioningBy(pushdownEligiblePredicate)); + + // Push down conjuncts from the inherited predicate that apply to common grouping symbols + PlanNode rewrittenNode = context.defaultRewrite(node, RowExpressionVariableInliner.inlineVariables(commonGroupingVariableMapping, logicalRowExpressions.combineConjuncts(conjuncts.get(true)))); + + // All other conjuncts, if any, will be in the filter node. + if (!conjuncts.get(false).isEmpty()) { + rewrittenNode = new FilterNode(idAllocator.getNextId(), rewrittenNode, logicalRowExpressions.combineConjuncts(conjuncts.get(false))); + } + + return rewrittenNode; + } + + @Override + public PlanNode visitMarkDistinct(MarkDistinctNode node, RewriteContext context) + { + Set pushDownableVariables = ImmutableSet.copyOf(node.getDistinctVariables()); + Map> conjuncts = extractConjuncts(context.get()).stream() + .collect(Collectors.partitioningBy(conjunct -> pushDownableVariables.containsAll(VariablesExtractor.extractUnique(conjunct)))); + + PlanNode rewrittenNode = context.defaultRewrite(node, logicalRowExpressions.combineConjuncts(conjuncts.get(true))); + + if (!conjuncts.get(false).isEmpty()) { + rewrittenNode = new FilterNode(idAllocator.getNextId(), rewrittenNode, logicalRowExpressions.combineConjuncts(conjuncts.get(false))); + } + return rewrittenNode; + } + + @Override + public PlanNode visitSort(SortNode node, RewriteContext context) + { + return context.defaultRewrite(node, context.get()); + } + + @Override + public PlanNode visitUnion(UnionNode node, RewriteContext context) + { + boolean modified = false; + ImmutableList.Builder builder = ImmutableList.builder(); + for (int i = 0; i < node.getSources().size(); i++) { + RowExpression sourcePredicate = RowExpressionVariableInliner.inlineVariables(node.sourceVariableMap(i), context.get()); + PlanNode source = node.getSources().get(i); + PlanNode rewrittenSource = context.rewrite(source, sourcePredicate); + if (rewrittenSource != source) { + modified = true; + } + builder.add(rewrittenSource); + } + + if (modified) { + return new UnionNode(node.getId(), builder.build(), node.getOutputVariables(), node.getVariableMapping()); + } + + return node; + } + + @Deprecated + @Override + public PlanNode visitFilter(FilterNode node, RewriteContext context) + { + PlanNode rewrittenPlan = context.rewrite(node.getSource(), logicalRowExpressions.combineConjuncts(node.getPredicate(), context.get())); + if (!(rewrittenPlan instanceof FilterNode)) { + return rewrittenPlan; + } + + FilterNode rewrittenFilterNode = (FilterNode) rewrittenPlan; + if (!areExpressionsEquivalent(rewrittenFilterNode.getPredicate(), node.getPredicate()) + || node.getSource() != rewrittenFilterNode.getSource()) { + return rewrittenPlan; + } + + return node; + } + + @Override + public PlanNode visitJoin(JoinNode node, RewriteContext context) + { + RowExpression inheritedPredicate = context.get(); + + // See if we can rewrite outer joins in terms of a plain inner join + node = tryNormalizeToOuterToInnerJoin(node, inheritedPredicate); + + RowExpression leftEffectivePredicate = effectivePredicateExtractor.extract(node.getLeft()); + RowExpression rightEffectivePredicate = effectivePredicateExtractor.extract(node.getRight()); + RowExpression joinPredicate = extractJoinPredicate(node); + + RowExpression leftPredicate; + RowExpression rightPredicate; + RowExpression postJoinPredicate; + RowExpression newJoinPredicate; + + switch (node.getType()) { + case INNER: + InnerJoinPushDownResult innerJoinPushDownResult = processInnerJoin(inheritedPredicate, + leftEffectivePredicate, + rightEffectivePredicate, + joinPredicate, + node.getLeft().getOutputVariables()); + leftPredicate = innerJoinPushDownResult.getLeftPredicate(); + rightPredicate = innerJoinPushDownResult.getRightPredicate(); + postJoinPredicate = innerJoinPushDownResult.getPostJoinPredicate(); + newJoinPredicate = innerJoinPushDownResult.getJoinPredicate(); + break; + case LEFT: + OuterJoinPushDownResult leftOuterJoinPushDownResult = processLimitedOuterJoin(inheritedPredicate, + leftEffectivePredicate, + rightEffectivePredicate, + joinPredicate, + node.getLeft().getOutputVariables()); + leftPredicate = leftOuterJoinPushDownResult.getOuterJoinPredicate(); + rightPredicate = leftOuterJoinPushDownResult.getInnerJoinPredicate(); + postJoinPredicate = leftOuterJoinPushDownResult.getPostJoinPredicate(); + newJoinPredicate = leftOuterJoinPushDownResult.getJoinPredicate(); + break; + case RIGHT: + OuterJoinPushDownResult rightOuterJoinPushDownResult = processLimitedOuterJoin(inheritedPredicate, + rightEffectivePredicate, + leftEffectivePredicate, + joinPredicate, + node.getRight().getOutputVariables()); + leftPredicate = rightOuterJoinPushDownResult.getInnerJoinPredicate(); + rightPredicate = rightOuterJoinPushDownResult.getOuterJoinPredicate(); + postJoinPredicate = rightOuterJoinPushDownResult.getPostJoinPredicate(); + newJoinPredicate = rightOuterJoinPushDownResult.getJoinPredicate(); + break; + case FULL: + leftPredicate = TRUE_CONSTANT; + rightPredicate = TRUE_CONSTANT; + postJoinPredicate = inheritedPredicate; + newJoinPredicate = joinPredicate; + break; + default: + throw new UnsupportedOperationException("Unsupported join type: " + node.getType()); + } + + newJoinPredicate = simplifyExpression(newJoinPredicate); + // TODO: find a better way to directly optimize FALSE LITERAL in join predicate + if (newJoinPredicate.equals(FALSE_CONSTANT)) { + newJoinPredicate = buildEqualsExpression(functionManager, constant(0L, BIGINT), constant(1L, BIGINT)); + } + + PlanNode leftSource = context.rewrite(node.getLeft(), leftPredicate); + PlanNode rightSource = context.rewrite(node.getRight(), rightPredicate); + + PlanNode output = node; + + // Create identity projections for all existing symbols + Assignments.Builder leftProjections = Assignments.builder() + .putAll(identityAssignments(node.getLeft().getOutputVariables())); + + Assignments.Builder rightProjections = Assignments.builder() + .putAll(identityAssignments(node.getRight().getOutputVariables())); + + // Create new projections for the new join clauses + List equiJoinClauses = new ArrayList<>(); + ImmutableList.Builder joinFilterBuilder = ImmutableList.builder(); + for (RowExpression conjunct : extractConjuncts(newJoinPredicate)) { + if (joinEqualityExpression(node.getLeft().getOutputVariables()).test(conjunct)) { + boolean alignedComparison = Iterables.all(VariablesExtractor.extractUnique(getLeft(conjunct)), in(node.getLeft().getOutputVariables())); + RowExpression leftExpression = (alignedComparison) ? getLeft(conjunct) : getRight(conjunct); + RowExpression rightExpression = (alignedComparison) ? getRight(conjunct) : getLeft(conjunct); + + VariableReferenceExpression leftVariable = variableForExpression(leftExpression); + if (!node.getLeft().getOutputVariables().contains(leftVariable)) { + leftProjections.put(leftVariable, leftExpression); + } + + VariableReferenceExpression rightVariable = variableForExpression(rightExpression); + if (!node.getRight().getOutputVariables().contains(rightVariable)) { + rightProjections.put(rightVariable, rightExpression); + } + + equiJoinClauses.add(new JoinNode.EquiJoinClause(leftVariable, rightVariable)); + } + else { + joinFilterBuilder.add(conjunct); + } + } + + Optional newJoinFilter = Optional.of(logicalRowExpressions.combineConjuncts(joinFilterBuilder.build())); + if (newJoinFilter.get() == TRUE_CONSTANT) { + newJoinFilter = Optional.empty(); + } + + if (node.getType() == INNER && newJoinFilter.isPresent() && equiJoinClauses.isEmpty()) { + // if we do not have any equi conjunct we do not pushdown non-equality condition into + // inner join, so we plan execution as nested-loops-join followed by filter instead + // hash join. + // todo: remove the code when we have support for filter function in nested loop join + postJoinPredicate = logicalRowExpressions.combineConjuncts(postJoinPredicate, newJoinFilter.get()); + newJoinFilter = Optional.empty(); + } + + boolean filtersEquivalent = + newJoinFilter.isPresent() == node.getFilter().isPresent() && + (!newJoinFilter.isPresent() || areExpressionsEquivalent(newJoinFilter.get(), node.getFilter().get())); + + if (leftSource != node.getLeft() || + rightSource != node.getRight() || + !filtersEquivalent || + !ImmutableSet.copyOf(equiJoinClauses).equals(ImmutableSet.copyOf(node.getCriteria()))) { + leftSource = new ProjectNode(idAllocator.getNextId(), leftSource, leftProjections.build()); + rightSource = new ProjectNode(idAllocator.getNextId(), rightSource, rightProjections.build()); + + // if the distribution type is already set, make sure that changes from PredicatePushDown + // don't make the join node invalid. + Optional distributionType = node.getDistributionType(); + if (node.getDistributionType().isPresent()) { + if (node.getType().mustPartition()) { + distributionType = Optional.of(PARTITIONED); + } + if (node.getType().mustReplicate(equiJoinClauses)) { + distributionType = Optional.of(REPLICATED); + } + } + + output = new JoinNode( + node.getId(), + node.getType(), + leftSource, + rightSource, + equiJoinClauses, + ImmutableList.builder() + .addAll(leftSource.getOutputVariables()) + .addAll(rightSource.getOutputVariables()) + .build(), + newJoinFilter, + node.getLeftHashVariable(), + node.getRightHashVariable(), + distributionType); + } + + if (!postJoinPredicate.equals(TRUE_CONSTANT)) { + output = new FilterNode(idAllocator.getNextId(), output, postJoinPredicate); + } + + if (!node.getOutputVariables().equals(output.getOutputVariables())) { + output = new ProjectNode(idAllocator.getNextId(), output, identityAssignments(node.getOutputVariables())); + } + + return output; + } + + private static RowExpression getLeft(RowExpression expression) + { + checkArgument(expression instanceof CallExpression && ((CallExpression) expression).getArguments().size() == 2, "must be binary call expression"); + return ((CallExpression) expression).getArguments().get(0); + } + + private static RowExpression getRight(RowExpression expression) + { + checkArgument(expression instanceof CallExpression && ((CallExpression) expression).getArguments().size() == 2, "must be binary call expression"); + return ((CallExpression) expression).getArguments().get(1); + } + + @Override + public PlanNode visitSpatialJoin(SpatialJoinNode node, RewriteContext context) + { + RowExpression inheritedPredicate = context.get(); + + // See if we can rewrite left join in terms of a plain inner join + if (node.getType() == SpatialJoinNode.Type.LEFT && canConvertOuterToInner(node.getRight().getOutputVariables(), inheritedPredicate)) { + node = new SpatialJoinNode( + node.getId(), + SpatialJoinNode.Type.INNER, + node.getLeft(), + node.getRight(), + node.getOutputVariables(), + node.getFilter(), + node.getLeftPartitionVariable(), + node.getRightPartitionVariable(), + node.getKdbTree()); + } + + RowExpression leftEffectivePredicate = effectivePredicateExtractor.extract(node.getLeft()); + RowExpression rightEffectivePredicate = effectivePredicateExtractor.extract(node.getRight()); + RowExpression joinPredicate = node.getFilter(); + + RowExpression leftPredicate; + RowExpression rightPredicate; + RowExpression postJoinPredicate; + RowExpression newJoinPredicate; + + switch (node.getType()) { + case INNER: + InnerJoinPushDownResult innerJoinPushDownResult = processInnerJoin( + inheritedPredicate, + leftEffectivePredicate, + rightEffectivePredicate, + joinPredicate, + node.getLeft().getOutputVariables()); + leftPredicate = innerJoinPushDownResult.getLeftPredicate(); + rightPredicate = innerJoinPushDownResult.getRightPredicate(); + postJoinPredicate = innerJoinPushDownResult.getPostJoinPredicate(); + newJoinPredicate = innerJoinPushDownResult.getJoinPredicate(); + break; + case LEFT: + OuterJoinPushDownResult leftOuterJoinPushDownResult = processLimitedOuterJoin( + inheritedPredicate, + leftEffectivePredicate, + rightEffectivePredicate, + joinPredicate, + node.getLeft().getOutputVariables()); + leftPredicate = leftOuterJoinPushDownResult.getOuterJoinPredicate(); + rightPredicate = leftOuterJoinPushDownResult.getInnerJoinPredicate(); + postJoinPredicate = leftOuterJoinPushDownResult.getPostJoinPredicate(); + newJoinPredicate = leftOuterJoinPushDownResult.getJoinPredicate(); + break; + default: + throw new IllegalArgumentException("Unsupported spatial join type: " + node.getType()); + } + + newJoinPredicate = simplifyExpression(newJoinPredicate); + verify(!newJoinPredicate.equals(FALSE_CONSTANT), "Spatial join predicate is missing"); + + PlanNode leftSource = context.rewrite(node.getLeft(), leftPredicate); + PlanNode rightSource = context.rewrite(node.getRight(), rightPredicate); + + PlanNode output = node; + if (leftSource != node.getLeft() || + rightSource != node.getRight() || + !areExpressionsEquivalent(newJoinPredicate, joinPredicate)) { + // Create identity projections for all existing symbols + Assignments.Builder leftProjections = Assignments.builder() + .putAll(identityAssignments(node.getLeft().getOutputVariables())); + + Assignments.Builder rightProjections = Assignments.builder() + .putAll(identityAssignments(node.getRight().getOutputVariables())); + + leftSource = new ProjectNode(idAllocator.getNextId(), leftSource, leftProjections.build()); + rightSource = new ProjectNode(idAllocator.getNextId(), rightSource, rightProjections.build()); + + output = new SpatialJoinNode( + node.getId(), + node.getType(), + leftSource, + rightSource, + node.getOutputVariables(), + newJoinPredicate, + node.getLeftPartitionVariable(), + node.getRightPartitionVariable(), + node.getKdbTree()); + } + + if (!postJoinPredicate.equals(TRUE_CONSTANT)) { + output = new FilterNode(idAllocator.getNextId(), output, postJoinPredicate); + } + + return output; + } + + private VariableReferenceExpression variableForExpression(RowExpression expression) + { + if (expression instanceof VariableReferenceExpression) { + return (VariableReferenceExpression) expression; + } + + return variableAllocator.newVariable(expression); + } + + private OuterJoinPushDownResult processLimitedOuterJoin(RowExpression inheritedPredicate, RowExpression outerEffectivePredicate, RowExpression innerEffectivePredicate, RowExpression joinPredicate, Collection outerVariables) + { + checkArgument(Iterables.all(VariablesExtractor.extractUnique(outerEffectivePredicate), in(outerVariables)), "outerEffectivePredicate must only contain variables from outerVariables"); + checkArgument(Iterables.all(VariablesExtractor.extractUnique(innerEffectivePredicate), not(in(outerVariables))), "innerEffectivePredicate must not contain variables from outerVariables"); + + ImmutableList.Builder outerPushdownConjuncts = ImmutableList.builder(); + ImmutableList.Builder innerPushdownConjuncts = ImmutableList.builder(); + ImmutableList.Builder postJoinConjuncts = ImmutableList.builder(); + ImmutableList.Builder joinConjuncts = ImmutableList.builder(); + + // Strip out non-deterministic conjuncts + postJoinConjuncts.addAll(filter(extractConjuncts(inheritedPredicate), not(determinismEvaluator::isDeterministic))); + inheritedPredicate = logicalRowExpressions.filterDeterministicConjuncts(inheritedPredicate); + + outerEffectivePredicate = logicalRowExpressions.filterDeterministicConjuncts(outerEffectivePredicate); + innerEffectivePredicate = logicalRowExpressions.filterDeterministicConjuncts(innerEffectivePredicate); + joinConjuncts.addAll(filter(extractConjuncts(joinPredicate), not(determinismEvaluator::isDeterministic))); + joinPredicate = logicalRowExpressions.filterDeterministicConjuncts(joinPredicate); + + // Generate equality inferences + RowExpressionEqualityInference inheritedInference = createEqualityInference(inheritedPredicate); + RowExpressionEqualityInference outerInference = createEqualityInference(inheritedPredicate, outerEffectivePredicate); + + RowExpressionEqualityInference.EqualityPartition equalityPartition = inheritedInference.generateEqualitiesPartitionedBy(in(outerVariables)); + RowExpression outerOnlyInheritedEqualities = logicalRowExpressions.combineConjuncts(equalityPartition.getScopeEqualities()); + RowExpressionEqualityInference potentialNullSymbolInference = createEqualityInference(outerOnlyInheritedEqualities, outerEffectivePredicate, innerEffectivePredicate, joinPredicate); + + // See if we can push inherited predicates down + for (RowExpression conjunct : nonInferrableConjuncts(inheritedPredicate)) { + RowExpression outerRewritten = outerInference.rewriteExpression(conjunct, in(outerVariables)); + if (outerRewritten != null) { + outerPushdownConjuncts.add(outerRewritten); + + // A conjunct can only be pushed down into an inner side if it can be rewritten in terms of the outer side + RowExpression innerRewritten = potentialNullSymbolInference.rewriteExpression(outerRewritten, not(in(outerVariables))); + if (innerRewritten != null) { + innerPushdownConjuncts.add(innerRewritten); + } + } + else { + postJoinConjuncts.add(conjunct); + } + } + // Add the equalities from the inferences back in + outerPushdownConjuncts.addAll(equalityPartition.getScopeEqualities()); + postJoinConjuncts.addAll(equalityPartition.getScopeComplementEqualities()); + postJoinConjuncts.addAll(equalityPartition.getScopeStraddlingEqualities()); + + // See if we can push down any outer effective predicates to the inner side + for (RowExpression conjunct : nonInferrableConjuncts(outerEffectivePredicate)) { + RowExpression rewritten = potentialNullSymbolInference.rewriteExpression(conjunct, not(in(outerVariables))); + if (rewritten != null) { + innerPushdownConjuncts.add(rewritten); + } + } + + // See if we can push down join predicates to the inner side + for (RowExpression conjunct : nonInferrableConjuncts(joinPredicate)) { + RowExpression innerRewritten = potentialNullSymbolInference.rewriteExpression(conjunct, not(in(outerVariables))); + if (innerRewritten != null) { + innerPushdownConjuncts.add(innerRewritten); + } + else { + joinConjuncts.add(conjunct); + } + } + + // Push outer and join equalities into the inner side. For example: + // SELECT * FROM nation LEFT OUTER JOIN region ON nation.regionkey = region.regionkey and nation.name = region.name WHERE nation.name = 'blah' + + RowExpressionEqualityInference potentialNullSymbolInferenceWithoutInnerInferred = createEqualityInference(outerOnlyInheritedEqualities, outerEffectivePredicate, joinPredicate); + innerPushdownConjuncts.addAll(potentialNullSymbolInferenceWithoutInnerInferred.generateEqualitiesPartitionedBy(not(in(outerVariables))).getScopeEqualities()); + + // TODO: we can further improve simplifying the equalities by considering other relationships from the outer side + RowExpressionEqualityInference.EqualityPartition joinEqualityPartition = createEqualityInference(joinPredicate).generateEqualitiesPartitionedBy(not(in(outerVariables))); + innerPushdownConjuncts.addAll(joinEqualityPartition.getScopeEqualities()); + joinConjuncts.addAll(joinEqualityPartition.getScopeComplementEqualities()) + .addAll(joinEqualityPartition.getScopeStraddlingEqualities()); + + return new OuterJoinPushDownResult(logicalRowExpressions.combineConjuncts(outerPushdownConjuncts.build()), + logicalRowExpressions.combineConjuncts(innerPushdownConjuncts.build()), + logicalRowExpressions.combineConjuncts(joinConjuncts.build()), + logicalRowExpressions.combineConjuncts(postJoinConjuncts.build())); + } + + private static class OuterJoinPushDownResult + { + private final RowExpression outerJoinPredicate; + private final RowExpression innerJoinPredicate; + private final RowExpression joinPredicate; + private final RowExpression postJoinPredicate; + + private OuterJoinPushDownResult(RowExpression outerJoinPredicate, RowExpression innerJoinPredicate, RowExpression joinPredicate, RowExpression postJoinPredicate) + { + this.outerJoinPredicate = outerJoinPredicate; + this.innerJoinPredicate = innerJoinPredicate; + this.joinPredicate = joinPredicate; + this.postJoinPredicate = postJoinPredicate; + } + + private RowExpression getOuterJoinPredicate() + { + return outerJoinPredicate; + } + + private RowExpression getInnerJoinPredicate() + { + return innerJoinPredicate; + } + + public RowExpression getJoinPredicate() + { + return joinPredicate; + } + + private RowExpression getPostJoinPredicate() + { + return postJoinPredicate; + } + } + + private InnerJoinPushDownResult processInnerJoin(RowExpression inheritedPredicate, RowExpression leftEffectivePredicate, RowExpression rightEffectivePredicate, RowExpression joinPredicate, Collection leftVariables) + { + checkArgument(Iterables.all(VariablesExtractor.extractUnique(leftEffectivePredicate), in(leftVariables)), "leftEffectivePredicate must only contain variables from leftVariables"); + checkArgument(Iterables.all(VariablesExtractor.extractUnique(rightEffectivePredicate), not(in(leftVariables))), "rightEffectivePredicate must not contain variables from leftVariables"); + + ImmutableList.Builder leftPushDownConjuncts = ImmutableList.builder(); + ImmutableList.Builder rightPushDownConjuncts = ImmutableList.builder(); + ImmutableList.Builder joinConjuncts = ImmutableList.builder(); + + // Strip out non-deterministic conjuncts + joinConjuncts.addAll(filter(extractConjuncts(inheritedPredicate), not(determinismEvaluator::isDeterministic))); + inheritedPredicate = logicalRowExpressions.filterDeterministicConjuncts(inheritedPredicate); + + joinConjuncts.addAll(filter(extractConjuncts(joinPredicate), not(determinismEvaluator::isDeterministic))); + joinPredicate = logicalRowExpressions.filterDeterministicConjuncts(joinPredicate); + + leftEffectivePredicate = logicalRowExpressions.filterDeterministicConjuncts(leftEffectivePredicate); + rightEffectivePredicate = logicalRowExpressions.filterDeterministicConjuncts(rightEffectivePredicate); + + // Generate equality inferences + RowExpressionEqualityInference allInference = new RowExpressionEqualityInference.Builder(functionManager, typeManager) + .addEqualityInference(inheritedPredicate, leftEffectivePredicate, rightEffectivePredicate, joinPredicate) + .build(); + RowExpressionEqualityInference allInferenceWithoutLeftInferred = new RowExpressionEqualityInference.Builder(functionManager, typeManager) + .addEqualityInference(inheritedPredicate, rightEffectivePredicate, joinPredicate) + .build(); + RowExpressionEqualityInference allInferenceWithoutRightInferred = new RowExpressionEqualityInference.Builder(functionManager, typeManager) + .addEqualityInference(inheritedPredicate, leftEffectivePredicate, joinPredicate) + .build(); + + // Sort through conjuncts in inheritedPredicate that were not used for inference + for (RowExpression conjunct : new RowExpressionEqualityInference.Builder(functionManager, typeManager).nonInferrableConjuncts(inheritedPredicate)) { + RowExpression leftRewrittenConjunct = allInference.rewriteExpression(conjunct, in(leftVariables)); + if (leftRewrittenConjunct != null) { + leftPushDownConjuncts.add(leftRewrittenConjunct); + } + + RowExpression rightRewrittenConjunct = allInference.rewriteExpression(conjunct, not(in(leftVariables))); + if (rightRewrittenConjunct != null) { + rightPushDownConjuncts.add(rightRewrittenConjunct); + } + + // Drop predicate after join only if unable to push down to either side + if (leftRewrittenConjunct == null && rightRewrittenConjunct == null) { + joinConjuncts.add(conjunct); + } + } + + // See if we can push the right effective predicate to the left side + for (RowExpression conjunct : new RowExpressionEqualityInference.Builder(functionManager, typeManager).nonInferrableConjuncts(rightEffectivePredicate)) { + RowExpression rewritten = allInference.rewriteExpression(conjunct, in(leftVariables)); + if (rewritten != null) { + leftPushDownConjuncts.add(rewritten); + } + } + + // See if we can push the left effective predicate to the right side + for (RowExpression conjunct : new RowExpressionEqualityInference.Builder(functionManager, typeManager).nonInferrableConjuncts(leftEffectivePredicate)) { + RowExpression rewritten = allInference.rewriteExpression(conjunct, not(in(leftVariables))); + if (rewritten != null) { + rightPushDownConjuncts.add(rewritten); + } + } + + // See if we can push any parts of the join predicates to either side + for (RowExpression conjunct : new RowExpressionEqualityInference.Builder(functionManager, typeManager).nonInferrableConjuncts(joinPredicate)) { + RowExpression leftRewritten = allInference.rewriteExpression(conjunct, in(leftVariables)); + if (leftRewritten != null) { + leftPushDownConjuncts.add(leftRewritten); + } + + RowExpression rightRewritten = allInference.rewriteExpression(conjunct, not(in(leftVariables))); + if (rightRewritten != null) { + rightPushDownConjuncts.add(rightRewritten); + } + + if (leftRewritten == null && rightRewritten == null) { + joinConjuncts.add(conjunct); + } + } + + // Add equalities from the inference back in + leftPushDownConjuncts.addAll(allInferenceWithoutLeftInferred.generateEqualitiesPartitionedBy(in(leftVariables)).getScopeEqualities()); + rightPushDownConjuncts.addAll(allInferenceWithoutRightInferred.generateEqualitiesPartitionedBy(not(in(leftVariables))).getScopeEqualities()); + joinConjuncts.addAll(allInference.generateEqualitiesPartitionedBy(in(leftVariables)::apply).getScopeStraddlingEqualities()); // scope straddling equalities get dropped in as part of the join predicate + + return new Rewriter.InnerJoinPushDownResult( + logicalRowExpressions.combineConjuncts(leftPushDownConjuncts.build()), + logicalRowExpressions.combineConjuncts(rightPushDownConjuncts.build()), + logicalRowExpressions.combineConjuncts(joinConjuncts.build()), TRUE_CONSTANT); + } + + private static class InnerJoinPushDownResult + { + private final RowExpression leftPredicate; + private final RowExpression rightPredicate; + private final RowExpression joinPredicate; + private final RowExpression postJoinPredicate; + + private InnerJoinPushDownResult(RowExpression leftPredicate, RowExpression rightPredicate, RowExpression joinPredicate, RowExpression postJoinPredicate) + { + this.leftPredicate = leftPredicate; + this.rightPredicate = rightPredicate; + this.joinPredicate = joinPredicate; + this.postJoinPredicate = postJoinPredicate; + } + + private RowExpression getLeftPredicate() + { + return leftPredicate; + } + + private RowExpression getRightPredicate() + { + return rightPredicate; + } + + private RowExpression getJoinPredicate() + { + return joinPredicate; + } + + private RowExpression getPostJoinPredicate() + { + return postJoinPredicate; + } + } + + private RowExpression extractJoinPredicate(JoinNode joinNode) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (JoinNode.EquiJoinClause equiJoinClause : joinNode.getCriteria()) { + builder.add(toRowExpression(equiJoinClause)); + } + joinNode.getFilter().ifPresent(builder::add); + return logicalRowExpressions.combineConjuncts(builder.build()); + } + + private RowExpression toRowExpression(JoinNode.EquiJoinClause equiJoinClause) + { + return buildEqualsExpression(functionManager, equiJoinClause.getLeft(), equiJoinClause.getRight()); + } + + private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, RowExpression inheritedPredicate) + { + checkArgument(EnumSet.of(INNER, RIGHT, LEFT, FULL).contains(node.getType()), "Unsupported join type: %s", node.getType()); + + if (node.getType() == JoinNode.Type.INNER) { + return node; + } + + if (node.getType() == JoinNode.Type.FULL) { + boolean canConvertToLeftJoin = canConvertOuterToInner(node.getLeft().getOutputVariables(), inheritedPredicate); + boolean canConvertToRightJoin = canConvertOuterToInner(node.getRight().getOutputVariables(), inheritedPredicate); + if (!canConvertToLeftJoin && !canConvertToRightJoin) { + return node; + } + if (canConvertToLeftJoin && canConvertToRightJoin) { + return new JoinNode(node.getId(), INNER, node.getLeft(), node.getRight(), node.getCriteria(), node.getOutputVariables(), node.getFilter(), node.getLeftHashVariable(), node.getRightHashVariable(), node.getDistributionType()); + } + else { + return new JoinNode(node.getId(), canConvertToLeftJoin ? LEFT : RIGHT, + node.getLeft(), node.getRight(), node.getCriteria(), node.getOutputVariables(), node.getFilter(), node.getLeftHashVariable(), node.getRightHashVariable(), node.getDistributionType()); + } + } + + if (node.getType() == JoinNode.Type.LEFT && !canConvertOuterToInner(node.getRight().getOutputVariables(), inheritedPredicate) || + node.getType() == JoinNode.Type.RIGHT && !canConvertOuterToInner(node.getLeft().getOutputVariables(), inheritedPredicate)) { + return node; + } + return new JoinNode(node.getId(), JoinNode.Type.INNER, node.getLeft(), node.getRight(), node.getCriteria(), node.getOutputVariables(), node.getFilter(), node.getLeftHashVariable(), node.getRightHashVariable(), node.getDistributionType()); + } + + private boolean canConvertOuterToInner(List innerVariablesForOuterJoin, RowExpression inheritedPredicate) + { + Set innerVariables = ImmutableSet.copyOf(innerVariablesForOuterJoin); + for (RowExpression conjunct : extractConjuncts(inheritedPredicate)) { + if (determinismEvaluator.isDeterministic(conjunct)) { + // Ignore a conjunct for this test if we can not deterministically get responses from it + RowExpression response = nullInputEvaluator(innerVariables, conjunct); + if (response == null || Expressions.isNull(response) || FALSE_CONSTANT.equals(response)) { + // If there is a single conjunct that returns FALSE or NULL given all NULL inputs for the inner side symbols of an outer join + // then this conjunct removes all effects of the outer join, and effectively turns this into an equivalent of an inner join. + // So, let's just rewrite this join as an INNER join + return true; + } + } + } + return false; + } + + // Temporary implementation for joins because the SimplifyExpressions optimizers can not run properly on join clauses + private RowExpression simplifyExpression(RowExpression expression) + { + return new RowExpressionOptimizer(metadata).optimize(expression, ExpressionOptimizer.Level.SERIALIZABLE, session.toConnectorSession()); + } + + private boolean areExpressionsEquivalent(RowExpression leftExpression, RowExpression rightExpression) + { + return expressionEquivalence.areExpressionsEquivalent(simplifyExpression(leftExpression), simplifyExpression(rightExpression)); + } + + /** + * Evaluates an expression's response to binding the specified input symbols to NULL + */ + private RowExpression nullInputEvaluator(final Collection nullSymbols, RowExpression expression) + { + expression = RowExpressionNodeInliner.replaceExpression(expression, nullSymbols.stream() + .collect(Collectors.toMap(identity(), variable -> constantNull(variable.getType())))); + return new RowExpressionOptimizer(metadata).optimize(expression, ExpressionOptimizer.Level.OPTIMIZED, session.toConnectorSession()); + } + + private Predicate joinEqualityExpression(final Collection leftVariables) + { + return expression -> { + // At this point in time, our join predicates need to be deterministic + if (determinismEvaluator.isDeterministic(expression) && isOperation(expression, EQUAL)) { + Set variables1 = VariablesExtractor.extractUnique(getLeft(expression)); + Set variables2 = VariablesExtractor.extractUnique(getRight(expression)); + if (variables1.isEmpty() || variables2.isEmpty()) { + return false; + } + return (Iterables.all(variables1, in(leftVariables)) && Iterables.all(variables2, not(in(leftVariables)))) || + (Iterables.all(variables2, in(leftVariables)) && Iterables.all(variables1, not(in(leftVariables)))); + } + return false; + }; + } + + private boolean isOperation(RowExpression expression, OperatorType type) + { + if (expression instanceof CallExpression) { + Optional operatorType = functionManager.getFunctionMetadata(((CallExpression) expression).getFunctionHandle()).getOperatorType(); + if (operatorType.isPresent()) { + return operatorType.get().equals(type); + } + } + return false; + } + + @Override + public PlanNode visitSemiJoin(SemiJoinNode node, RewriteContext context) + { + RowExpression inheritedPredicate = context.get(); + if (!extractConjuncts(inheritedPredicate).contains(node.getSemiJoinOutput())) { + return visitNonFilteringSemiJoin(node, context); + } + return visitFilteringSemiJoin(node, context); + } + + private PlanNode visitNonFilteringSemiJoin(SemiJoinNode node, RewriteContext context) + { + RowExpression inheritedPredicate = context.get(); + List sourceConjuncts = new ArrayList<>(); + List postJoinConjuncts = new ArrayList<>(); + + // TODO: see if there are predicates that can be inferred from the semi join output + + PlanNode rewrittenFilteringSource = context.defaultRewrite(node.getFilteringSource(), TRUE_CONSTANT); + + // Push inheritedPredicates down to the source if they don't involve the semi join output + RowExpressionEqualityInference inheritedInference = new RowExpressionEqualityInference.Builder(functionManager, typeManager) + .addEqualityInference(inheritedPredicate) + .build(); + for (RowExpression conjunct : new RowExpressionEqualityInference.Builder(functionManager, typeManager).nonInferrableConjuncts(inheritedPredicate)) { + RowExpression rewrittenConjunct = inheritedInference.rewriteExpressionAllowNonDeterministic(conjunct, in(node.getSource().getOutputVariables())); + // Since each source row is reflected exactly once in the output, ok to push non-deterministic predicates down + if (rewrittenConjunct != null) { + sourceConjuncts.add(rewrittenConjunct); + } + else { + postJoinConjuncts.add(conjunct); + } + } + + // Add the inherited equality predicates back in + RowExpressionEqualityInference.EqualityPartition equalityPartition = inheritedInference.generateEqualitiesPartitionedBy(in(node.getSource() + .getOutputVariables())::apply); + sourceConjuncts.addAll(equalityPartition.getScopeEqualities()); + postJoinConjuncts.addAll(equalityPartition.getScopeComplementEqualities()); + postJoinConjuncts.addAll(equalityPartition.getScopeStraddlingEqualities()); + + PlanNode rewrittenSource = context.rewrite(node.getSource(), logicalRowExpressions.combineConjuncts(sourceConjuncts)); + + PlanNode output = node; + if (rewrittenSource != node.getSource() || rewrittenFilteringSource != node.getFilteringSource()) { + output = new SemiJoinNode(node.getId(), rewrittenSource, rewrittenFilteringSource, node.getSourceJoinVariable(), node.getFilteringSourceJoinVariable(), node.getSemiJoinOutput(), node.getSourceHashVariable(), node.getFilteringSourceHashVariable(), node.getDistributionType()); + } + if (!postJoinConjuncts.isEmpty()) { + output = new FilterNode(idAllocator.getNextId(), output, logicalRowExpressions.combineConjuncts(postJoinConjuncts)); + } + return output; + } + + private PlanNode visitFilteringSemiJoin(SemiJoinNode node, RewriteContext context) + { + RowExpression inheritedPredicate = context.get(); + RowExpression deterministicInheritedPredicate = logicalRowExpressions.filterDeterministicConjuncts(inheritedPredicate); + RowExpression sourceEffectivePredicate = logicalRowExpressions.filterDeterministicConjuncts(effectivePredicateExtractor.extract(node.getSource())); + RowExpression filteringSourceEffectivePredicate = logicalRowExpressions.filterDeterministicConjuncts(effectivePredicateExtractor.extract(node.getFilteringSource())); + RowExpression joinExpression = buildEqualsExpression(functionManager, node.getSourceJoinVariable(), node.getFilteringSourceJoinVariable()); + + List sourceVariables = node.getSource().getOutputVariables(); + List filteringSourceVariables = node.getFilteringSource().getOutputVariables(); + + List sourceConjuncts = new ArrayList<>(); + List filteringSourceConjuncts = new ArrayList<>(); + List postJoinConjuncts = new ArrayList<>(); + + // Generate equality inferences + RowExpressionEqualityInference allInference = createEqualityInference(deterministicInheritedPredicate, sourceEffectivePredicate, filteringSourceEffectivePredicate, joinExpression); + RowExpressionEqualityInference allInferenceWithoutSourceInferred = createEqualityInference(deterministicInheritedPredicate, filteringSourceEffectivePredicate, joinExpression); + RowExpressionEqualityInference allInferenceWithoutFilteringSourceInferred = createEqualityInference(deterministicInheritedPredicate, sourceEffectivePredicate, joinExpression); + + // Push inheritedPredicates down to the source if they don't involve the semi join output + for (RowExpression conjunct : nonInferrableConjuncts(inheritedPredicate)) { + RowExpression rewrittenConjunct = allInference.rewriteExpressionAllowNonDeterministic(conjunct, in(sourceVariables)); + // Since each source row is reflected exactly once in the output, ok to push non-deterministic predicates down + if (rewrittenConjunct != null) { + sourceConjuncts.add(rewrittenConjunct); + } + else { + postJoinConjuncts.add(conjunct); + } + } + + // Push inheritedPredicates down to the filtering source if possible + for (RowExpression conjunct : nonInferrableConjuncts(deterministicInheritedPredicate)) { + RowExpression rewrittenConjunct = allInference.rewriteExpression(conjunct, in(filteringSourceVariables)); + // We cannot push non-deterministic predicates to filtering side. Each filtering side row have to be + // logically reevaluated for each source row. + if (rewrittenConjunct != null) { + filteringSourceConjuncts.add(rewrittenConjunct); + } + } + + // move effective predicate conjuncts source <-> filter + // See if we can push the filtering source effective predicate to the source side + for (RowExpression conjunct : nonInferrableConjuncts(filteringSourceEffectivePredicate)) { + RowExpression rewritten = allInference.rewriteExpression(conjunct, in(sourceVariables)); + if (rewritten != null) { + sourceConjuncts.add(rewritten); + } + } + + // See if we can push the source effective predicate to the filtering soruce side + for (RowExpression conjunct : nonInferrableConjuncts(sourceEffectivePredicate)) { + RowExpression rewritten = allInference.rewriteExpression(conjunct, in(filteringSourceVariables)); + if (rewritten != null) { + filteringSourceConjuncts.add(rewritten); + } + } + + // Add equalities from the inference back in + sourceConjuncts.addAll(allInferenceWithoutSourceInferred.generateEqualitiesPartitionedBy(in(sourceVariables)).getScopeEqualities()); + filteringSourceConjuncts.addAll(allInferenceWithoutFilteringSourceInferred.generateEqualitiesPartitionedBy(in(filteringSourceVariables)).getScopeEqualities()); + + PlanNode rewrittenSource = context.rewrite(node.getSource(), logicalRowExpressions.combineConjuncts(sourceConjuncts)); + PlanNode rewrittenFilteringSource = context.rewrite(node.getFilteringSource(), logicalRowExpressions.combineConjuncts(filteringSourceConjuncts)); + + PlanNode output = node; + if (rewrittenSource != node.getSource() || rewrittenFilteringSource != node.getFilteringSource()) { + output = new SemiJoinNode( + node.getId(), + rewrittenSource, + rewrittenFilteringSource, + node.getSourceJoinVariable(), + node.getFilteringSourceJoinVariable(), + node.getSemiJoinOutput(), + node.getSourceHashVariable(), + node.getFilteringSourceHashVariable(), + node.getDistributionType()); + } + if (!postJoinConjuncts.isEmpty()) { + output = new FilterNode(idAllocator.getNextId(), output, logicalRowExpressions.combineConjuncts(postJoinConjuncts)); + } + return output; + } + + private Iterable nonInferrableConjuncts(RowExpression inheritedPredicate) + { + return new RowExpressionEqualityInference.Builder(functionManager, typeManager) + .nonInferrableConjuncts(inheritedPredicate); + } + + private RowExpressionEqualityInference createEqualityInference(RowExpression... expressions) + { + return new RowExpressionEqualityInference.Builder(functionManager, typeManager) + .addEqualityInference(expressions) + .build(); + } + + @Override + public PlanNode visitAggregation(AggregationNode node, RewriteContext context) + { + if (node.hasEmptyGroupingSet()) { + // TODO: in case of grouping sets, we should be able to push the filters over grouping keys below the aggregation + // and also preserve the filter above the aggregation if it has an empty grouping set + return visitPlan(node, context); + } + + RowExpression inheritedPredicate = context.get(); + + RowExpressionEqualityInference equalityInference = createEqualityInference(inheritedPredicate); + + List pushdownConjuncts = new ArrayList<>(); + List postAggregationConjuncts = new ArrayList<>(); + + List groupingKeyVariables = node.getGroupingKeys(); + + // Strip out non-deterministic conjuncts + postAggregationConjuncts.addAll(ImmutableList.copyOf(filter(extractConjuncts(inheritedPredicate), not(determinismEvaluator::isDeterministic)))); + inheritedPredicate = logicalRowExpressions.filterDeterministicConjuncts(inheritedPredicate); + + // Sort non-equality predicates by those that can be pushed down and those that cannot + for (RowExpression conjunct : nonInferrableConjuncts(inheritedPredicate)) { + if (node.getGroupIdVariable().isPresent() && VariablesExtractor.extractUnique(conjunct).contains(node.getGroupIdVariable().get())) { + // aggregation operator synthesizes outputs for group ids corresponding to the global grouping set (i.e., ()), so we + // need to preserve any predicates that evaluate the group id to run after the aggregation + // TODO: we should be able to infer if conditions on grouping() correspond to global grouping sets to determine whether + // we need to do this for each specific case + postAggregationConjuncts.add(conjunct); + continue; + } + + RowExpression rewrittenConjunct = equalityInference.rewriteExpression(conjunct, in(groupingKeyVariables)); + if (rewrittenConjunct != null) { + pushdownConjuncts.add(rewrittenConjunct); + } + else { + postAggregationConjuncts.add(conjunct); + } + } + + // Add the equality predicates back in + RowExpressionEqualityInference.EqualityPartition equalityPartition = equalityInference.generateEqualitiesPartitionedBy(in(groupingKeyVariables)::apply); + pushdownConjuncts.addAll(equalityPartition.getScopeEqualities()); + postAggregationConjuncts.addAll(equalityPartition.getScopeComplementEqualities()); + postAggregationConjuncts.addAll(equalityPartition.getScopeStraddlingEqualities()); + + PlanNode rewrittenSource = context.rewrite(node.getSource(), logicalRowExpressions.combineConjuncts(pushdownConjuncts)); + + PlanNode output = node; + if (rewrittenSource != node.getSource()) { + output = new AggregationNode(node.getId(), + rewrittenSource, + node.getAggregations(), + node.getGroupingSets(), + ImmutableList.of(), + node.getStep(), + node.getHashVariable(), + node.getGroupIdVariable()); + } + if (!postAggregationConjuncts.isEmpty()) { + output = new FilterNode(idAllocator.getNextId(), output, logicalRowExpressions.combineConjuncts(postAggregationConjuncts)); + } + return output; + } + + @Override + public PlanNode visitUnnest(UnnestNode node, RewriteContext context) + { + RowExpression inheritedPredicate = context.get(); + + RowExpressionEqualityInference equalityInference = createEqualityInference(inheritedPredicate); + + List pushdownConjuncts = new ArrayList<>(); + List postUnnestConjuncts = new ArrayList<>(); + + // Strip out non-deterministic conjuncts + postUnnestConjuncts.addAll(ImmutableList.copyOf(filter(extractConjuncts(inheritedPredicate), not(determinismEvaluator::isDeterministic)))); + inheritedPredicate = logicalRowExpressions.filterDeterministicConjuncts(inheritedPredicate); + + // Sort non-equality predicates by those that can be pushed down and those that cannot + for (RowExpression conjunct : nonInferrableConjuncts(inheritedPredicate)) { + RowExpression rewrittenConjunct = equalityInference.rewriteExpression(conjunct, in(node.getReplicateVariables())); + if (rewrittenConjunct != null) { + pushdownConjuncts.add(rewrittenConjunct); + } + else { + postUnnestConjuncts.add(conjunct); + } + } + + // Add the equality predicates back in + RowExpressionEqualityInference.EqualityPartition equalityPartition = equalityInference.generateEqualitiesPartitionedBy(in(node.getReplicateVariables())::apply); + pushdownConjuncts.addAll(equalityPartition.getScopeEqualities()); + postUnnestConjuncts.addAll(equalityPartition.getScopeComplementEqualities()); + postUnnestConjuncts.addAll(equalityPartition.getScopeStraddlingEqualities()); + + PlanNode rewrittenSource = context.rewrite(node.getSource(), logicalRowExpressions.combineConjuncts(pushdownConjuncts)); + + PlanNode output = node; + if (rewrittenSource != node.getSource()) { + output = new UnnestNode(node.getId(), rewrittenSource, node.getReplicateVariables(), node.getUnnestVariables(), node.getOrdinalityVariable()); + } + if (!postUnnestConjuncts.isEmpty()) { + output = new FilterNode(idAllocator.getNextId(), output, logicalRowExpressions.combineConjuncts(postUnnestConjuncts)); + } + return output; + } + + @Override + public PlanNode visitSample(SampleNode node, RewriteContext context) + { + return context.defaultRewrite(node, context.get()); + } + + @Override + public PlanNode visitTableScan(TableScanNode node, RewriteContext context) + { + RowExpression predicate = simplifyExpression(context.get()); + + if (!TRUE_CONSTANT.equals(predicate)) { + return new FilterNode(idAllocator.getNextId(), node, predicate); + } + + return node; + } + + @Override + public PlanNode visitAssignUniqueId(AssignUniqueId node, RewriteContext context) + { + Set predicateVariables = VariablesExtractor.extractUnique(context.get()); + checkState(!predicateVariables.contains(node.getIdVariable()), "UniqueId in predicate is not yet supported"); + return context.defaultRewrite(node, context.get()); + } + + private static CallExpression buildEqualsExpression(FunctionManager functionManager, RowExpression left, RowExpression right) + { + return call( + EQUAL.getFunctionName().getFunctionName(), + functionManager.resolveOperator(EQUAL, fromTypes(left.getType(), right.getType())), + BOOLEAN, + left, + right); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ScalarAggregationToJoinRewriter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ScalarAggregationToJoinRewriter.java index e94f41ff67015..4179bd0855107 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ScalarAggregationToJoinRewriter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ScalarAggregationToJoinRewriter.java @@ -14,8 +14,12 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.BooleanType; @@ -23,14 +27,10 @@ import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.iterative.Lookup; import com.facebook.presto.sql.planner.optimizations.PlanNodeDecorrelator.DecorrelatedNode; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.tree.Expression; @@ -43,9 +43,9 @@ import java.util.Optional; import java.util.Set; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identitiesAsSymbolReferences; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignmentsAsSymbolReferences; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.asSymbolReference; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SetFlatteningOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SetFlatteningOptimizer.java index c04facc2bdee2..1969e98f8982a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SetFlatteningOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SetFlatteningOptimizer.java @@ -15,24 +15,26 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.ExceptNode; +import com.facebook.presto.spi.plan.IntersectNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.SetOperationNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.ExceptNode; -import com.facebook.presto.sql.planner.plan.IntersectNode; -import com.facebook.presto.sql.planner.plan.SetOperationNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; -import java.util.Collection; +import java.util.List; import java.util.Map; +import static com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils.fromListMultimap; import static java.util.Objects.requireNonNull; public class SetFlatteningOptimizer @@ -66,8 +68,9 @@ public PlanNode visitUnion(UnionNode node, RewriteContext context) ImmutableList.Builder flattenedSources = ImmutableList.builder(); ImmutableListMultimap.Builder flattenedVariableMap = ImmutableListMultimap.builder(); flattenSetOperation(node, context, flattenedSources, flattenedVariableMap); + ListMultimap mappings = flattenedVariableMap.build(); - return new UnionNode(node.getId(), flattenedSources.build(), flattenedVariableMap.build()); + return new UnionNode(node.getId(), flattenedSources.build(), ImmutableList.copyOf(mappings.keySet()), fromListMultimap(mappings)); } @Override @@ -76,8 +79,9 @@ public PlanNode visitIntersect(IntersectNode node, RewriteContext conte ImmutableList.Builder flattenedSources = ImmutableList.builder(); ImmutableListMultimap.Builder flattenedVariableMap = ImmutableListMultimap.builder(); flattenSetOperation(node, context, flattenedSources, flattenedVariableMap); + ListMultimap mappings = flattenedVariableMap.build(); - return new IntersectNode(node.getId(), flattenedSources.build(), flattenedVariableMap.build()); + return new IntersectNode(node.getId(), flattenedSources.build(), ImmutableList.copyOf(mappings.keySet()), fromListMultimap(mappings)); } @Override @@ -86,14 +90,15 @@ public PlanNode visitExcept(ExceptNode node, RewriteContext context) ImmutableList.Builder flattenedSources = ImmutableList.builder(); ImmutableListMultimap.Builder flattenedVariableMap = ImmutableListMultimap.builder(); flattenSetOperation(node, context, flattenedSources, flattenedVariableMap); + ListMultimap mappings = flattenedVariableMap.build(); - return new ExceptNode(node.getId(), flattenedSources.build(), flattenedVariableMap.build()); + return new ExceptNode(node.getId(), flattenedSources.build(), ImmutableList.copyOf(mappings.keySet()), fromListMultimap(mappings)); } private static void flattenSetOperation( SetOperationNode node, RewriteContext context, ImmutableList.Builder flattenedSources, - ImmutableListMultimap.Builder flattenedSymbolMap) + ImmutableListMultimap.Builder flattenedVariableMap) { for (int i = 0; i < node.getSources().size(); i++) { PlanNode subplan = node.getSources().get(i); @@ -105,15 +110,15 @@ private static void flattenSetOperation( // ExceptNodes can only flatten their first source because except is not associative SetOperationNode rewrittenSetOperation = (SetOperationNode) rewrittenSource; flattenedSources.addAll(rewrittenSetOperation.getSources()); - for (Map.Entry> entry : node.getVariableMapping().asMap().entrySet()) { + for (Map.Entry> entry : node.getVariableMapping().entrySet()) { VariableReferenceExpression inputVariable = Iterables.get(entry.getValue(), i); - flattenedSymbolMap.putAll(entry.getKey(), rewrittenSetOperation.getVariableMapping().get(inputVariable)); + flattenedVariableMap.putAll(entry.getKey(), rewrittenSetOperation.getVariableMapping().get(inputVariable)); } } else { flattenedSources.add(rewrittenSource); - for (Map.Entry> entry : node.getVariableMapping().asMap().entrySet()) { - flattenedSymbolMap.put(entry.getKey(), Iterables.get(entry.getValue(), i)); + for (Map.Entry> entry : node.getVariableMapping().entrySet()) { + flattenedVariableMap.put(entry.getKey(), Iterables.get(entry.getValue(), i)); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SetOperationNodeUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SetOperationNodeUtils.java new file mode 100644 index 0000000000000..903dd7ba753cc --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SetOperationNodeUtils.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.optimizations; + +import com.facebook.presto.spi.plan.UnionNode; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Multimap; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class SetOperationNodeUtils +{ + private SetOperationNodeUtils() {} + + public static Map> fromListMultimap(ListMultimap outputsToInputs) + { + Map> mapping = new LinkedHashMap<>(); + for (Map.Entry entry : outputsToInputs.entries()) { + if (!mapping.containsKey(entry.getKey())) { + List values = new ArrayList<>(); + values.add(entry.getValue()); + mapping.put(entry.getKey(), values); + } + else { + mapping.get(entry.getKey()).add(entry.getValue()); + } + } + + return mapping; + } + + /** + * Returns the input to output symbol mapping for the given source channel. + * A single input symbol can map to multiple output symbols, thus requiring a Multimap. + */ + public static Multimap outputMap(UnionNode node, int sourceIndex) + { + return FluentIterable.from(node.getOutputVariables()) + .toMap(output -> node.getVariableMapping().get(output).get(sourceIndex)) + .asMultimap() + .inverse(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StatsRecordingPlanOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StatsRecordingPlanOptimizer.java index 1153545ff5215..f820faba717e1 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StatsRecordingPlanOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StatsRecordingPlanOptimizer.java @@ -20,6 +20,7 @@ import com.facebook.presto.sql.planner.OptimizerStatsRecorder; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; +import com.google.common.annotations.VisibleForTesting; import static java.util.Objects.requireNonNull; @@ -36,6 +37,12 @@ public StatsRecordingPlanOptimizer(OptimizerStatsRecorder stats, PlanOptimizer d stats.register(delegate); } + @VisibleForTesting + public PlanOptimizer getDelegate() + { + return delegate; + } + public final PlanNode optimize( PlanNode plan, Session session, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java index b69045ca6735f..9f1a0628184be 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java @@ -18,16 +18,19 @@ import com.facebook.presto.metadata.TableLayout; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.LocalProperty; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.planner.Partitioning.ArgumentBinding; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.DeleteNode; @@ -41,10 +44,8 @@ import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; @@ -52,10 +53,9 @@ import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.tree.SymbolReference; @@ -91,6 +91,7 @@ import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; public final class StreamPropertyDerivations @@ -297,7 +298,7 @@ private Optional> getNonConstantVariables(Set inputProperties) { - if (node.getOrderingScheme().isPresent()) { + if (node.isEnsureSourceOrdering() || node.getOrderingScheme().isPresent()) { return StreamProperties.ordered(); } @@ -314,10 +315,13 @@ public StreamProperties visitExchange(ExchangeNode node, List if (node.getPartitioningScheme().getPartitioning().getHandle().equals(FIXED_ARBITRARY_DISTRIBUTION)) { return new StreamProperties(FIXED, Optional.empty(), false); } + checkArgument( + node.getPartitioningScheme().getPartitioning().getArguments().stream().allMatch(VariableReferenceExpression.class::isInstance), + format("Expect all partitioning arguments to be VariableReferenceExpression, but get %s", node.getPartitioningScheme().getPartitioning().getArguments())); return new StreamProperties( FIXED, Optional.of(node.getPartitioningScheme().getPartitioning().getArguments().stream() - .map(ArgumentBinding::getVariableReference) + .map(VariableReferenceExpression.class::cast) .collect(toImmutableList())), false); case REPLICATE: return new StreamProperties(MULTIPLE, Optional.empty(), false); @@ -422,6 +426,12 @@ public StreamProperties visitTableWriter(TableWriterNode node, List inputProperties) + { + return Iterables.getOnlyElement(inputProperties); + } + @Override public StreamProperties visitUnnest(UnnestNode node, List inputProperties) { @@ -731,7 +741,7 @@ public StreamProperties translate(Function> getPartitioningColumns() diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java index 034f0ddae5896..32ddfa43dd3a3 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java @@ -13,27 +13,29 @@ */ package com.facebook.presto.sql.planner.optimizations; +import com.facebook.presto.expressions.RowExpressionRewriter; +import com.facebook.presto.expressions.RowExpressionTreeRewriter; import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; -import com.facebook.presto.spi.relation.RowExpressionRewriter; -import com.facebook.presto.spi.relation.RowExpressionTreeRewriter; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.PartitioningScheme; import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.ExpressionRewriter; import com.facebook.presto.sql.tree.ExpressionTreeRewriter; @@ -48,7 +50,7 @@ import java.util.Map.Entry; import java.util.Set; -import static com.facebook.presto.sql.planner.plan.AggregationNode.groupingSets; +import static com.facebook.presto.spi.plan.AggregationNode.groupingSets; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.isExpression; @@ -134,12 +136,14 @@ public OrderingScheme map(OrderingScheme orderingScheme) // SymbolMapper inlines symbol with multiple level reference (SymbolInliner only inline single level). ImmutableList.Builder orderBy = ImmutableList.builder(); ImmutableMap.Builder ordering = ImmutableMap.builder(); - for (VariableReferenceExpression variable : orderingScheme.getOrderBy()) { + for (VariableReferenceExpression variable : orderingScheme.getOrderByVariables()) { VariableReferenceExpression translated = map(variable); orderBy.add(translated); ordering.put(translated, orderingScheme.getOrdering(variable)); } - return new OrderingScheme(orderBy.build(), ordering.build()); + + ImmutableMap orderingMap = ordering.build(); + return new OrderingScheme(orderBy.build().stream().map(variable -> new Ordering(variable, orderingMap.get(variable))).collect(toImmutableList())); } public AggregationNode map(AggregationNode node, PlanNode source) @@ -191,8 +195,8 @@ public TopNNode map(TopNNode node, PlanNode source, PlanNodeId newNodeId) { ImmutableList.Builder variables = ImmutableList.builder(); ImmutableMap.Builder orderings = ImmutableMap.builder(); - Set seenCanonicals = new HashSet<>(node.getOrderingScheme().getOrderBy().size()); - for (VariableReferenceExpression variable : node.getOrderingScheme().getOrderBy()) { + Set seenCanonicals = new HashSet<>(node.getOrderingScheme().getOrderByVariables().size()); + for (VariableReferenceExpression variable : node.getOrderingScheme().getOrderByVariables()) { VariableReferenceExpression canonical = map(variable); if (seenCanonicals.add(canonical)) { seenCanonicals.add(canonical); @@ -201,11 +205,12 @@ public TopNNode map(TopNNode node, PlanNode source, PlanNodeId newNodeId) } } + ImmutableMap orderingMap = orderings.build(); return new TopNNode( newNodeId, source, node.getCount(), - new OrderingScheme(variables.build(), orderings.build()), + new OrderingScheme(variables.build().stream().map(variable -> new Ordering(variable, orderingMap.get(variable))).collect(toImmutableList())), node.getStep()); } @@ -231,8 +236,7 @@ public TableWriterNode map(TableWriterNode node, PlanNode source, PlanNodeId new columns, node.getColumnNames(), node.getPartitioningScheme().map(partitioningScheme -> canonicalize(partitioningScheme, source)), - node.getStatisticsAggregation().map(this::map), - node.getStatisticsAggregationDescriptor().map(this::map)); + node.getStatisticsAggregation().map(this::map)); } public StatisticsWriterNode map(StatisticsWriterNode node, PlanNode source) @@ -240,7 +244,7 @@ public StatisticsWriterNode map(StatisticsWriterNode node, PlanNode source) return new StatisticsWriterNode( node.getId(), source, - node.getTarget(), + node.getTableHandle(), node.getRowCountVariable(), node.isRowCountEnabled(), node.getDescriptor().map(this::map)); @@ -257,10 +261,21 @@ public TableFinishNode map(TableFinishNode node, PlanNode source) node.getStatisticsAggregationDescriptor().map(descriptor -> descriptor.map(this::map))); } + public TableWriterMergeNode map(TableWriterMergeNode node, PlanNode source) + { + return new TableWriterMergeNode( + node.getId(), + source, + map(node.getRowCountVariable()), + map(node.getFragmentVariable()), + map(node.getTableCommitContextVariable()), + node.getStatisticsAggregation().map(this::map)); + } + private PartitioningScheme canonicalize(PartitioningScheme scheme, PlanNode source) { return new PartitioningScheme( - scheme.getPartitioning().translate(this::map), + scheme.getPartitioning().translateVariable(this::map), mapAndDistinctVariable(source.getOutputVariables()), scheme.getHashColumn().map(this::map), scheme.isReplicateNullsAndAny(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TransformQuantifiedComparisonApplyToLateralJoin.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TransformQuantifiedComparisonApplyToLateralJoin.java index 6ab6f66ff3307..07ed76ee17a5c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TransformQuantifiedComparisonApplyToLateralJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/TransformQuantifiedComparisonApplyToLateralJoin.java @@ -17,8 +17,12 @@ import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -28,12 +32,8 @@ import com.facebook.presto.sql.ExpressionUtils; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.planner.plan.ApplyNode; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.BooleanLiteral; @@ -55,9 +55,9 @@ import java.util.Optional; import java.util.function.Function; +import static com.facebook.presto.spi.plan.AggregationNode.globalAggregation; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.ExpressionUtils.combineConjuncts; -import static com.facebook.presto.sql.planner.plan.AggregationNode.globalAggregation; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identitiesAsSymbolReferences; import static com.facebook.presto.sql.planner.plan.SimplePlanRewriter.rewriteWith; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java index cbd3b28431462..35fca7fb2acb5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java @@ -15,58 +15,63 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.SetOperationNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.ExpressionDeterminismEvaluator; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.PartitioningScheme; import com.facebook.presto.sql.planner.PlanVariableAllocator; +import com.facebook.presto.sql.planner.RowExpressionVariableInliner; import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; -import com.facebook.presto.sql.planner.plan.ExceptNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; import com.facebook.presto.sql.planner.plan.GroupIdNode; import com.facebook.presto.sql.planner.plan.IndexJoinNode; import com.facebook.presto.sql.planner.plan.IndexSourceNode; -import com.facebook.presto.sql.planner.plan.IntersectNode; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; -import com.facebook.presto.sql.planner.plan.SetOperationNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; -import com.facebook.presto.sql.relational.OriginalExpressionUtils; +import com.facebook.presto.sql.relational.Expressions; +import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.ExpressionRewriter; import com.facebook.presto.sql.tree.ExpressionTreeRewriter; @@ -74,16 +79,14 @@ import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; -import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -114,6 +117,13 @@ public class UnaliasSymbolReferences implements PlanOptimizer { + private final FunctionManager functionManager; + + public UnaliasSymbolReferences(FunctionManager functionManager) + { + this.functionManager = requireNonNull(functionManager, "functionManager is null"); + } + @Override public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, PlanVariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { @@ -123,7 +133,7 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Pla requireNonNull(variableAllocator, "variableAllocator is null"); requireNonNull(idAllocator, "idAllocator is null"); - return SimplePlanRewriter.rewriteWith(new Rewriter(types), plan); + return SimplePlanRewriter.rewriteWith(new Rewriter(types, functionManager), plan); } private static class Rewriter @@ -131,10 +141,12 @@ private static class Rewriter { private final Map mapping = new HashMap<>(); private final TypeProvider types; + private final RowExpressionDeterminismEvaluator determinismEvaluator; - private Rewriter(TypeProvider types) + private Rewriter(TypeProvider types, FunctionManager functionManager) { this.types = types; + this.determinismEvaluator = new RowExpressionDeterminismEvaluator(functionManager); } @Override @@ -213,7 +225,8 @@ public PlanNode visitWindow(WindowNode node, RewriteContext context) callExpression.getFunctionHandle(), callExpression.getType(), rewrittenArguments), - canonicalFrame)); + canonicalFrame, + entry.getValue().isIgnoreNulls())); } return new WindowNode( @@ -231,7 +244,7 @@ private List canonicalizeCallExpression(CallExpression callExpres // TODO: arguments will be pure RowExpression once we introduce subquery expression for RowExpression. return callExpression.getArguments() .stream() - .map(argument -> castToRowExpression(canonicalize(castToExpression(argument)))) + .map(this::canonicalize) .collect(toImmutableList()); } @@ -280,7 +293,7 @@ public PlanNode visitExchange(ExchangeNode node, RewriteContext context) } PartitioningScheme partitioningScheme = new PartitioningScheme( - node.getPartitioningScheme().getPartitioning().translate(this::canonicalize), + node.getPartitioningScheme().getPartitioning().translateVariable(this::canonicalize), outputs.build(), canonicalize(node.getPartitioningScheme().getHashColumn()), node.getPartitioningScheme().isReplicateNullsAndAny(), @@ -288,7 +301,7 @@ public PlanNode visitExchange(ExchangeNode node, RewriteContext context) Optional orderingScheme = node.getOrderingScheme().map(this::canonicalizeAndDistinct); - return new ExchangeNode(node.getId(), node.getType(), node.getScope(), partitioningScheme, sources, inputs, orderingScheme); + return new ExchangeNode(node.getId(), node.getType(), node.getScope(), partitioningScheme, sources, inputs, node.isEnsureSourceOrdering(), orderingScheme); } private void mapExchangeNodeSymbols(ExchangeNode node) @@ -344,6 +357,7 @@ public PlanNode visitRemoteSource(RemoteSourceNode node, RewriteContext co node.getId(), node.getSourceFragmentIds(), canonicalizeAndDistinct(node.getOutputVariables()), + node.isEnsureSourceOrdering(), node.getOrderingScheme().map(this::canonicalizeAndDistinct), node.getExchangeType()); } @@ -371,12 +385,7 @@ public PlanNode visitValues(ValuesNode node, RewriteContext context) { List> canonicalizedRows = node.getRows().stream() .map(rowExpressions -> rowExpressions.stream() - .map(rowExpression -> { - if (isExpression(rowExpression)) { - return castToRowExpression(canonicalize(castToExpression(rowExpression))); - } - return rowExpression; - }) + .map(this::canonicalize) .collect(toImmutableList())) .collect(toImmutableList()); List canonicalizedOutputVariables = canonicalizeAndDistinct(node.getOutputVariables()); @@ -390,7 +399,7 @@ public PlanNode visitValues(ValuesNode node, RewriteContext context) @Override public PlanNode visitDelete(DeleteNode node, RewriteContext context) { - return new DeleteNode(node.getId(), context.rewrite(node.getSource()), node.getTarget(), canonicalize(node.getRowId()), node.getOutputVariables()); + return new DeleteNode(node.getId(), context.rewrite(node.getSource()), canonicalize(node.getRowId()), node.getOutputVariables()); } @Override @@ -433,7 +442,7 @@ public PlanNode visitFilter(FilterNode node, RewriteContext context) { PlanNode source = context.rewrite(node.getSource()); - return new FilterNode(node.getId(), source, castToRowExpression(canonicalize(castToExpression(node.getPredicate())))); + return new FilterNode(node.getId(), source, canonicalize(node.getPredicate())); } @Override @@ -504,7 +513,7 @@ public PlanNode visitSort(SortNode node, RewriteContext context) { PlanNode source = context.rewrite(node.getSource()); - return new SortNode(node.getId(), source, canonicalizeAndDistinct(node.getOrderingScheme())); + return new SortNode(node.getId(), source, canonicalizeAndDistinct(node.getOrderingScheme()), node.isPartial()); } @Override @@ -514,7 +523,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) PlanNode right = context.rewrite(node.getRight()); List canonicalCriteria = canonicalizeJoinCriteria(node.getCriteria()); - Optional canonicalFilter = node.getFilter().map(OriginalExpressionUtils::castToExpression).map(this::canonicalize); + Optional canonicalFilter = node.getFilter().map(this::canonicalize); Optional canonicalLeftHashVariable = canonicalize(node.getLeftHashVariable()); Optional canonicalRightHashVariable = canonicalize(node.getRightHashVariable()); @@ -532,7 +541,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) right, canonicalCriteria, canonicalizeAndDistinct(node.getOutputVariables()), - canonicalFilter.map(OriginalExpressionUtils::castToRowExpression), + canonicalFilter, canonicalLeftHashVariable, canonicalRightHashVariable, node.getDistributionType()); @@ -562,7 +571,7 @@ public PlanNode visitSpatialJoin(SpatialJoinNode node, RewriteContext cont PlanNode left = context.rewrite(node.getLeft()); PlanNode right = context.rewrite(node.getRight()); - return new SpatialJoinNode(node.getId(), node.getType(), left, right, canonicalizeAndDistinct(node.getOutputVariables()), castToRowExpression(canonicalize(castToExpression(node.getFilter()))), canonicalize(node.getLeftPartitionVariable()), canonicalize(node.getRightPartitionVariable()), node.getKdbTree()); + return new SpatialJoinNode(node.getId(), node.getType(), left, right, canonicalizeAndDistinct(node.getOutputVariables()), canonicalize(node.getFilter()), canonicalize(node.getLeftPartitionVariable()), canonicalize(node.getRightPartitionVariable()), node.getKdbTree()); } @Override @@ -583,19 +592,19 @@ public PlanNode visitIndexJoin(IndexJoinNode node, RewriteContext context) @Override public PlanNode visitUnion(UnionNode node, RewriteContext context) { - return new UnionNode(node.getId(), rewriteSources(node, context).build(), canonicalizeSetOperationVariableMap(node.getVariableMapping())); + return new UnionNode(node.getId(), rewriteSources(node, context).build(), canonicalizeSetOperationOutputVariables(node.getOutputVariables()), canonicalizeSetOperationVariableMap(node.getVariableMapping())); } @Override public PlanNode visitIntersect(IntersectNode node, RewriteContext context) { - return new IntersectNode(node.getId(), rewriteSources(node, context).build(), canonicalizeSetOperationVariableMap(node.getVariableMapping())); + return new IntersectNode(node.getId(), rewriteSources(node, context).build(), canonicalizeSetOperationOutputVariables(node.getOutputVariables()), canonicalizeSetOperationVariableMap(node.getVariableMapping())); } @Override public PlanNode visitExcept(ExceptNode node, RewriteContext context) { - return new ExceptNode(node.getId(), rewriteSources(node, context).build(), canonicalizeSetOperationVariableMap(node.getVariableMapping())); + return new ExceptNode(node.getId(), rewriteSources(node, context).build(), canonicalizeSetOperationOutputVariables(node.getOutputVariables()), canonicalizeSetOperationVariableMap(node.getVariableMapping())); } private static ImmutableList.Builder rewriteSources(SetOperationNode node, RewriteContext context) @@ -615,6 +624,14 @@ public PlanNode visitTableWriter(TableWriterNode node, RewriteContext cont return mapper.map(node, source); } + @Override + public PlanNode visitTableWriteMerge(TableWriterMergeNode node, RewriteContext context) + { + PlanNode source = context.rewrite(node.getSource()); + SymbolMapper mapper = new SymbolMapper(mapping, types); + return mapper.map(node, source); + } + @Override public PlanNode visitPlan(PlanNode node, RewriteContext context) { @@ -629,19 +646,25 @@ private void map(VariableReferenceExpression variable, VariableReferenceExpressi private Assignments canonicalize(Assignments oldAssignments) { - Map computedExpressions = new HashMap<>(); + Map computedExpressions = new HashMap<>(); Assignments.Builder assignments = Assignments.builder(); for (Map.Entry entry : oldAssignments.getMap().entrySet()) { - Expression expression = canonicalize(castToExpression(entry.getValue())); - - if (expression instanceof SymbolReference) { + RowExpression expression = canonicalize(entry.getValue()); + if (expression instanceof VariableReferenceExpression) { + // Always map a trivial variable projection + VariableReferenceExpression variable = (VariableReferenceExpression) expression; + if (!variable.getName().equals(entry.getKey().getName())) { + map(entry.getKey(), variable); + } + } + else if (isExpression(expression) && castToExpression(expression) instanceof SymbolReference) { // Always map a trivial symbol projection - VariableReferenceExpression variable = new VariableReferenceExpression(((SymbolReference) expression).getName(), types.get(expression)); + VariableReferenceExpression variable = new VariableReferenceExpression(Symbol.from(castToExpression(expression)).getName(), types.get(castToExpression(expression))); if (!variable.getName().equals(entry.getKey().getName())) { map(entry.getKey(), variable); } } - else if (ExpressionDeterminismEvaluator.isDeterministic(expression) && !(expression instanceof NullLiteral)) { + else if (!isNull(expression) && isDeterministic(expression)) { // Try to map same deterministic expressions within a projection into the same symbol // Omit NullLiterals since those have ambiguous types VariableReferenceExpression computedVariable = computedExpressions.get(expression); @@ -657,11 +680,27 @@ else if (ExpressionDeterminismEvaluator.isDeterministic(expression) && !(express } VariableReferenceExpression canonical = canonicalize(entry.getKey()); - assignments.put(canonical, castToRowExpression(expression)); + assignments.put(canonical, expression); } return assignments.build(); } + private boolean isDeterministic(RowExpression expression) + { + if (isExpression(expression)) { + return ExpressionDeterminismEvaluator.isDeterministic(castToExpression(expression)); + } + return determinismEvaluator.isDeterministic(expression); + } + + private static boolean isNull(RowExpression expression) + { + if (isExpression(expression)) { + return castToExpression(expression) instanceof NullLiteral; + } + return Expressions.isNull(expression); + } + private Symbol canonicalize(Symbol symbol) { String canonical = symbol.getName(); @@ -688,6 +727,15 @@ private Optional canonicalize(Optional() @@ -726,7 +774,7 @@ private OrderingScheme canonicalizeAndDistinct(OrderingScheme orderingScheme) Set added = new HashSet<>(); ImmutableList.Builder variables = ImmutableList.builder(); ImmutableMap.Builder orderings = ImmutableMap.builder(); - for (VariableReferenceExpression variable : orderingScheme.getOrderBy()) { + for (VariableReferenceExpression variable : orderingScheme.getOrderByVariables()) { VariableReferenceExpression canonical = canonicalize(variable); if (added.add(canonical)) { variables.add(canonical); @@ -734,7 +782,8 @@ private OrderingScheme canonicalizeAndDistinct(OrderingScheme orderingScheme) } } - return new OrderingScheme(variables.build(), orderings.build()); + ImmutableMap orderingsMap = orderings.build(); + return new OrderingScheme(variables.build().stream().map(variable -> new Ordering(variable, orderingsMap.get(variable))).collect(toImmutableList())); } private Set canonicalize(Set variables) @@ -764,16 +813,25 @@ private List canonicalizeIndexJoinCriteria(List canonicalizeSetOperationVariableMap(ListMultimap setOperationVariableMap) + private Map> canonicalizeSetOperationVariableMap(Map> setOperationVariableMap) { - ImmutableListMultimap.Builder builder = ImmutableListMultimap.builder(); - Set addedSymbols = new HashSet<>(); - for (Map.Entry> entry : setOperationVariableMap.asMap().entrySet()) { + LinkedHashMap> result = new LinkedHashMap<>(); + Set addVariables = new HashSet<>(); + for (Map.Entry> entry : setOperationVariableMap.entrySet()) { VariableReferenceExpression canonicalOutputVariable = canonicalize(entry.getKey()); - if (addedSymbols.add(canonicalOutputVariable)) { - builder.putAll(canonicalOutputVariable, Iterables.transform(entry.getValue(), this::canonicalize)); + if (addVariables.add(canonicalOutputVariable)) { + result.put(canonicalOutputVariable, ImmutableList.copyOf(Iterables.transform(entry.getValue(), this::canonicalize))); } } + return result; + } + + private List canonicalizeSetOperationOutputVariables(List setOperationOutputVariables) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (VariableReferenceExpression variable : setOperationOutputVariables) { + builder.add(canonicalize(variable)); + } return builder.build(); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java index 8bc4cc01a156b..af8456758d26e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java @@ -17,9 +17,10 @@ import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionMetadata; -import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.predicate.Domain; @@ -27,13 +28,11 @@ import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.predicate.ValueSet; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.sql.ExpressionUtils; import com.facebook.presto.sql.planner.ExpressionDomainTranslator; import com.facebook.presto.sql.planner.LiteralEncoder; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; @@ -47,10 +46,8 @@ import java.util.OptionalInt; import static com.facebook.presto.SystemSessionProperties.isOptimizeTopNRowNumber; -import static com.facebook.presto.spi.function.FunctionKind.WINDOW; import static com.facebook.presto.spi.predicate.Marker.Bound.BELOW; import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.sql.planner.ExpressionDomainTranslator.ExtractionResult; import static com.facebook.presto.sql.planner.ExpressionDomainTranslator.fromPredicate; import static com.facebook.presto.sql.planner.plan.ChildReplacer.replaceChildren; @@ -66,8 +63,6 @@ public class WindowFilterPushDown implements PlanOptimizer { - private static final Signature ROW_NUMBER_SIGNATURE = new Signature("row_number", WINDOW, parseTypeSignature(StandardTypes.BIGINT), ImmutableList.of()); - private final Metadata metadata; private final ExpressionDomainTranslator domainTranslator; @@ -281,15 +276,13 @@ private static boolean canOptimizeWindowFunction(WindowNode node, FunctionManage return false; } VariableReferenceExpression rowNumberVariable = getOnlyElement(node.getWindowFunctions().keySet()); - return isRowNumberMetadata(functionManager.getFunctionMetadata(node.getWindowFunctions().get(rowNumberVariable).getFunctionHandle())); + return isRowNumberMetadata(functionManager, functionManager.getFunctionMetadata(node.getWindowFunctions().get(rowNumberVariable).getFunctionHandle())); } - private static boolean isRowNumberMetadata(FunctionMetadata functionMetadata) + private static boolean isRowNumberMetadata(FunctionManager functionManager, FunctionMetadata functionMetadata) { - return functionMetadata.getName().equals(ROW_NUMBER_SIGNATURE.getName()) - && functionMetadata.getFunctionKind().equals(ROW_NUMBER_SIGNATURE.getKind()) - && functionMetadata.getArgumentTypes().equals(ROW_NUMBER_SIGNATURE.getArgumentTypes()) - && functionMetadata.getReturnType().equals(ROW_NUMBER_SIGNATURE.getReturnType()); + FunctionHandle rowNumberFunction = functionManager.lookupFunction("row_number", ImmutableList.of()); + return functionMetadata.equals(functionManager.getFunctionMetadata(rowNumberFunction)); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowNodeUtil.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowNodeUtil.java index 1d269c225308b..8c39f7a9fff0d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowNodeUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowNodeUtil.java @@ -45,7 +45,7 @@ private WindowNodeUtil() {} public static boolean dependsOn(WindowNode parent, WindowNode child, TypeProvider types) { return parent.getPartitionBy().stream().anyMatch(child.getCreatedVariable()::contains) - || (parent.getOrderingScheme().isPresent() && parent.getOrderingScheme().get().getOrderBy().stream() + || (parent.getOrderingScheme().isPresent() && parent.getOrderingScheme().get().getOrderByVariables().stream() .anyMatch(child.getCreatedVariable()::contains)) || parent.getWindowFunctions().values().stream() .map(function -> extractWindowFunctionUniqueVariables(function, types)) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/joins/JoinGraph.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/joins/JoinGraph.java index 0d053d47533c2..4a2211c846a00 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/joins/JoinGraph.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/joins/JoinGraph.java @@ -16,12 +16,12 @@ import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.GroupReference; import com.facebook.presto.sql.planner.iterative.Lookup; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.tree.Expression; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ApplyNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ApplyNode.java index 8364bb2457684..9ebe3240273f0 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ApplyNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ApplyNode.java @@ -13,11 +13,10 @@ */ package com.facebook.presto.sql.planner.plan; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.optimizations.ApplyNodeUtil; -import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -26,6 +25,7 @@ import java.util.List; +import static com.facebook.presto.sql.planner.optimizations.ApplyNodeUtil.verifySubquerySupported; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; @@ -77,22 +77,14 @@ public ApplyNode( @JsonProperty("originSubqueryError") String originSubqueryError) { super(id); - requireNonNull(input, "input is null"); - requireNonNull(subquery, "right is null"); - requireNonNull(subqueryAssignments, "assignments is null"); - requireNonNull(correlation, "correlation is null"); - requireNonNull(originSubqueryError, "originSubqueryError is null"); - checkArgument(input.getOutputVariables().containsAll(correlation), "Input does not contain symbols from correlation"); - checkArgument( - subqueryAssignments.getExpressions().stream().map(OriginalExpressionUtils::castToExpression).allMatch(ApplyNodeUtil::isSupportedSubqueryExpression), - "Unexpected expression used for subquery expression"); - - this.input = input; - this.subquery = subquery; - this.subqueryAssignments = subqueryAssignments; - this.correlation = ImmutableList.copyOf(correlation); - this.originSubqueryError = originSubqueryError; + verifySubquerySupported(subqueryAssignments); + + this.input = requireNonNull(input, "input is null"); + this.subquery = requireNonNull(subquery, "subquery is null"); + this.subqueryAssignments = requireNonNull(subqueryAssignments, "assignments is null"); + this.correlation = ImmutableList.copyOf(requireNonNull(correlation, "correlation is null")); + this.originSubqueryError = requireNonNull(originSubqueryError, "originSubqueryError is null"); } @JsonProperty diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/AssignmentUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/AssignmentUtils.java index 036e0f7758dd7..93181870140f5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/AssignmentUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/AssignmentUtils.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.planner.plan; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.tree.Expression; @@ -28,6 +29,7 @@ import static com.facebook.presto.sql.relational.OriginalExpressionUtils.asSymbolReference; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToExpression; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; +import static com.facebook.presto.sql.relational.OriginalExpressionUtils.isExpression; import static java.util.Arrays.asList; import static java.util.Collections.singletonMap; @@ -65,11 +67,20 @@ public static Assignments identityAssignments(Collection outputVariables; @@ -41,14 +39,12 @@ public class DeleteNode public DeleteNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("source") PlanNode source, - @JsonProperty("target") DeleteHandle target, @JsonProperty("rowId") VariableReferenceExpression rowId, @JsonProperty("outputVariables") List outputVariables) { super(id); this.source = requireNonNull(source, "source is null"); - this.target = requireNonNull(target, "target is null"); this.rowId = requireNonNull(rowId, "rowId is null"); this.outputVariables = ImmutableList.copyOf(requireNonNull(outputVariables, "outputVariables is null")); } @@ -59,12 +55,6 @@ public PlanNode getSource() return source; } - @JsonProperty - public DeleteHandle getTarget() - { - return target; - } - @JsonProperty public VariableReferenceExpression getRowId() { @@ -93,6 +83,6 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new DeleteNode(getId(), Iterables.getOnlyElement(newChildren), target, rowId, outputVariables); + return new DeleteNode(getId(), Iterables.getOnlyElement(newChildren), rowId, outputVariables); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java index b58191d829f9b..8b4cb7672a9be 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java @@ -13,12 +13,11 @@ */ package com.facebook.presto.sql.planner.plan; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.Partitioning; -import com.facebook.presto.sql.planner.Partitioning.ArgumentBinding; import com.facebook.presto.sql.planner.PartitioningHandle; import com.facebook.presto.sql.planner.PartitioningScheme; import com.fasterxml.jackson.annotation.JsonCreator; @@ -89,6 +88,7 @@ public boolean isLocal() // for each source, the list of inputs corresponding to each output private final List> inputs; + private final boolean ensureSourceOrdering; private final Optional orderingScheme; @JsonCreator @@ -99,6 +99,7 @@ public ExchangeNode( @JsonProperty("partitioningScheme") PartitioningScheme partitioningScheme, @JsonProperty("sources") List sources, @JsonProperty("inputs") List> inputs, + @JsonProperty("ensureSourceOrdering") boolean ensureSourceOrdering, @JsonProperty("orderingScheme") Optional orderingScheme) { super(id); @@ -117,7 +118,7 @@ public ExchangeNode( checkArgument(sources.get(i).getOutputVariables().containsAll(inputs.get(i)), "Source does not supply all required input variables"); } - checkArgument(!scope.isLocal() || partitioningScheme.getPartitioning().getArguments().stream().allMatch(ArgumentBinding::isVariable), + checkArgument(!scope.isLocal() || partitioningScheme.getPartitioning().getArguments().stream().allMatch(VariableReferenceExpression.class::isInstance), "local exchanges do not support constant partition function arguments"); checkArgument(!scope.isRemote() || type == REPARTITION || !partitioningScheme.isReplicateNullsAndAny(), "Only REPARTITION can replicate remotely"); @@ -127,13 +128,15 @@ public ExchangeNode( PartitioningHandle partitioningHandle = partitioningScheme.getPartitioning().getHandle(); checkArgument(!scope.isRemote() || partitioningHandle.equals(SINGLE_DISTRIBUTION), "remote merging exchange requires single distribution"); checkArgument(!scope.isLocal() || partitioningHandle.equals(FIXED_PASSTHROUGH_DISTRIBUTION), "local merging exchange requires passthrough distribution"); - checkArgument(partitioningScheme.getOutputLayout().containsAll(ordering.getOrderBy()), "Partitioning scheme does not supply all required ordering symbols"); + checkArgument(partitioningScheme.getOutputLayout().containsAll(ordering.getOrderByVariables()), "Partitioning scheme does not supply all required ordering symbols"); }); this.type = type; this.sources = sources; this.scope = scope; this.partitioningScheme = partitioningScheme; this.inputs = listOfListsCopy(inputs); + this.ensureSourceOrdering = ensureSourceOrdering; + orderingScheme.ifPresent(scheme -> checkArgument(ensureSourceOrdering, "if ordering scheme is present the exchange must ensure source ordering")); this.orderingScheme = orderingScheme; } @@ -184,6 +187,7 @@ public static ExchangeNode partitionedExchange(PlanNodeId id, Scope scope, PlanN partitioningScheme, ImmutableList.of(child), ImmutableList.of(partitioningScheme.getOutputLayout()), + false, Optional.empty()); } @@ -196,10 +200,21 @@ public static ExchangeNode replicatedExchange(PlanNodeId id, Scope scope, PlanNo new PartitioningScheme(Partitioning.create(FIXED_BROADCAST_DISTRIBUTION, ImmutableList.of()), child.getOutputVariables()), ImmutableList.of(child), ImmutableList.of(child.getOutputVariables()), + false, Optional.empty()); } public static ExchangeNode gatheringExchange(PlanNodeId id, Scope scope, PlanNode child) + { + return gatheringExchange(id, scope, child, false); + } + + public static ExchangeNode ensureSourceOrderingGatheringExchange(PlanNodeId id, Scope scope, PlanNode child) + { + return gatheringExchange(id, scope, child, true); + } + + private static ExchangeNode gatheringExchange(PlanNodeId id, Scope scope, PlanNode child, boolean ensureSourceOrdering) { return new ExchangeNode( id, @@ -208,6 +223,7 @@ public static ExchangeNode gatheringExchange(PlanNodeId id, Scope scope, PlanNod new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), child.getOutputVariables()), ImmutableList.of(child), ImmutableList.of(child.getOutputVariables()), + ensureSourceOrdering, Optional.empty()); } @@ -230,6 +246,7 @@ public static ExchangeNode mergingExchange(PlanNodeId id, Scope scope, PlanNode new PartitioningScheme(Partitioning.create(partitioningHandle, ImmutableList.of()), child.getOutputVariables()), ImmutableList.of(child), ImmutableList.of(child.getOutputVariables()), + true, Optional.of(orderingScheme)); } @@ -264,6 +281,12 @@ public PartitioningScheme getPartitioningScheme() return partitioningScheme; } + @JsonProperty + public boolean isEnsureSourceOrdering() + { + return ensureSourceOrdering; + } + @JsonProperty public Optional getOrderingScheme() { @@ -285,6 +308,6 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new ExchangeNode(getId(), type, scope, partitioningScheme, newChildren, inputs, orderingScheme); + return new ExchangeNode(getId(), type, scope, partitioningScheme, newChildren, inputs, ensureSourceOrdering, orderingScheme); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/InternalPlanVisitor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/InternalPlanVisitor.java index ded51cbee62d4..0054b48cce599 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/InternalPlanVisitor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/InternalPlanVisitor.java @@ -24,31 +24,11 @@ public R visitRemoteSource(RemoteSourceNode node, C context) return visitPlan(node, context); } - public R visitAggregation(AggregationNode node, C context) - { - return visitPlan(node, context); - } - - public R visitProject(ProjectNode node, C context) - { - return visitPlan(node, context); - } - - public R visitTopN(TopNNode node, C context) - { - return visitPlan(node, context); - } - public R visitOutput(OutputNode node, C context) { return visitPlan(node, context); } - public R visitLimit(LimitNode node, C context) - { - return visitPlan(node, context); - } - public R visitDistinctLimit(DistinctLimitNode node, C context) { return visitPlan(node, context); @@ -104,37 +84,27 @@ public R visitTableWriter(TableWriterNode node, C context) return visitPlan(node, context); } - public R visitDelete(DeleteNode node, C context) - { - return visitPlan(node, context); - } - - public R visitMetadataDelete(MetadataDeleteNode node, C context) - { - return visitPlan(node, context); - } - - public R visitTableFinish(TableFinishNode node, C context) + public R visitTableWriteMerge(TableWriterMergeNode node, C context) { return visitPlan(node, context); } - public R visitStatisticsWriterNode(StatisticsWriterNode node, C context) + public R visitDelete(DeleteNode node, C context) { return visitPlan(node, context); } - public R visitUnion(UnionNode node, C context) + public R visitMetadataDelete(MetadataDeleteNode node, C context) { return visitPlan(node, context); } - public R visitIntersect(IntersectNode node, C context) + public R visitTableFinish(TableFinishNode node, C context) { return visitPlan(node, context); } - public R visitExcept(ExceptNode node, C context) + public R visitStatisticsWriterNode(StatisticsWriterNode node, C context) { return visitPlan(node, context); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/LimitNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/LimitNode.java deleted file mode 100644 index 905ec39ba2204..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/LimitNode.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sql.planner.plan; - -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeId; -import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; - -import javax.annotation.concurrent.Immutable; - -import java.util.List; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -@Immutable -public class LimitNode - extends InternalPlanNode -{ - private final PlanNode source; - private final long count; - private final boolean partial; - - @JsonCreator - public LimitNode( - @JsonProperty("id") PlanNodeId id, - @JsonProperty("source") PlanNode source, - @JsonProperty("count") long count, - @JsonProperty("partial") boolean partial) - { - super(id); - this.partial = partial; - - requireNonNull(source, "source is null"); - checkArgument(count >= 0, "count must be greater than or equal to zero"); - - this.source = source; - this.count = count; - } - - @Override - public List getSources() - { - return ImmutableList.of(source); - } - - @JsonProperty - public PlanNode getSource() - { - return source; - } - - @JsonProperty - public long getCount() - { - return count; - } - - @JsonProperty - public boolean isPartial() - { - return partial; - } - - @Override - public List getOutputVariables() - { - return source.getOutputVariables(); - } - - @Override - public R accept(InternalPlanVisitor visitor, C context) - { - return visitor.visitLimit(this, context); - } - - @Override - public PlanNode replaceChildren(List newChildren) - { - return new LimitNode(getId(), Iterables.getOnlyElement(newChildren), count, isPartial()); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/MetadataDeleteNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/MetadataDeleteNode.java index d91a47576825c..78147550fee80 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/MetadataDeleteNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/MetadataDeleteNode.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner.plan; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.plan.TableWriterNode.DeleteHandle; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -31,25 +31,25 @@ public class MetadataDeleteNode extends InternalPlanNode { - private final DeleteHandle target; + private final TableHandle tableHandle; private final VariableReferenceExpression output; @JsonCreator public MetadataDeleteNode( @JsonProperty("id") PlanNodeId id, - @JsonProperty("target") DeleteHandle target, + @JsonProperty("target") TableHandle tableHandle, @JsonProperty("output") VariableReferenceExpression output) { super(id); - this.target = requireNonNull(target, "target is null"); + this.tableHandle = requireNonNull(tableHandle, "target is null"); this.output = requireNonNull(output, "output is null"); } @JsonProperty - public DeleteHandle getTarget() + public TableHandle getTableHandle() { - return target; + return tableHandle; } @JsonProperty @@ -79,6 +79,6 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new MetadataDeleteNode(getId(), target, output); + return new MetadataDeleteNode(getId(), tableHandle, output); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java index ba7c7c6658422..5eeac8d37eac8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java @@ -15,9 +15,14 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.matching.Property; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -138,6 +143,11 @@ public static Pattern tableWriterNode() return typeOf(TableWriterNode.class); } + public static Pattern tableWriterMergeNode() + { + return typeOf(TableWriterMergeNode.class); + } + public static Pattern topN() { return typeOf(TopNNode.class); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/RemoteSourceNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/RemoteSourceNode.java index 870effa798558..e18a8ed8b0c78 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/RemoteSourceNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/RemoteSourceNode.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner.plan; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -35,6 +35,7 @@ public class RemoteSourceNode { private final List sourceFragmentIds; private final List outputVariables; + private final boolean ensureSourceOrdering; private final Optional orderingScheme; private final ExchangeNode.Type exchangeType; // This is needed to "unfragment" to compute stats correctly. @@ -43,6 +44,7 @@ public RemoteSourceNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("sourceFragmentIds") List sourceFragmentIds, @JsonProperty("outputVariables") List outputVariables, + @JsonProperty("ensureSourceOrdering") boolean ensureSourceOrdering, @JsonProperty("orderingScheme") Optional orderingScheme, @JsonProperty("exchangeType") ExchangeNode.Type exchangeType) { @@ -50,6 +52,7 @@ public RemoteSourceNode( this.sourceFragmentIds = sourceFragmentIds; this.outputVariables = ImmutableList.copyOf(requireNonNull(outputVariables, "outputVariables is null")); + this.ensureSourceOrdering = ensureSourceOrdering; this.orderingScheme = requireNonNull(orderingScheme, "orderingScheme is null"); this.exchangeType = requireNonNull(exchangeType, "exchangeType is null"); } @@ -58,10 +61,11 @@ public RemoteSourceNode( PlanNodeId id, PlanFragmentId sourceFragmentId, List outputVariables, + boolean ensureSourceOrdering, Optional orderingScheme, ExchangeNode.Type exchangeType) { - this(id, ImmutableList.of(sourceFragmentId), outputVariables, orderingScheme, exchangeType); + this(id, ImmutableList.of(sourceFragmentId), outputVariables, ensureSourceOrdering, orderingScheme, exchangeType); } @Override @@ -83,6 +87,12 @@ public List getSourceFragmentIds() return sourceFragmentIds; } + @JsonProperty + public boolean isEnsureSourceOrdering() + { + return ensureSourceOrdering; + } + @JsonProperty public Optional getOrderingScheme() { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SetOperationNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SetOperationNode.java deleted file mode 100644 index 442d9c9fb885d..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SetOperationNode.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sql.planner.plan; - -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeId; -import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableListMultimap; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.common.collect.ListMultimap; -import com.google.common.collect.Multimap; - -import javax.annotation.concurrent.Immutable; - -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.util.Objects.requireNonNull; - -@Immutable -public abstract class SetOperationNode - extends InternalPlanNode -{ - private final List sources; - private final ListMultimap outputToInputs; - private final List outputVariables; - - @JsonCreator - protected SetOperationNode( - @JsonProperty("id") PlanNodeId id, - @JsonProperty("sources") List sources, - @JsonProperty("outputToInputs") ListMultimap outputToInputs) - { - super(id); - - requireNonNull(sources, "sources is null"); - checkArgument(!sources.isEmpty(), "Must have at least one source"); - requireNonNull(outputToInputs, "outputToInputs is null"); - - this.sources = ImmutableList.copyOf(sources); - this.outputToInputs = ImmutableListMultimap.copyOf(outputToInputs); - this.outputVariables = ImmutableList.copyOf(outputToInputs.keySet()); - - for (Collection inputs : this.outputToInputs.asMap().values()) { - checkArgument(inputs.size() == this.sources.size(), "Every source needs to map its symbols to an output %s operation symbol", this.getClass().getSimpleName()); - } - - // Make sure each source positionally corresponds to their Symbol values in the Multimap - for (int i = 0; i < sources.size(); i++) { - for (Collection expectedInputs : this.outputToInputs.asMap().values()) { - checkArgument(sources.get(i).getOutputVariables().contains(Iterables.get(expectedInputs, i)), "Source does not provide required symbols"); - } - } - } - - @Override - @JsonProperty - public List getSources() - { - return sources; - } - - @JsonProperty - public ListMultimap getVariableMapping() - { - return outputToInputs; - } - - @Override - public List getOutputVariables() - { - return outputVariables; - } - - public List sourceOutputLayout(int sourceIndex) - { - // Make sure the sourceOutputSymbolLayout symbols are listed in the same order as the corresponding output symbols - return getOutputVariables().stream() - .map(variable -> outputToInputs.get(variable).get(sourceIndex)) - .collect(toImmutableList()); - } - - /** - * Returns the output to input variable mapping for the given source channel - */ - public Map sourceVariableMap(int sourceIndex) - { - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (Map.Entry> entry : outputToInputs.asMap().entrySet()) { - builder.put(entry.getKey(), Iterables.get(entry.getValue(), sourceIndex)); - } - - return builder.build(); - } - /** - * Returns the input to output symbol mapping for the given source channel. - * A single input symbol can map to multiple output symbols, thus requiring a Multimap. - */ - public Multimap outputMap(int sourceIndex) - { - return FluentIterable.from(getOutputVariables()) - .toMap(output -> outputToInputs.get(output).get(sourceIndex)) - .asMultimap() - .inverse(); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SortNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SortNode.java index 6da65b26f2600..846440fa16531 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SortNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SortNode.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner.plan; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -31,11 +31,13 @@ public class SortNode { private final PlanNode source; private final OrderingScheme orderingScheme; + private final boolean isPartial; @JsonCreator public SortNode(@JsonProperty("id") PlanNodeId id, @JsonProperty("source") PlanNode source, - @JsonProperty("orderingScheme") OrderingScheme orderingScheme) + @JsonProperty("orderingScheme") OrderingScheme orderingScheme, + @JsonProperty("isPartial") boolean isPartial) { super(id); @@ -44,6 +46,7 @@ public SortNode(@JsonProperty("id") PlanNodeId id, this.source = source; this.orderingScheme = orderingScheme; + this.isPartial = isPartial; } @Override @@ -70,6 +73,12 @@ public OrderingScheme getOrderingScheme() return orderingScheme; } + @JsonProperty("isPartial") + public boolean isPartial() + { + return isPartial; + } + @Override public R accept(InternalPlanVisitor visitor, C context) { @@ -79,6 +88,6 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new SortNode(getId(), Iterables.getOnlyElement(newChildren), orderingScheme); + return new SortNode(getId(), Iterables.getOnlyElement(newChildren), orderingScheme, isPartial); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregations.java index 365d2a0adc252..619b91421afe1 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregations.java @@ -16,10 +16,10 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.operator.aggregation.InternalAggregationFunction; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.PlanVariableAllocator; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -29,8 +29,7 @@ import java.util.Map; import java.util.Optional; -import static com.facebook.presto.sql.relational.OriginalExpressionUtils.asSymbolReference; -import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; public class StatisticAggregations @@ -59,18 +58,28 @@ public List getGroupingVariables() return groupingVariables; } - public Parts createPartialAggregations(PlanVariableAllocator variableAllocator, FunctionManager functionManager) + public Parts splitIntoPartialAndFinal(PlanVariableAllocator variableAllocator, FunctionManager functionManager) { - ImmutableMap.Builder partialAggregation = ImmutableMap.builder(); - ImmutableMap.Builder finalAggregation = ImmutableMap.builder(); - ImmutableMap.Builder mappings = ImmutableMap.builder(); + return split(variableAllocator, functionManager, false); + } + + public Parts splitIntoPartialAndIntermediate(PlanVariableAllocator variableAllocator, FunctionManager functionManager) + { + return split(variableAllocator, functionManager, true); + } + + private Parts split(PlanVariableAllocator variableAllocator, FunctionManager functionManager, boolean intermediate) + { + ImmutableMap.Builder finalOrIntermediateAggregations = ImmutableMap.builder(); + ImmutableMap.Builder partialAggregations = ImmutableMap.builder(); for (Map.Entry entry : aggregations.entrySet()) { Aggregation originalAggregation = entry.getValue(); FunctionHandle functionHandle = originalAggregation.getFunctionHandle(); InternalAggregationFunction function = functionManager.getAggregateFunctionImplementation(functionHandle); - VariableReferenceExpression partialVariable = variableAllocator.newVariable(functionManager.getFunctionMetadata(functionHandle).getName(), function.getIntermediateType()); - mappings.put(entry.getKey(), partialVariable); - partialAggregation.put(partialVariable, new Aggregation( + + // create partial aggregation + VariableReferenceExpression partialVariable = variableAllocator.newVariable(functionManager.getFunctionMetadata(functionHandle).getName().getFunctionName(), function.getIntermediateType()); + partialAggregations.put(partialVariable, new Aggregation( new CallExpression( originalAggregation.getCall().getDisplayName(), functionHandle, @@ -80,51 +89,60 @@ public Parts createPartialAggregations(PlanVariableAllocator variableAllocator, originalAggregation.getOrderBy(), originalAggregation.isDistinct(), originalAggregation.getMask())); - finalAggregation.put(entry.getKey(), + + // create final aggregation + finalOrIntermediateAggregations.put(entry.getKey(), new Aggregation( new CallExpression( originalAggregation.getCall().getDisplayName(), functionHandle, - function.getFinalType(), - ImmutableList.of(castToRowExpression(asSymbolReference(partialVariable)))), + intermediate ? function.getIntermediateType() : function.getFinalType(), + ImmutableList.of(partialVariable)), Optional.empty(), Optional.empty(), false, Optional.empty())); } - groupingVariables.forEach(symbol -> mappings.put(symbol, symbol)); + + StatisticAggregations finalOrIntermediateAggregation = new StatisticAggregations(finalOrIntermediateAggregations.build(), groupingVariables); return new Parts( - new StatisticAggregations(partialAggregation.build(), groupingVariables), - new StatisticAggregations(finalAggregation.build(), groupingVariables), - mappings.build()); + intermediate ? Optional.empty() : Optional.of(finalOrIntermediateAggregation), + intermediate ? Optional.of(finalOrIntermediateAggregation) : Optional.empty(), + new StatisticAggregations(partialAggregations.build(), groupingVariables)); } public static class Parts { + private final Optional finalAggregation; + private final Optional intermediateAggregation; private final StatisticAggregations partialAggregation; - private final StatisticAggregations finalAggregation; - private final Map mappings; - public Parts(StatisticAggregations partialAggregation, StatisticAggregations finalAggregation, Map mappings) + public Parts( + Optional finalAggregation, + Optional intermediateAggregation, + StatisticAggregations partialAggregation) { - this.partialAggregation = requireNonNull(partialAggregation, "partialAggregation is null"); this.finalAggregation = requireNonNull(finalAggregation, "finalAggregation is null"); - this.mappings = ImmutableMap.copyOf(requireNonNull(mappings, "mappings is null")); + this.intermediateAggregation = requireNonNull(intermediateAggregation, "intermediateAggregation is null"); + checkArgument( + finalAggregation.isPresent() ^ intermediateAggregation.isPresent(), + "only final or only intermediate aggregation is expected to be present"); + this.partialAggregation = requireNonNull(partialAggregation, "partialAggregation is null"); } - public StatisticAggregations getPartialAggregation() + public StatisticAggregations getFinalAggregation() { - return partialAggregation; + return finalAggregation.orElseThrow(() -> new IllegalStateException("finalAggregation is not present")); } - public StatisticAggregations getFinalAggregation() + public StatisticAggregations getIntermediateAggregation() { - return finalAggregation; + return intermediateAggregation.orElseThrow(() -> new IllegalStateException("intermediateAggregation is not present")); } - public Map getMappings() + public StatisticAggregations getPartialAggregation() { - return mappings; + return partialAggregation; } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticsWriterNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticsWriterNode.java index dd75548144b6f..5b16638044028 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticsWriterNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticsWriterNode.java @@ -13,21 +13,16 @@ */ package com.facebook.presto.sql.planner.plan; -import com.facebook.presto.metadata.AnalyzeTableHandle; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import java.util.List; -import java.util.Objects; import static java.util.Objects.requireNonNull; @@ -35,8 +30,8 @@ public class StatisticsWriterNode extends InternalPlanNode { private final PlanNode source; + private final TableHandle tableHandle; private final VariableReferenceExpression rowCountVariable; - private final WriteStatisticsTarget target; private final boolean rowCountEnabled; private final StatisticAggregationsDescriptor descriptor; @@ -44,14 +39,14 @@ public class StatisticsWriterNode public StatisticsWriterNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("source") PlanNode source, - @JsonProperty("target") WriteStatisticsTarget target, + @JsonProperty("tableHandle") TableHandle tableHandle, @JsonProperty("rowCountVariable") VariableReferenceExpression rowCountVariable, @JsonProperty("rowCountEnabled") boolean rowCountEnabled, @JsonProperty("descriptor") StatisticAggregationsDescriptor descriptor) { super(id); this.source = requireNonNull(source, "source is null"); - this.target = requireNonNull(target, "target is null"); + this.tableHandle = requireNonNull(tableHandle, "tableHandle is null"); this.rowCountVariable = requireNonNull(rowCountVariable, "rowCountVariable is null"); this.rowCountEnabled = rowCountEnabled; this.descriptor = requireNonNull(descriptor, "descriptor is null"); @@ -64,9 +59,9 @@ public PlanNode getSource() } @JsonProperty - public WriteStatisticsTarget getTarget() + public TableHandle getTableHandle() { - return target; + return tableHandle; } @JsonProperty @@ -105,7 +100,7 @@ public PlanNode replaceChildren(List newChildren) return new StatisticsWriterNode( getId(), Iterables.getOnlyElement(newChildren), - target, + tableHandle, rowCountVariable, rowCountEnabled, descriptor); @@ -116,92 +111,4 @@ public R accept(InternalPlanVisitor visitor, C context) { return visitor.visitStatisticsWriterNode(this, context); } - - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "@type") - @JsonSubTypes({ - @JsonSubTypes.Type(value = WriteStatisticsHandle.class, name = "WriteStatisticsHandle"), - @JsonSubTypes.Type(value = TestWriteStatisticsHandle.class, name = "TestWriteStatisticsHandle")}) - @SuppressWarnings({"EmptyClass", "ClassMayBeInterface"}) - public abstract static class WriteStatisticsTarget - { - @Override - public abstract String toString(); - } - - public static class WriteStatisticsHandle - extends WriteStatisticsTarget - { - private final AnalyzeTableHandle handle; - - @JsonCreator - public WriteStatisticsHandle(@JsonProperty("handle") AnalyzeTableHandle handle) - { - this.handle = requireNonNull(handle, "handle is null"); - } - - @JsonProperty - public AnalyzeTableHandle getHandle() - { - return handle; - } - - @Override - public String toString() - { - return handle.toString(); - } - } - - // only used during planning -- will not be serialized - public static class WriteStatisticsReference - extends WriteStatisticsTarget - { - private final TableHandle handle; - - public WriteStatisticsReference(TableHandle handle) - { - this.handle = requireNonNull(handle, "handle is null"); - } - - public TableHandle getHandle() - { - return handle; - } - - @Override - public String toString() - { - return handle.toString(); - } - } - - // only used for testing - @VisibleForTesting - public static class TestWriteStatisticsHandle - extends WriteStatisticsTarget - { - @Override - public String toString() - { - return "test"; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - return true; - } - - @Override - public int hashCode() - { - return Objects.hashCode("test"); - } - } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableFinishNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableFinishNode.java index bf6dbe77498f0..e77471f82b2f7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableFinishNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableFinishNode.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -35,7 +36,7 @@ public class TableFinishNode extends InternalPlanNode { private final PlanNode source; - private final WriterTarget target; + private final Optional target; private final VariableReferenceExpression rowCountVariable; private final Optional statisticsAggregation; private final Optional> statisticsAggregationDescriptor; @@ -44,7 +45,7 @@ public class TableFinishNode public TableFinishNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("source") PlanNode source, - @JsonProperty("target") WriterTarget target, + @JsonProperty("target") Optional target, @JsonProperty("rowCountVariable") VariableReferenceExpression rowCountVariable, @JsonProperty("statisticsAggregation") Optional statisticsAggregation, @JsonProperty("statisticsAggregationDescriptor") Optional> statisticsAggregationDescriptor) @@ -66,8 +67,8 @@ public PlanNode getSource() return source; } - @JsonProperty - public WriterTarget getTarget() + @JsonIgnore + public Optional getTarget() { return target; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterMergeNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterMergeNode.java new file mode 100644 index 0000000000000..28c75087ef711 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterMergeNode.java @@ -0,0 +1,119 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.plan; + +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Optional; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static java.util.Objects.requireNonNull; + +public class TableWriterMergeNode + extends InternalPlanNode +{ + private final PlanNode source; + private final VariableReferenceExpression rowCountVariable; + private final VariableReferenceExpression fragmentVariable; + private final VariableReferenceExpression tableCommitContextVariable; + private final Optional statisticsAggregation; + private final List outputs; + + @JsonCreator + public TableWriterMergeNode( + @JsonProperty("id") PlanNodeId id, + @JsonProperty("source") PlanNode source, + @JsonProperty("rowCountVariable") VariableReferenceExpression rowCountVariable, + @JsonProperty("fragmentVariable") VariableReferenceExpression fragmentVariable, + @JsonProperty("tableCommitContextVariable") VariableReferenceExpression tableCommitContextVariable, + @JsonProperty("statisticsAggregation") Optional statisticsAggregation) + { + super(requireNonNull(id, "id is null")); + this.source = requireNonNull(source, "source is null"); + this.rowCountVariable = requireNonNull(rowCountVariable, "rowCountVariable is null"); + this.fragmentVariable = requireNonNull(fragmentVariable, "fragmentVariable is null"); + this.tableCommitContextVariable = requireNonNull(tableCommitContextVariable, "tableCommitContextVariable is null"); + this.statisticsAggregation = requireNonNull(statisticsAggregation, "statisticsAggregation is null"); + + ImmutableList.Builder outputs = ImmutableList.builder() + .add(rowCountVariable) + .add(fragmentVariable) + .add(tableCommitContextVariable); + statisticsAggregation.ifPresent(aggregation -> { + outputs.addAll(aggregation.getGroupingVariables()); + outputs.addAll(aggregation.getAggregations().keySet()); + }); + this.outputs = outputs.build(); + } + + @JsonProperty + public PlanNode getSource() + { + return source; + } + + @JsonProperty + public VariableReferenceExpression getRowCountVariable() + { + return rowCountVariable; + } + + @JsonProperty + public VariableReferenceExpression getFragmentVariable() + { + return fragmentVariable; + } + + @JsonProperty + public VariableReferenceExpression getTableCommitContextVariable() + { + return tableCommitContextVariable; + } + + @JsonProperty + public Optional getStatisticsAggregation() + { + return statisticsAggregation; + } + + @Override + public List getSources() + { + return ImmutableList.of(source); + } + + @Override + public List getOutputVariables() + { + return outputs; + } + + @Override + public PlanNode replaceChildren(List newChildren) + { + return new TableWriterMergeNode(getId(), getOnlyElement(newChildren), rowCountVariable, fragmentVariable, tableCommitContextVariable, statisticsAggregation); + } + + @Override + public R accept(InternalPlanVisitor visitor, C context) + { + return visitor.visitTableWriteMerge(this, context); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java index 8197a80975338..1675df9a23c05 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java @@ -13,9 +13,8 @@ */ package com.facebook.presto.sql.planner.plan; -import com.facebook.presto.metadata.InsertTableHandle; import com.facebook.presto.metadata.NewTableLayout; -import com.facebook.presto.metadata.OutputTableHandle; +import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableHandle; @@ -24,9 +23,8 @@ import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.PartitioningScheme; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -43,7 +41,7 @@ public class TableWriterNode extends InternalPlanNode { private final PlanNode source; - private final WriterTarget target; + private final Optional target; private final VariableReferenceExpression rowCountVariable; private final VariableReferenceExpression fragmentVariable; private final VariableReferenceExpression tableCommitContextVariable; @@ -51,22 +49,20 @@ public class TableWriterNode private final List columnNames; private final Optional partitioningScheme; private final Optional statisticsAggregation; - private final Optional> statisticsAggregationDescriptor; private final List outputs; @JsonCreator public TableWriterNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("source") PlanNode source, - @JsonProperty("target") WriterTarget target, + @JsonProperty("target") Optional target, @JsonProperty("rowCountVariable") VariableReferenceExpression rowCountVariable, @JsonProperty("fragmentVariable") VariableReferenceExpression fragmentVariable, @JsonProperty("tableCommitContextVariable") VariableReferenceExpression tableCommitContextVariable, @JsonProperty("columns") List columns, @JsonProperty("columnNames") List columnNames, @JsonProperty("partitioningScheme") Optional partitioningScheme, - @JsonProperty("statisticsAggregation") Optional statisticsAggregation, - @JsonProperty("statisticsAggregationDescriptor") Optional> statisticsAggregationDescriptor) + @JsonProperty("statisticsAggregation") Optional statisticsAggregation) { super(id); @@ -83,8 +79,6 @@ public TableWriterNode( this.columnNames = ImmutableList.copyOf(columnNames); this.partitioningScheme = requireNonNull(partitioningScheme, "partitioningScheme is null"); this.statisticsAggregation = requireNonNull(statisticsAggregation, "statisticsAggregation is null"); - this.statisticsAggregationDescriptor = requireNonNull(statisticsAggregationDescriptor, "statisticsAggregationDescriptor is null"); - checkArgument(statisticsAggregation.isPresent() == statisticsAggregationDescriptor.isPresent(), "statisticsAggregation and statisticsAggregationDescriptor must be either present or absent"); ImmutableList.Builder outputs = ImmutableList.builder() .add(rowCountVariable) @@ -103,8 +97,8 @@ public PlanNode getSource() return source; } - @JsonProperty - public WriterTarget getTarget() + @JsonIgnore + public Optional getTarget() { return target; } @@ -151,12 +145,6 @@ public Optional getStatisticsAggregation() return statisticsAggregation; } - @JsonProperty - public Optional> getStatisticsAggregationDescriptor() - { - return statisticsAggregationDescriptor; - } - @Override public List getSources() { @@ -178,39 +166,39 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new TableWriterNode(getId(), Iterables.getOnlyElement(newChildren), target, rowCountVariable, fragmentVariable, tableCommitContextVariable, columns, columnNames, partitioningScheme, statisticsAggregation, statisticsAggregationDescriptor); + return new TableWriterNode(getId(), Iterables.getOnlyElement(newChildren), target, rowCountVariable, fragmentVariable, tableCommitContextVariable, columns, columnNames, partitioningScheme, statisticsAggregation); } - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "@type") - @JsonSubTypes({ - @JsonSubTypes.Type(value = CreateHandle.class, name = "CreateHandle"), - @JsonSubTypes.Type(value = InsertHandle.class, name = "InsertHandle"), - @JsonSubTypes.Type(value = DeleteHandle.class, name = "DeleteHandle")}) + // only used during planning -- will not be serialized @SuppressWarnings({"EmptyClass", "ClassMayBeInterface"}) public abstract static class WriterTarget { + public abstract ConnectorId getConnectorId(); + + public abstract SchemaTableName getSchemaTableName(); + @Override public abstract String toString(); } - // only used during planning -- will not be serialized public static class CreateName extends WriterTarget { - private final String catalog; + private final ConnectorId connectorId; private final ConnectorTableMetadata tableMetadata; private final Optional layout; - public CreateName(String catalog, ConnectorTableMetadata tableMetadata, Optional layout) + public CreateName(ConnectorId connectorId, ConnectorTableMetadata tableMetadata, Optional layout) { - this.catalog = requireNonNull(catalog, "catalog is null"); + this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.tableMetadata = requireNonNull(tableMetadata, "tableMetadata is null"); this.layout = requireNonNull(layout, "layout is null"); } - public String getCatalog() + @Override + public ConnectorId getConnectorId() { - return catalog; + return connectorId; } public ConnectorTableMetadata getTableMetadata() @@ -224,55 +212,29 @@ public Optional getLayout() } @Override - public String toString() - { - return catalog + "." + tableMetadata.getTable(); - } - } - - public static class CreateHandle - extends WriterTarget - { - private final OutputTableHandle handle; - private final SchemaTableName schemaTableName; - - @JsonCreator - public CreateHandle( - @JsonProperty("handle") OutputTableHandle handle, - @JsonProperty("schemaTableName") SchemaTableName schemaTableName) - { - this.handle = requireNonNull(handle, "handle is null"); - this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null"); - } + public SchemaTableName getSchemaTableName() - @JsonProperty - public OutputTableHandle getHandle() { - return handle; - } - - @JsonProperty - public SchemaTableName getSchemaTableName() - { - return schemaTableName; + return tableMetadata.getTable(); } @Override public String toString() { - return handle.toString(); + return connectorId + "." + tableMetadata.getTable(); } } - // only used during planning -- will not be serialized public static class InsertReference extends WriterTarget { private final TableHandle handle; + private final SchemaTableName schemaTableName; - public InsertReference(TableHandle handle) + public InsertReference(TableHandle handle, SchemaTableName schemaTableName) { this.handle = requireNonNull(handle, "handle is null"); + this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null"); } public TableHandle getHandle() @@ -281,34 +243,12 @@ public TableHandle getHandle() } @Override - public String toString() - { - return handle.toString(); - } - } - - public static class InsertHandle - extends WriterTarget - { - private final InsertTableHandle handle; - private final SchemaTableName schemaTableName; - - @JsonCreator - public InsertHandle( - @JsonProperty("handle") InsertTableHandle handle, - @JsonProperty("schemaTableName") SchemaTableName schemaTableName) + public ConnectorId getConnectorId() { - this.handle = requireNonNull(handle, "handle is null"); - this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null"); - } - - @JsonProperty - public InsertTableHandle getHandle() - { - return handle; + return handle.getConnectorId(); } - @JsonProperty + @Override public SchemaTableName getSchemaTableName() { return schemaTableName; @@ -327,28 +267,31 @@ public static class DeleteHandle private final TableHandle handle; private final SchemaTableName schemaTableName; - @JsonCreator public DeleteHandle( - @JsonProperty("handle") TableHandle handle, - @JsonProperty("schemaTableName") SchemaTableName schemaTableName) + TableHandle handle, + SchemaTableName schemaTableName) { this.handle = requireNonNull(handle, "handle is null"); this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null"); } - @JsonProperty public TableHandle getHandle() { return handle; } - @JsonProperty + @Override + public ConnectorId getConnectorId() + { + return handle.getConnectorId(); + } + + @Override public SchemaTableName getSchemaTableName() { return schemaTableName; } - @Override public String toString() { return handle.toString(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TopNRowNumberNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TopNRowNumberNode.java index 4d09090008f0e..30fccd62fc0c4 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TopNRowNumberNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TopNRowNumberNode.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner.plan; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.plan.WindowNode.Specification; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/WindowNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/WindowNode.java index e7ed6558ed02c..7b3047adb849c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/WindowNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/WindowNode.java @@ -14,11 +14,11 @@ package com.facebook.presto.sql.planner.plan; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -67,7 +67,7 @@ public WindowNode( requireNonNull(hashVariable, "hashVariable is null"); checkArgument(specification.getPartitionBy().containsAll(prePartitionedInputs), "prePartitionedInputs must be contained in partitionBy"); Optional orderingScheme = specification.getOrderingScheme(); - checkArgument(preSortedOrderPrefix == 0 || (orderingScheme.isPresent() && preSortedOrderPrefix <= orderingScheme.get().getOrderBy().size()), "Cannot have sorted more symbols than those requested"); + checkArgument(preSortedOrderPrefix == 0 || (orderingScheme.isPresent() && preSortedOrderPrefix <= orderingScheme.get().getOrderByVariables().size()), "Cannot have sorted more symbols than those requested"); checkArgument(preSortedOrderPrefix == 0 || ImmutableSet.copyOf(prePartitionedInputs).equals(ImmutableSet.copyOf(specification.getPartitionBy())), "preSortedOrderPrefix can only be greater than zero if all partition symbols are pre-partitioned"); this.source = source; @@ -342,14 +342,17 @@ public static final class Function { private final CallExpression functionCall; private final Frame frame; + private final boolean ignoreNulls; @JsonCreator public Function( @JsonProperty("functionCall") CallExpression functionCall, - @JsonProperty("frame") Frame frame) + @JsonProperty("frame") Frame frame, + @JsonProperty("ignoreNulls") boolean ignoreNulls) { this.functionCall = requireNonNull(functionCall, "functionCall is null"); this.frame = requireNonNull(frame, "Frame is null"); + this.ignoreNulls = ignoreNulls; } @JsonProperty @@ -373,7 +376,7 @@ public Frame getFrame() @Override public int hashCode() { - return Objects.hash(functionCall, frame); + return Objects.hash(functionCall, frame, ignoreNulls); } @Override @@ -387,7 +390,13 @@ public boolean equals(Object obj) } Function other = (Function) obj; return Objects.equals(this.functionCall, other.functionCall) && - Objects.equals(this.frame, other.frame); + Objects.equals(this.frame, other.frame) && + Objects.equals(this.ignoreNulls, other.ignoreNulls); + } + + public boolean isIgnoreNulls() + { + return ignoreNulls; } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/HashCollisionPlanNodeStats.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/HashCollisionPlanNodeStats.java index ea7d9d9504a85..0c2eb6b439adc 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/HashCollisionPlanNodeStats.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/HashCollisionPlanNodeStats.java @@ -36,12 +36,14 @@ public HashCollisionPlanNodeStats( Duration planNodeCpuTime, long planNodeInputPositions, DataSize planNodeInputDataSize, + long planNodeRawInputPositions, + DataSize planNodeRawInputDataSize, long planNodeOutputPositions, DataSize planNodeOutputDataSize, Map operatorInputStats, Map operatorHashCollisionsStats) { - super(planNodeId, planNodeScheduledTime, planNodeCpuTime, planNodeInputPositions, planNodeInputDataSize, planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats); + super(planNodeId, planNodeScheduledTime, planNodeCpuTime, planNodeInputPositions, planNodeInputDataSize, planNodeRawInputPositions, planNodeRawInputDataSize, planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats); this.operatorHashCollisionsStats = requireNonNull(operatorHashCollisionsStats, "operatorHashCollisionsStats is null"); } @@ -92,6 +94,8 @@ public PlanNodeStats mergeWith(PlanNodeStats other) merged.getPlanNodeCpuTime(), merged.getPlanNodeInputPositions(), merged.getPlanNodeInputDataSize(), + merged.getPlanNodeRawInputPositions(), + merged.getPlanNodeRawInputDataSize(), merged.getPlanNodeOutputPositions(), merged.getPlanNodeOutputDataSize(), merged.operatorInputStats, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/IOPlanPrinter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/IOPlanPrinter.java index 592abe6ca609c..d892463fb5b44 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/IOPlanPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/IOPlanPrinter.java @@ -37,15 +37,11 @@ import com.facebook.presto.spi.type.VarcharType; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.TableFinishNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode.CreateHandle; -import com.facebook.presto.sql.planner.plan.TableWriterNode.CreateName; -import com.facebook.presto.sql.planner.plan.TableWriterNode.DeleteHandle; -import com.facebook.presto.sql.planner.plan.TableWriterNode.InsertHandle; -import com.facebook.presto.sql.planner.plan.TableWriterNode.InsertReference; import com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget; import com.facebook.presto.sql.planner.planPrinter.IOPlanPrinter.IOPlan.IOPlanBuilder; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableSet; import io.airlift.slice.Slice; @@ -55,12 +51,12 @@ import java.util.Optional; import java.util.Set; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.predicate.Marker.Bound.EXACTLY; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static io.airlift.json.JsonCodec.jsonCodec; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -486,34 +482,11 @@ public Void visitTableScan(TableScanNode node, IOPlanBuilder context) @Override public Void visitTableFinish(TableFinishNode node, IOPlanBuilder context) { - WriterTarget writerTarget = node.getTarget(); - if (writerTarget instanceof CreateHandle) { - CreateHandle createHandle = (CreateHandle) writerTarget; - context.setOutputTable(new CatalogSchemaTableName( - createHandle.getHandle().getConnectorId().getCatalogName(), - createHandle.getSchemaTableName().getSchemaName(), - createHandle.getSchemaTableName().getTableName())); - } - else if (writerTarget instanceof InsertHandle) { - InsertHandle insertHandle = (InsertHandle) writerTarget; - context.setOutputTable(new CatalogSchemaTableName( - insertHandle.getHandle().getConnectorId().getCatalogName(), - insertHandle.getSchemaTableName().getSchemaName(), - insertHandle.getSchemaTableName().getTableName())); - } - else if (writerTarget instanceof DeleteHandle) { - DeleteHandle deleteHandle = (DeleteHandle) writerTarget; - context.setOutputTable(new CatalogSchemaTableName( - deleteHandle.getHandle().getConnectorId().getCatalogName(), - deleteHandle.getSchemaTableName().getSchemaName(), - deleteHandle.getSchemaTableName().getTableName())); - } - else if (writerTarget instanceof CreateName || writerTarget instanceof InsertReference) { - throw new IllegalStateException(format("%s should not appear in final plan", writerTarget.getClass().getSimpleName())); - } - else { - throw new IllegalStateException(format("Unknown WriterTarget subclass %s", writerTarget.getClass().getSimpleName())); - } + WriterTarget writerTarget = node.getTarget().orElseThrow(() -> new VerifyException("target is absent")); + context.setOutputTable(new CatalogSchemaTableName( + writerTarget.getConnectorId().getCatalogName(), + writerTarget.getSchemaTableName().getSchemaName(), + writerTarget.getSchemaTableName().getTableName())); return processChildren(node, context); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/JsonRenderer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/JsonRenderer.java index beec3b5003874..82b2b1327a553 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/JsonRenderer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/JsonRenderer.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.sql.planner.planPrinter; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.sql.planner.plan.PlanFragmentId; import com.fasterxml.jackson.annotation.JsonProperty; -import io.airlift.json.JsonCodec; import java.util.List; import java.util.Optional; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStats.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStats.java index db6a736bca2af..6de2e9ef7d11c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStats.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStats.java @@ -39,6 +39,8 @@ public class PlanNodeStats private final Duration planNodeCpuTime; private final long planNodeInputPositions; private final DataSize planNodeInputDataSize; + private final long planNodeRawInputPositions; + private final DataSize planNodeRawInputDataSize; private final long planNodeOutputPositions; private final DataSize planNodeOutputDataSize; @@ -50,6 +52,8 @@ public class PlanNodeStats Duration planNodeCpuTime, long planNodeInputPositions, DataSize planNodeInputDataSize, + long planNodeRawInputPositions, + DataSize planNodeRawInputDataSize, long planNodeOutputPositions, DataSize planNodeOutputDataSize, Map operatorInputStats) @@ -60,6 +64,8 @@ public class PlanNodeStats this.planNodeCpuTime = requireNonNull(planNodeCpuTime, "planNodeCpuTime is null"); this.planNodeInputPositions = planNodeInputPositions; this.planNodeInputDataSize = planNodeInputDataSize; + this.planNodeRawInputPositions = planNodeRawInputPositions; + this.planNodeRawInputDataSize = planNodeRawInputDataSize; this.planNodeOutputPositions = planNodeOutputPositions; this.planNodeOutputDataSize = planNodeOutputDataSize; @@ -104,6 +110,16 @@ public DataSize getPlanNodeInputDataSize() return planNodeInputDataSize; } + public long getPlanNodeRawInputPositions() + { + return planNodeRawInputPositions; + } + + public DataSize getPlanNodeRawInputDataSize() + { + return planNodeRawInputDataSize; + } + public long getPlanNodeOutputPositions() { return planNodeOutputPositions; @@ -140,6 +156,8 @@ public PlanNodeStats mergeWith(PlanNodeStats other) long planNodeInputPositions = this.planNodeInputPositions + other.planNodeInputPositions; DataSize planNodeInputDataSize = succinctBytes(this.planNodeInputDataSize.toBytes() + other.planNodeInputDataSize.toBytes()); + long planNodeRawInputPositions = this.planNodeRawInputPositions + other.planNodeRawInputPositions; + DataSize planNodeRawInputDataSize = succinctBytes(this.planNodeRawInputDataSize.toBytes() + other.planNodeRawInputDataSize.toBytes()); long planNodeOutputPositions = this.planNodeOutputPositions + other.planNodeOutputPositions; DataSize planNodeOutputDataSize = succinctBytes(this.planNodeOutputDataSize.toBytes() + other.planNodeOutputDataSize.toBytes()); @@ -150,6 +168,7 @@ public PlanNodeStats mergeWith(PlanNodeStats other) new Duration(planNodeScheduledTime.toMillis() + other.getPlanNodeScheduledTime().toMillis(), MILLISECONDS), new Duration(planNodeCpuTime.toMillis() + other.getPlanNodeCpuTime().toMillis(), MILLISECONDS), planNodeInputPositions, planNodeInputDataSize, + planNodeRawInputPositions, planNodeRawInputDataSize, planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java index 6a078a1609476..b3090cfbc934d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java @@ -46,7 +46,7 @@ private PlanNodeStatsSummarizer() {} public static Map aggregateStageStats(List stageInfos) { return aggregateTaskStats(stageInfos.stream() - .flatMap(s -> s.getTasks().stream()) + .flatMap(s -> s.getLatestAttemptExecutionInfo().getTasks().stream()) .collect(toList())); } @@ -74,6 +74,8 @@ private static List getPlanNodeStats(TaskStats taskStats) Map planNodeInputPositions = new HashMap<>(); Map planNodeInputBytes = new HashMap<>(); + Map planNodeRawInputPositions = new HashMap<>(); + Map planNodeRawInputBytes = new HashMap<>(); Map planNodeOutputPositions = new HashMap<>(); Map planNodeOutputBytes = new HashMap<>(); Map planNodeScheduledMillis = new HashMap<>(); @@ -141,6 +143,10 @@ private static List getPlanNodeStats(TaskStats taskStats) planNodeInputPositions.merge(planNodeId, operatorStats.getInputPositions(), Long::sum); planNodeInputBytes.merge(planNodeId, operatorStats.getInputDataSize().toBytes(), Long::sum); + + planNodeRawInputPositions.merge(planNodeId, operatorStats.getRawInputPositions(), Long::sum); + planNodeRawInputBytes.merge(planNodeId, operatorStats.getRawInputDataSize().toBytes(), Long::sum); + processedNodes.add(planNodeId); } @@ -184,6 +190,8 @@ private static List getPlanNodeStats(TaskStats taskStats) new Duration(planNodeCpuMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctDataSize(planNodeInputBytes.get(planNodeId), BYTE), + planNodeRawInputPositions.get(planNodeId), + succinctDataSize(planNodeRawInputBytes.get(planNodeId), BYTE), outputPositions, succinctDataSize(planNodeOutputBytes.getOrDefault(planNodeId, 0L), BYTE), operatorInputStats.get(planNodeId), @@ -196,6 +204,8 @@ else if (windowNodeStats.containsKey(planNodeId)) { new Duration(planNodeCpuMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctDataSize(planNodeInputBytes.get(planNodeId), BYTE), + planNodeRawInputPositions.get(planNodeId), + succinctDataSize(planNodeRawInputBytes.get(planNodeId), BYTE), outputPositions, succinctDataSize(planNodeOutputBytes.getOrDefault(planNodeId, 0L), BYTE), operatorInputStats.get(planNodeId), @@ -208,6 +218,8 @@ else if (windowNodeStats.containsKey(planNodeId)) { new Duration(planNodeCpuMillis.get(planNodeId), MILLISECONDS), planNodeInputPositions.get(planNodeId), succinctDataSize(planNodeInputBytes.get(planNodeId), BYTE), + planNodeRawInputPositions.get(planNodeId), + succinctDataSize(planNodeRawInputBytes.get(planNodeId), BYTE), outputPositions, succinctDataSize(planNodeOutputBytes.getOrDefault(planNodeId, 0L), BYTE), operatorInputStats.get(planNodeId)); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java index 21ee9ad8fbbe2..688d3aac76c96 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java @@ -14,35 +14,43 @@ package com.facebook.presto.sql.planner.planPrinter; import com.facebook.presto.Session; -import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.cost.PlanCostEstimate; import com.facebook.presto.cost.PlanNodeStatsEstimate; import com.facebook.presto.cost.StatsAndCosts; +import com.facebook.presto.execution.StageExecutionStats; import com.facebook.presto.execution.StageInfo; -import com.facebook.presto.execution.StageStats; +import com.facebook.presto.execution.TaskInfo; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.OperatorNotFoundException; import com.facebook.presto.operator.StageExecutionDescriptor; import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.Marker; import com.facebook.presto.spi.predicate.Range; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.CallExpression; -import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.InterpretedFunctionInvoker; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.Partitioning; import com.facebook.presto.sql.planner.PartitioningScheme; import com.facebook.presto.sql.planner.PlanFragment; @@ -50,29 +58,23 @@ import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.GroupReference; import com.facebook.presto.sql.planner.optimizations.JoinNodeUtils; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; -import com.facebook.presto.sql.planner.plan.ExceptNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; import com.facebook.presto.sql.planner.plan.GroupIdNode; import com.facebook.presto.sql.planner.plan.IndexJoinNode; import com.facebook.presto.sql.planner.plan.IndexSourceNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; -import com.facebook.presto.sql.planner.plan.IntersectNode; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.MetadataDeleteNode; import com.facebook.presto.sql.planner.plan.OutputNode; import com.facebook.presto.sql.planner.plan.PlanFragmentId; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; @@ -82,10 +84,9 @@ import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.tree.ComparisonExpression; @@ -109,6 +110,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -134,7 +136,7 @@ public class PlanPrinter { private final PlanRepresentation representation; private final FunctionManager functionManager; - private final RowExpressionFormatter formatter; + private final Function formatter; private PlanPrinter( PlanNode planRoot, @@ -162,7 +164,10 @@ private PlanPrinter( .sum(), MILLISECONDS)); this.representation = new PlanRepresentation(planRoot, types, totalCpuTime, totalScheduledTime); - this.formatter = new RowExpressionFormatter(session.toConnectorSession(), functionManager); + + RowExpressionFormatter rowExpressionFormatter = new RowExpressionFormatter(functionManager); + ConnectorSession connectorSession = requireNonNull(session, "session is null").toConnectorSession(); + this.formatter = rowExpression -> rowExpressionFormatter.formatRowExpression(connectorSession, rowExpression); Visitor visitor = new Visitor(stageExecutionStrategy, types, estimatedStatsAndCosts, session, stats); planRoot.accept(visitor, null); @@ -250,22 +255,23 @@ private static String formatFragment(FunctionManager functionManager, Session se fragment.getPartitioning())); if (stageInfo.isPresent()) { - StageStats stageStats = stageInfo.get().getStageStats(); + StageExecutionStats stageExecutionStats = stageInfo.get().getLatestAttemptExecutionInfo().getStats(); + List tasks = stageInfo.get().getLatestAttemptExecutionInfo().getTasks(); - double avgPositionsPerTask = stageInfo.get().getTasks().stream().mapToLong(task -> task.getStats().getProcessedInputPositions()).average().orElse(Double.NaN); - double squaredDifferences = stageInfo.get().getTasks().stream().mapToDouble(task -> Math.pow(task.getStats().getProcessedInputPositions() - avgPositionsPerTask, 2)).sum(); - double sdAmongTasks = Math.sqrt(squaredDifferences / stageInfo.get().getTasks().size()); + double avgPositionsPerTask = tasks.stream().mapToLong(task -> task.getStats().getProcessedInputPositions()).average().orElse(Double.NaN); + double squaredDifferences = tasks.stream().mapToDouble(task -> Math.pow(task.getStats().getProcessedInputPositions() - avgPositionsPerTask, 2)).sum(); + double sdAmongTasks = Math.sqrt(squaredDifferences / tasks.size()); builder.append(indentString(1)) .append(format("CPU: %s, Scheduled: %s, Input: %s (%s); per task: avg.: %s std.dev.: %s, Output: %s (%s)\n", - stageStats.getTotalCpuTime().convertToMostSuccinctTimeUnit(), - stageStats.getTotalScheduledTime().convertToMostSuccinctTimeUnit(), - formatPositions(stageStats.getProcessedInputPositions()), - stageStats.getProcessedInputDataSize(), + stageExecutionStats.getTotalCpuTime().convertToMostSuccinctTimeUnit(), + stageExecutionStats.getTotalScheduledTime().convertToMostSuccinctTimeUnit(), + formatPositions(stageExecutionStats.getProcessedInputPositions()), + stageExecutionStats.getProcessedInputDataSize(), formatDouble(avgPositionsPerTask), formatDouble(sdAmongTasks), - formatPositions(stageStats.getOutputPositions()), - stageStats.getOutputDataSize())); + formatPositions(stageExecutionStats.getOutputPositions()), + stageExecutionStats.getOutputDataSize())); } PartitioningScheme partitioningScheme = fragment.getPartitioningScheme(); @@ -273,28 +279,18 @@ private static String formatFragment(FunctionManager functionManager, Session se .append(format("Output layout: [%s]\n", Joiner.on(", ").join(partitioningScheme.getOutputLayout()))); - boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny(); - List arguments = partitioningScheme.getPartitioning().getArguments().stream() - .map(argument -> { - if (argument.isConstant()) { - ConstantExpression constant = argument.getConstant(); - String printableValue = castToVarchar(constant.getType(), constant.getValue(), functionManager, session); - return constant.getType().getDisplayName() + "(" + printableValue + ")"; - } - return argument.getVariableReference().toString(); - }) - .collect(toImmutableList()); builder.append(indentString(1)); + boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny(); if (replicateNullsAndAny) { builder.append(format("Output partitioning: %s (replicate nulls and any) [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), - Joiner.on(", ").join(arguments), + Joiner.on(", ").join(partitioningScheme.getPartitioning().getArguments()), formatHash(partitioningScheme.getHashColumn()))); } else { builder.append(format("Output partitioning: %s [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), - Joiner.on(", ").join(arguments), + Joiner.on(", ").join(partitioningScheme.getPartitioning().getArguments()), formatHash(partitioningScheme.getHashColumn()))); } builder.append(indentString(1)).append(format("Stage Execution Strategy: %s\n", fragment.getStageExecutionDescriptor().getStageExecutionStrategy())); @@ -363,7 +359,7 @@ public Void visitJoin(JoinNode node, Void context) for (JoinNode.EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(JoinNodeUtils.toExpression(clause).toString()); } - node.getFilter().map(formatter::formatRowExpression).ifPresent(joinExpressions::add); + node.getFilter().map(formatter::apply).ifPresent(joinExpressions::add); NodeRepresentation nodeOutput; if (node.isCrossJoin()) { @@ -378,7 +374,7 @@ public Void visitJoin(JoinNode node, Void context) node.getDistributionType().ifPresent(distributionType -> nodeOutput.appendDetails("Distribution: %s", distributionType)); node.getSortExpressionContext(functionManager) - .ifPresent(sortContext -> nodeOutput.appendDetails("SortExpression[%s]", formatter.formatRowExpression(sortContext.getSortExpression()))); + .ifPresent(sortContext -> nodeOutput.appendDetails("SortExpression[%s]", formatter.apply(sortContext.getSortExpression()))); node.getLeft().accept(this, context); node.getRight().accept(this, context); @@ -390,7 +386,7 @@ public Void visitSpatialJoin(SpatialJoinNode node, Void context) { NodeRepresentation nodeOutput = addNode(node, node.getType().getJoinLabel(), - format("[%s]", formatter.formatRowExpression(node.getFilter()))); + format("[%s]", formatter.apply(node.getFilter()))); nodeOutput.appendDetailsLine("Distribution: %s", node.getDistributionType()); node.getLeft().accept(this, context); @@ -506,10 +502,10 @@ private String formatAggregation(AggregationNode.Aggregation aggregation) builder.append("*"); } else { - builder.append("(" + Joiner.on(",").join(aggregation.getArguments().stream().map(formatter::formatRowExpression).collect(toImmutableList())) + ")"); + builder.append("(" + Joiner.on(",").join(aggregation.getArguments().stream().map(formatter::apply).collect(toImmutableList())) + ")"); } builder.append(")"); - aggregation.getFilter().ifPresent(filter -> builder.append(" WHERE " + formatter.formatRowExpression(filter))); + aggregation.getFilter().ifPresent(filter -> builder.append(" WHERE " + formatter.apply(filter))); aggregation.getOrderBy().ifPresent(orderingScheme -> builder.append(" ORDER BY " + orderingScheme.toString())); aggregation.getMask().ifPresent(mask -> builder.append(" (mask = " + mask + ")")); return builder.toString(); @@ -576,10 +572,10 @@ public Void visitWindow(WindowNode node, Void context) if (node.getOrderingScheme().isPresent()) { OrderingScheme orderingScheme = node.getOrderingScheme().get(); args.add(format("order by (%s)", Stream.concat( - orderingScheme.getOrderBy().stream() + orderingScheme.getOrderByVariables().stream() .limit(node.getPreSortedOrderPrefix()) .map(symbol -> "<" + symbol + " " + orderingScheme.getOrdering(symbol) + ">"), - orderingScheme.getOrderBy().stream() + orderingScheme.getOrderByVariables().stream() .skip(node.getPreSortedOrderPrefix()) .map(symbol -> symbol + " " + orderingScheme.getOrdering(symbol))) .collect(Collectors.joining(", ")))); @@ -595,7 +591,7 @@ public Void visitWindow(WindowNode node, Void context) "%s := %s(%s) %s", entry.getKey(), call.getDisplayName(), - Joiner.on(", ").join(call.getArguments().stream().map(formatter::formatRowExpression).collect(toImmutableList())), + Joiner.on(", ").join(call.getArguments().stream().map(formatter::apply).collect(toImmutableList())), frameInfo); } return processChildren(node, context); @@ -608,7 +604,7 @@ public Void visitTopNRowNumber(TopNRowNumberNode node, Void context) .map(Functions.toStringFunction()) .collect(toImmutableList()); - List orderBy = node.getOrderingScheme().getOrderBy().stream() + List orderBy = node.getOrderingScheme().getOrderByVariables().stream() .map(input -> input + " " + node.getOrderingScheme().getOrdering(input)) .collect(toImmutableList()); @@ -659,7 +655,8 @@ public Void visitTableScan(TableScanNode node, Void context) else { nodeOutput = addNode(node, "TableScan", format("[%s]", table)); } - printTableScanInfo(nodeOutput, node); + PlanNodeStats nodeStats = stats.map(s -> s.get(node.getId())).orElse(null); + printTableScanInfo(nodeOutput, node, nodeStats); return null; } @@ -668,7 +665,7 @@ public Void visitValues(ValuesNode node, Void context) { NodeRepresentation nodeOutput = addNode(node, "Values"); for (List row : node.getRows()) { - nodeOutput.appendDetailsLine("(" + Joiner.on(", ").join(formatter.formatRowExpressions(row)) + ")"); + nodeOutput.appendDetailsLine("(" + row.stream().map(formatter::apply).collect(Collectors.joining(", ")) + ")"); } return null; } @@ -731,7 +728,7 @@ private Void visitScanFilterAndProjectInfo( if (filterNode.isPresent()) { operatorName += "Filter"; formatString += "filterPredicate = %s, "; - arguments.add(formatter.formatRowExpression(filterNode.get().getPredicate())); + arguments.add(formatter.apply(filterNode.get().getPredicate())); } if (formatString.length() > 1) { @@ -762,14 +759,8 @@ private Void visitScanFilterAndProjectInfo( } if (scanNode.isPresent()) { - printTableScanInfo(nodeOutput, scanNode.get()); PlanNodeStats nodeStats = stats.map(s -> s.get(node.getId())).orElse(null); - if (nodeStats != null) { - // Add to 'details' rather than 'statistics', since these stats are node-specific - nodeOutput.appendDetails("Input: %s (%s)", formatPositions(nodeStats.getPlanNodeInputPositions()), nodeStats.getPlanNodeInputDataSize().toString()); - double filtered = 100.0d * (nodeStats.getPlanNodeInputPositions() - nodeStats.getPlanNodeOutputPositions()) / nodeStats.getPlanNodeInputPositions(); - nodeOutput.appendDetailsLine(", Filtered: %s%%", formatDouble(filtered)); - } + printTableScanInfo(nodeOutput, scanNode.get(), nodeStats); return null; } @@ -777,7 +768,7 @@ private Void visitScanFilterAndProjectInfo( return null; } - private void printTableScanInfo(NodeRepresentation nodeOutput, TableScanNode node) + private void printTableScanInfo(NodeRepresentation nodeOutput, TableScanNode node, PlanNodeStats nodeStats) { TableHandle table = node.getTable(); @@ -815,6 +806,21 @@ private void printTableScanInfo(NodeRepresentation nodeOutput, TableScanNode nod }); } } + + if (nodeStats != null) { + // Add to 'details' rather than 'statistics', since these stats are node-specific + long inputPositions = nodeStats.getPlanNodeInputPositions(); + nodeOutput.appendDetails("Input: %s (%s)", formatPositions(inputPositions), nodeStats.getPlanNodeInputDataSize().toString()); + double filtered = 100.0d * (inputPositions - nodeStats.getPlanNodeOutputPositions()) / inputPositions; + nodeOutput.appendDetailsLine(", Filtered: %s%%", formatDouble(filtered)); + + long rawInputPositions = nodeStats.getPlanNodeRawInputPositions(); + if (rawInputPositions != inputPositions) { + nodeOutput.appendDetails("Raw input: %s (%s)", formatPositions(rawInputPositions), nodeStats.getPlanNodeRawInputDataSize().toString()); + double rawFiltered = 100.0d * (rawInputPositions - inputPositions) / rawInputPositions; + nodeOutput.appendDetailsLine(", Filtered: %s%%", formatDouble(rawFiltered)); + } + } } @Override @@ -843,7 +849,7 @@ public Void visitOutput(OutputNode node, Void context) @Override public Void visitTopN(TopNNode node, Void context) { - Iterable keys = Iterables.transform(node.getOrderingScheme().getOrderBy(), input -> input + " " + node.getOrderingScheme().getOrdering(input)); + Iterable keys = Iterables.transform(node.getOrderingScheme().getOrderByVariables(), input -> input + " " + node.getOrderingScheme().getOrdering(input)); addNode(node, format("TopN%s", node.getStep() == TopNNode.Step.PARTIAL ? "Partial" : ""), @@ -854,14 +860,10 @@ public Void visitTopN(TopNNode node, Void context) @Override public Void visitSort(SortNode node, Void context) { - Iterable keys = Iterables.transform(node.getOrderingScheme().getOrderBy(), input -> input + " " + node.getOrderingScheme().getOrdering(input)); - boolean isPartial = false; - if (SystemSessionProperties.isDistributedSortEnabled(session)) { - isPartial = true; - } + Iterable keys = Iterables.transform(node.getOrderingScheme().getOrderByVariables(), input -> input + " " + node.getOrderingScheme().getOrdering(input)); addNode(node, - format("%sSort", isPartial ? "Partial" : ""), + format("%sSort", node.isPartial() ? "Partial" : ""), format("[%s]", Joiner.on(", ").join(keys))); return processChildren(node, context); @@ -923,10 +925,17 @@ public Void visitTableWriter(TableWriterNode node, Void context) return processChildren(node, context); } + @Override + public Void visitTableWriteMerge(TableWriterMergeNode node, Void context) + { + addNode(node, "TableWriterMerge"); + return processChildren(node, context); + } + @Override public Void visitStatisticsWriterNode(StatisticsWriterNode node, Void context) { - addNode(node, "StatisticsWriter", format("[%s]", node.getTarget())); + addNode(node, "StatisticsWriter", format("[%s]", node.getTableHandle())); return processChildren(node, context); } @@ -950,7 +959,7 @@ public Void visitExchange(ExchangeNode node, Void context) { if (node.getOrderingScheme().isPresent()) { OrderingScheme orderingScheme = node.getOrderingScheme().get(); - List orderBy = orderingScheme.getOrderBy() + List orderBy = orderingScheme.getOrderByVariables() .stream() .map(input -> input + " " + orderingScheme.getOrdering(input)) .collect(toImmutableList()); @@ -982,7 +991,7 @@ else if (node.getScope().isLocal()) { @Override public Void visitDelete(DeleteNode node, Void context) { - addNode(node, "Delete", format("[%s]", node.getTarget())); + addNode(node, "Delete"); return processChildren(node, context); } @@ -990,7 +999,7 @@ public Void visitDelete(DeleteNode node, Void context) @Override public Void visitMetadataDelete(MetadataDeleteNode node, Void context) { - addNode(node, "MetadataDelete", format("[%s]", node.getTarget())); + addNode(node, "MetadataDelete", format("[%s]", node.getTableHandle())); return processChildren(node, context); } @@ -1058,7 +1067,7 @@ private void printAssignments(NodeRepresentation nodeOutput, Assignments assignm // skip identity assignments continue; } - nodeOutput.appendDetailsLine("%s := %s", entry.getKey(), formatter.formatRowExpression(entry.getValue())); + nodeOutput.appendDetailsLine("%s := %s", entry.getKey(), formatter.apply(entry.getValue())); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/RowExpressionFormatter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/RowExpressionFormatter.java index bae8e21231f26..11357fcde1a77 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/RowExpressionFormatter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/RowExpressionFormatter.java @@ -39,82 +39,80 @@ public final class RowExpressionFormatter { - private final ConnectorSession session; private final FunctionMetadataManager functionMetadataManager; private final StandardFunctionResolution standardFunctionResolution; - public RowExpressionFormatter(ConnectorSession session, FunctionManager functionManager) + public RowExpressionFormatter(FunctionManager functionManager) { - this.session = requireNonNull(session, "session is null"); this.functionMetadataManager = requireNonNull(functionManager, "function manager is null"); this.standardFunctionResolution = new FunctionResolution(functionManager); } - public String formatRowExpression(RowExpression expression) + public String formatRowExpression(ConnectorSession session, RowExpression expression) { - return expression.accept(new Formatter(), null); + return expression.accept(new Formatter(), requireNonNull(session, "session is null")); } - public List formatRowExpressions(List rowExpressions) + private List formatRowExpressions(ConnectorSession session, List rowExpressions) { - return rowExpressions.stream().map(this::formatRowExpression).collect(toList()); + return rowExpressions.stream().map(rowExpression -> formatRowExpression(session, rowExpression)).collect(toList()); } public class Formatter - implements RowExpressionVisitor + implements RowExpressionVisitor { @Override - public String visitCall(CallExpression node, Void context) + public String visitCall(CallExpression node, ConnectorSession session) { if (standardFunctionResolution.isArithmeticFunction(node.getFunctionHandle()) || standardFunctionResolution.isComparisonFunction(node.getFunctionHandle())) { String operation = functionMetadataManager.getFunctionMetadata(node.getFunctionHandle()).getOperatorType().get().getOperator(); - return String.join(" " + operation + " ", formatRowExpressions(node.getArguments()).stream().map(e -> "(" + e + ")").collect(toImmutableList())); + return String.join(" " + operation + " ", formatRowExpressions(session, node.getArguments()).stream().map(e -> "(" + e + ")").collect(toImmutableList())); } else if (standardFunctionResolution.isCastFunction(node.getFunctionHandle())) { - return String.format("CAST(%s AS %s)", formatRowExpression(node.getArguments().get(0)), node.getType().getDisplayName()); + return String.format("CAST(%s AS %s)", formatRowExpression(session, node.getArguments().get(0)), node.getType().getDisplayName()); } else if (standardFunctionResolution.isNegateFunction(node.getFunctionHandle())) { - return "-(" + formatRowExpression(node.getArguments().get(0)) + ")"; + return "-(" + formatRowExpression(session, node.getArguments().get(0)) + ")"; } else if (standardFunctionResolution.isSubscriptFunction(node.getFunctionHandle())) { - return formatRowExpression(node.getArguments().get(0)) + "[" + formatRowExpression(node.getArguments().get(1)) + "]"; + return formatRowExpression(session, node.getArguments().get(0)) + "[" + formatRowExpression(session, node.getArguments().get(1)) + "]"; } else if (standardFunctionResolution.isBetweenFunction(node.getFunctionHandle())) { - List formattedExpresions = formatRowExpressions(node.getArguments()); + List formattedExpresions = formatRowExpressions(session, node.getArguments()); return String.format("%s BETWEEN (%s) AND (%s)", formattedExpresions.get(0), formattedExpresions.get(1), formattedExpresions.get(2)); } - return node.getDisplayName() + "(" + String.join(", ", formatRowExpressions(node.getArguments())) + ")"; + return node.getDisplayName() + "(" + String.join(", ", formatRowExpressions(session, node.getArguments())) + ")"; } @Override - public String visitSpecialForm(SpecialFormExpression node, Void context) + public String visitSpecialForm(SpecialFormExpression node, ConnectorSession session) { if (node.getForm().equals(SpecialFormExpression.Form.AND) || node.getForm().equals(SpecialFormExpression.Form.OR)) { - return String.join(" " + node.getForm() + " ", formatRowExpressions(node.getArguments()).stream().map(e -> "(" + e + ")").collect(toImmutableList())); + return String.join(" " + node.getForm() + " ", formatRowExpressions(session, node.getArguments()).stream().map(e -> "(" + e + ")").collect(toImmutableList())); } - return node.getForm().name() + "(" + String.join(", ", formatRowExpressions(node.getArguments())) + ")"; + return node.getForm().name() + "(" + String.join(", ", formatRowExpressions(session, node.getArguments())) + ")"; } @Override - public String visitInputReference(InputReferenceExpression node, Void context) + public String visitInputReference(InputReferenceExpression node, ConnectorSession session) { return node.toString(); } @Override - public String visitLambda(LambdaDefinitionExpression node, Void context) + public String visitLambda(LambdaDefinitionExpression node, ConnectorSession session) { - return "(" + String.join(", ", node.getArguments()) + ") -> " + formatRowExpression(node.getBody()); + return "(" + String.join(", ", node.getArguments()) + ") -> " + formatRowExpression(session, node.getBody()); } @Override - public String visitVariableReference(VariableReferenceExpression node, Void context) + public String visitVariableReference(VariableReferenceExpression node, ConnectorSession session) { return node.getName(); } @Override - public String visitConstant(ConstantExpression node, Void context) + public String visitConstant(ConstantExpression node, ConnectorSession session) { Object value = LiteralInterpreter.evaluate(session, node); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/WindowPlanNodeStats.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/WindowPlanNodeStats.java index 374b945b65a97..5dfb5c63f9fd9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/WindowPlanNodeStats.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/WindowPlanNodeStats.java @@ -32,12 +32,14 @@ public WindowPlanNodeStats( Duration planNodeCpuTime, long planNodeInputPositions, DataSize planNodeInputDataSize, + long planNodeRawInputPositions, + DataSize planNodeRawInputDataSize, long planNodeOutputPositions, DataSize planNodeOutputDataSize, Map operatorInputStats, WindowOperatorStats windowOperatorStats) { - super(planNodeId, planNodeScheduledTime, planNodeCpuTime, planNodeInputPositions, planNodeInputDataSize, planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats); + super(planNodeId, planNodeScheduledTime, planNodeCpuTime, planNodeInputPositions, planNodeInputDataSize, planNodeRawInputPositions, planNodeRawInputDataSize, planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats); this.windowOperatorStats = windowOperatorStats; } @@ -58,6 +60,8 @@ public PlanNodeStats mergeWith(PlanNodeStats other) merged.getPlanNodeCpuTime(), merged.getPlanNodeInputPositions(), merged.getPlanNodeInputDataSize(), + merged.getPlanNodeRawInputPositions(), + merged.getPlanNodeRawInputDataSize(), merged.getPlanNodeOutputPositions(), merged.getPlanNodeOutputDataSize(), merged.operatorInputStats, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/TypeValidator.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/TypeValidator.java index 145b332a74e77..213de355ea017 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/TypeValidator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/TypeValidator.java @@ -18,7 +18,11 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -28,15 +32,10 @@ import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.SimplePlanVisitor; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; -import com.facebook.presto.sql.planner.plan.ProjectNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.NodeRef; import com.facebook.presto.sql.tree.SymbolReference; -import com.google.common.collect.ListMultimap; import java.util.List; import java.util.Map; @@ -143,9 +142,8 @@ public Void visitUnion(UnionNode node, Void context) { visitPlan(node, context); - ListMultimap variableMapping = node.getVariableMapping(); - for (VariableReferenceExpression keyVariable : variableMapping.keySet()) { - List valueVariables = variableMapping.get(keyVariable); + for (VariableReferenceExpression keyVariable : node.getOutputVariables()) { + List valueVariables = node.getVariableMapping().get(keyVariable); for (VariableReferenceExpression valueVariable : valueVariables) { verifyTypeSignature(keyVariable, valueVariable.getType().getTypeSignature()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateAggregationsWithDefaultValues.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateAggregationsWithDefaultValues.java index 21e0c933acc2f..bd5d4a334035e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateAggregationsWithDefaultValues.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateAggregationsWithDefaultValues.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.TypeProvider; @@ -23,7 +24,6 @@ import com.facebook.presto.sql.planner.optimizations.PropertyDerivations; import com.facebook.presto.sql.planner.optimizations.StreamPropertyDerivations; import com.facebook.presto.sql.planner.optimizations.StreamPropertyDerivations.StreamProperties; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.sanity.PlanSanityChecker.Checker; @@ -31,9 +31,9 @@ import java.util.List; import java.util.Optional; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.FINAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.INTERMEDIATE; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.INTERMEDIATE; +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; import static com.facebook.presto.util.Optionals.combine; import static com.google.common.base.Preconditions.checkArgument; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java index 0880819d48c6e..1bc83cbd0d230 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java @@ -16,53 +16,53 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.SetOperationNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.VariablesExtractor; import com.facebook.presto.sql.planner.optimizations.WindowNodeUtil; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; -import com.facebook.presto.sql.planner.plan.ExceptNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; import com.facebook.presto.sql.planner.plan.GroupIdNode; import com.facebook.presto.sql.planner.plan.IndexJoinNode; import com.facebook.presto.sql.planner.plan.IndexSourceNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; -import com.facebook.presto.sql.planner.plan.IntersectNode; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.MetadataDeleteNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; -import com.facebook.presto.sql.planner.plan.SetOperationNode; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.google.common.collect.ImmutableList; @@ -93,24 +93,22 @@ public final class ValidateDependenciesChecker @Override public void validate(PlanNode plan, Session session, Metadata metadata, SqlParser sqlParser, TypeProvider types, WarningCollector warningCollector) { - validate(plan, types, metadata.getTypeManager()); + validate(plan, types); } - public static void validate(PlanNode plan, TypeProvider types, TypeManager typeManager) + public static void validate(PlanNode plan, TypeProvider types) { - plan.accept(new Visitor(types, typeManager), ImmutableSet.of()); + plan.accept(new Visitor(types), ImmutableSet.of()); } private static class Visitor extends InternalPlanVisitor> { private final TypeProvider types; - private final TypeManager typeManager; - public Visitor(TypeProvider types, TypeManager typeManager) + public Visitor(TypeProvider types) { this.types = requireNonNull(types, "types is null"); - this.typeManager = requireNonNull(typeManager, "typeManager is null"); } @Override @@ -182,9 +180,9 @@ public Void visitWindow(WindowNode node, Set boundV if (node.getOrderingScheme().isPresent()) { checkDependencies( inputs, - node.getOrderingScheme().get().getOrderBy(), + node.getOrderingScheme().get().getOrderByVariables(), "Invalid node. Order by symbols (%s) not in source plan output (%s)", - node.getOrderingScheme().get().getOrderBy(), node.getSource().getOutputVariables()); + node.getOrderingScheme().get().getOrderByVariables(), node.getSource().getOutputVariables()); } ImmutableList.Builder bounds = ImmutableList.builder(); @@ -216,9 +214,9 @@ public Void visitTopNRowNumber(TopNRowNumberNode node, Set boundVaria checkDependencies(inputs, node.getOutputVariables(), "Invalid node. Output symbols (%s) not in source plan output (%s)", node.getOutputVariables(), node.getSource().getOutputVariables()); checkDependencies( inputs, - node.getOrderingScheme().getOrderBy(), + node.getOrderingScheme().getOrderByVariables(), "Invalid node. Order by dependencies (%s) not in source plan output (%s)", - node.getOrderingScheme().getOrderBy(), + node.getOrderingScheme().getOrderByVariables(), node.getSource().getOutputVariables()); return null; @@ -320,9 +318,9 @@ public Void visitSort(SortNode node, Set boundVaria checkDependencies(inputs, node.getOutputVariables(), "Invalid node. Output symbols (%s) not in source plan output (%s)", node.getOutputVariables(), node.getSource().getOutputVariables()); checkDependencies( inputs, - node.getOrderingScheme().getOrderBy(), + node.getOrderingScheme().getOrderByVariables(), "Invalid node. Order by dependencies (%s) not in source plan output (%s)", - node.getOrderingScheme().getOrderBy(), node.getSource().getOutputVariables()); + node.getOrderingScheme().getOrderByVariables(), node.getSource().getOutputVariables()); return null; } @@ -575,6 +573,15 @@ public Void visitTableWriter(TableWriterNode node, Set boundVariables) + { + PlanNode source = node.getSource(); + source.accept(this, boundVariables); // visit child + + return null; + } + @Override public Void visitDelete(DeleteNode node, Set boundVariables) { @@ -727,17 +734,7 @@ private static ImmutableSet createInputs(PlanNode s private void checkDependencies(Collection inputs, Collection required, String message, Object... parameters) { - // If a variable can be assigned into another type directly, CAST is usually implicitly removed. - // For example, we can assign input VARCHAR(3) to output VARCHAR(5) - // the reference variable in the assignment will have type VARCHAR(5) while the input is VARCHAR(3). - for (VariableReferenceExpression target : required) { - checkArgument( - inputs.stream() - .anyMatch(input -> input.getName().equalsIgnoreCase(target.getName()) && - typeManager.isTypeOnlyCoercion(input.getType(), target.getType())), - message, - parameters); - } + checkArgument(ImmutableSet.copyOf(inputs).containsAll(required), message, parameters); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateStreamingAggregations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateStreamingAggregations.java index 357a3c1a9ea1f..085b33c30b49b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateStreamingAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateStreamingAggregations.java @@ -18,13 +18,13 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.GroupingProperty; import com.facebook.presto.spi.LocalProperty; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.optimizations.LocalProperties; import com.facebook.presto.sql.planner.optimizations.StreamPropertyDerivations.StreamProperties; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.sanity.PlanSanityChecker.Checker; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/VerifyNoFilteredAggregations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/VerifyNoFilteredAggregations.java index 03384c2b20703..a1d621fc0b52b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/VerifyNoFilteredAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/VerifyNoFilteredAggregations.java @@ -16,10 +16,10 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/VerifyNoOriginalExpression.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/VerifyNoOriginalExpression.java index a39f3938a99dd..8727799d3c414 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/VerifyNoOriginalExpression.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/VerifyNoOriginalExpression.java @@ -16,20 +16,21 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.SimplePlanVisitor; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.WindowNode; @@ -141,6 +142,15 @@ public Void visitTableWriter(TableWriterNode node, Void context) return null; } + @Override + public Void visitTableWriteMerge(TableWriterMergeNode node, Void context) + { + visitPlan(node, context); + + checkStatisticsAggregation(node.getStatisticsAggregation()); + return null; + } + private static void checkStatisticsAggregation(Optional statisticsAggregation) { statisticsAggregation.ifPresent( diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java index 3606a3022b07f..35590a73e5290 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java @@ -13,11 +13,14 @@ */ package com.facebook.presto.sql.relational; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.PredicateCompiler; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionService; +import com.facebook.presto.sql.planner.planPrinter.RowExpressionFormatter; import static java.util.Objects.requireNonNull; @@ -28,13 +31,15 @@ public final class ConnectorRowExpressionService private final ExpressionOptimizer expressionOptimizer; private final PredicateCompiler predicateCompiler; private final DeterminismEvaluator determinismEvaluator; + private final RowExpressionFormatter rowExpressionFormatter; - public ConnectorRowExpressionService(DomainTranslator domainTranslator, ExpressionOptimizer expressionOptimizer, PredicateCompiler predicateCompiler, DeterminismEvaluator determinismEvaluator) + public ConnectorRowExpressionService(DomainTranslator domainTranslator, ExpressionOptimizer expressionOptimizer, PredicateCompiler predicateCompiler, DeterminismEvaluator determinismEvaluator, RowExpressionFormatter rowExpressionFormatter) { this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); this.predicateCompiler = requireNonNull(predicateCompiler, "predicateCompiler is null"); this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); + this.rowExpressionFormatter = requireNonNull(rowExpressionFormatter, "rowExpressionFormatter is null"); } @Override @@ -60,4 +65,10 @@ public DeterminismEvaluator getDeterminismEvaluator() { return determinismEvaluator; } + + @Override + public String formatRowExpression(ConnectorSession session, RowExpression expression) + { + return rowExpressionFormatter.formatRowExpression(session, expression); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/Expressions.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/Expressions.java index d85a9f59438b2..44d4fae418aa7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/Expressions.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/Expressions.java @@ -26,9 +26,11 @@ import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import java.util.Arrays; import java.util.List; +import java.util.Set; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -49,6 +51,11 @@ public static ConstantExpression constantNull(Type type) return new ConstantExpression(null, type); } + public static boolean isNull(RowExpression expression) + { + return expression instanceof ConstantExpression && ((ConstantExpression) expression).isNull(); + } + public static CallExpression call(String displayName, FunctionHandle functionHandle, Type returnType, RowExpression... arguments) { return new CallExpression(displayName, functionHandle, returnType, Arrays.asList(arguments)); @@ -85,6 +92,16 @@ public static SpecialFormExpression specialForm(Form form, Type returnType, List return new SpecialFormExpression(form, returnType, arguments); } + public static Set uniqueSubExpressions(RowExpression expression) + { + return ImmutableSet.copyOf(subExpressions(ImmutableList.of(expression))); + } + + public static List subExpressions(RowExpression expression) + { + return subExpressions(ImmutableList.of(expression)); + } + public static List subExpressions(Iterable expressions) { final ImmutableList.Builder builder = ImmutableList.builder(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java index aa82c31c4a7e5..9904ba9563a11 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java @@ -16,6 +16,7 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.Type; @@ -26,7 +27,7 @@ import java.util.List; import java.util.Optional; -import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.spi.function.OperatorType.ADD; import static com.facebook.presto.spi.function.OperatorType.BETWEEN; import static com.facebook.presto.spi.function.OperatorType.DIVIDE; @@ -87,7 +88,7 @@ public FunctionHandle likeCharFunction(Type valueType) public boolean isLikeFunction(FunctionHandle functionHandle) { - return functionManager.getFunctionMetadata(functionHandle).getName().toUpperCase().equals("LIKE"); + return functionManager.getFunctionMetadata(functionHandle).getName().equals(QualifiedFunctionName.of(DEFAULT_NAMESPACE, "LIKE")); } public FunctionHandle likePatternFunction() @@ -98,13 +99,18 @@ public FunctionHandle likePatternFunction() @Override public boolean isCastFunction(FunctionHandle functionHandle) { - return functionManager.getFunctionMetadata(functionHandle).getName().equals(mangleOperatorName(OperatorType.CAST.name())); + return functionManager.getFunctionMetadata(functionHandle).getOperatorType().equals(Optional.of(OperatorType.CAST)); + } + + public boolean isArrayConstructor(FunctionHandle functionHandle) + { + return functionManager.getFunctionMetadata(functionHandle).getName().equals(QualifiedFunctionName.of(DEFAULT_NAMESPACE, ARRAY_CONSTRUCTOR)); } @Override public FunctionHandle betweenFunction(Type valueType, Type lowerBoundType, Type upperBoundType) { - return functionManager.lookupFunction(BETWEEN.getFunctionName(), fromTypes(valueType, lowerBoundType, upperBoundType)); + return functionManager.lookupFunction(BETWEEN.getFunctionName().getFunctionName(), fromTypes(valueType, lowerBoundType, upperBoundType)); } @Override @@ -155,7 +161,7 @@ public boolean isArithmeticFunction(FunctionHandle functionHandle) @Override public FunctionHandle negateFunction(Type type) { - return functionManager.lookupFunction(NEGATION.getFunctionName(), fromTypes(type)); + return functionManager.lookupFunction(NEGATION.getFunctionName().getFunctionName(), fromTypes(type)); } @Override @@ -219,7 +225,7 @@ public boolean isComparisonFunction(FunctionHandle functionHandle) @Override public FunctionHandle subscriptFunction(Type baseType, Type indexType) { - return functionManager.lookupFunction(SUBSCRIPT.getFunctionName(), fromTypes(baseType, indexType)); + return functionManager.lookupFunction(SUBSCRIPT.getFunctionName().getFunctionName(), fromTypes(baseType, indexType)); } @Override @@ -230,18 +236,23 @@ public boolean isSubscriptFunction(FunctionHandle functionHandle) public FunctionHandle tryFunction(Type returnType) { - return functionManager.lookupFunction("TRY", fromTypes(returnType)); + return functionManager.lookupFunction("$internal$try", fromTypes(returnType)); } public boolean isTryFunction(FunctionHandle functionHandle) { - return functionManager.getFunctionMetadata(functionHandle).getName().equals("TRY"); + return functionManager.getFunctionMetadata(functionHandle).getName().equals("$internal$try"); + } + + public boolean isFailFunction(FunctionHandle functionHandle) + { + return functionManager.getFunctionMetadata(functionHandle).getName().equals(QualifiedFunctionName.of(DEFAULT_NAMESPACE, "fail")); } @Override public boolean isCountFunction(FunctionHandle functionHandle) { - return functionManager.getFunctionMetadata(functionHandle).getName().equalsIgnoreCase("count"); + return functionManager.getFunctionMetadata(functionHandle).getName().equals(QualifiedFunctionName.of(DEFAULT_NAMESPACE, "count")); } @Override @@ -259,7 +270,7 @@ public FunctionHandle countFunction(Type valueType) @Override public boolean isMaxFunction(FunctionHandle functionHandle) { - return functionManager.getFunctionMetadata(functionHandle).getName().equalsIgnoreCase("max"); + return functionManager.getFunctionMetadata(functionHandle).getName().equals(QualifiedFunctionName.of(DEFAULT_NAMESPACE, "max")); } @Override @@ -271,7 +282,7 @@ public FunctionHandle maxFunction(Type valueType) @Override public boolean isMinFunction(FunctionHandle functionHandle) { - return functionManager.getFunctionMetadata(functionHandle).getName().equalsIgnoreCase("min"); + return functionManager.getFunctionMetadata(functionHandle).getName().equals(QualifiedFunctionName.of(DEFAULT_NAMESPACE, "min")); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/ProjectNodeUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/ProjectNodeUtils.java index bcb65586bf35e..917f32ccc3fb0 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/ProjectNodeUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/ProjectNodeUtils.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.sql.relational; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.SymbolReference; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java index 10f92beb08f7f..33c4391652e97 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionDomainTranslator.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.relational; +import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.OperatorNotFoundException; @@ -38,7 +39,6 @@ import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.InputReferenceExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; -import com.facebook.presto.spi.relation.LogicalRowExpressions; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionVisitor; import com.facebook.presto.spi.relation.SpecialFormExpression; @@ -59,6 +59,10 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.presto.expressions.LogicalRowExpressions.FALSE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.and; +import static com.facebook.presto.expressions.LogicalRowExpressions.or; import static com.facebook.presto.metadata.CastType.CAST; import static com.facebook.presto.metadata.CastType.SATURATED_FLOOR_CAST; import static com.facebook.presto.spi.function.OperatorType.BETWEEN; @@ -69,10 +73,7 @@ import static com.facebook.presto.spi.function.OperatorType.LESS_THAN; import static com.facebook.presto.spi.function.OperatorType.LESS_THAN_OR_EQUAL; import static com.facebook.presto.spi.function.OperatorType.NOT_EQUAL; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.FALSE_CONSTANT; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.and; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.or; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IN; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IS_NULL; @@ -86,7 +87,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.collect.Iterators.peekingIterator; -import static java.util.Comparator.comparing; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.toList; @@ -104,20 +104,19 @@ public RowExpressionDomainTranslator(Metadata metadata) { this.metadata = requireNonNull(metadata, "metadata is null"); this.functionManager = metadata.getFunctionManager(); - this.logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(functionManager), new FunctionResolution(functionManager)); + this.logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(functionManager), new FunctionResolution(functionManager), functionManager); this.functionResolution = new FunctionResolution(functionManager); } @Override - public RowExpression toPredicate(TupleDomain tupleDomain) + public RowExpression toPredicate(TupleDomain tupleDomain) { if (tupleDomain.isNone()) { return FALSE_CONSTANT; } - Map domains = tupleDomain.getDomains().get(); + Map domains = tupleDomain.getDomains().get(); return domains.entrySet().stream() - .sorted(comparing(entry -> entry.getKey().getName())) .map(entry -> toPredicate(entry.getValue(), entry.getKey())) .collect(collectingAndThen(toImmutableList(), logicalRowExpressions::combineConjuncts)); } @@ -128,7 +127,7 @@ public ExtractionResult fromPredicate(ConnectorSession session, RowExpres return predicate.accept(new Visitor<>(metadata, session, columnExtractor), false); } - private RowExpression toPredicate(Domain domain, VariableReferenceExpression reference) + private RowExpression toPredicate(Domain domain, RowExpression reference) { if (domain.getValues().isNone()) { return domain.isNullAllowed() ? isNull(reference) : FALSE_CONSTANT; @@ -155,7 +154,7 @@ private RowExpression toPredicate(Domain domain, VariableReferenceExpression ref return logicalRowExpressions.combineDisjunctsWithDefault(disjuncts, TRUE_CONSTANT); } - private RowExpression processRange(Type type, Range range, VariableReferenceExpression reference) + private RowExpression processRange(Type type, Range range, RowExpression reference) { if (range.isAll()) { return TRUE_CONSTANT; @@ -206,7 +205,7 @@ private RowExpression processRange(Type type, Range range, VariableReferenceExpr return logicalRowExpressions.combineConjuncts(rangeConjuncts); } - private RowExpression combineRangeWithExcludedPoints(Type type, VariableReferenceExpression reference, Range range, List excludedPoints) + private RowExpression combineRangeWithExcludedPoints(Type type, RowExpression reference, Range range, List excludedPoints) { if (excludedPoints.isEmpty()) { return processRange(type, range, reference); @@ -220,7 +219,7 @@ private RowExpression combineRangeWithExcludedPoints(Type type, VariableReferenc return logicalRowExpressions.combineConjuncts(processRange(type, range, reference), excludedPointsExpression); } - private List extractDisjuncts(Type type, Ranges ranges, VariableReferenceExpression reference) + private List extractDisjuncts(Type type, Ranges ranges, RowExpression reference) { List disjuncts = new ArrayList<>(); List singleValues = new ArrayList<>(); @@ -263,7 +262,7 @@ else if (singleValues.size() > 1) { return disjuncts; } - private List extractDisjuncts(Type type, DiscreteValues discreteValues, VariableReferenceExpression reference) + private List extractDisjuncts(Type type, DiscreteValues discreteValues, RowExpression reference) { List values = discreteValues.getValues().stream() .map(object -> toRowExpression(object, type)) @@ -310,7 +309,7 @@ private Visitor(Metadata metadata, ConnectorSession session, ColumnExtractor this.metadata = metadata; this.session = session; this.functionManager = metadata.getFunctionManager(); - this.logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(functionManager), new FunctionResolution(functionManager)); + this.logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(functionManager), new FunctionResolution(functionManager), functionManager); this.determinismEvaluator = new RowExpressionDeterminismEvaluator(functionManager); this.resolution = new FunctionResolution(functionManager); this.columnExtractor = requireNonNull(columnExtractor, "columnExtractor is null"); @@ -347,12 +346,12 @@ public ExtractionResult visitSpecialForm(SpecialFormExpression node, Boolean } case IS_NULL: { RowExpression value = node.getArguments().get(0); - Optional column = columnExtractor.extract(value); + Domain domain = complementIfNecessary(Domain.onlyNull(value.getType()), complement); + Optional column = columnExtractor.extract(value, domain); if (!column.isPresent()) { return visitRowExpression(node, complement); } - Domain domain = complementIfNecessary(Domain.onlyNull(value.getType()), complement); return new ExtractionResult<>(TupleDomain.withColumnDomains(ImmutableMap.of(column.get(), domain)), TRUE_CONSTANT); } default: @@ -408,13 +407,17 @@ public ExtractionResult visitCall(CallExpression node, Boolean complement) NormalizedSimpleComparison normalized = optionalNormalized.get(); RowExpression expression = normalized.getExpression(); - Optional column = columnExtractor.extract(expression); + NullableValue value = normalized.getValue(); + Domain domain = createComparisonDomain(normalized.getComparisonOperator(), value.getType(), value.getValue(), complement); + Optional column = columnExtractor.extract(expression, domain); if (column.isPresent()) { - NullableValue value = normalized.getValue(); - Type type = value.getType(); // common type for symbol and value - return createComparisonExtractionResult(normalized.getComparisonOperator(), column.get(), type, value.getValue(), complement); + if (domain.isNone()) { + return new ExtractionResult<>(TupleDomain.none(), TRUE_CONSTANT); + } + return new ExtractionResult<>(TupleDomain.withColumnDomains(ImmutableMap.of(column.get(), domain)), TRUE_CONSTANT); } - else if (expression instanceof CallExpression && resolution.isCastFunction(((CallExpression) expression).getFunctionHandle())) { + + if (expression instanceof CallExpression && resolution.isCastFunction(((CallExpression) expression).getFunctionHandle())) { CallExpression castExpression = (CallExpression) expression; if (!isImplicitCoercion(castExpression)) { // @@ -636,13 +639,13 @@ private Optional toNormalizedSimpleComparison(Operat left = leftExpression; } else { - left = new RowExpressionInterpreter(leftExpression, metadata, session, true).optimize(); + left = new RowExpressionInterpreter(leftExpression, metadata, session, OPTIMIZED).optimize(); } if (rightExpression instanceof VariableReferenceExpression) { right = rightExpression; } else { - right = new RowExpressionInterpreter(rightExpression, metadata, session, true).optimize(); + right = new RowExpressionInterpreter(rightExpression, metadata, session, OPTIMIZED).optimize(); } if (left instanceof RowExpression == right instanceof RowExpression) { @@ -668,7 +671,7 @@ private Optional toNormalizedSimpleComparison(Operat return Optional.of(new NormalizedSimpleComparison(expression, comparisonOperator, value)); } - private static ExtractionResult createComparisonExtractionResult(OperatorType comparisonOperator, T column, Type type, @Nullable Object value, boolean complement) + private static Domain createComparisonDomain(OperatorType comparisonOperator, Type type, @Nullable Object value, boolean complement) { if (value == null) { switch (comparisonOperator) { @@ -678,13 +681,10 @@ private static ExtractionResult createComparisonExtractionResult(Operator case LESS_THAN: case LESS_THAN_OR_EQUAL: case NOT_EQUAL: - return new ExtractionResult<>(TupleDomain.none(), TRUE_CONSTANT); + return Domain.none(type); case IS_DISTINCT_FROM: - Domain domain = complementIfNecessary(Domain.notNull(type), complement); - return new ExtractionResult<>( - TupleDomain.withColumnDomains(ImmutableMap.of(column, domain)), - TRUE_CONSTANT); + return complementIfNecessary(Domain.notNull(type), complement); default: throw new AssertionError("Unhandled operator: " + comparisonOperator); @@ -702,7 +702,7 @@ else if (type.isComparable()) { throw new AssertionError("Type cannot be used in a comparison expression (should have been caught in analysis): " + type); } - return new ExtractionResult<>(TupleDomain.withColumnDomains(ImmutableMap.of(column, domain)), TRUE_CONSTANT); + return domain; } private static OperatorType flip(OperatorType operatorType) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java index 91c8f23df9f88..81df77a250c8d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java @@ -17,8 +17,12 @@ import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.RowExpressionInterpreter; +import java.util.function.Function; + +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.sql.planner.LiteralEncoder.toRowExpression; import static java.util.Objects.requireNonNull; @@ -35,12 +39,16 @@ public RowExpressionOptimizer(Metadata metadata) @Override public RowExpression optimize(RowExpression rowExpression, Level level, ConnectorSession session) { - if (level == Level.SERIALIZABLE) { - return toRowExpression(RowExpressionInterpreter.rowExpressionInterpreter(rowExpression, metadata, session).evaluate(), rowExpression.getType()); - } - if (level == Level.MOST_OPTIMIZED) { - return toRowExpression(new RowExpressionInterpreter(rowExpression, metadata, session, true).optimize(), rowExpression.getType()); + if (level.ordinal() <= OPTIMIZED.ordinal()) { + return toRowExpression(new RowExpressionInterpreter(rowExpression, metadata, session, level).optimize(), rowExpression.getType()); } - throw new IllegalArgumentException("Unrecognized optimization level: " + level); + throw new IllegalArgumentException("Not supported optimization level: " + level); + } + + @Override + public Object optimize(RowExpression expression, Level level, ConnectorSession session, Function variableResolver) + { + RowExpressionInterpreter interpreter = new RowExpressionInterpreter(expression, metadata, session, level); + return interpreter.optimize(variableResolver::apply); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlFunctionUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlFunctionUtils.java new file mode 100644 index 0000000000000..cc65596f1bbbf --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlFunctionUtils.java @@ -0,0 +1,249 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.relational; + +import com.facebook.presto.expressions.RowExpressionRewriter; +import com.facebook.presto.expressions.RowExpressionTreeRewriter; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.SqlFunctionProperties; +import com.facebook.presto.spi.function.SqlInvokedScalarFunctionImplementation; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.analyzer.ExpressionAnalysis; +import com.facebook.presto.sql.parser.ParsingOptions; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.planner.PlanVariableAllocator; +import com.facebook.presto.sql.planner.iterative.rule.LambdaCaptureDesugaringRewriter; +import com.facebook.presto.sql.tree.Cast; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.ExpressionRewriter; +import com.facebook.presto.sql.tree.ExpressionTreeRewriter; +import com.facebook.presto.sql.tree.Identifier; +import com.facebook.presto.sql.tree.LambdaArgumentDeclaration; +import com.facebook.presto.sql.tree.LambdaExpression; +import com.facebook.presto.sql.tree.NodeRef; +import com.facebook.presto.sql.tree.SymbolReference; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.spi.function.FunctionImplementationType.SQL; +import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.analyzeSqlFunctionExpression; +import static com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DECIMAL; +import static com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DOUBLE; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public final class SqlFunctionUtils +{ + private SqlFunctionUtils() {} + + public static Expression getSqlFunctionExpression(FunctionMetadata functionMetadata, SqlInvokedScalarFunctionImplementation implementation, SqlFunctionProperties sqlFunctionProperties, List arguments) + { + checkArgument(functionMetadata.getImplementationType().equals(SQL), format("Expect SQL function, get %s", functionMetadata.getImplementationType())); + Expression expression = parseSqlFunctionExpression(implementation, sqlFunctionProperties); + return SqlFunctionArgumentBinder.bindFunctionArguments(expression, functionMetadata.getArgumentNames().get(), arguments); + } + + public static RowExpression getSqlFunctionRowExpression(FunctionMetadata functionMetadata, SqlInvokedScalarFunctionImplementation functionImplementation, Metadata metadata, SqlFunctionProperties sqlFunctionProperties, List arguments) + { + Expression expression = coerceIfNecessary(functionMetadata, parseSqlFunctionExpression(functionImplementation, sqlFunctionProperties), sqlFunctionProperties, metadata); + + // Allocate variables for identifiers + PlanVariableAllocator variableAllocator = new PlanVariableAllocator(); + Map variables = buildIdentifierToVariableMap(functionMetadata, expression, sqlFunctionProperties, metadata, variableAllocator); + + // Rewrite expression with allocated variables + Expression rewritten = rewriteSqlFunctionExpressionWithVariables(expression, variables); + + // Desugar lambda capture + Expression lambdaCaptureDesugaredExpression = LambdaCaptureDesugaringRewriter.rewrite(rewritten, variableAllocator); + + // Translate to row expression + return SqlFunctionArgumentBinder.bindFunctionArguments( + SqlToRowExpressionTranslator.translate( + lambdaCaptureDesugaredExpression, + analyzeSqlFunctionExpression(metadata, sqlFunctionProperties, lambdaCaptureDesugaredExpression, variableAllocator.getTypes().allTypes()).getExpressionTypes(), + ImmutableMap.of(), + metadata.getFunctionManager(), + metadata.getTypeManager(), + Optional.empty(), + Optional.empty(), + sqlFunctionProperties), + functionMetadata.getArgumentNames().get().stream() + .map(Identifier::new) + .map(variables::get) + .map(Optional::ofNullable) + .map(variable -> variable.map(VariableReferenceExpression::getName)) + .collect(toImmutableList()), + arguments); + } + + private static Expression parseSqlFunctionExpression(SqlInvokedScalarFunctionImplementation functionImplementation, SqlFunctionProperties sqlFunctionProperties) + { + ParsingOptions parsingOptions = ParsingOptions.builder() + .setDecimalLiteralTreatment(sqlFunctionProperties.isParseDecimalLiteralAsDouble() ? AS_DOUBLE : AS_DECIMAL) + .build(); + return new SqlParser().createExpression(functionImplementation.getImplementation(), parsingOptions); + } + + private static Map getFunctionArgumentTypes(FunctionMetadata functionMetadata, Metadata metadata) + { + List argumentNames = functionMetadata.getArgumentNames().get(); + List argumentTypes = functionMetadata.getArgumentTypes().stream().map(metadata::getType).collect(toImmutableList()); + checkState(argumentNames.size() == argumentTypes.size(), format("Expect argumentNames (size %d) and argumentTypes (size %d) to be of the same size", argumentNames.size(), argumentTypes.size())); + ImmutableMap.Builder typeBuilder = ImmutableMap.builder(); + for (int i = 0; i < argumentNames.size(); i++) { + typeBuilder.put(argumentNames.get(i), argumentTypes.get(i)); + } + return typeBuilder.build(); + } + + private static Map buildIdentifierToVariableMap(FunctionMetadata functionMetadata, Expression sqlFunction, SqlFunctionProperties sqlFunctionProperties, Metadata metadata, PlanVariableAllocator variableAllocator) + { + // Allocate variables for identifiers + Map argumentTypes = getFunctionArgumentTypes(functionMetadata, metadata); + Map, Type> expressionTypes = analyzeSqlFunctionExpression(metadata, sqlFunctionProperties, sqlFunction, argumentTypes).getExpressionTypes(); + Map variables = new LinkedHashMap<>(); + for (Map.Entry, Type> entry : expressionTypes.entrySet()) { + Expression node = entry.getKey().getNode(); + if (node instanceof LambdaArgumentDeclaration) { + LambdaArgumentDeclaration lambdaArgumentDeclaration = (LambdaArgumentDeclaration) node; + if (!variables.containsKey(lambdaArgumentDeclaration.getName())) { + variables.put(lambdaArgumentDeclaration.getName(), variableAllocator.newVariable(lambdaArgumentDeclaration.getName(), entry.getValue())); + } + } + else if (node instanceof Identifier && argumentTypes.containsKey(((Identifier) node).getValue())) { + // input + if (!variables.containsKey(node)) { + variables.put((Identifier) node, variableAllocator.newVariable(node, entry.getValue())); + } + } + } + return variables; + } + + private static Expression rewriteSqlFunctionExpressionWithVariables(Expression sqlFunction, Map variableMap) + { + return ExpressionTreeRewriter.rewriteWith(new ExpressionRewriter>() + { + @Override + public Expression rewriteLambdaExpression(LambdaExpression node, Map context, ExpressionTreeRewriter> treeRewriter) + { + ImmutableList.Builder newArguments = ImmutableList.builder(); + for (LambdaArgumentDeclaration argument : node.getArguments()) { + VariableReferenceExpression variable = context.get(argument.getName()); + newArguments.add(new LambdaArgumentDeclaration(new Identifier(variable.getName()))); + } + return new LambdaExpression(newArguments.build(), treeRewriter.rewrite(node.getBody(), context)); + } + + @Override + public Expression rewriteIdentifier(Identifier node, Map context, ExpressionTreeRewriter> treeRewriter) + { + return new SymbolReference(context.get(node).getName()); + } + }, sqlFunction, variableMap); + } + + private static Expression coerceIfNecessary(FunctionMetadata functionMetadata, Expression sqlFunction, SqlFunctionProperties sqlFunctionProperties, Metadata metadata) + { + ExpressionAnalysis analysis = analyzeSqlFunctionExpression(metadata, sqlFunctionProperties, sqlFunction, getFunctionArgumentTypes(functionMetadata, metadata)); + + return ExpressionTreeRewriter.rewriteWith(new ExpressionRewriter() + { + @Override + public Expression rewriteExpression(Expression expression, ExpressionAnalysis context, ExpressionTreeRewriter treeRewriter) + { + Expression rewritten = treeRewriter.defaultRewrite(expression, null); + + Type coercion = analysis.getCoercion(expression); + if (coercion != null) { + return new Cast( + rewritten, + coercion.getTypeSignature().toString(), + false, + analysis.isTypeOnlyCoercion(expression)); + } + return rewritten; + } + }, sqlFunction, analysis); + } + + private static final class SqlFunctionArgumentBinder + { + private SqlFunctionArgumentBinder() {} + + public static Expression bindFunctionArguments(Expression function, List argumentNames, List argumentValues) + { + checkArgument(argumentNames.size() == argumentValues.size(), format("Expect same size for argumentNames (%d) and argumentValues (%d)", argumentNames.size(), argumentValues.size())); + ImmutableMap.Builder argumentBindings = ImmutableMap.builder(); + for (int i = 0; i < argumentNames.size(); i++) { + argumentBindings.put(argumentNames.get(i), argumentValues.get(i)); + } + return ExpressionTreeRewriter.rewriteWith(new ExpressionFunctionVisitor(argumentBindings.build()), function); + } + + public static RowExpression bindFunctionArguments(RowExpression function, List> argumentNames, List argumentValues) + { + checkArgument(argumentNames.size() == argumentValues.size(), format("Expect same size for argumentNames (%d) and argumentValues (%d)", argumentNames.size(), argumentValues.size())); + ImmutableMap.Builder argumentBindings = ImmutableMap.builder(); + for (int i = 0; i < argumentNames.size(); i++) { + if (argumentNames.get(i).isPresent()) { + argumentBindings.put(argumentNames.get(i).get(), argumentValues.get(i)); + } + } + return RowExpressionTreeRewriter.rewriteWith(new RowExpressionRewriter>() + { + @Override + public RowExpression rewriteVariableReference(VariableReferenceExpression variable, Map context, RowExpressionTreeRewriter> treeRewriter) + { + if (context.containsKey(variable.getName())) { + return context.get(variable.getName()); + } + return variable; + } + }, function, argumentBindings.build()); + } + + private static class ExpressionFunctionVisitor + extends ExpressionRewriter + { + private final Map argumentBindings; + + public ExpressionFunctionVisitor(Map argumentBindings) + { + this.argumentBindings = requireNonNull(argumentBindings, "argumentBindings is null"); + } + + @Override + public Expression rewriteIdentifier(Identifier node, Void context, ExpressionTreeRewriter treeRewriter) + { + if (argumentBindings.containsKey(node.getValue())) { + return argumentBindings.get(node.getValue()); + } + return node; + } + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java index 0afa457268596..4c8276ba0c581 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; @@ -28,8 +29,9 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.VarcharType; +import com.facebook.presto.sql.analyzer.SemanticErrorCode; +import com.facebook.presto.sql.analyzer.SemanticException; import com.facebook.presto.sql.analyzer.TypeSignatureProvider; -import com.facebook.presto.sql.relational.optimizer.ExpressionOptimizer; import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; import com.facebook.presto.sql.tree.ArithmeticUnaryExpression; import com.facebook.presto.sql.tree.ArrayConstructor; @@ -42,7 +44,6 @@ import com.facebook.presto.sql.tree.CharLiteral; import com.facebook.presto.sql.tree.CoalesceExpression; import com.facebook.presto.sql.tree.ComparisonExpression; -import com.facebook.presto.sql.tree.CurrentPath; import com.facebook.presto.sql.tree.CurrentUser; import com.facebook.presto.sql.tree.DecimalLiteral; import com.facebook.presto.sql.tree.DereferenceExpression; @@ -77,6 +78,7 @@ import com.facebook.presto.sql.tree.TimestampLiteral; import com.facebook.presto.sql.tree.TryExpression; import com.facebook.presto.sql.tree.WhenClause; +import com.facebook.presto.transaction.TransactionId; import com.facebook.presto.type.UnknownType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -85,10 +87,9 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.OptionalInt; -import static com.facebook.presto.SystemSessionProperties.isLegacyRowFieldOrdinalAccessEnabled; -import static com.facebook.presto.SystemSessionProperties.isLegacyTimestamp; import static com.facebook.presto.metadata.CastType.CAST; import static com.facebook.presto.metadata.CastType.TRY_CAST; import static com.facebook.presto.spi.function.OperatorType.BETWEEN; @@ -111,7 +112,9 @@ import static com.facebook.presto.spi.type.CharType.createCharType; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; import static com.facebook.presto.spi.type.TimeWithTimeZoneType.TIME_WITH_TIME_ZONE; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; @@ -135,6 +138,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.slice.SliceUtf8.countCodePoints; import static io.airlift.slice.Slices.utf8Slice; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; public final class SqlToRowExpressionTranslator @@ -147,24 +151,31 @@ public static RowExpression translate( Map layout, FunctionManager functionManager, TypeManager typeManager, - Session session, - boolean optimize) + Session session) + { + return translate(expression, types, layout, functionManager, typeManager, Optional.of(session.getUser()), session.getTransactionId(), session.getSqlFunctionProperties()); + } + + public static RowExpression translate( + Expression expression, + Map, Type> types, + Map layout, + FunctionManager functionManager, + TypeManager typeManager, + Optional user, + Optional transactionId, + SqlFunctionProperties sqlFunctionProperties) { Visitor visitor = new Visitor( types, layout, typeManager, functionManager, - session); + user, + transactionId, + sqlFunctionProperties); RowExpression result = visitor.process(expression, null); - requireNonNull(result, "translated expression is null"); - - if (optimize) { - ExpressionOptimizer optimizer = new ExpressionOptimizer(functionManager, session.toConnectorSession()); - return optimizer.optimize(result); - } - return result; } @@ -175,7 +186,9 @@ private static class Visitor private final Map layout; private final TypeManager typeManager; private final FunctionManager functionManager; - private final Session session; + private final Optional user; + private final Optional transactionId; + private final SqlFunctionProperties sqlFunctionProperties; private final FunctionResolution functionResolution; private Visitor( @@ -183,13 +196,17 @@ private Visitor( Map layout, TypeManager typeManager, FunctionManager functionManager, - Session session) + Optional user, + Optional transactionId, + SqlFunctionProperties sqlFunctionProperties) { this.types = ImmutableMap.copyOf(requireNonNull(types, "types is null")); this.layout = layout; this.typeManager = typeManager; this.functionManager = functionManager; - this.session = session; + this.user = user; + this.transactionId = transactionId; + this.sqlFunctionProperties = sqlFunctionProperties; this.functionResolution = new FunctionResolution(functionManager); } @@ -214,13 +231,7 @@ protected RowExpression visitIdentifier(Identifier node, Void context) @Override protected RowExpression visitCurrentUser(CurrentUser node, Void context) { - return constant(Slices.utf8Slice(session.getUser()), VARCHAR); - } - - @Override - protected RowExpression visitCurrentPath(CurrentPath node, Void context) - { - return constant(Slices.utf8Slice(session.getPath().toString()), VARCHAR); + return user.map(user -> constant(Slices.utf8Slice(user), VARCHAR)).orElseThrow(() -> new UnsupportedOperationException("Do not have current user")); } @Override @@ -292,6 +303,21 @@ protected RowExpression visitGenericLiteral(GenericLiteral node, Void context) throw new IllegalArgumentException("Unsupported type: " + node.getType()); } + try { + if (TINYINT.equals(type)) { + return constant((long) Byte.parseByte(node.getValue()), TINYINT); + } + else if (SMALLINT.equals(type)) { + return constant((long) Short.parseShort(node.getValue()), SMALLINT); + } + else if (BIGINT.equals(type)) { + return constant(Long.parseLong(node.getValue()), BIGINT); + } + } + catch (NumberFormatException e) { + throw new SemanticException(SemanticErrorCode.INVALID_LITERAL, node, format("Invalid formatted generic %s literal: %s", type, node)); + } + if (JSON.equals(type)) { return call( "json_parse", @@ -315,9 +341,9 @@ protected RowExpression visitTimeLiteral(TimeLiteral node, Void context) value = parseTimeWithTimeZone(node.getValue()); } else { - if (isLegacyTimestamp(session)) { + if (sqlFunctionProperties.isLegacyTimestamp()) { // parse in time zone of client - value = parseTimeWithoutTimeZone(session.getTimeZoneKey(), node.getValue()); + value = parseTimeWithoutTimeZone(sqlFunctionProperties.getTimeZoneKey(), node.getValue()); } else { value = parseTimeWithoutTimeZone(node.getValue()); @@ -330,8 +356,8 @@ protected RowExpression visitTimeLiteral(TimeLiteral node, Void context) protected RowExpression visitTimestampLiteral(TimestampLiteral node, Void context) { long value; - if (isLegacyTimestamp(session)) { - value = parseTimestampLiteral(session.getTimeZoneKey(), node.getValue()); + if (sqlFunctionProperties.isLegacyTimestamp()) { + value = parseTimestampLiteral(sqlFunctionProperties.getTimeZoneKey(), node.getValue()); } else { value = parseTimestampLiteral(node.getValue()); @@ -379,7 +405,7 @@ protected RowExpression visitFunctionCall(FunctionCall node, Void context) .map(TypeSignatureProvider::new) .collect(toImmutableList()); - return call(node.getName().toString(), functionManager.resolveFunction(session, node.getName(), argumentTypes), getType(node), arguments); + return call(node.getName().toString(), functionManager.resolveFunction(transactionId, node.getName(), argumentTypes), getType(node), arguments); } @Override @@ -576,7 +602,7 @@ protected RowExpression visitDereferenceExpression(DereferenceExpression node, V } } - if (isLegacyRowFieldOrdinalAccessEnabled(session) && index < 0) { + if (sqlFunctionProperties.isLegacyRowFieldOrdinalAccessEnabled() && index < 0) { OptionalInt rowIndex = parseAnonymousRowFieldOrdinalAccess(fieldName, fields); if (rowIndex.isPresent()) { index = rowIndex.getAsInt(); @@ -609,7 +635,7 @@ protected RowExpression visitIfExpression(IfExpression node, Void context) @Override protected RowExpression visitTryExpression(TryExpression node, Void context) { - return call("TRY", functionResolution.tryFunction(getType(node)), getType(node), process(node.getInnerExpression(), context)); + throw new UnsupportedOperationException("Must desugar TryExpression before translate it into RowExpression"); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java deleted file mode 100644 index c277c4af3dbc5..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/optimizer/ExpressionOptimizer.java +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sql.relational.optimizer; - -import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; -import com.facebook.presto.spi.ConnectorSession; -import com.facebook.presto.spi.function.FunctionHandle; -import com.facebook.presto.spi.function.FunctionMetadata; -import com.facebook.presto.spi.relation.CallExpression; -import com.facebook.presto.spi.relation.ConstantExpression; -import com.facebook.presto.spi.relation.InputReferenceExpression; -import com.facebook.presto.spi.relation.LambdaDefinitionExpression; -import com.facebook.presto.spi.relation.RowExpression; -import com.facebook.presto.spi.relation.RowExpressionVisitor; -import com.facebook.presto.spi.relation.SpecialFormExpression; -import com.facebook.presto.spi.relation.SpecialFormExpression.Form; -import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.spi.type.TypeSignature; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; - -import java.lang.invoke.MethodHandle; -import java.util.ArrayList; -import java.util.List; - -import static com.facebook.presto.metadata.CastType.CAST; -import static com.facebook.presto.metadata.CastType.JSON_TO_ARRAY_CAST; -import static com.facebook.presto.metadata.CastType.JSON_TO_MAP_CAST; -import static com.facebook.presto.metadata.CastType.JSON_TO_ROW_CAST; -import static com.facebook.presto.operator.scalar.TryCastFunction.TRY_CAST_NAME; -import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.BIND; -import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; -import static com.facebook.presto.spi.type.StandardTypes.ARRAY; -import static com.facebook.presto.spi.type.StandardTypes.MAP; -import static com.facebook.presto.spi.type.StandardTypes.ROW; -import static com.facebook.presto.spi.type.StandardTypes.VARCHAR; -import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.sql.relational.Expressions.call; -import static com.facebook.presto.sql.relational.Expressions.constant; -import static com.facebook.presto.sql.relational.Expressions.constantNull; -import static com.facebook.presto.type.JsonType.JSON; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Predicates.instanceOf; -import static com.google.common.collect.ImmutableList.toImmutableList; - -public class ExpressionOptimizer -{ - private final FunctionManager functionManager; - private final ConnectorSession session; - - public ExpressionOptimizer(FunctionManager functionManager, ConnectorSession session) - { - this.functionManager = functionManager; - this.session = session; - } - - public RowExpression optimize(RowExpression expression) - { - return expression.accept(new Visitor(), null); - } - - private class Visitor - implements RowExpressionVisitor - { - @Override - public RowExpression visitInputReference(InputReferenceExpression reference, Void context) - { - return reference; - } - - @Override - public RowExpression visitConstant(ConstantExpression literal, Void context) - { - return literal; - } - - @Override - public RowExpression visitCall(CallExpression call, Void context) - { - FunctionHandle functionHandle = call.getFunctionHandle(); - FunctionMetadata functionMetadata = functionManager.getFunctionMetadata(functionHandle); - if (functionMetadata.getName().equals(TRY_CAST_NAME)) { - List arguments = call.getArguments().stream() - .map(argument -> argument.accept(this, null)) - .collect(toImmutableList()); - return call(call.getDisplayName(), functionHandle, call.getType(), arguments); - } - if (functionMetadata.getName().equals(CAST.getCastName())) { - call = rewriteCast(call); - functionHandle = call.getFunctionHandle(); - } - - ScalarFunctionImplementation function = functionManager.getScalarFunctionImplementation(functionHandle); - List arguments = call.getArguments().stream() - .map(argument -> argument.accept(this, context)) - .collect(toImmutableList()); - - // TODO: optimize function calls with lambda arguments. For example, apply(x -> x + 2, 1) - if (Iterables.all(arguments, instanceOf(ConstantExpression.class)) && functionMetadata.isDeterministic()) { - MethodHandle method = function.getMethodHandle(); - - if (method.type().parameterCount() > 0 && method.type().parameterType(0) == ConnectorSession.class) { - method = method.bindTo(session); - } - - int index = 0; - List constantArguments = new ArrayList<>(); - for (RowExpression argument : arguments) { - Object value = ((ConstantExpression) argument).getValue(); - // if any argument is null, return null - if (value == null && !functionMetadata.isCalledOnNullInput()) { - return constantNull(call.getType()); - } - constantArguments.add(value); - index++; - } - - try { - return constant(method.invokeWithArguments(constantArguments), call.getType()); - } - catch (Throwable e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - // Do nothing. As a result, this specific tree will be left untouched. But irrelevant expressions will continue to get evaluated and optimized. - } - } - - return call(call.getDisplayName(), functionHandle, call.getType(), arguments); - } - - @Override - public RowExpression visitLambda(LambdaDefinitionExpression lambda, Void context) - { - return new LambdaDefinitionExpression(lambda.getArgumentTypes(), lambda.getArguments(), lambda.getBody().accept(this, context)); - } - - @Override - public RowExpression visitVariableReference(VariableReferenceExpression reference, Void context) - { - return reference; - } - - @Override - public RowExpression visitSpecialForm(SpecialFormExpression specialForm, Void context) - { - Form form = specialForm.getForm(); - switch (form) { - // TODO: optimize these special forms - case IF: { - checkState(specialForm.getArguments().size() == 3, "IF function should have 3 arguments. Get " + specialForm.getArguments().size()); - RowExpression optimizedOperand = specialForm.getArguments().get(0).accept(this, context); - if (optimizedOperand instanceof ConstantExpression) { - ConstantExpression constantOperand = (ConstantExpression) optimizedOperand; - checkState(constantOperand.getType().equals(BOOLEAN), "Operand of IF function should be BOOLEAN type. Get type " + constantOperand.getType().getDisplayName()); - if (Boolean.TRUE.equals(constantOperand.getValue())) { - return specialForm.getArguments().get(1).accept(this, context); - } - // FALSE and NULL - else { - return specialForm.getArguments().get(2).accept(this, context); - } - } - List arguments = specialForm.getArguments().stream() - .map(argument -> argument.accept(this, null)) - .collect(toImmutableList()); - return new SpecialFormExpression(form, specialForm.getType(), arguments); - } - case BIND: { - checkState(specialForm.getArguments().size() >= 1, BIND.name() + " function should have at least 1 argument. Got " + specialForm.getArguments().size()); - - boolean allConstantExpression = true; - ImmutableList.Builder optimizedArgumentsBuilder = ImmutableList.builder(); - for (RowExpression argument : specialForm.getArguments()) { - RowExpression optimizedArgument = argument.accept(this, context); - if (!(optimizedArgument instanceof ConstantExpression)) { - allConstantExpression = false; - } - optimizedArgumentsBuilder.add(optimizedArgument); - } - if (allConstantExpression) { - // Here, optimizedArguments should be merged together into a new ConstantExpression. - // It's not implemented because it would be dead code anyways because visitLambda does not produce ConstantExpression. - throw new UnsupportedOperationException(); - } - return new SpecialFormExpression(form, specialForm.getType(), optimizedArgumentsBuilder.build()); - } - case NULL_IF: - case SWITCH: - case WHEN: - case IS_NULL: - case COALESCE: - case AND: - case OR: - case IN: - case DEREFERENCE: - case ROW_CONSTRUCTOR: { - List arguments = specialForm.getArguments().stream() - .map(argument -> argument.accept(this, null)) - .collect(toImmutableList()); - return new SpecialFormExpression(form, specialForm.getType(), arguments); - } - default: - throw new IllegalArgumentException("Unsupported special form " + specialForm.getForm()); - } - } - - private CallExpression rewriteCast(CallExpression call) - { - if (call.getArguments().get(0) instanceof CallExpression) { - // Optimization for CAST(JSON_PARSE(...) AS ARRAY/MAP/ROW) - CallExpression innerCall = (CallExpression) call.getArguments().get(0); - if (functionManager.getFunctionMetadata(innerCall.getFunctionHandle()).getName().equals("json_parse")) { - checkArgument(innerCall.getType().equals(JSON)); - checkArgument(innerCall.getArguments().size() == 1); - TypeSignature returnType = functionManager.getFunctionMetadata(call.getFunctionHandle()).getReturnType(); - if (returnType.getBase().equals(ARRAY)) { - return call( - JSON_TO_ARRAY_CAST.name(), - functionManager.lookupCast( - JSON_TO_ARRAY_CAST, - parseTypeSignature(VARCHAR), - returnType), - call.getType(), - innerCall.getArguments()); - } - if (returnType.getBase().equals(MAP)) { - return call( - JSON_TO_MAP_CAST.name(), - functionManager.lookupCast( - JSON_TO_MAP_CAST, - parseTypeSignature(VARCHAR), - returnType), - call.getType(), - innerCall.getArguments()); - } - if (returnType.getBase().equals(ROW)) { - return call( - JSON_TO_ROW_CAST.name(), - functionManager.lookupCast( - JSON_TO_ROW_CAST, - parseTypeSignature(VARCHAR), - returnType), - call.getType(), - innerCall.getArguments()); - } - } - } - - return call( - CAST.name(), - functionManager.lookupCast( - CAST, - call.getArguments().get(0).getType().getTypeSignature(), - call.getType().getTypeSignature()), - call.getType(), - call.getArguments()); - } - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java index 3b66f4bd887c0..b66a648285663 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java @@ -18,7 +18,6 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.metadata.SessionPropertyManager.SessionPropertyValue; -import com.facebook.presto.metadata.SqlFunction; import com.facebook.presto.metadata.ViewDefinition; import com.facebook.presto.security.AccessControl; import com.facebook.presto.spi.CatalogSchemaName; @@ -29,6 +28,7 @@ import com.facebook.presto.spi.StandardErrorCode; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.SqlFunction; import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.PrincipalType; import com.facebook.presto.spi.session.PropertyMetadata; @@ -89,6 +89,7 @@ import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.TABLE_SCHEMATA; import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.TABLE_TABLES; import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.TABLE_TABLE_PRIVILEGES; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.metadata.MetadataListing.listCatalogs; import static com.facebook.presto.metadata.MetadataListing.listSchemas; import static com.facebook.presto.metadata.MetadataUtil.createCatalogSchemaName; @@ -525,7 +526,9 @@ protected Node visitShowFunctions(ShowFunctions node, Void context) ImmutableList.Builder rows = ImmutableList.builder(); for (SqlFunction function : metadata.listFunctions()) { rows.add(row( - new StringLiteral(function.getSignature().getName()), + function.getSignature().getName().getFunctionNamespace().equals(DEFAULT_NAMESPACE) ? + new StringLiteral(function.getSignature().getNameSuffix()) : + new StringLiteral(function.getSignature().getName().toString()), new StringLiteral(function.getSignature().getReturnType().toString()), new StringLiteral(Joiner.on(", ").join(function.getSignature().getArgumentTypes())), new StringLiteral(getFunctionType(function)), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowStatsRewrite.java b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowStatsRewrite.java index b9d7155e013a2..a505a94b39ec7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowStatsRewrite.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowStatsRewrite.java @@ -165,11 +165,11 @@ private void validateShowStatsSubquery(ShowStats node, Query query, QuerySpecifi private Node rewriteShowStats(ShowStats node, Table table, Constraint constraint) { TableHandle tableHandle = getTableHandle(node, table.getName()); - TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, constraint); + Map columnHandles = metadata.getColumnHandles(session, tableHandle); + TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, ImmutableList.copyOf(columnHandles.values()), constraint); List statsColumnNames = buildColumnsNames(); List selectItems = buildSelectItems(statsColumnNames); TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle); - Map columnHandles = metadata.getColumnHandles(session, tableHandle); List resultRows = buildStatisticsRows(tableMetadata, columnHandles, tableStatistics); return simpleQuery(selectAll(selectItems), diff --git a/presto-main/src/main/java/com/facebook/presto/testing/InMemoryFunctionNamespaceManager.java b/presto-main/src/main/java/com/facebook/presto/testing/InMemoryFunctionNamespaceManager.java new file mode 100644 index 0000000000000..605b5820905c1 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/testing/InMemoryFunctionNamespaceManager.java @@ -0,0 +1,118 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.testing; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.function.ScalarFunctionImplementation; +import com.facebook.presto.spi.function.SqlFunctionHandle; +import com.facebook.presto.spi.function.SqlFunctionId; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.facebook.presto.spi.type.TypeSignature; +import com.facebook.presto.sqlfunction.AbstractSqlInvokedFunctionNamespaceManager; +import com.facebook.presto.sqlfunction.SqlInvokedFunctionNamespaceManagerConfig; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.MoreCollectors.onlyElement; +import static java.lang.String.format; + +@ThreadSafe +public class InMemoryFunctionNamespaceManager + extends AbstractSqlInvokedFunctionNamespaceManager +{ + private final Map latestFunctions = new ConcurrentHashMap<>(); + + public InMemoryFunctionNamespaceManager(SqlInvokedFunctionNamespaceManagerConfig config) + { + super(config); + } + + @Override + public synchronized void createFunction(SqlInvokedFunction function, boolean replace) + { + SqlFunctionId functionId = function.getFunctionId(); + if (!replace && latestFunctions.containsKey(function.getFunctionId())) { + throw new PrestoException(GENERIC_USER_ERROR, format("Function '%s' already exists", functionId.getId())); + } + + SqlInvokedFunction replacedFunction = latestFunctions.get(functionId); + long version = 1; + if (replacedFunction != null) { + version = replacedFunction.getRequiredVersion() + 1; + } + latestFunctions.put(functionId, function.withVersion(version)); + } + + @Override + public synchronized void dropFunction(QualifiedFunctionName functionName, Optional> parameterTypes, boolean exists) + { + throw new PrestoException(NOT_SUPPORTED, "Drop Function is not supported in InMemoryFunctionNamespaceManager"); + } + + @Override + public Collection listFunctions() + { + return latestFunctions.values(); + } + + @Override + public Collection fetchFunctionsDirect(QualifiedFunctionName name) + { + return latestFunctions.values().stream() + .filter(function -> function.getSignature().getName().equals(name)) + .map(InMemoryFunctionNamespaceManager::copyFunction) + .collect(toImmutableList()); + } + + @Override + public FunctionMetadata fetchFunctionMetadataDirect(SqlFunctionHandle functionHandle) + { + return fetchFunctionsDirect(functionHandle.getFunctionId().getFunctionName()).stream() + .filter(function -> function.getRequiredFunctionHandle().equals(functionHandle)) + .map(AbstractSqlInvokedFunctionNamespaceManager::sqlInvokedFunctionToMetadata) + .collect(onlyElement()); + } + + @Override + protected ScalarFunctionImplementation fetchFunctionImplementationDirect(SqlFunctionHandle functionHandle) + { + return fetchFunctionsDirect(functionHandle.getFunctionId().getFunctionName()).stream() + .filter(function -> function.getRequiredFunctionHandle().equals(functionHandle)) + .map(AbstractSqlInvokedFunctionNamespaceManager::sqlInvokedFunctionToImplementation) + .collect(onlyElement()); + } + + private static SqlInvokedFunction copyFunction(SqlInvokedFunction function) + { + return new SqlInvokedFunction( + function.getSignature().getName(), + function.getParameters(), + function.getSignature().getReturnType(), + function.getDescription(), + function.getRoutineCharacteristics(), + function.getBody(), + function.getVersion()); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index c56243f21386a..90d8ef187eb12 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.testing; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.GroupByHashPageIndexerFactory; import com.facebook.presto.PagesIndexPageSorter; import com.facebook.presto.Session; @@ -32,14 +33,19 @@ import com.facebook.presto.cost.CostCalculatorUsingExchanges; import com.facebook.presto.cost.CostCalculatorWithEstimatedExchanges; import com.facebook.presto.cost.CostComparator; +import com.facebook.presto.cost.FilterStatsCalculator; +import com.facebook.presto.cost.ScalarStatsCalculator; import com.facebook.presto.cost.StatsCalculator; +import com.facebook.presto.cost.StatsNormalizer; import com.facebook.presto.cost.TaskCountEstimator; import com.facebook.presto.eventlistener.EventListenerManager; import com.facebook.presto.execution.CommitTask; +import com.facebook.presto.execution.CreateFunctionTask; import com.facebook.presto.execution.CreateTableTask; import com.facebook.presto.execution.CreateViewTask; import com.facebook.presto.execution.DataDefinitionTask; import com.facebook.presto.execution.DeallocateTask; +import com.facebook.presto.execution.DropFunctionTask; import com.facebook.presto.execution.DropTableTask; import com.facebook.presto.execution.DropViewTask; import com.facebook.presto.execution.Lifespan; @@ -53,7 +59,6 @@ import com.facebook.presto.execution.ResetSessionTask; import com.facebook.presto.execution.RollbackTask; import com.facebook.presto.execution.ScheduledSplit; -import com.facebook.presto.execution.SetPathTask; import com.facebook.presto.execution.SetSessionTask; import com.facebook.presto.execution.StartTransactionTask; import com.facebook.presto.execution.TaskManagerConfig; @@ -62,6 +67,8 @@ import com.facebook.presto.execution.scheduler.LegacyNetworkTopology; import com.facebook.presto.execution.scheduler.NodeScheduler; import com.facebook.presto.execution.scheduler.NodeSchedulerConfig; +import com.facebook.presto.execution.scheduler.SqlQueryScheduler.StreamingPlanSection; +import com.facebook.presto.execution.scheduler.SqlQueryScheduler.StreamingSubPlan; import com.facebook.presto.execution.warnings.DefaultWarningCollector; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.execution.warnings.WarningCollectorConfig; @@ -135,22 +142,22 @@ import com.facebook.presto.sql.planner.LocalExecutionPlanner.LocalExecutionPlan; import com.facebook.presto.sql.planner.LogicalPlanner; import com.facebook.presto.sql.planner.NodePartitioningManager; +import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.PlanFragmenter; import com.facebook.presto.sql.planner.PlanOptimizers; -import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.SubPlan; -import com.facebook.presto.sql.planner.iterative.IterativeOptimizer; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; -import com.facebook.presto.sql.planner.optimizations.TranslateExpressions; import com.facebook.presto.sql.planner.planPrinter.PlanPrinter; import com.facebook.presto.sql.planner.sanity.PlanSanityChecker; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.facebook.presto.sql.tree.Commit; +import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateView; import com.facebook.presto.sql.tree.Deallocate; +import com.facebook.presto.sql.tree.DropFunction; import com.facebook.presto.sql.tree.DropTable; import com.facebook.presto.sql.tree.DropView; import com.facebook.presto.sql.tree.Explain; @@ -159,7 +166,6 @@ import com.facebook.presto.sql.tree.RenameTable; import com.facebook.presto.sql.tree.ResetSession; import com.facebook.presto.sql.tree.Rollback; -import com.facebook.presto.sql.tree.SetPath; import com.facebook.presto.sql.tree.SetSession; import com.facebook.presto.sql.tree.StartTransaction; import com.facebook.presto.sql.tree.Statement; @@ -173,7 +179,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.io.Closer; -import io.airlift.node.NodeInfo; import io.airlift.units.Duration; import org.intellij.lang.annotations.Language; import org.weakref.jmx.MBeanExporter; @@ -196,7 +201,12 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.cost.StatsCalculatorModule.createNewStatsCalculator; +import static com.facebook.presto.execution.scheduler.SqlQueryScheduler.extractStreamingSections; +import static com.facebook.presto.execution.scheduler.TableWriteInfo.createTableWriteInfo; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.GROUPED_SCHEDULING; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.REWINDABLE_GROUPED_SCHEDULING; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING; @@ -210,9 +220,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.json.JsonCodec.jsonCodec; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; @@ -232,6 +239,9 @@ public class LocalQueryRunner private final PageSorter pageSorter; private final PageIndexerFactory pageIndexerFactory; private final MetadataManager metadata; + private final ScalarStatsCalculator scalarStatsCalculator; + private final StatsNormalizer statsNormalizer; + private final FilterStatsCalculator filterStatsCalculator; private final StatsCalculator statsCalculator; private final CostCalculator costCalculator; private final CostCalculator estimatedExchangesCostCalculator; @@ -241,6 +251,7 @@ public class LocalQueryRunner private final BlockEncodingManager blockEncodingManager; private final PageSourceManager pageSourceManager; private final IndexManager indexManager; + private final PartitioningProviderManager partitioningProviderManager; private final NodePartitioningManager nodePartitioningManager; private final ConnectorPlanOptimizerManager planOptimizerManager; private final PageSinkManager pageSinkManager; @@ -309,7 +320,8 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, yieldExecutor, catalogManager, notificationExecutor); - this.nodePartitioningManager = new NodePartitioningManager(nodeScheduler); + this.partitioningProviderManager = new PartitioningProviderManager(); + this.nodePartitioningManager = new NodePartitioningManager(nodeScheduler, partitioningProviderManager); this.planOptimizerManager = new ConnectorPlanOptimizerManager(); this.blockEncodingManager = new BlockEncodingManager(typeRegistry); @@ -327,7 +339,10 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, this.planFragmenter = new PlanFragmenter(this.metadata, this.nodePartitioningManager, new QueryManagerConfig(), sqlParser); this.joinCompiler = new JoinCompiler(metadata, featuresConfig); this.pageIndexerFactory = new GroupByHashPageIndexerFactory(joinCompiler); - this.statsCalculator = createNewStatsCalculator(metadata); + this.statsNormalizer = new StatsNormalizer(); + this.scalarStatsCalculator = new ScalarStatsCalculator(metadata); + this.filterStatsCalculator = new FilterStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer); + this.statsCalculator = createNewStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer, filterStatsCalculator); this.taskCountEstimator = new TaskCountEstimator(() -> nodeCountForStats); this.costCalculator = new CostCalculatorUsingExchanges(taskCountEstimator); this.estimatedExchangesCostCalculator = new CostCalculatorWithEstimatedExchanges(costCalculator, taskCountEstimator); @@ -346,7 +361,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, splitManager, pageSourceManager, indexManager, - nodePartitioningManager, + partitioningProviderManager, planOptimizerManager, pageSinkManager, new HandleResolver(), @@ -358,7 +373,8 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, transactionManager, new RowExpressionDomainTranslator(metadata), new RowExpressionPredicateCompiler(metadata), - new RowExpressionDeterminismEvaluator(metadata.getFunctionManager())); + new RowExpressionDeterminismEvaluator(metadata.getFunctionManager()), + new FilterStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer)); GlobalSystemConnectorFactory globalSystemConnectorFactory = new GlobalSystemConnectorFactory(ImmutableSet.of( new NodeSystemTable(nodeManager), @@ -398,7 +414,6 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, defaultSession.getSource(), defaultSession.getCatalog(), defaultSession.getSchema(), - defaultSession.getPath(), defaultSession.getTraceToken(), defaultSession.getTimeZoneKey(), defaultSession.getLocale(), @@ -406,7 +421,6 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, defaultSession.getUserAgent(), defaultSession.getClientInfo(), defaultSession.getClientTags(), - defaultSession.getClientCapabilities(), defaultSession.getResourceEstimates(), defaultSession.getStartTime(), defaultSession.getSystemProperties(), @@ -418,6 +432,8 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, dataDefinitionTask = ImmutableMap., DataDefinitionTask>builder() .put(CreateTable.class, new CreateTableTask()) .put(CreateView.class, new CreateViewTask(jsonCodec(ViewDefinition.class), sqlParser, new FeaturesConfig())) + .put(CreateFunction.class, new CreateFunctionTask(sqlParser)) + .put(DropFunction.class, new DropFunctionTask(sqlParser)) .put(DropTable.class, new DropTableTask()) .put(DropView.class, new DropViewTask()) .put(RenameColumn.class, new RenameColumnTask()) @@ -429,7 +445,6 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, .put(StartTransaction.class, new StartTransactionTask()) .put(Commit.class, new CommitTask()) .put(Rollback.class, new RollbackTask()) - .put(SetPath.class, new SetPathTask()) .build(); SpillerStats spillerStats = new SpillerStats(); @@ -729,10 +744,10 @@ public PlanNodeId getNextId() LocalExecutionPlanner executionPlanner = new LocalExecutionPlanner( metadata, - sqlParser, Optional.empty(), pageSourceManager, indexManager, + partitioningProviderManager, nodePartitioningManager, pageSinkManager, expressionCompiler, @@ -752,6 +767,9 @@ public PlanNodeId getNextId() // plan query StageExecutionDescriptor stageExecutionDescriptor = subplan.getFragment().getStageExecutionDescriptor(); + StreamingPlanSection streamingPlanSection = extractStreamingSections(subplan); + checkState(streamingPlanSection.getChildren().isEmpty(), "expected no materialized exchanges"); + StreamingSubPlan streamingSubPlan = streamingPlanSection.getPlan(); LocalExecutionPlan localExecutionPlan = executionPlanner.plan( taskContext, stageExecutionDescriptor, @@ -762,7 +780,8 @@ public PlanNodeId getNextId() outputFactory, new TaskExchangeClientManager(ignored -> { throw new UnsupportedOperationException(); - })); + }), + createTableWriteInfo(streamingSubPlan, metadata, session)); // generate sources List sources = new ArrayList<>(); @@ -859,7 +878,6 @@ public List getPlanOptimizers(boolean forceSingleNode) return new PlanOptimizers( metadata, sqlParser, - featuresConfig, forceSingleNode, new MBeanExporter(new TestingMBeanServer()), splitManager, @@ -914,16 +932,4 @@ private static List findTableScanNodes(PlanNode node) .where(TableScanNode.class::isInstance) .findAll(); } - - public PlanOptimizer translateExpressions() - { - // Translate all OriginalExpression in planNodes to RowExpression so that we can do plan pattern asserting and printing on RowExpression only. - return new IterativeOptimizer( - new RuleStatsRecorder(), - getStatsCalculator(), - getCostCalculator(), - new ImmutableSet.Builder() - .addAll(new TranslateExpressions(getMetadata(), getSqlParser()).rules()) - .build()); - } } diff --git a/presto-main/src/main/java/com/facebook/presto/testing/MaterializedResult.java b/presto-main/src/main/java/com/facebook/presto/testing/MaterializedResult.java index cfe230ed3e02f..04133501d4b45 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/MaterializedResult.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/MaterializedResult.java @@ -419,6 +419,9 @@ public static MaterializedResult materializeSourceDataStream(ConnectorSession se if (outputPage == null) { break; } + if (outputPage.getPositionCount() == 0) { + continue; + } builder.page(outputPage); } return builder.build(); diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java index b1e87c8cb4762..c3b638669a278 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java @@ -17,27 +17,35 @@ import com.facebook.presto.PagesIndexPageSorter; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.connector.ConnectorAwareNodeManager; +import com.facebook.presto.cost.ConnectorFilterStatsCalculatorService; +import com.facebook.presto.cost.FilterStatsCalculator; +import com.facebook.presto.cost.ScalarStatsCalculator; +import com.facebook.presto.cost.StatsNormalizer; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.InMemoryNodeManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.operator.PagesIndex; import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.FilterStatsCalculatorService; import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.PredicateCompiler; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.gen.RowExpressionPredicateCompiler; +import com.facebook.presto.sql.planner.planPrinter.RowExpressionFormatter; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; @@ -57,6 +65,7 @@ public class TestingConnectorContext private final DomainTranslator domainTranslator = new RowExpressionDomainTranslator(metadata); private final PredicateCompiler predicateCompiler = new RowExpressionPredicateCompiler(metadata); private final DeterminismEvaluator determinismEvaluator = new RowExpressionDeterminismEvaluator(functionManager); + private final FilterStatsCalculatorService filterStatsCalculatorService = new ConnectorFilterStatsCalculatorService(new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer())); @Override public NodeManager getNodeManager() @@ -122,6 +131,18 @@ public DeterminismEvaluator getDeterminismEvaluator() { return determinismEvaluator; } + + @Override + public String formatRowExpression(ConnectorSession session, RowExpression expression) + { + return new RowExpressionFormatter(functionManager).formatRowExpression(session, expression); + } }; } + + @Override + public FilterStatsCalculatorService getFilterStatsCalculatorService() + { + return filterStatsCalculatorService; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorSession.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorSession.java index 43778a877f1b1..5215d54bb3950 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorSession.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorSession.java @@ -16,6 +16,7 @@ import com.facebook.presto.execution.QueryIdGenerator; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.security.ConnectorIdentity; import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spi.type.TimeZoneKey; @@ -44,17 +45,17 @@ public class TestingConnectorSession private final String queryId; private final ConnectorIdentity identity; private final Optional source; - private final TimeZoneKey timeZoneKey; private final Locale locale; private final Optional traceToken; private final long startTime; private final Map> properties; private final Map propertyValues; - private final boolean isLegacyTimestamp; + private final Optional clientInfo; + private final SqlFunctionProperties sqlFunctionProperties; public TestingConnectorSession(List> properties) { - this("user", Optional.of("test"), Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, ImmutableMap.of(), new FeaturesConfig().isLegacyTimestamp()); + this("user", Optional.of("test"), Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, ImmutableMap.of(), new FeaturesConfig().isLegacyTimestamp(), Optional.empty()); } public TestingConnectorSession( @@ -66,18 +67,22 @@ public TestingConnectorSession( long startTime, List> propertyMetadatas, Map propertyValues, - boolean isLegacyTimestamp) + boolean isLegacyTimestamp, + Optional clientInfo) { this.queryId = queryIdGenerator.createNextQueryId().toString(); this.identity = new ConnectorIdentity(requireNonNull(user, "user is null"), Optional.empty(), Optional.empty()); this.source = requireNonNull(source, "source is null"); this.traceToken = requireNonNull(traceToken, "traceToken is null"); - this.timeZoneKey = requireNonNull(timeZoneKey, "timeZoneKey is null"); this.locale = requireNonNull(locale, "locale is null"); this.startTime = startTime; this.properties = Maps.uniqueIndex(propertyMetadatas, PropertyMetadata::getName); this.propertyValues = ImmutableMap.copyOf(propertyValues); - this.isLegacyTimestamp = isLegacyTimestamp; + this.clientInfo = clientInfo; + this.sqlFunctionProperties = SqlFunctionProperties.builder() + .setTimeZoneKey(requireNonNull(timeZoneKey, "timeZoneKey is null")) + .setLegacyTimestamp(isLegacyTimestamp) + .build(); } @Override @@ -101,7 +106,7 @@ public ConnectorIdentity getIdentity() @Override public TimeZoneKey getTimeZoneKey() { - return timeZoneKey; + return sqlFunctionProperties.getTimeZoneKey(); } @Override @@ -122,10 +127,22 @@ public Optional getTraceToken() return traceToken; } + @Override + public Optional getClientInfo() + { + return clientInfo; + } + @Override public boolean isLegacyTimestamp() { - return isLegacyTimestamp; + return sqlFunctionProperties.isLegacyTimestamp(); + } + + @Override + public SqlFunctionProperties getSqlFunctionProperties() + { + return SqlFunctionProperties.builder().setTimeZoneKey(UTC_KEY).build(); } @Override @@ -149,10 +166,11 @@ public String toString() .add("user", getUser()) .add("source", source.orElse(null)) .add("traceToken", traceToken.orElse(null)) - .add("timeZoneKey", timeZoneKey) .add("locale", locale) .add("startTime", startTime) + .add("sqlFunctionProperties", sqlFunctionProperties) .add("properties", propertyValues) + .add("clientInfo", clientInfo) .omitNullValues() .toString(); } diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingMetadata.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingMetadata.java index c06fd10fc333b..289d404f3077b 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingMetadata.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingMetadata.java @@ -321,6 +321,25 @@ public SchemaTableName getTableName() { return tableName; } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof TestingTableHandle)) { + return false; + } + TestingTableHandle other = (TestingTableHandle) o; + return Objects.equals(tableName, other.tableName); + } + + @Override + public int hashCode() + { + return Objects.hash(tableName); + } } public static class TestingColumnHandle diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingSession.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingSession.java index e8e3b0b982682..ba94a88c3957e 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingSession.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingSession.java @@ -28,7 +28,6 @@ import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.transaction.IsolationLevel; import com.facebook.presto.spi.type.TimeZoneKey; -import com.facebook.presto.sql.SqlPath; import com.google.common.collect.ImmutableSet; import java.util.Optional; @@ -66,7 +65,6 @@ public static SessionBuilder testSessionBuilder(SessionPropertyManager sessionPr .setSource("test") .setCatalog("catalog") .setSchema("schema") - .setPath(new SqlPath(Optional.of("path"))) .setTimeZoneKey(DEFAULT_TIME_ZONE_KEY) .setLocale(ENGLISH) .setRemoteUserAddress("address") diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingTaskContext.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingTaskContext.java index 763849b4ce6a3..a6b5dbca44e69 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingTaskContext.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingTaskContext.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.testing; +import com.facebook.airlift.stats.GcMonitor; +import com.facebook.airlift.stats.TestingGcMonitor; import com.facebook.presto.Session; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.TaskStateMachine; @@ -22,8 +24,6 @@ import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.memory.MemoryPoolId; import com.facebook.presto.spiller.SpillSpaceTracker; -import io.airlift.stats.GcMonitor; -import io.airlift.stats.TestingGcMonitor; import io.airlift.units.DataSize; import java.util.OptionalInt; @@ -61,7 +61,7 @@ public static TaskContext createTaskContext(Executor notificationExecutor, Sched public static TaskContext createTaskContext(QueryContext queryContext, Executor executor, Session session) { - return createTaskContext(queryContext, session, new TaskStateMachine(new TaskId("query", 0, 0), executor)); + return createTaskContext(queryContext, session, new TaskStateMachine(new TaskId("query", 0, 0, 0), executor)); } private static TaskContext createTaskContext(QueryContext queryContext, Session session, TaskStateMachine taskStateMachine) @@ -98,7 +98,7 @@ private Builder(Executor notificationExecutor, ScheduledExecutorService yieldExe this.notificationExecutor = notificationExecutor; this.yieldExecutor = yieldExecutor; this.session = session; - this.taskStateMachine = new TaskStateMachine(new TaskId("query", 0, 0), notificationExecutor); + this.taskStateMachine = new TaskStateMachine(new TaskId("query", 0, 0, 0), notificationExecutor); } public Builder setTaskStateMachine(TaskStateMachine taskStateMachine) diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingWarningCollectorConfig.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingWarningCollectorConfig.java index ff4c02a9f43d7..3711e5e8a1b45 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingWarningCollectorConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingWarningCollectorConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.testing; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import static com.google.common.base.Preconditions.checkArgument; diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingWarningCollectorModule.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingWarningCollectorModule.java index 265d7c1585404..6f3eb93e80c50 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingWarningCollectorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingWarningCollectorModule.java @@ -20,7 +20,7 @@ import com.google.inject.Provides; import com.google.inject.Singleton; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; public class TestingWarningCollectorModule diff --git a/presto-main/src/main/java/com/facebook/presto/transaction/DelegatingTransactionManager.java b/presto-main/src/main/java/com/facebook/presto/transaction/DelegatingTransactionManager.java new file mode 100644 index 0000000000000..bae513607222d --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/transaction/DelegatingTransactionManager.java @@ -0,0 +1,166 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.transaction; + +import com.facebook.presto.Session; +import com.facebook.presto.metadata.CatalogMetadata; +import com.facebook.presto.security.AccessControl; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.function.FunctionNamespaceManager; +import com.facebook.presto.spi.function.FunctionNamespaceTransactionHandle; +import com.facebook.presto.spi.transaction.IsolationLevel; +import com.google.common.util.concurrent.ListenableFuture; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class DelegatingTransactionManager + implements TransactionManager +{ + private final TransactionManager delegate; + + public TransactionManager getDelegate() + { + return delegate; + } + + public DelegatingTransactionManager(TransactionManager delegate) + { + this.delegate = requireNonNull(delegate, "delegate is null"); + } + + @Override + public TransactionInfo getTransactionInfo(TransactionId transactionId) + { + return delegate.getTransactionInfo(transactionId); + } + + @Override + public Optional getOptionalTransactionInfo(TransactionId transactionId) + { + return delegate.getOptionalTransactionInfo(transactionId); + } + + @Override + public List getAllTransactionInfos() + { + return delegate.getAllTransactionInfos(); + } + + @Override + public TransactionId beginTransaction(boolean autoCommitContext) + { + return delegate.beginTransaction(autoCommitContext); + } + + @Override + public TransactionId beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommitContext) + { + return delegate.beginTransaction(isolationLevel, readOnly, autoCommitContext); + } + + @Override + public Map getCatalogNames(TransactionId transactionId) + { + return delegate.getCatalogNames(transactionId); + } + + @Override + public Optional getOptionalCatalogMetadata(TransactionId transactionId, String catalogName) + { + return delegate.getOptionalCatalogMetadata(transactionId, catalogName); + } + + @Override + public CatalogMetadata getCatalogMetadata(TransactionId transactionId, ConnectorId connectorId) + { + return delegate.getCatalogMetadata(transactionId, connectorId); + } + + @Override + public CatalogMetadata getCatalogMetadataForWrite(TransactionId transactionId, ConnectorId connectorId) + { + return delegate.getCatalogMetadataForWrite(transactionId, connectorId); + } + + @Override + public CatalogMetadata getCatalogMetadataForWrite(TransactionId transactionId, String catalogName) + { + return delegate.getCatalogMetadataForWrite(transactionId, catalogName); + } + + @Override + public ConnectorTransactionHandle getConnectorTransaction(TransactionId transactionId, ConnectorId connectorId) + { + return delegate.getConnectorTransaction(transactionId, connectorId); + } + + @Override + public void checkAndSetActive(TransactionId transactionId) + { + delegate.checkAndSetActive(transactionId); + } + + @Override + public void trySetActive(TransactionId transactionId) + { + delegate.trySetActive(transactionId); + } + + @Override + public void trySetInactive(TransactionId transactionId) + { + delegate.trySetInactive(transactionId); + } + + @Override + public ListenableFuture asyncCommit(TransactionId transactionId) + { + return delegate.asyncCommit(transactionId); + } + + @Override + public ListenableFuture asyncAbort(TransactionId transactionId) + { + return delegate.asyncAbort(transactionId); + } + + @Override + public void fail(TransactionId transactionId) + { + delegate.fail(transactionId); + } + + @Override + public void activateTransaction(Session session, boolean transactionControl, AccessControl accessControl) + { + delegate.activateTransaction(session, transactionControl, accessControl); + } + + @Override + public void registerFunctionNamespaceManager(String functionNamespaceManagerName, FunctionNamespaceManager functionNamespaceManager) + { + delegate.registerFunctionNamespaceManager(functionNamespaceManagerName, functionNamespaceManager); + } + + @Override + public FunctionNamespaceTransactionHandle getFunctionNamespaceTransaction(TransactionId transactionId, String functionNamespaceManagerName) + { + return delegate.getFunctionNamespaceTransaction(transactionId, functionNamespaceManagerName); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/transaction/InMemoryTransactionManager.java b/presto-main/src/main/java/com/facebook/presto/transaction/InMemoryTransactionManager.java index f41e40a0de863..02f18f5cc2c9c 100644 --- a/presto-main/src/main/java/com/facebook/presto/transaction/InMemoryTransactionManager.java +++ b/presto-main/src/main/java/com/facebook/presto/transaction/InMemoryTransactionManager.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.transaction; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.concurrent.ExecutorServiceAdapter; +import com.facebook.airlift.log.Logger; import com.facebook.presto.metadata.Catalog; import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.metadata.CatalogMetadata; @@ -21,15 +24,14 @@ import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.function.FunctionNamespaceManager; +import com.facebook.presto.spi.function.FunctionNamespaceTransactionHandle; import com.facebook.presto.spi.transaction.IsolationLevel; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.concurrent.ExecutorServiceAdapter; -import io.airlift.log.Logger; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -51,7 +53,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; +import java.util.stream.Stream; +import static com.facebook.airlift.concurrent.MoreFutures.addExceptionCallback; import static com.facebook.presto.spi.StandardErrorCode.AUTOCOMMIT_WRITE_CONFLICT; import static com.facebook.presto.spi.StandardErrorCode.MULTI_CATALOG_WRITE_CONFLICT; import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; @@ -67,7 +71,6 @@ import static com.google.common.util.concurrent.Futures.nonCancellationPropagating; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; -import static io.airlift.concurrent.MoreFutures.addExceptionCallback; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -86,7 +89,10 @@ public class InMemoryTransactionManager private final CatalogManager catalogManager; private final Executor finishingExecutor; - private InMemoryTransactionManager(Duration idleTimeout, int maxFinishingConcurrency, CatalogManager catalogManager, Executor finishingExecutor) + private final Map> functionNamespaceManagers = new HashMap<>(); + private final Map companionCatalogs; + + private InMemoryTransactionManager(Duration idleTimeout, int maxFinishingConcurrency, CatalogManager catalogManager, Executor finishingExecutor, Map companionCatalogs) { this.catalogManager = catalogManager; requireNonNull(idleTimeout, "idleTimeout is null"); @@ -96,6 +102,7 @@ private InMemoryTransactionManager(Duration idleTimeout, int maxFinishingConcurr this.idleTimeout = idleTimeout; this.maxFinishingConcurrency = maxFinishingConcurrency; this.finishingExecutor = finishingExecutor; + this.companionCatalogs = ImmutableMap.copyOf(companionCatalogs); } public static TransactionManager create( @@ -104,7 +111,7 @@ public static TransactionManager create( CatalogManager catalogManager, ExecutorService finishingExecutor) { - InMemoryTransactionManager transactionManager = new InMemoryTransactionManager(config.getIdleTimeout(), config.getMaxFinishingConcurrency(), catalogManager, finishingExecutor); + InMemoryTransactionManager transactionManager = new InMemoryTransactionManager(config.getIdleTimeout(), config.getMaxFinishingConcurrency(), catalogManager, finishingExecutor, config.getCompanionCatalogs()); transactionManager.scheduleIdleChecks(config.getIdleCheckInterval(), idleCheckExecutor); return transactionManager; } @@ -117,7 +124,7 @@ public static TransactionManager createTestTransactionManager() public static TransactionManager createTestTransactionManager(CatalogManager catalogManager) { // No idle checks needed - return new InMemoryTransactionManager(new Duration(1, TimeUnit.DAYS), 1, catalogManager, directExecutor()); + return new InMemoryTransactionManager(new Duration(1, TimeUnit.DAYS), 1, catalogManager, directExecutor(), ImmutableMap.of()); } private void scheduleIdleChecks(Duration idleCheckInterval, ScheduledExecutorService idleCheckExecutor) @@ -132,7 +139,7 @@ private void scheduleIdleChecks(Duration idleCheckInterval, ScheduledExecutorSer }, idleCheckInterval.toMillis(), idleCheckInterval.toMillis(), MILLISECONDS); } - private synchronized void cleanUpExpiredTransactions() + synchronized void cleanUpExpiredTransactions() { Iterator> iterator = transactions.entrySet().iterator(); while (iterator.hasNext()) { @@ -146,15 +153,15 @@ private synchronized void cleanUpExpiredTransactions() } @Override - public boolean transactionExists(TransactionId transactionId) + public TransactionInfo getTransactionInfo(TransactionId transactionId) { - return tryGetTransactionMetadata(transactionId).isPresent(); + return getTransactionMetadata(transactionId).getTransactionInfo(); } @Override - public TransactionInfo getTransactionInfo(TransactionId transactionId) + public Optional getOptionalTransactionInfo(TransactionId transactionId) { - return getTransactionMetadata(transactionId).getTransactionInfo(); + return tryGetTransactionMetadata(transactionId).map(TransactionMetadata::getTransactionInfo); } @Override @@ -176,7 +183,7 @@ public TransactionId beginTransaction(IsolationLevel isolationLevel, boolean rea { TransactionId transactionId = TransactionId.create(); BoundedExecutor executor = new BoundedExecutor(finishingExecutor, maxFinishingConcurrency); - TransactionMetadata transactionMetadata = new TransactionMetadata(transactionId, isolationLevel, readOnly, autoCommitContext, catalogManager, executor); + TransactionMetadata transactionMetadata = new TransactionMetadata(transactionId, isolationLevel, readOnly, autoCommitContext, catalogManager, executor, functionNamespaceManagers, companionCatalogs); checkState(transactions.put(transactionId, transactionMetadata) == null, "Duplicate transaction ID: %s", transactionId); return transactionId; } @@ -227,6 +234,19 @@ public ConnectorTransactionHandle getConnectorTransaction(TransactionId transact return getCatalogMetadata(transactionId, connectorId).getTransactionHandleFor(connectorId); } + @Override + public synchronized void registerFunctionNamespaceManager(String functionNamespaceManagerId, FunctionNamespaceManager functionNamespaceManager) + { + checkArgument(!functionNamespaceManagers.containsKey(functionNamespaceManagerId), "FunctionNamespaceManager %s is already registered", functionNamespaceManagerId); + functionNamespaceManagers.put(functionNamespaceManagerId, functionNamespaceManager); + } + + @Override + public FunctionNamespaceTransactionHandle getFunctionNamespaceTransaction(TransactionId transactionId, String functionNamespaceManagerId) + { + return getTransactionMetadata(transactionId).getFunctionNamespaceTransaction(functionNamespaceManagerId).getTransactionHandle(); + } + private void checkConnectorWrite(TransactionId transactionId, ConnectorId connectorId) { getTransactionMetadata(transactionId).checkConnectorWrite(connectorId); @@ -323,13 +343,20 @@ private static class TransactionMetadata @GuardedBy("this") private final Map catalogMetadata = new ConcurrentHashMap<>(); + private final Map> functionNamespaceManagers; + @GuardedBy("this") + private final Map functionNamespaceTransactions = new ConcurrentHashMap<>(); + private final Map companionCatalogs; + public TransactionMetadata( TransactionId transactionId, IsolationLevel isolationLevel, boolean readOnly, boolean autoCommitContext, CatalogManager catalogManager, - Executor finishingExecutor) + Executor finishingExecutor, + Map> functionNamespaceManagers, + Map companionCatalogs) { this.transactionId = requireNonNull(transactionId, "transactionId is null"); this.isolationLevel = requireNonNull(isolationLevel, "isolationLevel is null"); @@ -337,6 +364,8 @@ public TransactionMetadata( this.autoCommitContext = autoCommitContext; this.catalogManager = requireNonNull(catalogManager, "catalogManager is null"); this.finishingExecutor = listeningDecorator(ExecutorServiceAdapter.from(requireNonNull(finishingExecutor, "finishingExecutor is null"))); + this.functionNamespaceManagers = requireNonNull(functionNamespaceManagers, "functionNamespaceManagers is null"); + this.companionCatalogs = requireNonNull(companionCatalogs, "companionCatalogs is null"); } public void setActive() @@ -393,6 +422,12 @@ private synchronized Optional getConnectorId(String catalogName) if (catalog.isPresent()) { registerCatalog(catalog.get()); } + + if (companionCatalogs.containsKey(catalogName)) { + Optional companionCatalog = catalogManager.getCatalog(companionCatalogs.get(catalogName)); + checkArgument(companionCatalog.isPresent(), format("Invalid config, no catalog exists for catalog name %s: %s", catalogName, companionCatalogs.get(catalogName))); + registerCatalog(companionCatalog.get()); + } } return catalog.map(Catalog::getConnectorId); } @@ -437,6 +472,19 @@ private synchronized CatalogMetadata getTransactionCatalogMetadata(ConnectorId c return catalogMetadata; } + private synchronized FunctionNamespaceTransactionMetadata getFunctionNamespaceTransaction(String functionNamespaceManagerId) + { + checkOpenTransaction(); + + return functionNamespaceTransactions.computeIfAbsent( + functionNamespaceManagerId, id -> { + verify(id != null, "Unknown function namespace manager: %s", id); + FunctionNamespaceManager functionNamespaceManager = functionNamespaceManagers.get(id); + FunctionNamespaceTransactionHandle transactionHandle = functionNamespaceManager.beginTransaction(); + return new FunctionNamespaceTransactionMetadata(functionNamespaceManager, transactionHandle); + }); + } + public synchronized ConnectorTransactionMetadata createConnectorTransactionMetadata(ConnectorId connectorId, Catalog catalog) { Connector connector = catalog.getConnector(connectorId); @@ -482,16 +530,23 @@ public synchronized ListenableFuture asyncCommit() return immediateFailedFuture(new PrestoException(TRANSACTION_ALREADY_ABORTED, "Current transaction has already been aborted")); } + ListenableFuture functionNamespaceFuture = Futures.allAsList(functionNamespaceTransactions.values().stream() + .map(transactionMetadata -> finishingExecutor.submit(transactionMetadata::commit)) + .collect(toImmutableList())); + ConnectorId writeConnectorId = this.writtenConnectorId.get(); if (writeConnectorId == null) { - ListenableFuture future = Futures.allAsList(connectorIdToMetadata.values().stream() - .map(transactionMetadata -> finishingExecutor.submit(transactionMetadata::commit)) - .collect(toList())); - addExceptionCallback(future, throwable -> { - abortInternal(); - log.error(throwable, "Read-only connector should not throw exception on commit"); - }); - return nonCancellationPropagating(future); + Supplier> commitReadOnlyConnectors = () -> { + ListenableFuture> future = Futures.allAsList(connectorIdToMetadata.values().stream() + .map(transactionMetadata -> finishingExecutor.submit(transactionMetadata::commit)) + .collect(toList())); + addExceptionCallback(future, throwable -> log.error(throwable, "Read-only connector should not throw exception on commit")); + return future; + }; + + ListenableFuture readOnlyCommitFuture = Futures.transformAsync(functionNamespaceFuture, ignored -> commitReadOnlyConnectors.get(), directExecutor()); + addExceptionCallback(readOnlyCommitFuture, this::abortInternal); + return nonCancellationPropagating(readOnlyCommitFuture); } Supplier> commitReadOnlyConnectors = () -> { @@ -505,9 +560,11 @@ public synchronized ListenableFuture asyncCommit() }; ConnectorTransactionMetadata writeConnector = connectorIdToMetadata.get(writeConnectorId); - ListenableFuture commitFuture = finishingExecutor.submit(writeConnector::commit); + Supplier commitFunctionNamespaceTransactions = () -> functionNamespaceFuture; + ListenableFuture commitFuture = Futures.transformAsync(finishingExecutor.submit(writeConnector::commit), ignored -> commitFunctionNamespaceTransactions.get(), directExecutor()); ListenableFuture readOnlyCommitFuture = Futures.transformAsync(commitFuture, ignored -> commitReadOnlyConnectors.get(), directExecutor()); addExceptionCallback(readOnlyCommitFuture, this::abortInternal); + return nonCancellationPropagating(readOnlyCommitFuture); } @@ -527,8 +584,11 @@ public synchronized ListenableFuture asyncAbort() private synchronized ListenableFuture abortInternal() { // the callbacks in statement performed on another thread so are safe - return nonCancellationPropagating(Futures.allAsList(connectorIdToMetadata.values().stream() - .map(connection -> finishingExecutor.submit(() -> safeAbort(connection))) + return nonCancellationPropagating(Futures.allAsList(Stream.concat( + functionNamespaceTransactions.values().stream() + .map(transactionMetadata -> finishingExecutor.submit(() -> safeAbort(transactionMetadata))), + connectorIdToMetadata.values().stream() + .map(connection -> finishingExecutor.submit(() -> safeAbort(connection)))) .collect(toList()))); } @@ -542,6 +602,16 @@ private static void safeAbort(ConnectorTransactionMetadata connection) } } + private static void safeAbort(FunctionNamespaceTransactionMetadata transactionMetadata) + { + try { + transactionMetadata.abort(); + } + catch (Exception e) { + log.error(e, "Function namespace transaction threw exception on abort"); + } + } + public TransactionInfo getTransactionInfo() { Duration idleTime = Optional.ofNullable(idleStartTime.get()) @@ -609,5 +679,42 @@ public void abort() } } } + + private static class FunctionNamespaceTransactionMetadata + { + private final FunctionNamespaceManager functionNamespaceManager; + private final FunctionNamespaceTransactionHandle transactionHandle; + private final AtomicBoolean finished = new AtomicBoolean(); + + public FunctionNamespaceTransactionMetadata(FunctionNamespaceManager functionNamespaceManager, FunctionNamespaceTransactionHandle transactionHandle) + { + this.functionNamespaceManager = requireNonNull(functionNamespaceManager, "functionNamespaceManager is null"); + this.transactionHandle = requireNonNull(transactionHandle, "transactionHandle is null"); + } + + public FunctionNamespaceManager getFunctionNamespaceManager() + { + return functionNamespaceManager; + } + + public FunctionNamespaceTransactionHandle getTransactionHandle() + { + return transactionHandle; + } + + public void commit() + { + if (finished.compareAndSet(false, true)) { + functionNamespaceManager.commit(transactionHandle); + } + } + + public void abort() + { + if (finished.compareAndSet(false, true)) { + functionNamespaceManager.abort(transactionHandle); + } + } + } } } diff --git a/presto-main/src/main/java/com/facebook/presto/transaction/NoOpTransactionManager.java b/presto-main/src/main/java/com/facebook/presto/transaction/NoOpTransactionManager.java index 89700104fe238..d989f57d9f4b7 100644 --- a/presto-main/src/main/java/com/facebook/presto/transaction/NoOpTransactionManager.java +++ b/presto-main/src/main/java/com/facebook/presto/transaction/NoOpTransactionManager.java @@ -16,6 +16,8 @@ import com.facebook.presto.metadata.CatalogMetadata; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.function.FunctionNamespaceManager; +import com.facebook.presto.spi.function.FunctionNamespaceTransactionHandle; import com.facebook.presto.spi.transaction.IsolationLevel; import com.google.common.util.concurrent.ListenableFuture; @@ -30,13 +32,13 @@ public class NoOpTransactionManager implements TransactionManager { @Override - public boolean transactionExists(TransactionId transactionId) + public TransactionInfo getTransactionInfo(TransactionId transactionId) { throw new UnsupportedOperationException(); } @Override - public TransactionInfo getTransactionInfo(TransactionId transactionId) + public Optional getOptionalTransactionInfo(TransactionId transactionId) { throw new UnsupportedOperationException(); } @@ -95,6 +97,17 @@ public ConnectorTransactionHandle getConnectorTransaction(TransactionId transact throw new UnsupportedOperationException(); } + @Override + public void registerFunctionNamespaceManager(String functionNamespaceManagerName, FunctionNamespaceManager functionNamespaceManager) + { + } + + @Override + public FunctionNamespaceTransactionHandle getFunctionNamespaceTransaction(TransactionId transactionId, String functionNamespaceManagerName) + { + throw new UnsupportedOperationException(); + } + @Override public void checkAndSetActive(TransactionId transactionId) { diff --git a/presto-main/src/main/java/com/facebook/presto/transaction/TransactionBuilder.java b/presto-main/src/main/java/com/facebook/presto/transaction/TransactionBuilder.java index 9a86569f38f60..d6d20e484a35c 100644 --- a/presto-main/src/main/java/com/facebook/presto/transaction/TransactionBuilder.java +++ b/presto-main/src/main/java/com/facebook/presto/transaction/TransactionBuilder.java @@ -20,8 +20,8 @@ import java.util.function.Consumer; import java.util.function.Function; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static java.util.Objects.requireNonNull; public class TransactionBuilder @@ -152,7 +152,7 @@ public T execute(Session session, Function callback) return result; } finally { - if (managedTransaction && transactionManager.transactionExists(transactionSession.getTransactionId().get())) { + if (managedTransaction && transactionSession.getTransactionId().flatMap(transactionManager::getOptionalTransactionInfo).isPresent()) { if (success) { getFutureValue(transactionManager.asyncCommit(transactionSession.getTransactionId().get())); } diff --git a/presto-main/src/main/java/com/facebook/presto/transaction/TransactionManager.java b/presto-main/src/main/java/com/facebook/presto/transaction/TransactionManager.java index bc99b944d6789..43e143160f490 100644 --- a/presto-main/src/main/java/com/facebook/presto/transaction/TransactionManager.java +++ b/presto-main/src/main/java/com/facebook/presto/transaction/TransactionManager.java @@ -18,6 +18,8 @@ import com.facebook.presto.security.AccessControl; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.function.FunctionNamespaceManager; +import com.facebook.presto.spi.function.FunctionNamespaceTransactionHandle; import com.facebook.presto.spi.transaction.IsolationLevel; import com.google.common.util.concurrent.ListenableFuture; @@ -30,15 +32,10 @@ public interface TransactionManager IsolationLevel DEFAULT_ISOLATION = IsolationLevel.READ_UNCOMMITTED; boolean DEFAULT_READ_ONLY = false; - boolean transactionExists(TransactionId transactionId); - - default boolean isAutoCommit(TransactionId transactionId) - { - return getTransactionInfo(transactionId).isAutoCommitContext(); - } - TransactionInfo getTransactionInfo(TransactionId transactionId); + Optional getOptionalTransactionInfo(TransactionId transactionId); + List getAllTransactionInfos(); TransactionId beginTransaction(boolean autoCommitContext); @@ -57,6 +54,10 @@ default boolean isAutoCommit(TransactionId transactionId) ConnectorTransactionHandle getConnectorTransaction(TransactionId transactionId, ConnectorId connectorId); + void registerFunctionNamespaceManager(String functionNamespaceManagerName, FunctionNamespaceManager functionNamespaceManager); + + FunctionNamespaceTransactionHandle getFunctionNamespaceTransaction(TransactionId transactionId, String functionNamespaceManagerName); + void checkAndSetActive(TransactionId transactionId); void trySetActive(TransactionId transactionId); diff --git a/presto-main/src/main/java/com/facebook/presto/transaction/TransactionManagerConfig.java b/presto-main/src/main/java/com/facebook/presto/transaction/TransactionManagerConfig.java index 1c96833434d05..2b38c16ea4d1b 100644 --- a/presto-main/src/main/java/com/facebook/presto/transaction/TransactionManagerConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/transaction/TransactionManagerConfig.java @@ -13,21 +13,29 @@ */ package com.facebook.presto.transaction; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableMap; import io.airlift.units.Duration; import io.airlift.units.MinDuration; import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; +import java.util.Map; import java.util.concurrent.TimeUnit; +import static com.google.common.base.Strings.nullToEmpty; + public class TransactionManagerConfig { + private static final Splitter.MapSplitter MAP_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings().withKeyValueSeparator('='); + private Duration idleCheckInterval = new Duration(1, TimeUnit.MINUTES); private Duration idleTimeout = new Duration(5, TimeUnit.MINUTES); private int maxFinishingConcurrency = 1; + private Map companionCatalogs = ImmutableMap.of(); @MinDuration("1ms") @NotNull @@ -72,4 +80,18 @@ public TransactionManagerConfig setMaxFinishingConcurrency(int maxFinishingConcu this.maxFinishingConcurrency = maxFinishingConcurrency; return this; } + + @NotNull + public Map getCompanionCatalogs() + { + return this.companionCatalogs; + } + + @Config("transaction.companion-catalogs") + @ConfigDescription("Companion catalogs: catalog_name1=catalog_name2,catalog_name3=catalog_name4,...") + public TransactionManagerConfig setCompanionCatalogs(String extraAccessibleCatalogs) + { + this.companionCatalogs = MAP_SPLITTER.split(nullToEmpty(extraAccessibleCatalogs)); + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/type/DecimalInequalityOperators.java b/presto-main/src/main/java/com/facebook/presto/type/DecimalInequalityOperators.java index e2a1eb37bdffa..2fe144687c410 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/DecimalInequalityOperators.java +++ b/presto-main/src/main/java/com/facebook/presto/type/DecimalInequalityOperators.java @@ -30,9 +30,9 @@ import java.util.Optional; import static com.facebook.presto.metadata.PolymorphicScalarFunctionBuilder.constant; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.OperatorType.BETWEEN; diff --git a/presto-main/src/main/java/com/facebook/presto/type/HyperLogLogOperators.java b/presto-main/src/main/java/com/facebook/presto/type/HyperLogLogOperators.java index fa85108d26d91..7040a122880e1 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/HyperLogLogOperators.java +++ b/presto-main/src/main/java/com/facebook/presto/type/HyperLogLogOperators.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.type; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.spi.function.ScalarOperator; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; import io.airlift.slice.Slice; -import io.airlift.stats.cardinality.HyperLogLog; import static com.facebook.presto.spi.function.OperatorType.CAST; diff --git a/presto-main/src/main/java/com/facebook/presto/type/IpPrefixOperators.java b/presto-main/src/main/java/com/facebook/presto/type/IpPrefixOperators.java new file mode 100644 index 0000000000000..50066d111353c --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/type/IpPrefixOperators.java @@ -0,0 +1,273 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.type; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.function.BlockIndex; +import com.facebook.presto.spi.function.BlockPosition; +import com.facebook.presto.spi.function.IsNull; +import com.facebook.presto.spi.function.LiteralParameters; +import com.facebook.presto.spi.function.ScalarOperator; +import com.facebook.presto.spi.function.SqlNullable; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.type.StandardTypes; +import com.google.common.net.InetAddresses; +import io.airlift.slice.Slice; +import io.airlift.slice.XxHash64; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; +import static com.facebook.presto.spi.function.OperatorType.BETWEEN; +import static com.facebook.presto.spi.function.OperatorType.CAST; +import static com.facebook.presto.spi.function.OperatorType.EQUAL; +import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; +import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN_OR_EQUAL; +import static com.facebook.presto.spi.function.OperatorType.HASH_CODE; +import static com.facebook.presto.spi.function.OperatorType.INDETERMINATE; +import static com.facebook.presto.spi.function.OperatorType.IS_DISTINCT_FROM; +import static com.facebook.presto.spi.function.OperatorType.LESS_THAN; +import static com.facebook.presto.spi.function.OperatorType.LESS_THAN_OR_EQUAL; +import static com.facebook.presto.spi.function.OperatorType.NOT_EQUAL; +import static com.facebook.presto.spi.function.OperatorType.XX_HASH_64; +import static com.facebook.presto.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.type.IpPrefixType.IPPREFIX; +import static io.airlift.slice.Slices.utf8Slice; +import static io.airlift.slice.Slices.wrappedBuffer; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.lang.System.arraycopy; + +public final class IpPrefixOperators +{ + private IpPrefixOperators() + { + } + + @ScalarOperator(EQUAL) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean equal(@SqlType(StandardTypes.IPPREFIX) Slice left, @SqlType(StandardTypes.IPPREFIX) Slice right) + { + return left.equals(right); + } + + @ScalarOperator(NOT_EQUAL) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean notEqual(@SqlType(StandardTypes.IPPREFIX) Slice left, @SqlType(StandardTypes.IPPREFIX) Slice right) + { + return !left.equals(right); + } + + @ScalarOperator(LESS_THAN) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean lessThan(@SqlType(StandardTypes.IPPREFIX) Slice left, @SqlType(StandardTypes.IPPREFIX) Slice right) + { + return left.compareTo(right) < 0; + } + + @ScalarOperator(LESS_THAN_OR_EQUAL) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean lessThanOrEqual(@SqlType(StandardTypes.IPPREFIX) Slice left, @SqlType(StandardTypes.IPPREFIX) Slice right) + { + return left.compareTo(right) <= 0; + } + + @ScalarOperator(GREATER_THAN) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean greaterThan(@SqlType(StandardTypes.IPPREFIX) Slice left, @SqlType(StandardTypes.IPPREFIX) Slice right) + { + return left.compareTo(right) > 0; + } + + @ScalarOperator(GREATER_THAN_OR_EQUAL) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean greaterThanOrEqual(@SqlType(StandardTypes.IPPREFIX) Slice left, @SqlType(StandardTypes.IPPREFIX) Slice right) + { + return left.compareTo(right) >= 0; + } + + @ScalarOperator(BETWEEN) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean between(@SqlType(StandardTypes.IPPREFIX) Slice value, @SqlType(StandardTypes.IPPREFIX) Slice min, @SqlType(StandardTypes.IPPREFIX) Slice max) + { + return min.compareTo(value) <= 0 && value.compareTo(max) <= 0; + } + + @ScalarOperator(HASH_CODE) + @SqlType(StandardTypes.BIGINT) + public static long hashCode(@SqlType(StandardTypes.IPPREFIX) Slice value) + { + return XxHash64.hash(value); + } + + @ScalarOperator(XX_HASH_64) + @SqlType(StandardTypes.BIGINT) + public static long xxHash64(@SqlType(StandardTypes.IPPREFIX) Slice value) + { + return XxHash64.hash(value); + } + + @LiteralParameters("x") + @ScalarOperator(CAST) + @SqlType(StandardTypes.IPPREFIX) + public static Slice castFromVarcharToIpPrefix(@SqlType("varchar(x)") Slice slice) + { + byte[] address; + int subnetSize; + String string = slice.toStringUtf8(); + if (!string.contains("/")) { + throw new PrestoException(INVALID_CAST_ARGUMENT, "Cannot cast value to IPPREFIX: " + slice.toStringUtf8()); + } + String[] parts = string.split("/"); + try { + address = InetAddresses.forString(parts[0]).getAddress(); + } + catch (IllegalArgumentException e) { + throw new PrestoException(INVALID_CAST_ARGUMENT, "Cannot cast value to IPPREFIX: " + slice.toStringUtf8()); + } + subnetSize = Integer.parseInt(parts[1]); + + byte[] bytes = new byte[IPPREFIX.getFixedSize()]; + if (address.length == 4) { + if (subnetSize < 0 || 32 < subnetSize) { + throw new PrestoException(INVALID_CAST_ARGUMENT, "Cannot cast value to IPPREFIX: " + slice.toStringUtf8()); + } + + for (int i = 0; i < 4; i++) { + address[3 - i] &= -0x1 << min(max((32 - subnetSize) - 8 * i, 0), 8); + } + bytes[10] = (byte) 0xff; + bytes[11] = (byte) 0xff; + arraycopy(address, 0, bytes, 12, 4); + bytes[IPPREFIX.getFixedSize() - 1] = (byte) subnetSize; + } + else if (address.length == 16) { + if (subnetSize < 0 || 128 < subnetSize) { + throw new PrestoException(INVALID_CAST_ARGUMENT, "Cannot cast value to IPPREFIX: " + slice.toStringUtf8()); + } + + for (int i = 0; i < 16; i++) { + address[15 - i] &= (byte) -0x1 << min(max((128 - subnetSize) - 8 * i, 0), 8); + } + arraycopy(address, 0, bytes, 0, 16); + bytes[IPPREFIX.getFixedSize() - 1] = (byte) subnetSize; + } + else { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Invalid InetAddress length: " + address.length); + } + + return wrappedBuffer(bytes); + } + + @ScalarOperator(CAST) + @SqlType(StandardTypes.VARCHAR) + public static Slice castFromIpPrefixToVarchar(@SqlType(StandardTypes.IPPREFIX) Slice slice) + { + try { + String addrString = InetAddresses.toAddrString(InetAddress.getByAddress(slice.getBytes(0, 2 * Long.BYTES))); + String prefixString = Integer.toString(slice.getByte(2 * Long.BYTES) & 0xff); + return utf8Slice(addrString + "/" + prefixString); + } + catch (UnknownHostException e) { + throw new PrestoException(INVALID_CAST_ARGUMENT, "Invalid IP address binary length: " + slice.length(), e); + } + } + + @ScalarOperator(CAST) + @SqlType(StandardTypes.IPADDRESS) + public static Slice castFromIpPrefixToIpAddress(@SqlType(StandardTypes.IPPREFIX) Slice slice) + { + return wrappedBuffer(slice.getBytes(0, IPADDRESS.getFixedSize())); + } + + @ScalarOperator(CAST) + @SqlType(StandardTypes.IPPREFIX) + public static Slice castFromIpAddressToIpPrefix(@SqlType(StandardTypes.IPADDRESS) Slice slice) + { + byte[] address; + try { + address = InetAddress.getByAddress(slice.getBytes()).getAddress(); + } + catch (UnknownHostException e) { + throw new PrestoException(INVALID_CAST_ARGUMENT, "Invalid IP address binary: " + slice.toStringUtf8(), e); + } + + byte[] bytes = new byte[IPPREFIX.getFixedSize()]; + arraycopy(slice.getBytes(), 0, bytes, 0, IPPREFIX.getFixedSize() - 1); + if (address.length == 4) { + bytes[IPPREFIX.getFixedSize() - 1] = (byte) 32; + } + else if (address.length == 16) { + bytes[IPPREFIX.getFixedSize() - 1] = (byte) 128; + } + else { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Invalid InetAddress length: " + address.length); + } + + return wrappedBuffer(bytes); + } + + @ScalarOperator(IS_DISTINCT_FROM) + public static class IpPrefixDistinctFromOperator + { + @SqlType(StandardTypes.BOOLEAN) + public static boolean isDistinctFrom( + @SqlType(StandardTypes.IPPREFIX) Slice left, + @IsNull boolean leftNull, + @SqlType(StandardTypes.IPPREFIX) Slice right, + @IsNull boolean rightNull) + { + if (leftNull != rightNull) { + return true; + } + if (leftNull) { + return false; + } + return notEqual(left, right); + } + + @SqlType(StandardTypes.BOOLEAN) + public static boolean isDistinctFrom( + @BlockPosition @SqlType(value = StandardTypes.IPPREFIX, nativeContainerType = Slice.class) Block left, + @BlockIndex int leftPosition, + @BlockPosition @SqlType(value = StandardTypes.IPPREFIX, nativeContainerType = Slice.class) Block right, + @BlockIndex int rightPosition) + { + if (left.isNull(leftPosition) != right.isNull(rightPosition)) { + return true; + } + if (left.isNull(leftPosition)) { + return false; + } + return left.compareTo(leftPosition, 0, IPPREFIX.getFixedSize(), right, rightPosition, 0, IPPREFIX.getFixedSize()) != 0; + } + } + + @ScalarOperator(INDETERMINATE) + @SqlType(StandardTypes.BOOLEAN) + public static boolean indeterminate(@SqlType(StandardTypes.IPPREFIX) Slice value, @IsNull boolean isNull) + { + return isNull; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/type/IpPrefixType.java b/presto-main/src/main/java/com/facebook/presto/type/IpPrefixType.java new file mode 100644 index 0000000000000..b5982abcdce56 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/type/IpPrefixType.java @@ -0,0 +1,158 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.type; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.block.BlockBuilderStatus; +import com.facebook.presto.spi.block.PageBuilderStatus; +import com.facebook.presto.spi.block.VariableWidthBlockBuilder; +import com.facebook.presto.spi.type.AbstractType; +import com.facebook.presto.spi.type.FixedWidthType; +import com.facebook.presto.spi.type.StandardTypes; +import com.google.common.net.InetAddresses; +import io.airlift.slice.Slice; +import io.airlift.slice.XxHash64; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; + +public class IpPrefixType + extends AbstractType + implements FixedWidthType +{ + public static final IpPrefixType IPPREFIX = new IpPrefixType(); + + private IpPrefixType() + { + super(parseTypeSignature(StandardTypes.IPPREFIX), Slice.class); + } + + @Override + public int getFixedSize() + { + return Long.BYTES * 2 + 1; + } + + @Override + public BlockBuilder createBlockBuilder(BlockBuilderStatus blockBuilderStatus, int expectedEntries, int expectedBytesPerEntry) + { + int maxBlockSizeInBytes; + if (blockBuilderStatus == null) { + maxBlockSizeInBytes = PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES; + } + else { + maxBlockSizeInBytes = blockBuilderStatus.getMaxPageSizeInBytes(); + } + return new VariableWidthBlockBuilder( + blockBuilderStatus, + Math.min(expectedEntries, maxBlockSizeInBytes / getFixedSize()), + Math.min(expectedEntries * getFixedSize(), maxBlockSizeInBytes)); + } + + @Override + public BlockBuilder createBlockBuilder(BlockBuilderStatus blockBuilderStatus, int expectedEntries) + { + return createBlockBuilder(blockBuilderStatus, expectedEntries, getFixedSize()); + } + + @Override + public BlockBuilder createFixedSizeBlockBuilder(int positionCount) + { + return createBlockBuilder(null, positionCount); + } + + @Override + public boolean isComparable() + { + return true; + } + + @Override + public boolean isOrderable() + { + return true; + } + + @Override + public boolean equalTo(Block leftBlock, int leftPosition, Block rightBlock, int rightPosition) + { + return leftBlock.equals(leftPosition, 0, rightBlock, rightPosition, 0, getFixedSize()); + } + + @Override + public int compareTo(Block leftBlock, int leftPosition, Block rightBlock, int rightPosition) + { + return leftBlock.compareTo(leftPosition, 0, getFixedSize(), rightBlock, rightPosition, 0, getFixedSize()); + } + + @Override + public long hash(Block block, int position) + { + return XxHash64.hash(getSlice(block, position)); + } + + @Override + public Object getObjectValue(ConnectorSession session, Block block, int position) + { + if (block.isNull(position)) { + return null; + } + try { + String addrString = InetAddresses.toAddrString(InetAddress.getByAddress(getSlice(block, position).getBytes(0, 2 * Long.BYTES))); + String prefixString = Integer.toString(getSlice(block, position).getByte(2 * Long.BYTES) & 0xff); + return addrString + "/" + prefixString; + } + catch (UnknownHostException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public void appendTo(Block block, int position, BlockBuilder blockBuilder) + { + if (block.isNull(position)) { + blockBuilder.appendNull(); + } + else { + blockBuilder.writeBytes(block.getSlice(position, 0, getFixedSize()), 0, getFixedSize()); + blockBuilder.closeEntry(); + } + } + + @Override + public void writeSlice(BlockBuilder blockBuilder, Slice value) + { + writeSlice(blockBuilder, value, 0, value.length()); + } + + @Override + public void writeSlice(BlockBuilder blockBuilder, Slice value, int offset, int length) + { + if (length != getFixedSize()) { + throw new IllegalStateException("Expected entry size to be exactly " + getFixedSize() + " but was " + length); + } + blockBuilder.writeBytes(value, 0, length); + blockBuilder.closeEntry(); + } + + @Override + public final Slice getSlice(Block block, int position) + { + return block.getSlice(position, 0, getFixedSize()); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/type/Re2JRegexp.java b/presto-main/src/main/java/com/facebook/presto/type/Re2JRegexp.java index 270e2265ae150..7c83185143d5b 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/Re2JRegexp.java +++ b/presto-main/src/main/java/com/facebook/presto/type/Re2JRegexp.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.type; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.google.re2j.Matcher; import com.google.re2j.Options; import com.google.re2j.Pattern; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; diff --git a/presto-main/src/main/java/com/facebook/presto/type/TypeRegistry.java b/presto-main/src/main/java/com/facebook/presto/type/TypeRegistry.java index a779ca8e0364c..9238c1f5da6d4 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/TypeRegistry.java +++ b/presto-main/src/main/java/com/facebook/presto/type/TypeRegistry.java @@ -80,6 +80,7 @@ import static com.facebook.presto.type.IntervalDayTimeType.INTERVAL_DAY_TIME; import static com.facebook.presto.type.IntervalYearMonthType.INTERVAL_YEAR_MONTH; import static com.facebook.presto.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.type.JoniRegexpType.JONI_REGEXP; import static com.facebook.presto.type.JsonPathType.JSON_PATH; import static com.facebook.presto.type.JsonType.JSON; @@ -148,6 +149,7 @@ public TypeRegistry(Set types, FeaturesConfig featuresConfig) addType(JSON); addType(CODE_POINTS); addType(IPADDRESS); + addType(IPPREFIX); addParametricType(VarcharParametricType.VARCHAR); addParametricType(CharParametricType.CHAR); addParametricType(DecimalParametricType.DECIMAL); @@ -663,7 +665,7 @@ public static boolean isCovariantTypeBase(String typeBase) public MethodHandle resolveOperator(OperatorType operatorType, List argumentTypes) { requireNonNull(functionManager, "functionManager is null"); - return functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(operatorType, fromTypes(argumentTypes))).getMethodHandle(); + return functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(operatorType, fromTypes(argumentTypes))).getMethodHandle(); } public static class TypeCompatibility diff --git a/presto-main/src/main/java/com/facebook/presto/type/setdigest/SetDigest.java b/presto-main/src/main/java/com/facebook/presto/type/setdigest/SetDigest.java index dc142a6d44e98..666f6b4a37589 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/setdigest/SetDigest.java +++ b/presto-main/src/main/java/com/facebook/presto/type/setdigest/SetDigest.java @@ -14,6 +14,7 @@ package com.facebook.presto.type.setdigest; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import com.google.common.primitives.Shorts; @@ -23,7 +24,6 @@ import io.airlift.slice.SliceInput; import io.airlift.slice.SliceOutput; import io.airlift.slice.Slices; -import io.airlift.stats.cardinality.HyperLogLog; import it.unimi.dsi.fastutil.longs.Long2ShortRBTreeMap; import it.unimi.dsi.fastutil.longs.Long2ShortSortedMap; import it.unimi.dsi.fastutil.longs.LongBidirectionalIterator; diff --git a/presto-main/src/main/java/com/facebook/presto/type/setdigest/SetDigestFunctions.java b/presto-main/src/main/java/com/facebook/presto/type/setdigest/SetDigestFunctions.java index 4df229f060298..5d977c346c056 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/setdigest/SetDigestFunctions.java +++ b/presto-main/src/main/java/com/facebook/presto/type/setdigest/SetDigestFunctions.java @@ -14,6 +14,7 @@ package com.facebook.presto.type.setdigest; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.ScalarFunction; @@ -23,7 +24,6 @@ import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import io.airlift.json.ObjectMapperProvider; import io.airlift.slice.Slice; import io.airlift.slice.Slices; diff --git a/presto-main/src/main/java/com/facebook/presto/util/CompilerUtils.java b/presto-main/src/main/java/com/facebook/presto/util/CompilerUtils.java index 19ae9d8607263..1098b5f8e955f 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/CompilerUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/util/CompilerUtils.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.util; +import com.facebook.airlift.log.Logger; import com.facebook.presto.bytecode.ClassDefinition; import com.facebook.presto.bytecode.DynamicClassLoader; import com.facebook.presto.bytecode.ParameterizedType; -import io.airlift.log.Logger; import java.lang.invoke.MethodHandle; import java.time.Instant; diff --git a/presto-main/src/main/java/com/facebook/presto/util/FastutilSetHelper.java b/presto-main/src/main/java/com/facebook/presto/util/FastutilSetHelper.java index d1e1364545f93..bd0b91b99d140 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/FastutilSetHelper.java +++ b/presto-main/src/main/java/com/facebook/presto/util/FastutilSetHelper.java @@ -94,8 +94,8 @@ private static final class LongStrategy private LongStrategy(FunctionManager functionManager, Type type) { - hashCodeHandle = functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(HASH_CODE, fromTypes(type))).getMethodHandle(); - equalsHandle = functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(EQUAL, fromTypes(type, type))).getMethodHandle(); + hashCodeHandle = functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(HASH_CODE, fromTypes(type))).getMethodHandle(); + equalsHandle = functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(EQUAL, fromTypes(type, type))).getMethodHandle(); } @Override @@ -136,8 +136,8 @@ private static final class DoubleStrategy private DoubleStrategy(FunctionManager functionManager, Type type) { - hashCodeHandle = functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(HASH_CODE, fromTypes(type))).getMethodHandle(); - equalsHandle = functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(EQUAL, fromTypes(type, type))).getMethodHandle(); + hashCodeHandle = functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(HASH_CODE, fromTypes(type))).getMethodHandle(); + equalsHandle = functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(EQUAL, fromTypes(type, type))).getMethodHandle(); } @Override @@ -178,10 +178,10 @@ private static final class ObjectStrategy private ObjectStrategy(FunctionManager functionManager, Type type) { - hashCodeHandle = functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(HASH_CODE, fromTypes(type))) + hashCodeHandle = functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(HASH_CODE, fromTypes(type))) .getMethodHandle() .asType(MethodType.methodType(long.class, Object.class)); - equalsHandle = functionManager.getScalarFunctionImplementation(functionManager.resolveOperator(EQUAL, fromTypes(type, type))) + equalsHandle = functionManager.getBuiltInScalarFunctionImplementation(functionManager.resolveOperator(EQUAL, fromTypes(type, type))) .getMethodHandle() .asType(MethodType.methodType(Boolean.class, Object.class, Object.class)); } diff --git a/presto-main/src/main/java/com/facebook/presto/util/FinalizerService.java b/presto-main/src/main/java/com/facebook/presto/util/FinalizerService.java index e23a587e017d2..bfa8fd3b52a9f 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/FinalizerService.java +++ b/presto-main/src/main/java/com/facebook/presto/util/FinalizerService.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.util; +import com.facebook.airlift.log.Logger; import com.google.common.collect.Sets; -import io.airlift.log.Logger; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -28,7 +28,7 @@ import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicBoolean; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newSingleThreadExecutor; diff --git a/presto-main/src/main/java/com/facebook/presto/util/GcStatusMonitor.java b/presto-main/src/main/java/com/facebook/presto/util/GcStatusMonitor.java new file mode 100644 index 0000000000000..1def2225937cf --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/util/GcStatusMonitor.java @@ -0,0 +1,237 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.util; + +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.GarbageCollectionNotificationInfo; +import com.facebook.presto.execution.SqlTask; +import com.facebook.presto.execution.SqlTaskIoStats; +import com.facebook.presto.execution.SqlTaskManager; +import com.facebook.presto.execution.TaskInfo; +import com.facebook.presto.execution.TaskStatus; +import com.facebook.presto.operator.TaskStats; +import com.facebook.presto.spi.QueryId; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ListMultimap; +import io.airlift.units.DataSize; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.inject.Inject; +import javax.management.JMException; +import javax.management.Notification; +import javax.management.NotificationListener; +import javax.management.ObjectName; +import javax.management.openmbean.CompositeData; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.util.Comparator; +import java.util.List; +import java.util.function.Function; + +import static com.facebook.presto.util.StringTableUtils.getTableStrings; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableListMultimap.toImmutableListMultimap; +import static java.util.AbstractMap.SimpleEntry; +import static java.util.Map.Entry; +import static java.util.Objects.requireNonNull; + +public class GcStatusMonitor +{ + private static final Logger log = Logger.get(GcStatusMonitor.class); + private static final String GC_NOTIFICATION_TYPE = "com.sun.management.gc.notification"; + private final NotificationListener notificationListener = (notification, ignored) -> onNotification(notification); + private final SqlTaskManager sqlTaskManager; + + @Inject + public GcStatusMonitor(SqlTaskManager sqlTaskManager) + { + this.sqlTaskManager = requireNonNull(sqlTaskManager, "sqlTaskManager must not be null"); + } + + @PostConstruct + public void start() + { + for (GarbageCollectorMXBean mbean : ManagementFactory.getGarbageCollectorMXBeans()) { + if (mbean.getName().equals("TestingMBeanServer")) { + continue; + } + + ObjectName objectName = mbean.getObjectName(); + try { + ManagementFactory.getPlatformMBeanServer().addNotificationListener( + objectName, + notificationListener, + null, + null); + } + catch (JMException e) { + throw new RuntimeException("Unable to add GC listener", e); + } + } + } + + @PreDestroy + public void stop() + { + for (GarbageCollectorMXBean mbean : ManagementFactory.getGarbageCollectorMXBeans()) { + ObjectName objectName = mbean.getObjectName(); + try { + ManagementFactory.getPlatformMBeanServer().removeNotificationListener(objectName, notificationListener); + } + catch (JMException ignored) { + } + } + } + + private void onNotification(Notification notification) + { + if (GC_NOTIFICATION_TYPE.equals(notification.getType())) { + GarbageCollectionNotificationInfo info = new GarbageCollectionNotificationInfo((CompositeData) notification.getUserData()); + if (info.isMajorGc()) { + onMajorGc(); + } + } + } + + private void onMajorGc() + { + try { + logActiveTasks(); + } + catch (Throwable throwable) { + log.error(throwable); + } + } + + private void logActiveTasks() + { + // We only care about active tasks + List activeSqlTasks = getActiveSqlTasks(); + ListMultimap activeQueriesToTasksMap = activeSqlTasks.stream() + .collect(toImmutableListMultimap(task -> task.getQueryContext().getQueryId(), Function.identity())); + logQueriesAndTasks(activeQueriesToTasksMap); + } + + private static void logQueriesAndTasks(ListMultimap queryIDToSqlTaskMap) + { + Comparator> comparator = Comparator.comparingLong(Entry::getValue); + //Print summary of queries sorted from most memory hungry to least memory hungry + List queryIdsSortedByMemoryUsage = queryIDToSqlTaskMap.asMap().entrySet().stream() + //Convert to Entry, QueryId -> Total Memory Reservation + .map(entry -> + new SimpleEntry<>(entry.getKey(), entry.getValue().stream() + .map(SqlTask::getTaskInfo) + .map(TaskInfo::getStats) + .mapToLong(stats -> stats.getUserMemoryReservation().toBytes() + stats.getSystemMemoryReservation().toBytes()) + .sum()) + ).sorted(comparator.reversed()) + .map(Entry::getKey) + .collect(toImmutableList()); + logQueriesAndTasks(queryIdsSortedByMemoryUsage, queryIDToSqlTaskMap); + logTaskStats(queryIdsSortedByMemoryUsage, queryIDToSqlTaskMap); + } + + private List getActiveSqlTasks() + { + return sqlTaskManager.getAllTasks().stream() + .filter(task -> !task.getTaskInfo().getTaskStatus().getState().isDone()) + .collect(toImmutableList()); + } + + private static void logQueriesAndTasks(List queryIds, ListMultimap tasksByQueryId) + { + List> rows = queryIds.stream().map(queryId -> { + List sqlTasks = tasksByQueryId.get(queryId); + long userMemoryReservation = sqlTasks.stream() + .map(SqlTask::getTaskInfo) + .map(TaskInfo::getStats) + .map(TaskStats::getUserMemoryReservation) + .mapToLong(DataSize::toBytes) + .sum(); + long systemMemoryReservation = sqlTasks.stream() + .map(SqlTask::getTaskInfo) + .map(TaskInfo::getStats) + .map(TaskStats::getSystemMemoryReservation) + .mapToLong(DataSize::toBytes) + .sum(); + return ImmutableList.of( + queryId.toString(), + Long.toString(userMemoryReservation + systemMemoryReservation), + Long.toString(userMemoryReservation), + Long.toString(systemMemoryReservation)); + }).collect(toImmutableList()); + logInfoTable(ImmutableList.>builder().add( + ImmutableList.of( + "Query ID", + "Total Memory Reservation", + "User Memory Reservation", + "System Memory Reservation")) + .addAll(rows) + .build()); + } + + private static void logTaskStats(List queryIds, ListMultimap tasksByQueryId) + { + List> rows = queryIds.stream().flatMap(queryId -> { + List sqlTasks = tasksByQueryId.get(queryId); + Comparator comparator = (first, second) -> { + TaskStats aTaskStats = first.getTaskInfo().getStats(); + TaskStats bTaskStats = second.getTaskInfo().getStats(); + return Long.compare(aTaskStats.getUserMemoryReservation().toBytes() + aTaskStats.getSystemMemoryReservation().toBytes(), + bTaskStats.getUserMemoryReservation().toBytes() + bTaskStats.getSystemMemoryReservation().toBytes()); + }; + return sqlTasks.stream() + .sorted(comparator.reversed()) + .map(task -> { + TaskInfo taskInfo = task.getTaskInfo(); + SqlTaskIoStats taskIOStats = task.getIoStats(); + TaskStatus taskStatus = taskInfo.getTaskStatus(); + TaskStats taskStats = taskInfo.getStats(); + return ImmutableList.of( + task.getQueryContext().getQueryId().toString(), + task.getTaskId().toString(), + taskStatus.getState().toString(), + taskStats.getCreateTime().toString(), + taskStats.getUserMemoryReservation().toString(), + taskStats.getSystemMemoryReservation().toString(), + Long.toString(taskIOStats.getInputDataSize().getTotalCount()), + Long.toString(taskIOStats.getOutputDataSize().getTotalCount()), + Long.toString(taskIOStats.getInputPositions().getTotalCount()), + Long.toString(taskIOStats.getOutputPositions().getTotalCount())); + }); + }).collect(toImmutableList()); + + logInfoTable(ImmutableList.>builder() + .add(ImmutableList.of( + "Query ID", + "Task ID", + "State", + "Created Ts", + "User Memory", + "System Memory", + "Input Bytes", + "Output Bytes", + "Input Row Count", + "Output Row Count")) + .addAll(rows) + .build()); + } + + private static void logInfoTable(List> table) + { + getTableStrings(table).stream().forEach(log::info); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java b/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java index adb31a4671ce5..1912508a393b9 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java @@ -15,17 +15,22 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.SubPlan; import com.facebook.presto.sql.planner.optimizations.JoinNodeUtils; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; @@ -37,11 +42,9 @@ import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.OutputNode; import com.facebook.presto.sql.planner.plan.PlanFragmentId; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; @@ -50,10 +53,9 @@ import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; +import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.planner.planPrinter.RowExpressionFormatter; @@ -71,12 +73,14 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Maps.immutableEnumMap; import static java.lang.String.format; +import static java.util.Objects.requireNonNull; public final class GraphvizPrinter { @@ -99,6 +103,7 @@ private enum NodeType SAMPLE, MARK_DISTINCT, TABLE_WRITER, + TABLE_WRITER_MERGE, TABLE_FINISH, INDEX_SOURCE, UNNEST, @@ -122,6 +127,7 @@ private enum NodeType .put(NodeType.UNION, "turquoise4") .put(NodeType.MARK_DISTINCT, "violet") .put(NodeType.TABLE_WRITER, "cyan") + .put(NodeType.TABLE_WRITER_MERGE, "cyan4") .put(NodeType.TABLE_FINISH, "hotpink") .put(NodeType.INDEX_SOURCE, "dodgerblue3") .put(NodeType.UNNEST, "crimson") @@ -213,13 +219,15 @@ private static class NodePrinter private static final int MAX_NAME_WIDTH = 100; private final StringBuilder output; private final PlanNodeIdGenerator idGenerator; - private final RowExpressionFormatter formatter; + private final Function formatter; public NodePrinter(StringBuilder output, PlanNodeIdGenerator idGenerator, Session session, FunctionManager functionManager) { this.output = output; this.idGenerator = idGenerator; - this.formatter = new RowExpressionFormatter(session.toConnectorSession(), functionManager); + RowExpressionFormatter rowExpressionFormatter = new RowExpressionFormatter(functionManager); + ConnectorSession connectorSession = requireNonNull(session, "session is null").toConnectorSession(); + this.formatter = rowExpression -> rowExpressionFormatter.formatRowExpression(connectorSession, rowExpression); } @Override @@ -239,6 +247,13 @@ public Void visitTableWriter(TableWriterNode node, Void context) return node.getSource().accept(this, context); } + @Override + public Void visitTableWriteMerge(TableWriterMergeNode node, Void context) + { + printNode(node, "TableWriterMerge", NODE_COLORS.get(NodeType.TABLE_WRITER_MERGE)); + return node.getSource().accept(this, context); + } + @Override public Void visitStatisticsWriterNode(StatisticsWriterNode node, Void context) { @@ -263,7 +278,7 @@ public Void visitSample(SampleNode node, Void context) @Override public Void visitSort(SortNode node, Void context) { - printNode(node, format("Sort[%s]", Joiner.on(", ").join(node.getOrderingScheme().getOrderBy())), NODE_COLORS.get(NodeType.SORT)); + printNode(node, format("Sort[%s]", Joiner.on(", ").join(node.getOrderingScheme().getOrderByVariables())), NODE_COLORS.get(NodeType.SORT)); return node.getSource().accept(this, context); } @@ -280,7 +295,7 @@ public Void visitWindow(WindowNode node, Void context) printNode(node, "Window", format("partition by = %s|order by = %s", Joiner.on(", ").join(node.getPartitionBy()), node.getOrderingScheme() - .map(orderingScheme -> Joiner.on(", ").join(orderingScheme.getOrderBy())) + .map(orderingScheme -> Joiner.on(", ").join(orderingScheme.getOrderByVariables())) .orElse("")), NODE_COLORS.get(NodeType.WINDOW)); return node.getSource().accept(this, context); @@ -303,7 +318,7 @@ public Void visitTopNRowNumber(TopNRowNumberNode node, Void context) "TopNRowNumber", format("partition by = %s|order by = %s|n = %s", Joiner.on(", ").join(node.getPartitionBy()), - Joiner.on(", ").join(node.getOrderingScheme().getOrderBy()), node.getMaxRowCountPerPartition()), + Joiner.on(", ").join(node.getOrderingScheme().getOrderByVariables()), node.getMaxRowCountPerPartition()), NODE_COLORS.get(NodeType.WINDOW)); return node.getSource().accept(this, context); } @@ -382,7 +397,7 @@ public Void visitGroupId(GroupIdNode node, Void context) @Override public Void visitFilter(FilterNode node, Void context) { - String expression = formatter.formatRowExpression(node.getPredicate()); + String expression = formatter.apply(node.getPredicate()); printNode(node, "Filter", expression, NODE_COLORS.get(NodeType.FILTER)); return node.getSource().accept(this, context); } @@ -397,7 +412,7 @@ public Void visitProject(ProjectNode node, Void context) // skip identity assignments continue; } - builder.append(format("%s := %s\\n", entry.getKey(), formatter.formatRowExpression(entry.getValue()))); + builder.append(format("%s := %s\\n", entry.getKey(), formatter.apply(entry.getValue()))); } printNode(node, "Project", builder.toString(), NODE_COLORS.get(NodeType.PROJECT)); @@ -419,7 +434,7 @@ public Void visitUnnest(UnnestNode node, Void context) @Override public Void visitTopN(final TopNNode node, Void context) { - Iterable keys = Iterables.transform(node.getOrderingScheme().getOrderBy(), input -> input + " " + node.getOrderingScheme().getOrdering(input)); + Iterable keys = Iterables.transform(node.getOrderingScheme().getOrderByVariables(), input -> input + " " + node.getOrderingScheme().getOrdering(input)); printNode(node, format("TopN[%s]", node.getCount()), Joiner.on(", ").join(keys), NODE_COLORS.get(NodeType.TOPN)); return node.getSource().accept(this, context); } @@ -498,7 +513,7 @@ public Void visitSemiJoin(SemiJoinNode node, Void context) @Override public Void visitSpatialJoin(SpatialJoinNode node, Void context) { - printNode(node, node.getType().getJoinLabel(), formatter.formatRowExpression(node.getFilter()), NODE_COLORS.get(NodeType.JOIN)); + printNode(node, node.getType().getJoinLabel(), formatter.apply(node.getFilter()), NODE_COLORS.get(NodeType.JOIN)); node.getLeft().accept(this, context); node.getRight().accept(this, context); diff --git a/presto-main/src/main/java/com/facebook/presto/util/SpatialJoinUtils.java b/presto-main/src/main/java/com/facebook/presto/util/SpatialJoinUtils.java index 86db95654bb9d..c83992fdb1167 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/SpatialJoinUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/util/SpatialJoinUtils.java @@ -13,10 +13,11 @@ */ package com.facebook.presto.util; +import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.relation.CallExpression; -import com.facebook.presto.spi.relation.LogicalRowExpressions; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.ComparisonExpression; @@ -24,17 +25,32 @@ import com.facebook.presto.sql.tree.FunctionCall; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.sql.ExpressionUtils.extractConjuncts; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Locale.ENGLISH; public class SpatialJoinUtils { - public static final String ST_CONTAINS = "st_contains"; - public static final String ST_WITHIN = "st_within"; - public static final String ST_INTERSECTS = "st_intersects"; - public static final String ST_DISTANCE = "st_distance"; + public static final QualifiedFunctionName ST_CONTAINS = QualifiedFunctionName.of(DEFAULT_NAMESPACE, "st_contains"); + public static final QualifiedFunctionName ST_CROSSES = QualifiedFunctionName.of(DEFAULT_NAMESPACE, "st_crosses"); + public static final QualifiedFunctionName ST_EQUALS = QualifiedFunctionName.of(DEFAULT_NAMESPACE, "st_equals"); + public static final QualifiedFunctionName ST_INTERSECTS = QualifiedFunctionName.of(DEFAULT_NAMESPACE, "st_intersects"); + public static final QualifiedFunctionName ST_OVERLAPS = QualifiedFunctionName.of(DEFAULT_NAMESPACE, "st_overlaps"); + public static final QualifiedFunctionName ST_TOUCHES = QualifiedFunctionName.of(DEFAULT_NAMESPACE, "st_touches"); + public static final QualifiedFunctionName ST_WITHIN = QualifiedFunctionName.of(DEFAULT_NAMESPACE, "st_within"); + public static final QualifiedFunctionName ST_DISTANCE = QualifiedFunctionName.of(DEFAULT_NAMESPACE, "st_distance"); + + private static final Set ALLOWED_SPATIAL_JOIN_FUNCTIONS = Stream.of( + ST_CONTAINS, ST_CROSSES, ST_EQUALS, ST_INTERSECTS, ST_OVERLAPS, ST_TOUCHES, ST_WITHIN) + .map(QualifiedFunctionName::getFunctionName) + .map(String::toLowerCase) + .collect(Collectors.toSet()); private SpatialJoinUtils() {} @@ -66,16 +82,14 @@ public static List extractSupportedSpatialFunctions(RowExpressio private static boolean isSupportedSpatialFunction(FunctionCall functionCall) { - String functionName = functionCall.getName().toString(); - return functionName.equalsIgnoreCase(ST_CONTAINS) || functionName.equalsIgnoreCase(ST_WITHIN) - || functionName.equalsIgnoreCase(ST_INTERSECTS); + String functionName = functionCall.getName().getSuffix().toLowerCase(ENGLISH); + return ALLOWED_SPATIAL_JOIN_FUNCTIONS.contains(functionName); } private static boolean isSupportedSpatialFunction(CallExpression call, FunctionManager functionManager) { - String functionName = functionManager.getFunctionMetadata(call.getFunctionHandle()).getName(); - return functionName.equalsIgnoreCase(ST_CONTAINS) || functionName.equalsIgnoreCase(ST_WITHIN) - || functionName.equalsIgnoreCase(ST_INTERSECTS); + String functionName = functionManager.getFunctionMetadata(call.getFunctionHandle()).getName().getFunctionName().toLowerCase(ENGLISH); + return ALLOWED_SPATIAL_JOIN_FUNCTIONS.contains(functionName); } /** @@ -140,7 +154,7 @@ private static boolean isSupportedSpatialComparison(CallExpression expression, F private static boolean isSTDistance(Expression expression) { if (expression instanceof FunctionCall) { - return ((FunctionCall) expression).getName().toString().equalsIgnoreCase(ST_DISTANCE); + return ((FunctionCall) expression).getName().getSuffix().equalsIgnoreCase(ST_DISTANCE.getFunctionName()); } return false; @@ -149,7 +163,7 @@ private static boolean isSTDistance(Expression expression) private static boolean isSTDistance(RowExpression expression, FunctionManager functionManager) { if (expression instanceof CallExpression) { - return functionManager.getFunctionMetadata(((CallExpression) expression).getFunctionHandle()).getName().equalsIgnoreCase(ST_DISTANCE); + return functionManager.getFunctionMetadata(((CallExpression) expression).getFunctionHandle()).getName().equals(ST_DISTANCE); } return false; diff --git a/presto-main/src/main/java/com/facebook/presto/util/StatementUtils.java b/presto-main/src/main/java/com/facebook/presto/util/StatementUtils.java index 44373780790ed..410e88d6f58ae 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/StatementUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/util/StatementUtils.java @@ -18,6 +18,7 @@ import com.facebook.presto.sql.tree.Analyze; import com.facebook.presto.sql.tree.Call; import com.facebook.presto.sql.tree.Commit; +import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateRole; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; @@ -28,6 +29,7 @@ import com.facebook.presto.sql.tree.DescribeInput; import com.facebook.presto.sql.tree.DescribeOutput; import com.facebook.presto.sql.tree.DropColumn; +import com.facebook.presto.sql.tree.DropFunction; import com.facebook.presto.sql.tree.DropRole; import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; @@ -45,7 +47,6 @@ import com.facebook.presto.sql.tree.Revoke; import com.facebook.presto.sql.tree.RevokeRoles; import com.facebook.presto.sql.tree.Rollback; -import com.facebook.presto.sql.tree.SetPath; import com.facebook.presto.sql.tree.SetRole; import com.facebook.presto.sql.tree.SetSession; import com.facebook.presto.sql.tree.ShowCatalogs; @@ -110,6 +111,8 @@ private StatementUtils() {} builder.put(DropTable.class, QueryType.DATA_DEFINITION); builder.put(CreateView.class, QueryType.DATA_DEFINITION); builder.put(DropView.class, QueryType.DATA_DEFINITION); + builder.put(CreateFunction.class, QueryType.DATA_DEFINITION); + builder.put(DropFunction.class, QueryType.DATA_DEFINITION); builder.put(Use.class, QueryType.DATA_DEFINITION); builder.put(SetSession.class, QueryType.DATA_DEFINITION); builder.put(ResetSession.class, QueryType.DATA_DEFINITION); @@ -126,7 +129,6 @@ private StatementUtils() {} builder.put(Revoke.class, QueryType.DATA_DEFINITION); builder.put(Prepare.class, QueryType.DATA_DEFINITION); builder.put(Deallocate.class, QueryType.DATA_DEFINITION); - builder.put(SetPath.class, QueryType.DATA_DEFINITION); STATEMENT_QUERY_TYPES = builder.build(); } diff --git a/presto-main/src/main/java/com/facebook/presto/util/StringTableUtils.java b/presto-main/src/main/java/com/facebook/presto/util/StringTableUtils.java new file mode 100644 index 0000000000000..08c6d37b10b6d --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/util/StringTableUtils.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.util; + +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.Math.max; +import static java.lang.String.format; + +public class StringTableUtils +{ + private StringTableUtils() {} + + public static String getShortestTableStringFormat(List> table) + { + if (table.size() == 0) { + throw new IllegalArgumentException("Table must include at least one row"); + } + + int tableWidth = table.get(0).size(); + int[] lengthTracker = new int[tableWidth]; + for (List row : table) { + if (row.size() != tableWidth) { + String errorString = format("All rows in the table are expected to have exactly same number of columns: %s != %s", + tableWidth, row.size()); + throw new IllegalArgumentException(errorString); + } + + for (int i = 0; i < row.size(); i++) { + lengthTracker[i] = max(row.get(i).length(), lengthTracker[i]); + } + } + + StringBuilder sb = new StringBuilder(); + sb.append('|'); + for (int maxLen : lengthTracker) { + sb.append(" %-") + .append(maxLen) + .append("s |"); + } + + return sb.toString(); + } + + public static List getTableStrings(List> table) + { + String formatString = getShortestTableStringFormat(table); + return table.stream() + .map(List::toArray) + .map(line -> format(formatString, line)) + .collect(toImmutableList()); + } +} diff --git a/presto-main/src/main/resources/webapp/assets/presto.css b/presto-main/src/main/resources/webapp/assets/presto.css index 1fb42afe51917..165b027f41f94 100644 --- a/presto-main/src/main/resources/webapp/assets/presto.css +++ b/presto-main/src/main/resources/webapp/assets/presto.css @@ -578,6 +578,10 @@ pre { text-align: right; } +#stage-list #tasks { + text-align: left; +} + .stage-id { font-size: 90px; color: #b5b5b5; diff --git a/presto-main/src/main/resources/webapp/dist/embedded_plan.js b/presto-main/src/main/resources/webapp/dist/embedded_plan.js index c091aaf04b9ae..46a15117d6408 100644 --- a/presto-main/src/main/resources/webapp/dist/embedded_plan.js +++ b/presto-main/src/main/resources/webapp/dist/embedded_plan.js @@ -94,7 +94,7 @@ /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.LivePlan = undefined;\n\nvar _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/index.js\");\n\nvar _react2 = _interopRequireDefault(_react);\n\nvar _server = __webpack_require__(/*! react-dom/server */ \"./node_modules/react-dom/server.browser.js\");\n\nvar _server2 = _interopRequireDefault(_server);\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nvar _utils = __webpack_require__(/*! ../utils */ \"./utils.js\");\n\nvar _QueryHeader = __webpack_require__(/*! ./QueryHeader */ \"./components/QueryHeader.jsx\");\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return call && (typeof call === \"object\" || typeof call === \"function\") ? call : self; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function, not \" + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } /*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nvar StageStatistics = function (_React$Component) {\n _inherits(StageStatistics, _React$Component);\n\n function StageStatistics() {\n _classCallCheck(this, StageStatistics);\n\n return _possibleConstructorReturn(this, (StageStatistics.__proto__ || Object.getPrototypeOf(StageStatistics)).apply(this, arguments));\n }\n\n _createClass(StageStatistics, [{\n key: \"render\",\n value: function render() {\n var stage = this.props.stage;\n var stats = this.props.stage.stageStats;\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"h3\",\n { className: \"margin-top: 0\" },\n \"Stage \",\n stage.id\n ),\n stage.state,\n _react2.default.createElement(\"hr\", null),\n \"CPU: \",\n stats.totalCpuTime,\n _react2.default.createElement(\"br\", null),\n \"Buffered: \",\n stats.bufferedDataSize,\n _react2.default.createElement(\"br\", null),\n stats.fullyBlocked ? _react2.default.createElement(\n \"div\",\n { style: { color: '#ff0000' } },\n \"Blocked: \",\n stats.totalBlockedTime,\n \" \"\n ) : _react2.default.createElement(\n \"div\",\n null,\n \"Blocked: \",\n stats.totalBlockedTime,\n \" \"\n ),\n \"Memory: \",\n stats.userMemoryReservation,\n _react2.default.createElement(\"br\", null),\n \"Splits: \",\n \"Q:\" + stats.queuedDrivers + \", R:\" + stats.runningDrivers + \", F:\" + stats.completedDrivers,\n _react2.default.createElement(\"br\", null),\n \"Lifespans: \",\n stats.completedLifespans + \" / \" + stats.totalLifespans,\n _react2.default.createElement(\"hr\", null),\n \"Input: \",\n stats.rawInputDataSize + \" / \" + (0, _utils.formatRows)(stats.rawInputPositions)\n )\n );\n }\n }], [{\n key: \"getStages\",\n value: function getStages(queryInfo) {\n var stages = new Map();\n StageStatistics.flattenStage(queryInfo.outputStage, stages);\n return stages;\n }\n }, {\n key: \"flattenStage\",\n value: function flattenStage(stageInfo, result) {\n stageInfo.subStages.forEach(function (stage) {\n StageStatistics.flattenStage(stage, result);\n });\n\n var nodes = new Map();\n StageStatistics.flattenNode(result, stageInfo.plan.root, JSON.parse(stageInfo.plan.jsonRepresentation), nodes);\n\n result.set(stageInfo.plan.id, {\n stageId: stageInfo.stageId,\n id: stageInfo.plan.id,\n root: stageInfo.plan.root.id,\n distribution: stageInfo.plan.distribution,\n stageStats: stageInfo.stageStats,\n state: stageInfo.state,\n nodes: nodes\n });\n }\n }, {\n key: \"flattenNode\",\n value: function flattenNode(stages, rootNodeInfo, node, result) {\n result.set(node.id, {\n id: node.id,\n name: node['name'],\n identifier: node['identifier'],\n details: node['details'],\n sources: node.children.map(function (node) {\n return node.id;\n }),\n remoteSources: node.remoteSources\n });\n\n node.children.forEach(function (child) {\n StageStatistics.flattenNode(stages, rootNodeInfo, child, result);\n });\n }\n }]);\n\n return StageStatistics;\n}(_react2.default.Component);\n\nvar PlanNode = function (_React$Component2) {\n _inherits(PlanNode, _React$Component2);\n\n function PlanNode(props) {\n _classCallCheck(this, PlanNode);\n\n return _possibleConstructorReturn(this, (PlanNode.__proto__ || Object.getPrototypeOf(PlanNode)).call(this, props));\n }\n\n _createClass(PlanNode, [{\n key: \"render\",\n value: function render() {\n return _react2.default.createElement(\n \"div\",\n { style: { color: \"#000\" }, \"data-toggle\": \"tooltip\", \"data-placement\": \"bottom\", \"data-container\": \"body\", \"data-html\": \"true\",\n title: \"

\" + this.props.name + \"

\" + this.props.identifier },\n _react2.default.createElement(\n \"strong\",\n null,\n this.props.name\n ),\n _react2.default.createElement(\n \"div\",\n null,\n (0, _utils.truncateString)(this.props.identifier, 35)\n )\n );\n }\n }]);\n\n return PlanNode;\n}(_react2.default.Component);\n\nvar LivePlan = exports.LivePlan = function (_React$Component3) {\n _inherits(LivePlan, _React$Component3);\n\n function LivePlan(props) {\n _classCallCheck(this, LivePlan);\n\n var _this3 = _possibleConstructorReturn(this, (LivePlan.__proto__ || Object.getPrototypeOf(LivePlan)).call(this, props));\n\n _this3.state = {\n initialized: false,\n ended: false,\n\n query: null,\n\n graph: (0, _utils.initializeGraph)(),\n svg: null,\n render: new dagreD3.render()\n };\n return _this3;\n }\n\n _createClass(LivePlan, [{\n key: \"resetTimer\",\n value: function resetTimer() {\n clearTimeout(this.timeoutId);\n // stop refreshing when query finishes or fails\n if (this.state.query === null || !this.state.ended) {\n this.timeoutId = setTimeout(this.refreshLoop.bind(this), 1000);\n }\n }\n }, {\n key: \"refreshLoop\",\n value: function refreshLoop() {\n var _this4 = this;\n\n clearTimeout(this.timeoutId); // to stop multiple series of refreshLoop from going on simultaneously\n fetch('/v1/query/' + this.props.queryId).then(function (response) {\n return response.json();\n }).then(function (query) {\n _this4.setState({\n query: query,\n\n initialized: true,\n ended: query.finalQueryInfo\n });\n _this4.resetTimer();\n }).catch(function () {\n _this4.setState({\n initialized: true\n });\n _this4.resetTimer();\n });\n }\n }, {\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.refreshLoop.bind(this)();\n }\n }, {\n key: \"updateD3Stage\",\n value: function updateD3Stage(stage, graph, allStages) {\n var clusterId = stage.stageId;\n var stageRootNodeId = \"stage-\" + stage.id + \"-root\";\n var color = (0, _utils.getStageStateColor)(stage);\n\n graph.setNode(clusterId, { style: 'fill: ' + color, labelStyle: 'fill: #fff' });\n\n // this is a non-standard use of ReactDOMServer, but it's the cleanest way to unify DagreD3 with React\n var html = _server2.default.renderToString(_react2.default.createElement(StageStatistics, { key: stage.id, stage: stage }));\n\n graph.setNode(stageRootNodeId, { class: \"stage-stats\", label: html, labelType: \"html\" });\n graph.setParent(stageRootNodeId, clusterId);\n graph.setEdge(\"node-\" + stage.root, stageRootNodeId, { style: \"visibility: hidden\" });\n\n stage.nodes.forEach(function (node) {\n var nodeId = \"node-\" + node.id;\n var nodeHtml = _server2.default.renderToString(_react2.default.createElement(PlanNode, node));\n\n graph.setNode(nodeId, { label: nodeHtml, style: 'fill: #fff', labelType: \"html\" });\n graph.setParent(nodeId, clusterId);\n\n node.sources.forEach(function (source) {\n graph.setEdge(\"node-\" + source, nodeId, { class: \"plan-edge\", arrowheadClass: \"plan-arrowhead\" });\n });\n\n if (node.remoteSources.length > 0) {\n graph.setNode(nodeId, { label: '', shape: \"circle\" });\n\n node.remoteSources.forEach(function (sourceId) {\n var source = allStages.get(sourceId);\n if (source) {\n var sourceStats = source.stageStats;\n graph.setEdge(\"stage-\" + sourceId + \"-root\", nodeId, {\n class: \"plan-edge\",\n style: \"stroke-width: 4px\",\n arrowheadClass: \"plan-arrowhead\",\n label: sourceStats.outputDataSize + \" / \" + (0, _utils.formatRows)(sourceStats.outputPositions),\n labelStyle: \"color: #fff; font-weight: bold; font-size: 24px;\",\n labelType: \"html\"\n });\n }\n });\n }\n });\n }\n }, {\n key: \"updateD3Graph\",\n value: function updateD3Graph() {\n var _this5 = this;\n\n if (!this.state.svg) {\n this.setState({\n svg: (0, _utils.initializeSvg)(\"#plan-canvas\")\n });\n return;\n }\n\n if (!this.state.query) {\n return;\n }\n\n var graph = this.state.graph;\n var stages = StageStatistics.getStages(this.state.query);\n stages.forEach(function (stage) {\n _this5.updateD3Stage(stage, graph, stages);\n });\n\n var inner = d3.select(\"#plan-canvas g\");\n this.state.render(inner, graph);\n\n var svg = this.state.svg;\n svg.selectAll(\"g.cluster\").on(\"click\", LivePlan.handleStageClick);\n\n var width = parseInt(window.getComputedStyle(document.getElementById(\"live-plan\"), null).getPropertyValue(\"width\").replace(/px/, \"\")) - 50;\n var height = parseInt(window.getComputedStyle(document.getElementById(\"live-plan\"), null).getPropertyValue(\"height\").replace(/px/, \"\")) - 50;\n\n var graphHeight = graph.graph().height + 100;\n var graphWidth = graph.graph().width + 100;\n if (this.state.ended) {\n // Zoom doesn't deal well with DOM changes\n var initialScale = Math.min(width / graphWidth, height / graphHeight);\n var zoom = d3.zoom().scaleExtent([initialScale, 1]).on(\"zoom\", function () {\n inner.attr(\"transform\", d3.event.transform);\n });\n\n svg.call(zoom);\n svg.call(zoom.transform, d3.zoomIdentity.translate((width - graph.graph().width * initialScale) / 2, 20).scale(initialScale));\n svg.attr('height', height);\n svg.attr('width', width);\n } else {\n svg.attr('height', graphHeight);\n svg.attr('width', graphWidth);\n }\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n this.updateD3Graph();\n //$FlowFixMe\n $('[data-toggle=\"tooltip\"]').tooltip();\n }\n }, {\n key: \"render\",\n value: function render() {\n var query = this.state.query;\n\n if (query === null || this.state.initialized === false) {\n var label = _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n );\n if (this.state.initialized) {\n label = \"Query not found\";\n }\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n label\n )\n )\n );\n }\n\n var loadingMessage = null;\n if (query && !query.outputStage) {\n loadingMessage = _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Live plan graph will appear automatically when query starts running.\"\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n )\n )\n );\n }\n\n // TODO: Refactor components to move refreshLoop to parent rather than using this property\n var queryHeader = this.props.isEmbedded ? null : _react2.default.createElement(_QueryHeader.QueryHeader, { query: query });\n return _react2.default.createElement(\n \"div\",\n null,\n queryHeader,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n loadingMessage,\n _react2.default.createElement(\n \"div\",\n { id: \"live-plan\", className: \"graph-container\" },\n _react2.default.createElement(\n \"div\",\n { className: \"pull-right\" },\n this.state.ended ? \"Scroll to zoom.\" : \"Zoom disabled while query is running.\",\n \" Click stage to view additional statistics\"\n ),\n _react2.default.createElement(\"svg\", { id: \"plan-canvas\" })\n )\n )\n )\n );\n }\n }], [{\n key: \"handleStageClick\",\n value: function handleStageClick(stageCssId) {\n window.open(\"stage.html?\" + stageCssId, '_blank');\n }\n }]);\n\n return LivePlan;\n}(_react2.default.Component);\n\n//# sourceURL=webpack:///./components/LivePlan.jsx?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.LivePlan = undefined;\n\nvar _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/index.js\");\n\nvar _react2 = _interopRequireDefault(_react);\n\nvar _server = __webpack_require__(/*! react-dom/server */ \"./node_modules/react-dom/server.browser.js\");\n\nvar _server2 = _interopRequireDefault(_server);\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nvar _utils = __webpack_require__(/*! ../utils */ \"./utils.js\");\n\nvar _QueryHeader = __webpack_require__(/*! ./QueryHeader */ \"./components/QueryHeader.jsx\");\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return call && (typeof call === \"object\" || typeof call === \"function\") ? call : self; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function, not \" + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } /*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nvar StageStatistics = function (_React$Component) {\n _inherits(StageStatistics, _React$Component);\n\n function StageStatistics() {\n _classCallCheck(this, StageStatistics);\n\n return _possibleConstructorReturn(this, (StageStatistics.__proto__ || Object.getPrototypeOf(StageStatistics)).apply(this, arguments));\n }\n\n _createClass(StageStatistics, [{\n key: \"render\",\n value: function render() {\n var stage = this.props.stage;\n var stats = this.props.stage.stageStats;\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"h3\",\n { className: \"margin-top: 0\" },\n \"Stage \",\n stage.id\n ),\n stage.state,\n _react2.default.createElement(\"hr\", null),\n \"CPU: \",\n stats.totalCpuTime,\n _react2.default.createElement(\"br\", null),\n \"Buffered: \",\n stats.bufferedDataSize,\n _react2.default.createElement(\"br\", null),\n stats.fullyBlocked ? _react2.default.createElement(\n \"div\",\n { style: { color: '#ff0000' } },\n \"Blocked: \",\n stats.totalBlockedTime,\n \" \"\n ) : _react2.default.createElement(\n \"div\",\n null,\n \"Blocked: \",\n stats.totalBlockedTime,\n \" \"\n ),\n \"Memory: \",\n stats.userMemoryReservation,\n _react2.default.createElement(\"br\", null),\n \"Splits: \",\n \"Q:\" + stats.queuedDrivers + \", R:\" + stats.runningDrivers + \", F:\" + stats.completedDrivers,\n _react2.default.createElement(\"br\", null),\n \"Lifespans: \",\n stats.completedLifespans + \" / \" + stats.totalLifespans,\n _react2.default.createElement(\"hr\", null),\n \"Input: \",\n stats.rawInputDataSize + \" / \" + (0, _utils.formatRows)(stats.rawInputPositions)\n )\n );\n }\n }], [{\n key: \"getStages\",\n value: function getStages(queryInfo) {\n var stages = new Map();\n StageStatistics.flattenStage(queryInfo.outputStage, stages);\n return stages;\n }\n }, {\n key: \"flattenStage\",\n value: function flattenStage(stageInfo, result) {\n stageInfo.subStages.forEach(function (stage) {\n StageStatistics.flattenStage(stage, result);\n });\n\n var nodes = new Map();\n StageStatistics.flattenNode(result, stageInfo.plan.root, JSON.parse(stageInfo.plan.jsonRepresentation), nodes);\n\n result.set(stageInfo.plan.id, {\n stageId: stageInfo.stageId,\n id: stageInfo.plan.id,\n root: stageInfo.plan.root.id,\n distribution: stageInfo.plan.distribution,\n stageStats: stageInfo.latestAttemptExecutionInfo.stats,\n state: stageInfo.latestAttemptExecutionInfo.state,\n nodes: nodes\n });\n }\n }, {\n key: \"flattenNode\",\n value: function flattenNode(stages, rootNodeInfo, node, result) {\n result.set(node.id, {\n id: node.id,\n name: node['name'],\n identifier: node['identifier'],\n details: node['details'],\n sources: node.children.map(function (node) {\n return node.id;\n }),\n remoteSources: node.remoteSources\n });\n\n node.children.forEach(function (child) {\n StageStatistics.flattenNode(stages, rootNodeInfo, child, result);\n });\n }\n }]);\n\n return StageStatistics;\n}(_react2.default.Component);\n\nvar PlanNode = function (_React$Component2) {\n _inherits(PlanNode, _React$Component2);\n\n function PlanNode(props) {\n _classCallCheck(this, PlanNode);\n\n return _possibleConstructorReturn(this, (PlanNode.__proto__ || Object.getPrototypeOf(PlanNode)).call(this, props));\n }\n\n _createClass(PlanNode, [{\n key: \"render\",\n value: function render() {\n return _react2.default.createElement(\n \"div\",\n { style: { color: \"#000\" }, \"data-toggle\": \"tooltip\", \"data-placement\": \"bottom\", \"data-container\": \"body\", \"data-html\": \"true\",\n title: \"

\" + this.props.name + \"

\" + this.props.identifier },\n _react2.default.createElement(\n \"strong\",\n null,\n this.props.name\n ),\n _react2.default.createElement(\n \"div\",\n null,\n (0, _utils.truncateString)(this.props.identifier, 35)\n )\n );\n }\n }]);\n\n return PlanNode;\n}(_react2.default.Component);\n\nvar LivePlan = exports.LivePlan = function (_React$Component3) {\n _inherits(LivePlan, _React$Component3);\n\n function LivePlan(props) {\n _classCallCheck(this, LivePlan);\n\n var _this3 = _possibleConstructorReturn(this, (LivePlan.__proto__ || Object.getPrototypeOf(LivePlan)).call(this, props));\n\n _this3.state = {\n initialized: false,\n ended: false,\n\n query: null,\n\n graph: (0, _utils.initializeGraph)(),\n svg: null,\n render: new dagreD3.render()\n };\n return _this3;\n }\n\n _createClass(LivePlan, [{\n key: \"resetTimer\",\n value: function resetTimer() {\n clearTimeout(this.timeoutId);\n // stop refreshing when query finishes or fails\n if (this.state.query === null || !this.state.ended) {\n this.timeoutId = setTimeout(this.refreshLoop.bind(this), 1000);\n }\n }\n }, {\n key: \"refreshLoop\",\n value: function refreshLoop() {\n var _this4 = this;\n\n clearTimeout(this.timeoutId); // to stop multiple series of refreshLoop from going on simultaneously\n fetch('/v1/query/' + this.props.queryId).then(function (response) {\n return response.json();\n }).then(function (query) {\n _this4.setState({\n query: query,\n\n initialized: true,\n ended: query.finalQueryInfo\n });\n _this4.resetTimer();\n }).catch(function () {\n _this4.setState({\n initialized: true\n });\n _this4.resetTimer();\n });\n }\n }, {\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.refreshLoop.bind(this)();\n }\n }, {\n key: \"updateD3Stage\",\n value: function updateD3Stage(stage, graph, allStages) {\n var clusterId = stage.stageId;\n var stageRootNodeId = \"stage-\" + stage.id + \"-root\";\n var color = (0, _utils.getStageStateColor)(stage);\n\n graph.setNode(clusterId, { style: 'fill: ' + color, labelStyle: 'fill: #fff' });\n\n // this is a non-standard use of ReactDOMServer, but it's the cleanest way to unify DagreD3 with React\n var html = _server2.default.renderToString(_react2.default.createElement(StageStatistics, { key: stage.id, stage: stage }));\n\n graph.setNode(stageRootNodeId, { class: \"stage-stats\", label: html, labelType: \"html\" });\n graph.setParent(stageRootNodeId, clusterId);\n graph.setEdge(\"node-\" + stage.root, stageRootNodeId, { style: \"visibility: hidden\" });\n\n stage.nodes.forEach(function (node) {\n var nodeId = \"node-\" + node.id;\n var nodeHtml = _server2.default.renderToString(_react2.default.createElement(PlanNode, node));\n\n graph.setNode(nodeId, { label: nodeHtml, style: 'fill: #fff', labelType: \"html\" });\n graph.setParent(nodeId, clusterId);\n\n node.sources.forEach(function (source) {\n graph.setEdge(\"node-\" + source, nodeId, { class: \"plan-edge\", arrowheadClass: \"plan-arrowhead\" });\n });\n\n if (node.remoteSources.length > 0) {\n graph.setNode(nodeId, { label: '', shape: \"circle\" });\n\n node.remoteSources.forEach(function (sourceId) {\n var source = allStages.get(sourceId);\n if (source) {\n var sourceStats = source.stageStats;\n graph.setEdge(\"stage-\" + sourceId + \"-root\", nodeId, {\n class: \"plan-edge\",\n style: \"stroke-width: 4px\",\n arrowheadClass: \"plan-arrowhead\",\n label: sourceStats.outputDataSize + \" / \" + (0, _utils.formatRows)(sourceStats.outputPositions),\n labelStyle: \"color: #fff; font-weight: bold; font-size: 24px;\",\n labelType: \"html\"\n });\n }\n });\n }\n });\n }\n }, {\n key: \"updateD3Graph\",\n value: function updateD3Graph() {\n var _this5 = this;\n\n if (!this.state.svg) {\n this.setState({\n svg: (0, _utils.initializeSvg)(\"#plan-canvas\")\n });\n return;\n }\n\n if (!this.state.query) {\n return;\n }\n\n var graph = this.state.graph;\n var stages = StageStatistics.getStages(this.state.query);\n stages.forEach(function (stage) {\n _this5.updateD3Stage(stage, graph, stages);\n });\n\n var inner = d3.select(\"#plan-canvas g\");\n this.state.render(inner, graph);\n\n var svg = this.state.svg;\n svg.selectAll(\"g.cluster\").on(\"click\", LivePlan.handleStageClick);\n\n var width = parseInt(window.getComputedStyle(document.getElementById(\"live-plan\"), null).getPropertyValue(\"width\").replace(/px/, \"\")) - 50;\n var height = parseInt(window.getComputedStyle(document.getElementById(\"live-plan\"), null).getPropertyValue(\"height\").replace(/px/, \"\")) - 50;\n\n var graphHeight = graph.graph().height + 100;\n var graphWidth = graph.graph().width + 100;\n if (this.state.ended) {\n // Zoom doesn't deal well with DOM changes\n var initialScale = Math.min(width / graphWidth, height / graphHeight);\n var zoom = d3.zoom().scaleExtent([initialScale, 1]).on(\"zoom\", function () {\n inner.attr(\"transform\", d3.event.transform);\n });\n\n svg.call(zoom);\n svg.call(zoom.transform, d3.zoomIdentity.translate((width - graph.graph().width * initialScale) / 2, 20).scale(initialScale));\n svg.attr('height', height);\n svg.attr('width', width);\n } else {\n svg.attr('height', graphHeight);\n svg.attr('width', graphWidth);\n }\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n this.updateD3Graph();\n //$FlowFixMe\n $('[data-toggle=\"tooltip\"]').tooltip();\n }\n }, {\n key: \"render\",\n value: function render() {\n var query = this.state.query;\n\n if (query === null || this.state.initialized === false) {\n var label = _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n );\n if (this.state.initialized) {\n label = \"Query not found\";\n }\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n label\n )\n )\n );\n }\n\n var loadingMessage = null;\n if (query && !query.outputStage) {\n loadingMessage = _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Live plan graph will appear automatically when query starts running.\"\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n )\n )\n );\n }\n\n // TODO: Refactor components to move refreshLoop to parent rather than using this property\n var queryHeader = this.props.isEmbedded ? null : _react2.default.createElement(_QueryHeader.QueryHeader, { query: query });\n return _react2.default.createElement(\n \"div\",\n null,\n queryHeader,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n loadingMessage,\n _react2.default.createElement(\n \"div\",\n { id: \"live-plan\", className: \"graph-container\" },\n _react2.default.createElement(\n \"div\",\n { className: \"pull-right\" },\n this.state.ended ? \"Scroll to zoom.\" : \"Zoom disabled while query is running.\",\n \" Click stage to view additional statistics\"\n ),\n _react2.default.createElement(\"svg\", { id: \"plan-canvas\" })\n )\n )\n )\n );\n }\n }], [{\n key: \"handleStageClick\",\n value: function handleStageClick(stageCssId) {\n window.open(\"stage.html?\" + stageCssId, '_blank');\n }\n }]);\n\n return LivePlan;\n}(_react2.default.Component);\n\n//# sourceURL=webpack:///./components/LivePlan.jsx?"); /***/ }), diff --git a/presto-main/src/main/resources/webapp/dist/plan.js b/presto-main/src/main/resources/webapp/dist/plan.js index 8334101085ca2..db89ccaef42cb 100644 --- a/presto-main/src/main/resources/webapp/dist/plan.js +++ b/presto-main/src/main/resources/webapp/dist/plan.js @@ -94,7 +94,7 @@ /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.LivePlan = undefined;\n\nvar _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/index.js\");\n\nvar _react2 = _interopRequireDefault(_react);\n\nvar _server = __webpack_require__(/*! react-dom/server */ \"./node_modules/react-dom/server.browser.js\");\n\nvar _server2 = _interopRequireDefault(_server);\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nvar _utils = __webpack_require__(/*! ../utils */ \"./utils.js\");\n\nvar _QueryHeader = __webpack_require__(/*! ./QueryHeader */ \"./components/QueryHeader.jsx\");\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return call && (typeof call === \"object\" || typeof call === \"function\") ? call : self; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function, not \" + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } /*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nvar StageStatistics = function (_React$Component) {\n _inherits(StageStatistics, _React$Component);\n\n function StageStatistics() {\n _classCallCheck(this, StageStatistics);\n\n return _possibleConstructorReturn(this, (StageStatistics.__proto__ || Object.getPrototypeOf(StageStatistics)).apply(this, arguments));\n }\n\n _createClass(StageStatistics, [{\n key: \"render\",\n value: function render() {\n var stage = this.props.stage;\n var stats = this.props.stage.stageStats;\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"h3\",\n { className: \"margin-top: 0\" },\n \"Stage \",\n stage.id\n ),\n stage.state,\n _react2.default.createElement(\"hr\", null),\n \"CPU: \",\n stats.totalCpuTime,\n _react2.default.createElement(\"br\", null),\n \"Buffered: \",\n stats.bufferedDataSize,\n _react2.default.createElement(\"br\", null),\n stats.fullyBlocked ? _react2.default.createElement(\n \"div\",\n { style: { color: '#ff0000' } },\n \"Blocked: \",\n stats.totalBlockedTime,\n \" \"\n ) : _react2.default.createElement(\n \"div\",\n null,\n \"Blocked: \",\n stats.totalBlockedTime,\n \" \"\n ),\n \"Memory: \",\n stats.userMemoryReservation,\n _react2.default.createElement(\"br\", null),\n \"Splits: \",\n \"Q:\" + stats.queuedDrivers + \", R:\" + stats.runningDrivers + \", F:\" + stats.completedDrivers,\n _react2.default.createElement(\"br\", null),\n \"Lifespans: \",\n stats.completedLifespans + \" / \" + stats.totalLifespans,\n _react2.default.createElement(\"hr\", null),\n \"Input: \",\n stats.rawInputDataSize + \" / \" + (0, _utils.formatRows)(stats.rawInputPositions)\n )\n );\n }\n }], [{\n key: \"getStages\",\n value: function getStages(queryInfo) {\n var stages = new Map();\n StageStatistics.flattenStage(queryInfo.outputStage, stages);\n return stages;\n }\n }, {\n key: \"flattenStage\",\n value: function flattenStage(stageInfo, result) {\n stageInfo.subStages.forEach(function (stage) {\n StageStatistics.flattenStage(stage, result);\n });\n\n var nodes = new Map();\n StageStatistics.flattenNode(result, stageInfo.plan.root, JSON.parse(stageInfo.plan.jsonRepresentation), nodes);\n\n result.set(stageInfo.plan.id, {\n stageId: stageInfo.stageId,\n id: stageInfo.plan.id,\n root: stageInfo.plan.root.id,\n distribution: stageInfo.plan.distribution,\n stageStats: stageInfo.stageStats,\n state: stageInfo.state,\n nodes: nodes\n });\n }\n }, {\n key: \"flattenNode\",\n value: function flattenNode(stages, rootNodeInfo, node, result) {\n result.set(node.id, {\n id: node.id,\n name: node['name'],\n identifier: node['identifier'],\n details: node['details'],\n sources: node.children.map(function (node) {\n return node.id;\n }),\n remoteSources: node.remoteSources\n });\n\n node.children.forEach(function (child) {\n StageStatistics.flattenNode(stages, rootNodeInfo, child, result);\n });\n }\n }]);\n\n return StageStatistics;\n}(_react2.default.Component);\n\nvar PlanNode = function (_React$Component2) {\n _inherits(PlanNode, _React$Component2);\n\n function PlanNode(props) {\n _classCallCheck(this, PlanNode);\n\n return _possibleConstructorReturn(this, (PlanNode.__proto__ || Object.getPrototypeOf(PlanNode)).call(this, props));\n }\n\n _createClass(PlanNode, [{\n key: \"render\",\n value: function render() {\n return _react2.default.createElement(\n \"div\",\n { style: { color: \"#000\" }, \"data-toggle\": \"tooltip\", \"data-placement\": \"bottom\", \"data-container\": \"body\", \"data-html\": \"true\",\n title: \"

\" + this.props.name + \"

\" + this.props.identifier },\n _react2.default.createElement(\n \"strong\",\n null,\n this.props.name\n ),\n _react2.default.createElement(\n \"div\",\n null,\n (0, _utils.truncateString)(this.props.identifier, 35)\n )\n );\n }\n }]);\n\n return PlanNode;\n}(_react2.default.Component);\n\nvar LivePlan = exports.LivePlan = function (_React$Component3) {\n _inherits(LivePlan, _React$Component3);\n\n function LivePlan(props) {\n _classCallCheck(this, LivePlan);\n\n var _this3 = _possibleConstructorReturn(this, (LivePlan.__proto__ || Object.getPrototypeOf(LivePlan)).call(this, props));\n\n _this3.state = {\n initialized: false,\n ended: false,\n\n query: null,\n\n graph: (0, _utils.initializeGraph)(),\n svg: null,\n render: new dagreD3.render()\n };\n return _this3;\n }\n\n _createClass(LivePlan, [{\n key: \"resetTimer\",\n value: function resetTimer() {\n clearTimeout(this.timeoutId);\n // stop refreshing when query finishes or fails\n if (this.state.query === null || !this.state.ended) {\n this.timeoutId = setTimeout(this.refreshLoop.bind(this), 1000);\n }\n }\n }, {\n key: \"refreshLoop\",\n value: function refreshLoop() {\n var _this4 = this;\n\n clearTimeout(this.timeoutId); // to stop multiple series of refreshLoop from going on simultaneously\n fetch('/v1/query/' + this.props.queryId).then(function (response) {\n return response.json();\n }).then(function (query) {\n _this4.setState({\n query: query,\n\n initialized: true,\n ended: query.finalQueryInfo\n });\n _this4.resetTimer();\n }).catch(function () {\n _this4.setState({\n initialized: true\n });\n _this4.resetTimer();\n });\n }\n }, {\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.refreshLoop.bind(this)();\n }\n }, {\n key: \"updateD3Stage\",\n value: function updateD3Stage(stage, graph, allStages) {\n var clusterId = stage.stageId;\n var stageRootNodeId = \"stage-\" + stage.id + \"-root\";\n var color = (0, _utils.getStageStateColor)(stage);\n\n graph.setNode(clusterId, { style: 'fill: ' + color, labelStyle: 'fill: #fff' });\n\n // this is a non-standard use of ReactDOMServer, but it's the cleanest way to unify DagreD3 with React\n var html = _server2.default.renderToString(_react2.default.createElement(StageStatistics, { key: stage.id, stage: stage }));\n\n graph.setNode(stageRootNodeId, { class: \"stage-stats\", label: html, labelType: \"html\" });\n graph.setParent(stageRootNodeId, clusterId);\n graph.setEdge(\"node-\" + stage.root, stageRootNodeId, { style: \"visibility: hidden\" });\n\n stage.nodes.forEach(function (node) {\n var nodeId = \"node-\" + node.id;\n var nodeHtml = _server2.default.renderToString(_react2.default.createElement(PlanNode, node));\n\n graph.setNode(nodeId, { label: nodeHtml, style: 'fill: #fff', labelType: \"html\" });\n graph.setParent(nodeId, clusterId);\n\n node.sources.forEach(function (source) {\n graph.setEdge(\"node-\" + source, nodeId, { class: \"plan-edge\", arrowheadClass: \"plan-arrowhead\" });\n });\n\n if (node.remoteSources.length > 0) {\n graph.setNode(nodeId, { label: '', shape: \"circle\" });\n\n node.remoteSources.forEach(function (sourceId) {\n var source = allStages.get(sourceId);\n if (source) {\n var sourceStats = source.stageStats;\n graph.setEdge(\"stage-\" + sourceId + \"-root\", nodeId, {\n class: \"plan-edge\",\n style: \"stroke-width: 4px\",\n arrowheadClass: \"plan-arrowhead\",\n label: sourceStats.outputDataSize + \" / \" + (0, _utils.formatRows)(sourceStats.outputPositions),\n labelStyle: \"color: #fff; font-weight: bold; font-size: 24px;\",\n labelType: \"html\"\n });\n }\n });\n }\n });\n }\n }, {\n key: \"updateD3Graph\",\n value: function updateD3Graph() {\n var _this5 = this;\n\n if (!this.state.svg) {\n this.setState({\n svg: (0, _utils.initializeSvg)(\"#plan-canvas\")\n });\n return;\n }\n\n if (!this.state.query) {\n return;\n }\n\n var graph = this.state.graph;\n var stages = StageStatistics.getStages(this.state.query);\n stages.forEach(function (stage) {\n _this5.updateD3Stage(stage, graph, stages);\n });\n\n var inner = d3.select(\"#plan-canvas g\");\n this.state.render(inner, graph);\n\n var svg = this.state.svg;\n svg.selectAll(\"g.cluster\").on(\"click\", LivePlan.handleStageClick);\n\n var width = parseInt(window.getComputedStyle(document.getElementById(\"live-plan\"), null).getPropertyValue(\"width\").replace(/px/, \"\")) - 50;\n var height = parseInt(window.getComputedStyle(document.getElementById(\"live-plan\"), null).getPropertyValue(\"height\").replace(/px/, \"\")) - 50;\n\n var graphHeight = graph.graph().height + 100;\n var graphWidth = graph.graph().width + 100;\n if (this.state.ended) {\n // Zoom doesn't deal well with DOM changes\n var initialScale = Math.min(width / graphWidth, height / graphHeight);\n var zoom = d3.zoom().scaleExtent([initialScale, 1]).on(\"zoom\", function () {\n inner.attr(\"transform\", d3.event.transform);\n });\n\n svg.call(zoom);\n svg.call(zoom.transform, d3.zoomIdentity.translate((width - graph.graph().width * initialScale) / 2, 20).scale(initialScale));\n svg.attr('height', height);\n svg.attr('width', width);\n } else {\n svg.attr('height', graphHeight);\n svg.attr('width', graphWidth);\n }\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n this.updateD3Graph();\n //$FlowFixMe\n $('[data-toggle=\"tooltip\"]').tooltip();\n }\n }, {\n key: \"render\",\n value: function render() {\n var query = this.state.query;\n\n if (query === null || this.state.initialized === false) {\n var label = _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n );\n if (this.state.initialized) {\n label = \"Query not found\";\n }\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n label\n )\n )\n );\n }\n\n var loadingMessage = null;\n if (query && !query.outputStage) {\n loadingMessage = _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Live plan graph will appear automatically when query starts running.\"\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n )\n )\n );\n }\n\n // TODO: Refactor components to move refreshLoop to parent rather than using this property\n var queryHeader = this.props.isEmbedded ? null : _react2.default.createElement(_QueryHeader.QueryHeader, { query: query });\n return _react2.default.createElement(\n \"div\",\n null,\n queryHeader,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n loadingMessage,\n _react2.default.createElement(\n \"div\",\n { id: \"live-plan\", className: \"graph-container\" },\n _react2.default.createElement(\n \"div\",\n { className: \"pull-right\" },\n this.state.ended ? \"Scroll to zoom.\" : \"Zoom disabled while query is running.\",\n \" Click stage to view additional statistics\"\n ),\n _react2.default.createElement(\"svg\", { id: \"plan-canvas\" })\n )\n )\n )\n );\n }\n }], [{\n key: \"handleStageClick\",\n value: function handleStageClick(stageCssId) {\n window.open(\"stage.html?\" + stageCssId, '_blank');\n }\n }]);\n\n return LivePlan;\n}(_react2.default.Component);\n\n//# sourceURL=webpack:///./components/LivePlan.jsx?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.LivePlan = undefined;\n\nvar _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/index.js\");\n\nvar _react2 = _interopRequireDefault(_react);\n\nvar _server = __webpack_require__(/*! react-dom/server */ \"./node_modules/react-dom/server.browser.js\");\n\nvar _server2 = _interopRequireDefault(_server);\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nvar _utils = __webpack_require__(/*! ../utils */ \"./utils.js\");\n\nvar _QueryHeader = __webpack_require__(/*! ./QueryHeader */ \"./components/QueryHeader.jsx\");\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return call && (typeof call === \"object\" || typeof call === \"function\") ? call : self; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function, not \" + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } /*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\nvar StageStatistics = function (_React$Component) {\n _inherits(StageStatistics, _React$Component);\n\n function StageStatistics() {\n _classCallCheck(this, StageStatistics);\n\n return _possibleConstructorReturn(this, (StageStatistics.__proto__ || Object.getPrototypeOf(StageStatistics)).apply(this, arguments));\n }\n\n _createClass(StageStatistics, [{\n key: \"render\",\n value: function render() {\n var stage = this.props.stage;\n var stats = this.props.stage.stageStats;\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"h3\",\n { className: \"margin-top: 0\" },\n \"Stage \",\n stage.id\n ),\n stage.state,\n _react2.default.createElement(\"hr\", null),\n \"CPU: \",\n stats.totalCpuTime,\n _react2.default.createElement(\"br\", null),\n \"Buffered: \",\n stats.bufferedDataSize,\n _react2.default.createElement(\"br\", null),\n stats.fullyBlocked ? _react2.default.createElement(\n \"div\",\n { style: { color: '#ff0000' } },\n \"Blocked: \",\n stats.totalBlockedTime,\n \" \"\n ) : _react2.default.createElement(\n \"div\",\n null,\n \"Blocked: \",\n stats.totalBlockedTime,\n \" \"\n ),\n \"Memory: \",\n stats.userMemoryReservation,\n _react2.default.createElement(\"br\", null),\n \"Splits: \",\n \"Q:\" + stats.queuedDrivers + \", R:\" + stats.runningDrivers + \", F:\" + stats.completedDrivers,\n _react2.default.createElement(\"br\", null),\n \"Lifespans: \",\n stats.completedLifespans + \" / \" + stats.totalLifespans,\n _react2.default.createElement(\"hr\", null),\n \"Input: \",\n stats.rawInputDataSize + \" / \" + (0, _utils.formatRows)(stats.rawInputPositions)\n )\n );\n }\n }], [{\n key: \"getStages\",\n value: function getStages(queryInfo) {\n var stages = new Map();\n StageStatistics.flattenStage(queryInfo.outputStage, stages);\n return stages;\n }\n }, {\n key: \"flattenStage\",\n value: function flattenStage(stageInfo, result) {\n stageInfo.subStages.forEach(function (stage) {\n StageStatistics.flattenStage(stage, result);\n });\n\n var nodes = new Map();\n StageStatistics.flattenNode(result, stageInfo.plan.root, JSON.parse(stageInfo.plan.jsonRepresentation), nodes);\n\n result.set(stageInfo.plan.id, {\n stageId: stageInfo.stageId,\n id: stageInfo.plan.id,\n root: stageInfo.plan.root.id,\n distribution: stageInfo.plan.distribution,\n stageStats: stageInfo.latestAttemptExecutionInfo.stats,\n state: stageInfo.latestAttemptExecutionInfo.state,\n nodes: nodes\n });\n }\n }, {\n key: \"flattenNode\",\n value: function flattenNode(stages, rootNodeInfo, node, result) {\n result.set(node.id, {\n id: node.id,\n name: node['name'],\n identifier: node['identifier'],\n details: node['details'],\n sources: node.children.map(function (node) {\n return node.id;\n }),\n remoteSources: node.remoteSources\n });\n\n node.children.forEach(function (child) {\n StageStatistics.flattenNode(stages, rootNodeInfo, child, result);\n });\n }\n }]);\n\n return StageStatistics;\n}(_react2.default.Component);\n\nvar PlanNode = function (_React$Component2) {\n _inherits(PlanNode, _React$Component2);\n\n function PlanNode(props) {\n _classCallCheck(this, PlanNode);\n\n return _possibleConstructorReturn(this, (PlanNode.__proto__ || Object.getPrototypeOf(PlanNode)).call(this, props));\n }\n\n _createClass(PlanNode, [{\n key: \"render\",\n value: function render() {\n return _react2.default.createElement(\n \"div\",\n { style: { color: \"#000\" }, \"data-toggle\": \"tooltip\", \"data-placement\": \"bottom\", \"data-container\": \"body\", \"data-html\": \"true\",\n title: \"

\" + this.props.name + \"

\" + this.props.identifier },\n _react2.default.createElement(\n \"strong\",\n null,\n this.props.name\n ),\n _react2.default.createElement(\n \"div\",\n null,\n (0, _utils.truncateString)(this.props.identifier, 35)\n )\n );\n }\n }]);\n\n return PlanNode;\n}(_react2.default.Component);\n\nvar LivePlan = exports.LivePlan = function (_React$Component3) {\n _inherits(LivePlan, _React$Component3);\n\n function LivePlan(props) {\n _classCallCheck(this, LivePlan);\n\n var _this3 = _possibleConstructorReturn(this, (LivePlan.__proto__ || Object.getPrototypeOf(LivePlan)).call(this, props));\n\n _this3.state = {\n initialized: false,\n ended: false,\n\n query: null,\n\n graph: (0, _utils.initializeGraph)(),\n svg: null,\n render: new dagreD3.render()\n };\n return _this3;\n }\n\n _createClass(LivePlan, [{\n key: \"resetTimer\",\n value: function resetTimer() {\n clearTimeout(this.timeoutId);\n // stop refreshing when query finishes or fails\n if (this.state.query === null || !this.state.ended) {\n this.timeoutId = setTimeout(this.refreshLoop.bind(this), 1000);\n }\n }\n }, {\n key: \"refreshLoop\",\n value: function refreshLoop() {\n var _this4 = this;\n\n clearTimeout(this.timeoutId); // to stop multiple series of refreshLoop from going on simultaneously\n fetch('/v1/query/' + this.props.queryId).then(function (response) {\n return response.json();\n }).then(function (query) {\n _this4.setState({\n query: query,\n\n initialized: true,\n ended: query.finalQueryInfo\n });\n _this4.resetTimer();\n }).catch(function () {\n _this4.setState({\n initialized: true\n });\n _this4.resetTimer();\n });\n }\n }, {\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.refreshLoop.bind(this)();\n }\n }, {\n key: \"updateD3Stage\",\n value: function updateD3Stage(stage, graph, allStages) {\n var clusterId = stage.stageId;\n var stageRootNodeId = \"stage-\" + stage.id + \"-root\";\n var color = (0, _utils.getStageStateColor)(stage);\n\n graph.setNode(clusterId, { style: 'fill: ' + color, labelStyle: 'fill: #fff' });\n\n // this is a non-standard use of ReactDOMServer, but it's the cleanest way to unify DagreD3 with React\n var html = _server2.default.renderToString(_react2.default.createElement(StageStatistics, { key: stage.id, stage: stage }));\n\n graph.setNode(stageRootNodeId, { class: \"stage-stats\", label: html, labelType: \"html\" });\n graph.setParent(stageRootNodeId, clusterId);\n graph.setEdge(\"node-\" + stage.root, stageRootNodeId, { style: \"visibility: hidden\" });\n\n stage.nodes.forEach(function (node) {\n var nodeId = \"node-\" + node.id;\n var nodeHtml = _server2.default.renderToString(_react2.default.createElement(PlanNode, node));\n\n graph.setNode(nodeId, { label: nodeHtml, style: 'fill: #fff', labelType: \"html\" });\n graph.setParent(nodeId, clusterId);\n\n node.sources.forEach(function (source) {\n graph.setEdge(\"node-\" + source, nodeId, { class: \"plan-edge\", arrowheadClass: \"plan-arrowhead\" });\n });\n\n if (node.remoteSources.length > 0) {\n graph.setNode(nodeId, { label: '', shape: \"circle\" });\n\n node.remoteSources.forEach(function (sourceId) {\n var source = allStages.get(sourceId);\n if (source) {\n var sourceStats = source.stageStats;\n graph.setEdge(\"stage-\" + sourceId + \"-root\", nodeId, {\n class: \"plan-edge\",\n style: \"stroke-width: 4px\",\n arrowheadClass: \"plan-arrowhead\",\n label: sourceStats.outputDataSize + \" / \" + (0, _utils.formatRows)(sourceStats.outputPositions),\n labelStyle: \"color: #fff; font-weight: bold; font-size: 24px;\",\n labelType: \"html\"\n });\n }\n });\n }\n });\n }\n }, {\n key: \"updateD3Graph\",\n value: function updateD3Graph() {\n var _this5 = this;\n\n if (!this.state.svg) {\n this.setState({\n svg: (0, _utils.initializeSvg)(\"#plan-canvas\")\n });\n return;\n }\n\n if (!this.state.query) {\n return;\n }\n\n var graph = this.state.graph;\n var stages = StageStatistics.getStages(this.state.query);\n stages.forEach(function (stage) {\n _this5.updateD3Stage(stage, graph, stages);\n });\n\n var inner = d3.select(\"#plan-canvas g\");\n this.state.render(inner, graph);\n\n var svg = this.state.svg;\n svg.selectAll(\"g.cluster\").on(\"click\", LivePlan.handleStageClick);\n\n var width = parseInt(window.getComputedStyle(document.getElementById(\"live-plan\"), null).getPropertyValue(\"width\").replace(/px/, \"\")) - 50;\n var height = parseInt(window.getComputedStyle(document.getElementById(\"live-plan\"), null).getPropertyValue(\"height\").replace(/px/, \"\")) - 50;\n\n var graphHeight = graph.graph().height + 100;\n var graphWidth = graph.graph().width + 100;\n if (this.state.ended) {\n // Zoom doesn't deal well with DOM changes\n var initialScale = Math.min(width / graphWidth, height / graphHeight);\n var zoom = d3.zoom().scaleExtent([initialScale, 1]).on(\"zoom\", function () {\n inner.attr(\"transform\", d3.event.transform);\n });\n\n svg.call(zoom);\n svg.call(zoom.transform, d3.zoomIdentity.translate((width - graph.graph().width * initialScale) / 2, 20).scale(initialScale));\n svg.attr('height', height);\n svg.attr('width', width);\n } else {\n svg.attr('height', graphHeight);\n svg.attr('width', graphWidth);\n }\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n this.updateD3Graph();\n //$FlowFixMe\n $('[data-toggle=\"tooltip\"]').tooltip();\n }\n }, {\n key: \"render\",\n value: function render() {\n var query = this.state.query;\n\n if (query === null || this.state.initialized === false) {\n var label = _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n );\n if (this.state.initialized) {\n label = \"Query not found\";\n }\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n label\n )\n )\n );\n }\n\n var loadingMessage = null;\n if (query && !query.outputStage) {\n loadingMessage = _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Live plan graph will appear automatically when query starts running.\"\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n )\n )\n );\n }\n\n // TODO: Refactor components to move refreshLoop to parent rather than using this property\n var queryHeader = this.props.isEmbedded ? null : _react2.default.createElement(_QueryHeader.QueryHeader, { query: query });\n return _react2.default.createElement(\n \"div\",\n null,\n queryHeader,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n loadingMessage,\n _react2.default.createElement(\n \"div\",\n { id: \"live-plan\", className: \"graph-container\" },\n _react2.default.createElement(\n \"div\",\n { className: \"pull-right\" },\n this.state.ended ? \"Scroll to zoom.\" : \"Zoom disabled while query is running.\",\n \" Click stage to view additional statistics\"\n ),\n _react2.default.createElement(\"svg\", { id: \"plan-canvas\" })\n )\n )\n )\n );\n }\n }], [{\n key: \"handleStageClick\",\n value: function handleStageClick(stageCssId) {\n window.open(\"stage.html?\" + stageCssId, '_blank');\n }\n }]);\n\n return LivePlan;\n}(_react2.default.Component);\n\n//# sourceURL=webpack:///./components/LivePlan.jsx?"); /***/ }), diff --git a/presto-main/src/main/resources/webapp/dist/query.js b/presto-main/src/main/resources/webapp/dist/query.js index 5de8741dd7c97..c8dd417f2a3fb 100644 --- a/presto-main/src/main/resources/webapp/dist/query.js +++ b/presto-main/src/main/resources/webapp/dist/query.js @@ -106,7 +106,7 @@ eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n}); /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.QueryDetail = undefined;\n\nvar _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/index.js\");\n\nvar _react2 = _interopRequireDefault(_react);\n\nvar _reactable = __webpack_require__(/*! reactable */ \"./node_modules/reactable/lib/reactable.js\");\n\nvar _reactable2 = _interopRequireDefault(_reactable);\n\nvar _utils = __webpack_require__(/*! ../utils */ \"./utils.js\");\n\nvar _QueryHeader = __webpack_require__(/*! ./QueryHeader */ \"./components/QueryHeader.jsx\");\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return call && (typeof call === \"object\" || typeof call === \"function\") ? call : self; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function, not \" + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } /*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nvar Table = _reactable2.default.Table,\n Thead = _reactable2.default.Thead,\n Th = _reactable2.default.Th,\n Tr = _reactable2.default.Tr,\n Td = _reactable2.default.Td;\n\nvar TaskList = function (_React$Component) {\n _inherits(TaskList, _React$Component);\n\n function TaskList() {\n _classCallCheck(this, TaskList);\n\n return _possibleConstructorReturn(this, (TaskList.__proto__ || Object.getPrototypeOf(TaskList)).apply(this, arguments));\n }\n\n _createClass(TaskList, [{\n key: \"render\",\n value: function render() {\n var tasks = this.props.tasks;\n\n if (tasks === undefined || tasks.length === 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"No threads in the selected group\"\n )\n )\n );\n }\n\n var showPortNumbers = TaskList.showPortNumbers(tasks);\n\n var renderedTasks = tasks.map(function (task) {\n var elapsedTime = (0, _utils.parseDuration)(task.stats.elapsedTime);\n if (elapsedTime === 0) {\n elapsedTime = Date.now() - Date.parse(task.stats.createTime);\n }\n\n return _react2.default.createElement(\n Tr,\n { key: task.taskStatus.taskId },\n _react2.default.createElement(\n Td,\n { column: \"id\", value: task.taskStatus.taskId },\n _react2.default.createElement(\n \"a\",\n { href: task.taskStatus.self + \"?pretty\" },\n (0, _utils.getTaskIdSuffix)(task.taskStatus.taskId)\n )\n ),\n _react2.default.createElement(\n Td,\n { column: \"host\", value: (0, _utils.getHostname)(task.taskStatus.self) },\n _react2.default.createElement(\n \"a\",\n { href: \"worker.html?\" + task.taskStatus.nodeId, className: \"font-light\", target: \"_blank\" },\n showPortNumbers ? (0, _utils.getHostAndPort)(task.taskStatus.self) : (0, _utils.getHostname)(task.taskStatus.self)\n )\n ),\n _react2.default.createElement(\n Td,\n { column: \"state\", value: TaskList.formatState(task.taskStatus.state, task.stats.fullyBlocked) },\n TaskList.formatState(task.taskStatus.state, task.stats.fullyBlocked)\n ),\n _react2.default.createElement(\n Td,\n { column: \"rows\", value: task.stats.rawInputPositions },\n (0, _utils.formatCount)(task.stats.rawInputPositions)\n ),\n _react2.default.createElement(\n Td,\n { column: \"rowsSec\", value: (0, _utils.computeRate)(task.stats.rawInputPositions, elapsedTime) },\n (0, _utils.formatCount)((0, _utils.computeRate)(task.stats.rawInputPositions, elapsedTime))\n ),\n _react2.default.createElement(\n Td,\n { column: \"bytes\", value: (0, _utils.parseDataSize)(task.stats.rawInputDataSize) },\n (0, _utils.formatDataSizeBytes)((0, _utils.parseDataSize)(task.stats.rawInputDataSize))\n ),\n _react2.default.createElement(\n Td,\n { column: \"bytesSec\", value: (0, _utils.computeRate)((0, _utils.parseDataSize)(task.stats.rawInputDataSize), elapsedTime) },\n (0, _utils.formatDataSizeBytes)((0, _utils.computeRate)((0, _utils.parseDataSize)(task.stats.rawInputDataSize), elapsedTime))\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsPending\", value: task.stats.queuedDrivers },\n task.stats.queuedDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsRunning\", value: task.stats.runningDrivers },\n task.stats.runningDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsBlocked\", value: task.stats.blockedDrivers },\n task.stats.blockedDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsDone\", value: task.stats.completedDrivers },\n task.stats.completedDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"elapsedTime\", value: (0, _utils.parseDuration)(task.stats.elapsedTime) },\n task.stats.elapsedTime\n ),\n _react2.default.createElement(\n Td,\n { column: \"cpuTime\", value: (0, _utils.parseDuration)(task.stats.totalCpuTime) },\n task.stats.totalCpuTime\n ),\n _react2.default.createElement(\n Td,\n { column: \"bufferedBytes\", value: task.outputBuffers.totalBufferedBytes },\n (0, _utils.formatDataSizeBytes)(task.outputBuffers.totalBufferedBytes)\n )\n );\n });\n\n return _react2.default.createElement(\n Table,\n { id: \"tasks\", className: \"table table-striped sortable\", sortable: [{\n column: 'id',\n sortFunction: TaskList.compareTaskId\n }, 'host', 'state', 'splitsPending', 'splitsRunning', 'splitsBlocked', 'splitsDone', 'rows', 'rowsSec', 'bytes', 'bytesSec', 'elapsedTime', 'cpuTime', 'bufferedBytes'],\n defaultSort: { column: 'id', direction: 'asc' } },\n _react2.default.createElement(\n Thead,\n null,\n _react2.default.createElement(\n Th,\n { column: \"id\" },\n \"ID\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"host\" },\n \"Host\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"state\" },\n \"State\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsPending\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-pause\", style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Pending splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsRunning\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-play\", style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Running splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsBlocked\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-bookmark\", style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Blocked splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsDone\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-ok\", style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Completed splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"rows\" },\n \"Rows\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"rowsSec\" },\n \"Rows/s\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"bytes\" },\n \"Bytes\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"bytesSec\" },\n \"Bytes/s\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"elapsedTime\" },\n \"Elapsed\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"cpuTime\" },\n \"CPU Time\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"bufferedBytes\" },\n \"Buffered\"\n )\n ),\n renderedTasks\n );\n }\n }], [{\n key: \"removeQueryId\",\n value: function removeQueryId(id) {\n var pos = id.indexOf('.');\n if (pos !== -1) {\n return id.substring(pos + 1);\n }\n return id;\n }\n }, {\n key: \"compareTaskId\",\n value: function compareTaskId(taskA, taskB) {\n var taskIdArrA = TaskList.removeQueryId(taskA).split(\".\");\n var taskIdArrB = TaskList.removeQueryId(taskB).split(\".\");\n\n if (taskIdArrA.length > taskIdArrB.length) {\n return 1;\n }\n for (var i = 0; i < taskIdArrA.length; i++) {\n var anum = Number.parseInt(taskIdArrA[i]);\n var bnum = Number.parseInt(taskIdArrB[i]);\n if (anum !== bnum) {\n return anum > bnum ? 1 : -1;\n }\n }\n\n return 0;\n }\n }, {\n key: \"showPortNumbers\",\n value: function showPortNumbers(tasks) {\n // check if any host has multiple port numbers\n var hostToPortNumber = {};\n for (var i = 0; i < tasks.length; i++) {\n var taskUri = tasks[i].taskStatus.self;\n var hostname = (0, _utils.getHostname)(taskUri);\n var port = (0, _utils.getPort)(taskUri);\n if (hostname in hostToPortNumber && hostToPortNumber[hostname] !== port) {\n return true;\n }\n hostToPortNumber[hostname] = port;\n }\n\n return false;\n }\n }, {\n key: \"formatState\",\n value: function formatState(state, fullyBlocked) {\n if (fullyBlocked && state === \"RUNNING\") {\n return \"BLOCKED\";\n } else {\n return state;\n }\n }\n }]);\n\n return TaskList;\n}(_react2.default.Component);\n\nvar BAR_CHART_WIDTH = 800;\n\nvar BAR_CHART_PROPERTIES = {\n type: 'bar',\n barSpacing: '0',\n height: '80px',\n barColor: '#747F96',\n zeroColor: '#8997B3',\n chartRangeMin: 0,\n tooltipClassname: 'sparkline-tooltip',\n tooltipFormat: 'Task {{offset:offset}} - {{value}}',\n disableHiddenCheck: true\n};\n\nvar HISTOGRAM_WIDTH = 175;\n\nvar HISTOGRAM_PROPERTIES = {\n type: 'bar',\n barSpacing: '0',\n height: '80px',\n barColor: '#747F96',\n zeroColor: '#747F96',\n zeroAxis: true,\n chartRangeMin: 0,\n tooltipClassname: 'sparkline-tooltip',\n tooltipFormat: '{{offset:offset}} -- {{value}} tasks',\n disableHiddenCheck: true\n};\n\nvar StageSummary = function (_React$Component2) {\n _inherits(StageSummary, _React$Component2);\n\n function StageSummary(props) {\n _classCallCheck(this, StageSummary);\n\n var _this2 = _possibleConstructorReturn(this, (StageSummary.__proto__ || Object.getPrototypeOf(StageSummary)).call(this, props));\n\n _this2.state = {\n expanded: false,\n lastRender: null\n };\n return _this2;\n }\n\n _createClass(StageSummary, [{\n key: \"getExpandedIcon\",\n value: function getExpandedIcon() {\n return this.state.expanded ? \"glyphicon-chevron-up\" : \"glyphicon-chevron-down\";\n }\n }, {\n key: \"getExpandedStyle\",\n value: function getExpandedStyle() {\n return this.state.expanded ? {} : { display: \"none\" };\n }\n }, {\n key: \"toggleExpanded\",\n value: function toggleExpanded() {\n this.setState({\n expanded: !this.state.expanded\n });\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n var stage = this.props.stage;\n var numTasks = stage.tasks.length;\n\n // sort the x-axis\n stage.tasks.sort(function (taskA, taskB) {\n return (0, _utils.getTaskNumber)(taskA.taskStatus.taskId) - (0, _utils.getTaskNumber)(taskB.taskStatus.taskId);\n });\n\n var scheduledTimes = stage.tasks.map(function (task) {\n return (0, _utils.parseDuration)(task.stats.totalScheduledTime);\n });\n var cpuTimes = stage.tasks.map(function (task) {\n return (0, _utils.parseDuration)(task.stats.totalCpuTime);\n });\n\n // prevent multiple calls to componentDidUpdate (resulting from calls to setState or otherwise) within the refresh interval from re-rendering sparklines/charts\n if (this.state.lastRender === null || Date.now() - this.state.lastRender >= 1000) {\n var renderTimestamp = Date.now();\n var stageId = (0, _utils.getStageNumber)(stage.stageId);\n\n StageSummary.renderHistogram('#scheduled-time-histogram-' + stageId, scheduledTimes, _utils.formatDuration);\n StageSummary.renderHistogram('#cpu-time-histogram-' + stageId, cpuTimes, _utils.formatDuration);\n\n if (this.state.expanded) {\n // this needs to be a string otherwise it will also be passed to numberFormatter\n var tooltipValueLookups = { 'offset': {} };\n for (var i = 0; i < numTasks; i++) {\n tooltipValueLookups['offset'][i] = (0, _utils.getStageNumber)(stage.stageId) + \".\" + i;\n }\n\n var stageBarChartProperties = $.extend({}, BAR_CHART_PROPERTIES, { barWidth: BAR_CHART_WIDTH / numTasks, tooltipValueLookups: tooltipValueLookups });\n\n $('#scheduled-time-bar-chart-' + stageId).sparkline(scheduledTimes, $.extend({}, stageBarChartProperties, { numberFormatter: _utils.formatDuration }));\n $('#cpu-time-bar-chart-' + stageId).sparkline(cpuTimes, $.extend({}, stageBarChartProperties, { numberFormatter: _utils.formatDuration }));\n }\n\n this.setState({\n lastRender: renderTimestamp\n });\n }\n }\n }, {\n key: \"render\",\n value: function render() {\n var stage = this.props.stage;\n if (stage === undefined || !stage.hasOwnProperty('plan')) {\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Information about this stage is unavailable.\"\n )\n );\n }\n\n var totalBufferedBytes = stage.tasks.map(function (task) {\n return task.outputBuffers.totalBufferedBytes;\n }).reduce(function (a, b) {\n return a + b;\n }, 0);\n\n var stageId = (0, _utils.getStageNumber)(stage.stageId);\n\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-id\" },\n _react2.default.createElement(\n \"div\",\n { className: \"stage-state-color\", style: { borderLeftColor: (0, _utils.getStageStateColor)(stage) } },\n stageId\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"table single-stage-table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table stage-table-time\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-stat-header\" },\n \"Time\"\n ),\n _react2.default.createElement(\"th\", null)\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Scheduled\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.stageStats.totalScheduledTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Blocked\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.stageStats.totalBlockedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"CPU\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.stageStats.totalCpuTime\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table stage-table-memory\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-stat-header\" },\n \"Memory\"\n ),\n _react2.default.createElement(\"th\", null)\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Cumulative\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n (0, _utils.formatDataSizeBytes)(stage.stageStats.cumulativeUserMemory / 1000)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Current\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.stageStats.userMemoryReservation\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Buffers\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n (0, _utils.formatDataSize)(totalBufferedBytes)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Peak\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.stageStats.peakUserMemoryReservation\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table stage-table-tasks\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-stat-header\" },\n \"Tasks\"\n ),\n _react2.default.createElement(\"th\", null)\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Pending\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.tasks.filter(function (task) {\n return task.taskStatus.state === \"PLANNED\";\n }).length\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Running\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.tasks.filter(function (task) {\n return task.taskStatus.state === \"RUNNING\";\n }).length\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Blocked\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.tasks.filter(function (task) {\n return task.stats.fullyBlocked;\n }).length\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Total\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.tasks.length\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table histogram-table\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-chart-header\" },\n \"Scheduled Time Skew\"\n )\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"histogram-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"histogram\", id: \"scheduled-time-histogram-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table histogram-table\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-chart-header\" },\n \"CPU Time Skew\"\n )\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"histogram-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"histogram\", id: \"cpu-time-histogram-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"expand-charts-container\" },\n _react2.default.createElement(\n \"a\",\n { onClick: this.toggleExpanded.bind(this), className: \"expand-charts-button\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon \" + this.getExpandedIcon(), style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\", title: \"More\" })\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { style: this.getExpandedStyle() },\n _react2.default.createElement(\n \"td\",\n { colSpan: \"6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"expanded-chart\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title expanded-chart-title\" },\n \"Task Scheduled Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"bar-chart-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"bar-chart\", id: \"scheduled-time-bar-chart-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { style: this.getExpandedStyle() },\n _react2.default.createElement(\n \"td\",\n { colSpan: \"6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"expanded-chart\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title expanded-chart-title\" },\n \"Task CPU Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"bar-chart-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"bar-chart\", id: \"cpu-time-bar-chart-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n )\n )\n )\n )\n )\n );\n }\n }], [{\n key: \"renderHistogram\",\n value: function renderHistogram(histogramId, inputData, numberFormatter) {\n var numBuckets = Math.min(HISTOGRAM_WIDTH, Math.sqrt(inputData.length));\n var dataMin = Math.min.apply(null, inputData);\n var dataMax = Math.max.apply(null, inputData);\n var bucketSize = (dataMax - dataMin) / numBuckets;\n\n var histogramData = [];\n if (bucketSize === 0) {\n histogramData = [inputData.length];\n } else {\n for (var i = 0; i < numBuckets + 1; i++) {\n histogramData.push(0);\n }\n\n for (var _i in inputData) {\n var dataPoint = inputData[_i];\n var bucket = Math.floor((dataPoint - dataMin) / bucketSize);\n histogramData[bucket] = histogramData[bucket] + 1;\n }\n }\n\n var tooltipValueLookups = { 'offset': {} };\n for (var _i2 = 0; _i2 < histogramData.length; _i2++) {\n tooltipValueLookups['offset'][_i2] = numberFormatter(dataMin + _i2 * bucketSize) + \"-\" + numberFormatter(dataMin + (_i2 + 1) * bucketSize);\n }\n\n var stageHistogramProperties = $.extend({}, HISTOGRAM_PROPERTIES, { barWidth: HISTOGRAM_WIDTH / histogramData.length, tooltipValueLookups: tooltipValueLookups });\n $(histogramId).sparkline(histogramData, stageHistogramProperties);\n }\n }]);\n\n return StageSummary;\n}(_react2.default.Component);\n\nvar StageList = function (_React$Component3) {\n _inherits(StageList, _React$Component3);\n\n function StageList() {\n _classCallCheck(this, StageList);\n\n return _possibleConstructorReturn(this, (StageList.__proto__ || Object.getPrototypeOf(StageList)).apply(this, arguments));\n }\n\n _createClass(StageList, [{\n key: \"getStages\",\n value: function getStages(stage) {\n if (stage === undefined || !stage.hasOwnProperty('subStages')) {\n return [];\n }\n\n return [].concat.apply(stage, stage.subStages.map(this.getStages, this));\n }\n }, {\n key: \"render\",\n value: function render() {\n var stages = this.getStages(this.props.outputStage);\n\n if (stages === undefined || stages.length === 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n \"No stage information available.\"\n )\n );\n }\n\n var renderedStages = stages.map(function (stage) {\n return _react2.default.createElement(StageSummary, { key: stage.stageId, stage: stage });\n });\n\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\", id: \"stage-list\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n renderedStages\n )\n )\n )\n );\n }\n }]);\n\n return StageList;\n}(_react2.default.Component);\n\nvar SMALL_SPARKLINE_PROPERTIES = {\n width: '100%',\n height: '57px',\n fillColor: '#3F4552',\n lineColor: '#747F96',\n spotColor: '#1EDCFF',\n tooltipClassname: 'sparkline-tooltip',\n disableHiddenCheck: true\n};\n\nvar TASK_FILTER = {\n NONE: {\n text: \"None\",\n predicate: function predicate() {\n return false;\n }\n },\n ALL: {\n text: \"All\",\n predicate: function predicate() {\n return true;\n }\n },\n PLANNED: {\n text: \"Planned\",\n predicate: function predicate(state) {\n return state === 'PLANNED';\n }\n },\n RUNNING: {\n text: \"Running\",\n predicate: function predicate(state) {\n return state === 'RUNNING';\n }\n },\n FINISHED: {\n text: \"Finished\",\n predicate: function predicate(state) {\n return state === 'FINISHED';\n }\n },\n FAILED: {\n text: \"Aborted/Canceled/Failed\",\n predicate: function predicate(state) {\n return state === 'FAILED' || state === 'ABORTED' || state === 'CANCELED';\n }\n }\n};\n\nvar QueryDetail = exports.QueryDetail = function (_React$Component4) {\n _inherits(QueryDetail, _React$Component4);\n\n function QueryDetail(props) {\n _classCallCheck(this, QueryDetail);\n\n var _this4 = _possibleConstructorReturn(this, (QueryDetail.__proto__ || Object.getPrototypeOf(QueryDetail)).call(this, props));\n\n _this4.state = {\n query: null,\n lastSnapshotStages: null,\n lastSnapshotTasks: null,\n\n lastScheduledTime: 0,\n lastCpuTime: 0,\n lastRowInput: 0,\n lastByteInput: 0,\n\n scheduledTimeRate: [],\n cpuTimeRate: [],\n rowInputRate: [],\n byteInputRate: [],\n\n reservedMemory: [],\n\n initialized: false,\n ended: false,\n\n lastRefresh: null,\n lastRender: null,\n\n stageRefresh: true,\n taskRefresh: true,\n\n taskFilter: TASK_FILTER.NONE\n };\n\n _this4.refreshLoop = _this4.refreshLoop.bind(_this4);\n return _this4;\n }\n\n _createClass(QueryDetail, [{\n key: \"resetTimer\",\n value: function resetTimer() {\n clearTimeout(this.timeoutId);\n // stop refreshing when query finishes or fails\n if (this.state.query === null || !this.state.ended) {\n // task.info-update-interval is set to 3 seconds by default\n this.timeoutId = setTimeout(this.refreshLoop, 3000);\n }\n }\n }, {\n key: \"refreshLoop\",\n value: function refreshLoop() {\n var _this5 = this;\n\n clearTimeout(this.timeoutId); // to stop multiple series of refreshLoop from going on simultaneously\n var queryId = (0, _utils.getFirstParameter)(window.location.search);\n $.get('/v1/query/' + queryId, function (query) {\n var lastSnapshotStages = this.state.lastSnapshotStage;\n if (this.state.stageRefresh) {\n lastSnapshotStages = query.outputStage;\n }\n var lastSnapshotTasks = this.state.lastSnapshotTasks;\n if (this.state.taskRefresh) {\n lastSnapshotTasks = query.outputStage;\n }\n\n var lastRefresh = this.state.lastRefresh;\n var lastScheduledTime = this.state.lastScheduledTime;\n var lastCpuTime = this.state.lastCpuTime;\n var lastRowInput = this.state.lastRowInput;\n var lastByteInput = this.state.lastByteInput;\n var alreadyEnded = this.state.ended;\n var nowMillis = Date.now();\n\n this.setState({\n query: query,\n lastSnapshotStage: lastSnapshotStages,\n lastSnapshotTasks: lastSnapshotTasks,\n\n lastScheduledTime: (0, _utils.parseDuration)(query.queryStats.totalScheduledTime),\n lastCpuTime: (0, _utils.parseDuration)(query.queryStats.totalCpuTime),\n lastRowInput: query.queryStats.processedInputPositions,\n lastByteInput: (0, _utils.parseDataSize)(query.queryStats.processedInputDataSize),\n\n initialized: true,\n ended: query.finalQueryInfo,\n\n lastRefresh: nowMillis\n });\n\n // i.e. don't show sparklines if we've already decided not to update or if we don't have one previous measurement\n if (alreadyEnded || lastRefresh === null && query.state === \"RUNNING\") {\n this.resetTimer();\n return;\n }\n\n if (lastRefresh === null) {\n lastRefresh = nowMillis - (0, _utils.parseDuration)(query.queryStats.elapsedTime);\n }\n\n var elapsedSecsSinceLastRefresh = (nowMillis - lastRefresh) / 1000.0;\n if (elapsedSecsSinceLastRefresh >= 0) {\n var currentScheduledTimeRate = ((0, _utils.parseDuration)(query.queryStats.totalScheduledTime) - lastScheduledTime) / (elapsedSecsSinceLastRefresh * 1000);\n var currentCpuTimeRate = ((0, _utils.parseDuration)(query.queryStats.totalCpuTime) - lastCpuTime) / (elapsedSecsSinceLastRefresh * 1000);\n var currentRowInputRate = (query.queryStats.processedInputPositions - lastRowInput) / elapsedSecsSinceLastRefresh;\n var currentByteInputRate = ((0, _utils.parseDataSize)(query.queryStats.processedInputDataSize) - lastByteInput) / elapsedSecsSinceLastRefresh;\n this.setState({\n scheduledTimeRate: (0, _utils.addToHistory)(currentScheduledTimeRate, this.state.scheduledTimeRate),\n cpuTimeRate: (0, _utils.addToHistory)(currentCpuTimeRate, this.state.cpuTimeRate),\n rowInputRate: (0, _utils.addToHistory)(currentRowInputRate, this.state.rowInputRate),\n byteInputRate: (0, _utils.addToHistory)(currentByteInputRate, this.state.byteInputRate),\n reservedMemory: (0, _utils.addToHistory)((0, _utils.parseDataSize)(query.queryStats.userMemoryReservation), this.state.reservedMemory)\n });\n }\n this.resetTimer();\n }.bind(this)).error(function () {\n _this5.setState({\n initialized: true\n });\n _this5.resetTimer();\n });\n }\n }, {\n key: \"handleTaskRefreshClick\",\n value: function handleTaskRefreshClick() {\n if (this.state.taskRefresh) {\n this.setState({\n taskRefresh: false,\n lastSnapshotTasks: this.state.query.outputStage\n });\n } else {\n this.setState({\n taskRefresh: true\n });\n }\n }\n }, {\n key: \"renderTaskRefreshButton\",\n value: function renderTaskRefreshButton() {\n if (this.state.taskRefresh) {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleTaskRefreshClick.bind(this) },\n \"Auto-Refresh: On\"\n );\n } else {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleTaskRefreshClick.bind(this) },\n \"Auto-Refresh: Off\"\n );\n }\n }\n }, {\n key: \"handleStageRefreshClick\",\n value: function handleStageRefreshClick() {\n if (this.state.stageRefresh) {\n this.setState({\n stageRefresh: false,\n lastSnapshotStages: this.state.query.outputStage\n });\n } else {\n this.setState({\n stageRefresh: true\n });\n }\n }\n }, {\n key: \"renderStageRefreshButton\",\n value: function renderStageRefreshButton() {\n if (this.state.stageRefresh) {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleStageRefreshClick.bind(this) },\n \"Auto-Refresh: On\"\n );\n } else {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleStageRefreshClick.bind(this) },\n \"Auto-Refresh: Off\"\n );\n }\n }\n }, {\n key: \"renderTaskFilterListItem\",\n value: function renderTaskFilterListItem(taskFilter) {\n return _react2.default.createElement(\n \"li\",\n null,\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: this.state.taskFilter === taskFilter ? \"selected\" : \"\", onClick: this.handleTaskFilterClick.bind(this, taskFilter) },\n taskFilter.text\n )\n );\n }\n }, {\n key: \"handleTaskFilterClick\",\n value: function handleTaskFilterClick(filter, event) {\n this.setState({\n taskFilter: filter\n });\n event.preventDefault();\n }\n }, {\n key: \"getTasksFromStage\",\n value: function getTasksFromStage(stage) {\n if (stage === undefined || !stage.hasOwnProperty('subStages') || !stage.hasOwnProperty('tasks')) {\n return [];\n }\n\n return [].concat.apply(stage.tasks, stage.subStages.map(this.getTasksFromStage, this));\n }\n }, {\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.refreshLoop();\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n // prevent multiple calls to componentDidUpdate (resulting from calls to setState or otherwise) within the refresh interval from re-rendering sparklines/charts\n if (this.state.lastRender === null || Date.now() - this.state.lastRender >= 1000) {\n var renderTimestamp = Date.now();\n $('#scheduled-time-rate-sparkline').sparkline(this.state.scheduledTimeRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, {\n chartRangeMin: 0,\n numberFormatter: _utils.precisionRound\n }));\n $('#cpu-time-rate-sparkline').sparkline(this.state.cpuTimeRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { chartRangeMin: 0, numberFormatter: _utils.precisionRound }));\n $('#row-input-rate-sparkline').sparkline(this.state.rowInputRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatCount }));\n $('#byte-input-rate-sparkline').sparkline(this.state.byteInputRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatDataSize }));\n $('#reserved-memory-sparkline').sparkline(this.state.reservedMemory, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatDataSize }));\n\n if (this.state.lastRender === null) {\n $('#query').each(function (i, block) {\n hljs.highlightBlock(block);\n });\n }\n\n this.setState({\n lastRender: renderTimestamp\n });\n }\n\n $('[data-toggle=\"tooltip\"]').tooltip();\n new Clipboard('.copy-button');\n }\n }, {\n key: \"renderTasks\",\n value: function renderTasks() {\n var _this6 = this;\n\n if (this.state.lastSnapshotTasks === null) {\n return;\n }\n\n var tasks = [];\n if (this.state.taskFilter !== TASK_FILTER.NONE) {\n tasks = this.getTasksFromStage(this.state.lastSnapshotTasks).filter(function (task) {\n return _this6.state.taskFilter.predicate(task.taskStatus.state);\n }, this);\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Tasks\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"header-inline-links\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"input-group-btn text-right\" },\n _react2.default.createElement(\n \"button\",\n { type: \"button\", className: \"btn btn-default dropdown-toggle pull-right text-right\", \"data-toggle\": \"dropdown\", \"aria-haspopup\": \"true\",\n \"aria-expanded\": \"false\" },\n \"Show: \",\n this.state.taskFilter.text,\n \" \",\n _react2.default.createElement(\"span\", { className: \"caret\" })\n ),\n _react2.default.createElement(\n \"ul\",\n { className: \"dropdown-menu\" },\n this.renderTaskFilterListItem(TASK_FILTER.NONE),\n this.renderTaskFilterListItem(TASK_FILTER.ALL),\n this.renderTaskFilterListItem(TASK_FILTER.PLANNED),\n this.renderTaskFilterListItem(TASK_FILTER.RUNNING),\n this.renderTaskFilterListItem(TASK_FILTER.FINISHED),\n this.renderTaskFilterListItem(TASK_FILTER.FAILED)\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n \"\\xA0\\xA0\",\n this.renderTaskRefreshButton()\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(TaskList, { key: this.state.query.queryId, tasks: tasks })\n )\n )\n );\n }\n }, {\n key: \"renderStages\",\n value: function renderStages() {\n if (this.state.lastSnapshotStage === null) {\n return;\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-9\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Stages\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-3\" },\n _react2.default.createElement(\n \"table\",\n { className: \"header-inline-links\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n this.renderStageRefreshButton()\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(StageList, { key: this.state.query.queryId, outputStage: this.state.lastSnapshotStage })\n )\n )\n );\n }\n }, {\n key: \"renderSessionProperties\",\n value: function renderSessionProperties() {\n var query = this.state.query;\n\n var properties = [];\n for (var property in query.session.systemProperties) {\n if (query.session.systemProperties.hasOwnProperty(property)) {\n properties.push(_react2.default.createElement(\n \"span\",\n null,\n \"- \",\n property + \"=\" + query.session.systemProperties[property],\n \" \",\n _react2.default.createElement(\"br\", null)\n ));\n }\n }\n\n for (var catalog in query.session.catalogProperties) {\n if (query.session.catalogProperties.hasOwnProperty(catalog)) {\n for (var _property in query.session.catalogProperties[catalog]) {\n if (query.session.catalogProperties[catalog].hasOwnProperty(_property)) {\n properties.push(_react2.default.createElement(\n \"span\",\n null,\n \"- \",\n catalog + \".\" + _property + \"=\" + query.session.catalogProperties[catalog][_property],\n \" \",\n _react2.default.createElement(\"br\", null)\n ));\n }\n }\n }\n }\n\n return properties;\n }\n }, {\n key: \"renderResourceEstimates\",\n value: function renderResourceEstimates() {\n var query = this.state.query;\n var estimates = query.session.resourceEstimates;\n var renderedEstimates = [];\n\n for (var resource in estimates) {\n if (estimates.hasOwnProperty(resource)) {\n var upperChars = resource.match(/([A-Z])/g) || [];\n var snakeCased = resource;\n for (var i = 0, n = upperChars.length; i < n; i++) {\n snakeCased = snakeCased.replace(new RegExp(upperChars[i]), '_' + upperChars[i].toLowerCase());\n }\n\n renderedEstimates.push(_react2.default.createElement(\n \"span\",\n null,\n \"- \",\n snakeCased + \"=\" + query.session.resourceEstimates[resource],\n \" \",\n _react2.default.createElement(\"br\", null)\n ));\n }\n }\n\n return renderedEstimates;\n }\n }, {\n key: \"renderWarningInfo\",\n value: function renderWarningInfo() {\n var query = this.state.query;\n if (query.warnings.length > 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Warnings\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\", id: \"warnings-table\" },\n query.warnings.map(function (warning) {\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n warning.warningCode.name\n ),\n _react2.default.createElement(\n \"td\",\n null,\n warning.message\n )\n );\n })\n )\n )\n );\n } else {\n return null;\n }\n }\n }, {\n key: \"renderFailureInfo\",\n value: function renderFailureInfo() {\n var query = this.state.query;\n if (query.failureInfo) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Error Information\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Error Type\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.errorType\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Error Code\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.errorCode.name + \" (\" + this.state.query.errorCode.code + \")\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Stack Trace\",\n _react2.default.createElement(\n \"a\",\n { className: \"btn copy-button\", \"data-clipboard-target\": \"#stack-trace\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n _react2.default.createElement(\n \"pre\",\n { id: \"stack-trace\" },\n QueryDetail.formatStackTrace(query.failureInfo)\n )\n )\n )\n )\n )\n )\n );\n } else {\n return \"\";\n }\n }\n }, {\n key: \"render\",\n value: function render() {\n var query = this.state.query;\n\n if (query === null || this.state.initialized === false) {\n var label = _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n );\n if (this.state.initialized) {\n label = \"Query not found\";\n }\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n label\n )\n )\n );\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(_QueryHeader.QueryHeader, { query: query }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Session\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"User\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"query-user\" },\n query.session.user\n ),\n \"\\xA0\\xA0\",\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: \"copy-button\", \"data-clipboard-target\": \"#query-user\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Principal\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n query.session.principal\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Source\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n query.session.source\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Catalog\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.catalog\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Schema\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.schema\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Client Address\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.remoteUserAddress\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Client Tags\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.clientTags.join(\", \")\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Session Properties\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n this.renderSessionProperties()\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Resource Estimates\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n this.renderResourceEstimates()\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Execution\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Resource Group\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n query.resourceGroupId ? query.resourceGroupId.join(\".\") : \"n/a\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Submission Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatShortDateTime)(new Date(query.queryStats.createTime))\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Completion Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.endTime ? (0, _utils.formatShortDateTime)(new Date(query.queryStats.endTime)) : \"\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Elapsed Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.elapsedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Queued Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.queuedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Planning Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.totalPlanningTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Execution Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.executionTime\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Resource Utilization Summary\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"CPU Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.totalCpuTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Scheduled Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.totalScheduledTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Blocked Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.totalBlockedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.processedInputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.processedInputDataSize\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Raw Input Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.rawInputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Raw Input Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.rawInputDataSize\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Peak User Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.peakUserMemoryReservation\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Peak Total Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.peakTotalMemoryReservation\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Memory Pool\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.memoryPool\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Cumulative User Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatDataSizeBytes)(query.queryStats.cumulativeUserMemory / 1000.0) + \" seconds\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Output Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.outputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Output Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.outputDataSize\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Written Output Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.writtenOutputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Written Output Logical Data Size\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.writtenOutputLogicalDataSize\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Written Output Physical Data Size\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.writtenOutputPhysicalDataSize\n )\n ),\n (0, _utils.parseDataSize)(query.queryStats.spilledDataSize) > 0 && _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Spilled Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.spilledDataSize\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Timeline\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Parallelism\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"cpu-time-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.cpuTimeRate[this.state.cpuTimeRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Scheduled Time/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"scheduled-time-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.scheduledTimeRate[this.state.scheduledTimeRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Rows/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"row-input-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.rowInputRate[this.state.rowInputRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Bytes/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"byte-input-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatDataSize)(this.state.byteInputRate[this.state.byteInputRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Memory Utilization\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"reserved-memory-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatDataSize)(this.state.reservedMemory[this.state.reservedMemory.length - 1])\n )\n )\n )\n )\n )\n )\n )\n ),\n this.renderWarningInfo(),\n this.renderFailureInfo(),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Query\",\n _react2.default.createElement(\n \"a\",\n { className: \"btn copy-button\", \"data-clipboard-target\": \"#query-text\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n ),\n _react2.default.createElement(\n \"pre\",\n { id: \"query\" },\n _react2.default.createElement(\n \"code\",\n { className: \"lang-sql\", id: \"query-text\" },\n query.query\n )\n )\n )\n ),\n this.renderStages(),\n this.renderTasks()\n );\n }\n }], [{\n key: \"formatStackTrace\",\n value: function formatStackTrace(info) {\n return QueryDetail.formatStackTraceHelper(info, [], \"\", \"\");\n }\n }, {\n key: \"formatStackTraceHelper\",\n value: function formatStackTraceHelper(info, parentStack, prefix, linePrefix) {\n var s = linePrefix + prefix + QueryDetail.failureInfoToString(info) + \"\\n\";\n\n if (info.stack) {\n var sharedStackFrames = 0;\n if (parentStack !== null) {\n sharedStackFrames = QueryDetail.countSharedStackFrames(info.stack, parentStack);\n }\n\n for (var i = 0; i < info.stack.length - sharedStackFrames; i++) {\n s += linePrefix + \"\\tat \" + info.stack[i] + \"\\n\";\n }\n if (sharedStackFrames !== 0) {\n s += linePrefix + \"\\t... \" + sharedStackFrames + \" more\" + \"\\n\";\n }\n }\n\n if (info.suppressed) {\n for (var _i3 = 0; _i3 < info.suppressed.length; _i3++) {\n s += QueryDetail.formatStackTraceHelper(info.suppressed[_i3], info.stack, \"Suppressed: \", linePrefix + \"\\t\");\n }\n }\n\n if (info.cause) {\n s += QueryDetail.formatStackTraceHelper(info.cause, info.stack, \"Caused by: \", linePrefix);\n }\n\n return s;\n }\n }, {\n key: \"countSharedStackFrames\",\n value: function countSharedStackFrames(stack, parentStack) {\n var n = 0;\n var minStackLength = Math.min(stack.length, parentStack.length);\n while (n < minStackLength && stack[stack.length - 1 - n] === parentStack[parentStack.length - 1 - n]) {\n n++;\n }\n return n;\n }\n }, {\n key: \"failureInfoToString\",\n value: function failureInfoToString(t) {\n return t.message !== null ? t.type + \": \" + t.message : t.type;\n }\n }]);\n\n return QueryDetail;\n}(_react2.default.Component);\n\n//# sourceURL=webpack:///./components/QueryDetail.jsx?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.QueryDetail = undefined;\n\nvar _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/index.js\");\n\nvar _react2 = _interopRequireDefault(_react);\n\nvar _reactable = __webpack_require__(/*! reactable */ \"./node_modules/reactable/lib/reactable.js\");\n\nvar _reactable2 = _interopRequireDefault(_reactable);\n\nvar _utils = __webpack_require__(/*! ../utils */ \"./utils.js\");\n\nvar _QueryHeader = __webpack_require__(/*! ./QueryHeader */ \"./components/QueryHeader.jsx\");\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return call && (typeof call === \"object\" || typeof call === \"function\") ? call : self; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function, not \" + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } /*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nvar Table = _reactable2.default.Table,\n Thead = _reactable2.default.Thead,\n Th = _reactable2.default.Th,\n Tr = _reactable2.default.Tr,\n Td = _reactable2.default.Td;\n\nvar TaskList = function (_React$Component) {\n _inherits(TaskList, _React$Component);\n\n function TaskList() {\n _classCallCheck(this, TaskList);\n\n return _possibleConstructorReturn(this, (TaskList.__proto__ || Object.getPrototypeOf(TaskList)).apply(this, arguments));\n }\n\n _createClass(TaskList, [{\n key: \"render\",\n value: function render() {\n var tasks = this.props.tasks;\n\n if (tasks === undefined || tasks.length === 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"No threads in the selected group\"\n )\n )\n );\n }\n\n var showPortNumbers = TaskList.showPortNumbers(tasks);\n\n var renderedTasks = tasks.map(function (task) {\n var elapsedTime = (0, _utils.parseDuration)(task.stats.elapsedTime);\n if (elapsedTime === 0) {\n elapsedTime = Date.now() - Date.parse(task.stats.createTime);\n }\n\n return _react2.default.createElement(\n Tr,\n { key: task.taskStatus.taskId },\n _react2.default.createElement(\n Td,\n { column: \"id\", value: task.taskStatus.taskId },\n _react2.default.createElement(\n \"a\",\n { href: task.taskStatus.self + \"?pretty\" },\n (0, _utils.getTaskIdSuffix)(task.taskStatus.taskId)\n )\n ),\n _react2.default.createElement(\n Td,\n { column: \"host\", value: (0, _utils.getHostname)(task.taskStatus.self) },\n _react2.default.createElement(\n \"a\",\n { href: \"worker.html?\" + task.taskStatus.nodeId, className: \"font-light\", target: \"_blank\" },\n showPortNumbers ? (0, _utils.getHostAndPort)(task.taskStatus.self) : (0, _utils.getHostname)(task.taskStatus.self)\n )\n ),\n _react2.default.createElement(\n Td,\n { column: \"state\", value: TaskList.formatState(task.taskStatus.state, task.stats.fullyBlocked) },\n TaskList.formatState(task.taskStatus.state, task.stats.fullyBlocked)\n ),\n _react2.default.createElement(\n Td,\n { column: \"rows\", value: task.stats.rawInputPositions },\n (0, _utils.formatCount)(task.stats.rawInputPositions)\n ),\n _react2.default.createElement(\n Td,\n { column: \"rowsSec\", value: (0, _utils.computeRate)(task.stats.rawInputPositions, elapsedTime) },\n (0, _utils.formatCount)((0, _utils.computeRate)(task.stats.rawInputPositions, elapsedTime))\n ),\n _react2.default.createElement(\n Td,\n { column: \"bytes\", value: (0, _utils.parseDataSize)(task.stats.rawInputDataSize) },\n (0, _utils.formatDataSizeBytes)((0, _utils.parseDataSize)(task.stats.rawInputDataSize))\n ),\n _react2.default.createElement(\n Td,\n { column: \"bytesSec\", value: (0, _utils.computeRate)((0, _utils.parseDataSize)(task.stats.rawInputDataSize), elapsedTime) },\n (0, _utils.formatDataSizeBytes)((0, _utils.computeRate)((0, _utils.parseDataSize)(task.stats.rawInputDataSize), elapsedTime))\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsPending\", value: task.stats.queuedDrivers },\n task.stats.queuedDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsRunning\", value: task.stats.runningDrivers },\n task.stats.runningDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsBlocked\", value: task.stats.blockedDrivers },\n task.stats.blockedDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"splitsDone\", value: task.stats.completedDrivers },\n task.stats.completedDrivers\n ),\n _react2.default.createElement(\n Td,\n { column: \"elapsedTime\", value: (0, _utils.parseDuration)(task.stats.elapsedTime) },\n task.stats.elapsedTime\n ),\n _react2.default.createElement(\n Td,\n { column: \"cpuTime\", value: (0, _utils.parseDuration)(task.stats.totalCpuTime) },\n task.stats.totalCpuTime\n ),\n _react2.default.createElement(\n Td,\n { column: \"bufferedBytes\", value: task.outputBuffers.totalBufferedBytes },\n (0, _utils.formatDataSizeBytes)(task.outputBuffers.totalBufferedBytes)\n )\n );\n });\n\n return _react2.default.createElement(\n Table,\n { id: \"tasks\", className: \"table table-striped sortable\", sortable: [{\n column: 'id',\n sortFunction: TaskList.compareTaskId\n }, 'host', 'state', 'splitsPending', 'splitsRunning', 'splitsBlocked', 'splitsDone', 'rows', 'rowsSec', 'bytes', 'bytesSec', 'elapsedTime', 'cpuTime', 'bufferedBytes'],\n defaultSort: { column: 'id', direction: 'asc' } },\n _react2.default.createElement(\n Thead,\n null,\n _react2.default.createElement(\n Th,\n { column: \"id\" },\n \"ID\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"host\" },\n \"Host\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"state\" },\n \"State\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsPending\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-pause\", style: _utils.GLYPHICON_HIGHLIGHT,\n \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Pending splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsRunning\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-play\", style: _utils.GLYPHICON_HIGHLIGHT,\n \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Running splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsBlocked\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-bookmark\",\n style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\",\n \"data-placement\": \"top\",\n title: \"Blocked splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"splitsDone\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-ok\", style: _utils.GLYPHICON_HIGHLIGHT,\n \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Completed splits\" })\n ),\n _react2.default.createElement(\n Th,\n { column: \"rows\" },\n \"Rows\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"rowsSec\" },\n \"Rows/s\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"bytes\" },\n \"Bytes\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"bytesSec\" },\n \"Bytes/s\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"elapsedTime\" },\n \"Elapsed\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"cpuTime\" },\n \"CPU Time\"\n ),\n _react2.default.createElement(\n Th,\n { column: \"bufferedBytes\" },\n \"Buffered\"\n )\n ),\n renderedTasks\n );\n }\n }], [{\n key: \"removeQueryId\",\n value: function removeQueryId(id) {\n var pos = id.indexOf('.');\n if (pos !== -1) {\n return id.substring(pos + 1);\n }\n return id;\n }\n }, {\n key: \"compareTaskId\",\n value: function compareTaskId(taskA, taskB) {\n var taskIdArrA = TaskList.removeQueryId(taskA).split(\".\");\n var taskIdArrB = TaskList.removeQueryId(taskB).split(\".\");\n\n if (taskIdArrA.length > taskIdArrB.length) {\n return 1;\n }\n for (var i = 0; i < taskIdArrA.length; i++) {\n var anum = Number.parseInt(taskIdArrA[i]);\n var bnum = Number.parseInt(taskIdArrB[i]);\n if (anum !== bnum) {\n return anum > bnum ? 1 : -1;\n }\n }\n\n return 0;\n }\n }, {\n key: \"showPortNumbers\",\n value: function showPortNumbers(tasks) {\n // check if any host has multiple port numbers\n var hostToPortNumber = {};\n for (var i = 0; i < tasks.length; i++) {\n var taskUri = tasks[i].taskStatus.self;\n var hostname = (0, _utils.getHostname)(taskUri);\n var port = (0, _utils.getPort)(taskUri);\n if (hostname in hostToPortNumber && hostToPortNumber[hostname] !== port) {\n return true;\n }\n hostToPortNumber[hostname] = port;\n }\n\n return false;\n }\n }, {\n key: \"formatState\",\n value: function formatState(state, fullyBlocked) {\n if (fullyBlocked && state === \"RUNNING\") {\n return \"BLOCKED\";\n } else {\n return state;\n }\n }\n }]);\n\n return TaskList;\n}(_react2.default.Component);\n\nvar BAR_CHART_WIDTH = 800;\n\nvar BAR_CHART_PROPERTIES = {\n type: 'bar',\n barSpacing: '0',\n height: '80px',\n barColor: '#747F96',\n zeroColor: '#8997B3',\n chartRangeMin: 0,\n tooltipClassname: 'sparkline-tooltip',\n tooltipFormat: 'Task {{offset:offset}} - {{value}}',\n disableHiddenCheck: true\n};\n\nvar HISTOGRAM_WIDTH = 175;\n\nvar HISTOGRAM_PROPERTIES = {\n type: 'bar',\n barSpacing: '0',\n height: '80px',\n barColor: '#747F96',\n zeroColor: '#747F96',\n zeroAxis: true,\n chartRangeMin: 0,\n tooltipClassname: 'sparkline-tooltip',\n tooltipFormat: '{{offset:offset}} -- {{value}} tasks',\n disableHiddenCheck: true\n};\n\nvar StageSummary = function (_React$Component2) {\n _inherits(StageSummary, _React$Component2);\n\n function StageSummary(props) {\n _classCallCheck(this, StageSummary);\n\n var _this2 = _possibleConstructorReturn(this, (StageSummary.__proto__ || Object.getPrototypeOf(StageSummary)).call(this, props));\n\n _this2.state = {\n expanded: false,\n lastRender: null,\n taskFilter: TASK_FILTER.ALL\n };\n return _this2;\n }\n\n _createClass(StageSummary, [{\n key: \"getExpandedIcon\",\n value: function getExpandedIcon() {\n return this.state.expanded ? \"glyphicon-chevron-up\" : \"glyphicon-chevron-down\";\n }\n }, {\n key: \"getExpandedStyle\",\n value: function getExpandedStyle() {\n return this.state.expanded ? {} : { display: \"none\" };\n }\n }, {\n key: \"toggleExpanded\",\n value: function toggleExpanded() {\n this.setState({\n expanded: !this.state.expanded\n });\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n var stage = this.props.stage;\n var numTasks = stage.latestAttemptExecutionInfo.tasks.length;\n\n // sort the x-axis\n stage.latestAttemptExecutionInfo.tasks.sort(function (taskA, taskB) {\n return (0, _utils.getTaskNumber)(taskA.taskStatus.taskId) - (0, _utils.getTaskNumber)(taskB.taskStatus.taskId);\n });\n\n var scheduledTimes = stage.latestAttemptExecutionInfo.tasks.map(function (task) {\n return (0, _utils.parseDuration)(task.stats.totalScheduledTime);\n });\n var cpuTimes = stage.latestAttemptExecutionInfo.tasks.map(function (task) {\n return (0, _utils.parseDuration)(task.stats.totalCpuTime);\n });\n\n // prevent multiple calls to componentDidUpdate (resulting from calls to setState or otherwise) within the refresh interval from re-rendering sparklines/charts\n if (this.state.lastRender === null || Date.now() - this.state.lastRender >= 1000) {\n var renderTimestamp = Date.now();\n var stageId = (0, _utils.getStageNumber)(stage.stageId);\n\n StageSummary.renderHistogram('#scheduled-time-histogram-' + stageId, scheduledTimes, _utils.formatDuration);\n StageSummary.renderHistogram('#cpu-time-histogram-' + stageId, cpuTimes, _utils.formatDuration);\n\n if (this.state.expanded) {\n // this needs to be a string otherwise it will also be passed to numberFormatter\n var tooltipValueLookups = { 'offset': {} };\n for (var i = 0; i < numTasks; i++) {\n tooltipValueLookups['offset'][i] = (0, _utils.getStageNumber)(stage.stageId) + \".\" + i;\n }\n\n var stageBarChartProperties = $.extend({}, BAR_CHART_PROPERTIES, { barWidth: BAR_CHART_WIDTH / numTasks, tooltipValueLookups: tooltipValueLookups });\n\n $('#scheduled-time-bar-chart-' + stageId).sparkline(scheduledTimes, $.extend({}, stageBarChartProperties, { numberFormatter: _utils.formatDuration }));\n $('#cpu-time-bar-chart-' + stageId).sparkline(cpuTimes, $.extend({}, stageBarChartProperties, { numberFormatter: _utils.formatDuration }));\n }\n\n this.setState({\n lastRender: renderTimestamp\n });\n }\n }\n }, {\n key: \"renderStageExecutionAttemptsTasks\",\n value: function renderStageExecutionAttemptsTasks(attempts) {\n var _this3 = this;\n\n return attempts.map(function (attempt) {\n return _this3.renderTaskList(attempt.tasks);\n });\n }\n }, {\n key: \"renderTaskList\",\n value: function renderTaskList(tasks) {\n var _this4 = this;\n\n tasks = this.state.expanded ? tasks : [];\n tasks = tasks.filter(function (task) {\n return _this4.state.taskFilter.predicate(task.taskStatus.state);\n }, this);\n return _react2.default.createElement(\n \"tr\",\n { style: this.getExpandedStyle() },\n _react2.default.createElement(\n \"td\",\n { colSpan: \"6\" },\n _react2.default.createElement(TaskList, { tasks: tasks })\n )\n );\n }\n }, {\n key: \"renderTaskFilterListItem\",\n value: function renderTaskFilterListItem(taskFilter) {\n return _react2.default.createElement(\n \"li\",\n null,\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: this.state.taskFilter === taskFilter ? \"selected\" : \"\",\n onClick: this.handleTaskFilterClick.bind(this, taskFilter) },\n taskFilter.text\n )\n );\n }\n }, {\n key: \"handleTaskFilterClick\",\n value: function handleTaskFilterClick(filter, event) {\n this.setState({\n taskFilter: filter\n });\n event.preventDefault();\n }\n }, {\n key: \"renderTaskFilter\",\n value: function renderTaskFilter() {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Tasks\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"header-inline-links\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"input-group-btn text-right\" },\n _react2.default.createElement(\n \"button\",\n { type: \"button\", className: \"btn btn-default dropdown-toggle pull-right text-right\",\n \"data-toggle\": \"dropdown\", \"aria-haspopup\": \"true\",\n \"aria-expanded\": \"false\" },\n \"Show: \",\n this.state.taskFilter.text,\n \" \",\n _react2.default.createElement(\"span\", { className: \"caret\" })\n ),\n _react2.default.createElement(\n \"ul\",\n { className: \"dropdown-menu\" },\n this.renderTaskFilterListItem(TASK_FILTER.ALL),\n this.renderTaskFilterListItem(TASK_FILTER.PLANNED),\n this.renderTaskFilterListItem(TASK_FILTER.RUNNING),\n this.renderTaskFilterListItem(TASK_FILTER.FINISHED),\n this.renderTaskFilterListItem(TASK_FILTER.FAILED)\n )\n )\n )\n )\n )\n )\n )\n );\n }\n }, {\n key: \"render\",\n value: function render() {\n var stage = this.props.stage;\n if (stage === undefined || !stage.hasOwnProperty('plan')) {\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Information about this stage is unavailable.\"\n )\n );\n }\n\n var totalBufferedBytes = stage.latestAttemptExecutionInfo.tasks.map(function (task) {\n return task.outputBuffers.totalBufferedBytes;\n }).reduce(function (a, b) {\n return a + b;\n }, 0);\n\n var stageId = (0, _utils.getStageNumber)(stage.stageId);\n\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-id\" },\n _react2.default.createElement(\n \"div\",\n { className: \"stage-state-color\", style: { borderLeftColor: (0, _utils.getStageStateColor)(stage) } },\n stageId\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"table single-stage-table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table stage-table-time\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-stat-header\" },\n \"Time\"\n ),\n _react2.default.createElement(\"th\", null)\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Scheduled\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.latestAttemptExecutionInfo.stats.totalScheduledTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Blocked\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.latestAttemptExecutionInfo.stats.totalBlockedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"CPU\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.latestAttemptExecutionInfo.stats.totalCpuTime\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table stage-table-memory\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-stat-header\" },\n \"Memory\"\n ),\n _react2.default.createElement(\"th\", null)\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Cumulative\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n (0, _utils.formatDataSizeBytes)(stage.latestAttemptExecutionInfo.stats.cumulativeUserMemory / 1000)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Current\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.latestAttemptExecutionInfo.stats.userMemoryReservation\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Buffers\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n (0, _utils.formatDataSize)(totalBufferedBytes)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Peak\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.latestAttemptExecutionInfo.stats.peakUserMemoryReservation\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table stage-table-tasks\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-stat-header\" },\n \"Tasks\"\n ),\n _react2.default.createElement(\"th\", null)\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Pending\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.latestAttemptExecutionInfo.tasks.filter(function (task) {\n return task.taskStatus.state === \"PLANNED\";\n }).length\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Running\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.latestAttemptExecutionInfo.tasks.filter(function (task) {\n return task.taskStatus.state === \"RUNNING\";\n }).length\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Blocked\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.latestAttemptExecutionInfo.tasks.filter(function (task) {\n return task.stats.fullyBlocked;\n }).length\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title\" },\n \"Total\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-text\" },\n stage.latestAttemptExecutionInfo.tasks.length\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table histogram-table\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-chart-header\" },\n \"Scheduled Time Skew\"\n )\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"histogram-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"histogram\", id: \"scheduled-time-histogram-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"stage-table histogram-table\" },\n _react2.default.createElement(\n \"thead\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"th\",\n { className: \"stage-table-stat-title stage-table-chart-header\" },\n \"CPU Time Skew\"\n )\n )\n ),\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"histogram-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"histogram\", id: \"cpu-time-histogram-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"expand-charts-container\" },\n _react2.default.createElement(\n \"a\",\n { onClick: this.toggleExpanded.bind(this), className: \"expand-charts-button\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon \" + this.getExpandedIcon(), style: _utils.GLYPHICON_HIGHLIGHT, \"data-toggle\": \"tooltip\", \"data-placement\": \"top\", title: \"More\" })\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { style: this.getExpandedStyle() },\n _react2.default.createElement(\n \"td\",\n { colSpan: \"6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"expanded-chart\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title expanded-chart-title\" },\n \"Task Scheduled Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"bar-chart-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"bar-chart\", id: \"scheduled-time-bar-chart-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { style: this.getExpandedStyle() },\n _react2.default.createElement(\n \"td\",\n { colSpan: \"6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"expanded-chart\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"stage-table-stat-title expanded-chart-title\" },\n \"Task CPU Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"bar-chart-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"bar-chart\", id: \"cpu-time-bar-chart-\" + stageId },\n _react2.default.createElement(\"div\", { className: \"loader\" })\n )\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { style: this.getExpandedStyle() },\n _react2.default.createElement(\n \"td\",\n { colSpan: \"6\" },\n this.renderTaskFilter()\n )\n ),\n this.renderStageExecutionAttemptsTasks([stage.latestAttemptExecutionInfo]),\n this.renderStageExecutionAttemptsTasks(stage.previousAttemptsExecutionInfos)\n )\n )\n )\n );\n }\n }], [{\n key: \"renderHistogram\",\n value: function renderHistogram(histogramId, inputData, numberFormatter) {\n var numBuckets = Math.min(HISTOGRAM_WIDTH, Math.sqrt(inputData.length));\n var dataMin = Math.min.apply(null, inputData);\n var dataMax = Math.max.apply(null, inputData);\n var bucketSize = (dataMax - dataMin) / numBuckets;\n\n var histogramData = [];\n if (bucketSize === 0) {\n histogramData = [inputData.length];\n } else {\n for (var i = 0; i < numBuckets + 1; i++) {\n histogramData.push(0);\n }\n\n for (var _i in inputData) {\n var dataPoint = inputData[_i];\n var bucket = Math.floor((dataPoint - dataMin) / bucketSize);\n histogramData[bucket] = histogramData[bucket] + 1;\n }\n }\n\n var tooltipValueLookups = { 'offset': {} };\n for (var _i2 = 0; _i2 < histogramData.length; _i2++) {\n tooltipValueLookups['offset'][_i2] = numberFormatter(dataMin + _i2 * bucketSize) + \"-\" + numberFormatter(dataMin + (_i2 + 1) * bucketSize);\n }\n\n var stageHistogramProperties = $.extend({}, HISTOGRAM_PROPERTIES, { barWidth: HISTOGRAM_WIDTH / histogramData.length, tooltipValueLookups: tooltipValueLookups });\n $(histogramId).sparkline(histogramData, stageHistogramProperties);\n }\n }]);\n\n return StageSummary;\n}(_react2.default.Component);\n\nvar StageList = function (_React$Component3) {\n _inherits(StageList, _React$Component3);\n\n function StageList() {\n _classCallCheck(this, StageList);\n\n return _possibleConstructorReturn(this, (StageList.__proto__ || Object.getPrototypeOf(StageList)).apply(this, arguments));\n }\n\n _createClass(StageList, [{\n key: \"getStages\",\n value: function getStages(stage) {\n if (stage === undefined || !stage.hasOwnProperty('subStages')) {\n return [];\n }\n\n return [].concat.apply(stage, stage.subStages.map(this.getStages, this));\n }\n }, {\n key: \"render\",\n value: function render() {\n var stages = this.getStages(this.props.outputStage);\n\n if (stages === undefined || stages.length === 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n \"No stage information available.\"\n )\n );\n }\n\n var renderedStages = stages.map(function (stage) {\n return _react2.default.createElement(StageSummary, { key: stage.stageId, stage: stage });\n });\n\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\", id: \"stage-list\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n renderedStages\n )\n )\n )\n );\n }\n }]);\n\n return StageList;\n}(_react2.default.Component);\n\nvar SMALL_SPARKLINE_PROPERTIES = {\n width: '100%',\n height: '57px',\n fillColor: '#3F4552',\n lineColor: '#747F96',\n spotColor: '#1EDCFF',\n tooltipClassname: 'sparkline-tooltip',\n disableHiddenCheck: true\n};\n\nvar TASK_FILTER = {\n ALL: {\n text: \"All\",\n predicate: function predicate() {\n return true;\n }\n },\n PLANNED: {\n text: \"Planned\",\n predicate: function predicate(state) {\n return state === 'PLANNED';\n }\n },\n RUNNING: {\n text: \"Running\",\n predicate: function predicate(state) {\n return state === 'RUNNING';\n }\n },\n FINISHED: {\n text: \"Finished\",\n predicate: function predicate(state) {\n return state === 'FINISHED';\n }\n },\n FAILED: {\n text: \"Aborted/Canceled/Failed\",\n predicate: function predicate(state) {\n return state === 'FAILED' || state === 'ABORTED' || state === 'CANCELED';\n }\n }\n};\n\nvar QueryDetail = exports.QueryDetail = function (_React$Component4) {\n _inherits(QueryDetail, _React$Component4);\n\n function QueryDetail(props) {\n _classCallCheck(this, QueryDetail);\n\n var _this6 = _possibleConstructorReturn(this, (QueryDetail.__proto__ || Object.getPrototypeOf(QueryDetail)).call(this, props));\n\n _this6.state = {\n query: null,\n lastSnapshotStages: null,\n\n lastScheduledTime: 0,\n lastCpuTime: 0,\n lastRowInput: 0,\n lastByteInput: 0,\n\n scheduledTimeRate: [],\n cpuTimeRate: [],\n rowInputRate: [],\n byteInputRate: [],\n\n reservedMemory: [],\n\n initialized: false,\n ended: false,\n\n lastRefresh: null,\n lastRender: null,\n\n stageRefresh: true\n };\n\n _this6.refreshLoop = _this6.refreshLoop.bind(_this6);\n return _this6;\n }\n\n _createClass(QueryDetail, [{\n key: \"resetTimer\",\n value: function resetTimer() {\n clearTimeout(this.timeoutId);\n // stop refreshing when query finishes or fails\n if (this.state.query === null || !this.state.ended) {\n // task.info-update-interval is set to 3 seconds by default\n this.timeoutId = setTimeout(this.refreshLoop, 3000);\n }\n }\n }, {\n key: \"refreshLoop\",\n value: function refreshLoop() {\n var _this7 = this;\n\n clearTimeout(this.timeoutId); // to stop multiple series of refreshLoop from going on simultaneously\n var queryId = (0, _utils.getFirstParameter)(window.location.search);\n $.get('/v1/query/' + queryId, function (query) {\n var lastSnapshotStages = this.state.lastSnapshotStage;\n if (this.state.stageRefresh) {\n lastSnapshotStages = query.outputStage;\n }\n\n var lastRefresh = this.state.lastRefresh;\n var lastScheduledTime = this.state.lastScheduledTime;\n var lastCpuTime = this.state.lastCpuTime;\n var lastRowInput = this.state.lastRowInput;\n var lastByteInput = this.state.lastByteInput;\n var alreadyEnded = this.state.ended;\n var nowMillis = Date.now();\n\n this.setState({\n query: query,\n lastSnapshotStage: lastSnapshotStages,\n\n lastScheduledTime: (0, _utils.parseDuration)(query.queryStats.totalScheduledTime),\n lastCpuTime: (0, _utils.parseDuration)(query.queryStats.totalCpuTime),\n lastRowInput: query.queryStats.processedInputPositions,\n lastByteInput: (0, _utils.parseDataSize)(query.queryStats.processedInputDataSize),\n\n initialized: true,\n ended: query.finalQueryInfo,\n\n lastRefresh: nowMillis\n });\n\n // i.e. don't show sparklines if we've already decided not to update or if we don't have one previous measurement\n if (alreadyEnded || lastRefresh === null && query.state === \"RUNNING\") {\n this.resetTimer();\n return;\n }\n\n if (lastRefresh === null) {\n lastRefresh = nowMillis - (0, _utils.parseDuration)(query.queryStats.elapsedTime);\n }\n\n var elapsedSecsSinceLastRefresh = (nowMillis - lastRefresh) / 1000.0;\n if (elapsedSecsSinceLastRefresh >= 0) {\n var currentScheduledTimeRate = ((0, _utils.parseDuration)(query.queryStats.totalScheduledTime) - lastScheduledTime) / (elapsedSecsSinceLastRefresh * 1000);\n var currentCpuTimeRate = ((0, _utils.parseDuration)(query.queryStats.totalCpuTime) - lastCpuTime) / (elapsedSecsSinceLastRefresh * 1000);\n var currentRowInputRate = (query.queryStats.processedInputPositions - lastRowInput) / elapsedSecsSinceLastRefresh;\n var currentByteInputRate = ((0, _utils.parseDataSize)(query.queryStats.processedInputDataSize) - lastByteInput) / elapsedSecsSinceLastRefresh;\n this.setState({\n scheduledTimeRate: (0, _utils.addToHistory)(currentScheduledTimeRate, this.state.scheduledTimeRate),\n cpuTimeRate: (0, _utils.addToHistory)(currentCpuTimeRate, this.state.cpuTimeRate),\n rowInputRate: (0, _utils.addToHistory)(currentRowInputRate, this.state.rowInputRate),\n byteInputRate: (0, _utils.addToHistory)(currentByteInputRate, this.state.byteInputRate),\n reservedMemory: (0, _utils.addToHistory)((0, _utils.parseDataSize)(query.queryStats.userMemoryReservation), this.state.reservedMemory)\n });\n }\n this.resetTimer();\n }.bind(this)).error(function () {\n _this7.setState({\n initialized: true\n });\n _this7.resetTimer();\n });\n }\n }, {\n key: \"handleStageRefreshClick\",\n value: function handleStageRefreshClick() {\n if (this.state.stageRefresh) {\n this.setState({\n stageRefresh: false,\n lastSnapshotStages: this.state.query.outputStage\n });\n } else {\n this.setState({\n stageRefresh: true\n });\n }\n }\n }, {\n key: \"renderStageRefreshButton\",\n value: function renderStageRefreshButton() {\n if (this.state.stageRefresh) {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleStageRefreshClick.bind(this) },\n \"Auto-Refresh: On\"\n );\n } else {\n return _react2.default.createElement(\n \"button\",\n { className: \"btn btn-info live-button\", onClick: this.handleStageRefreshClick.bind(this) },\n \"Auto-Refresh: Off\"\n );\n }\n }\n }, {\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.refreshLoop();\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n // prevent multiple calls to componentDidUpdate (resulting from calls to setState or otherwise) within the refresh interval from re-rendering sparklines/charts\n if (this.state.lastRender === null || Date.now() - this.state.lastRender >= 1000) {\n var renderTimestamp = Date.now();\n $('#scheduled-time-rate-sparkline').sparkline(this.state.scheduledTimeRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, {\n chartRangeMin: 0,\n numberFormatter: _utils.precisionRound\n }));\n $('#cpu-time-rate-sparkline').sparkline(this.state.cpuTimeRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { chartRangeMin: 0, numberFormatter: _utils.precisionRound }));\n $('#row-input-rate-sparkline').sparkline(this.state.rowInputRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatCount }));\n $('#byte-input-rate-sparkline').sparkline(this.state.byteInputRate, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatDataSize }));\n $('#reserved-memory-sparkline').sparkline(this.state.reservedMemory, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { numberFormatter: _utils.formatDataSize }));\n\n if (this.state.lastRender === null) {\n $('#query').each(function (i, block) {\n hljs.highlightBlock(block);\n });\n }\n\n this.setState({\n lastRender: renderTimestamp\n });\n }\n\n $('[data-toggle=\"tooltip\"]').tooltip();\n new Clipboard('.copy-button');\n }\n }, {\n key: \"renderStages\",\n value: function renderStages() {\n if (this.state.lastSnapshotStage === null) {\n return;\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-9\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Stages\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-3\" },\n _react2.default.createElement(\n \"table\",\n { className: \"header-inline-links\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n this.renderStageRefreshButton()\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(StageList, { key: this.state.query.queryId, outputStage: this.state.lastSnapshotStage })\n )\n )\n );\n }\n }, {\n key: \"renderSessionProperties\",\n value: function renderSessionProperties() {\n var query = this.state.query;\n\n var properties = [];\n for (var property in query.session.systemProperties) {\n if (query.session.systemProperties.hasOwnProperty(property)) {\n properties.push(_react2.default.createElement(\n \"span\",\n null,\n \"- \",\n property + \"=\" + query.session.systemProperties[property],\n \" \",\n _react2.default.createElement(\"br\", null)\n ));\n }\n }\n\n for (var catalog in query.session.catalogProperties) {\n if (query.session.catalogProperties.hasOwnProperty(catalog)) {\n for (var _property in query.session.catalogProperties[catalog]) {\n if (query.session.catalogProperties[catalog].hasOwnProperty(_property)) {\n properties.push(_react2.default.createElement(\n \"span\",\n null,\n \"- \",\n catalog + \".\" + _property + \"=\" + query.session.catalogProperties[catalog][_property],\n \" \",\n _react2.default.createElement(\"br\", null)\n ));\n }\n }\n }\n }\n\n return properties;\n }\n }, {\n key: \"renderResourceEstimates\",\n value: function renderResourceEstimates() {\n var query = this.state.query;\n var estimates = query.session.resourceEstimates;\n var renderedEstimates = [];\n\n for (var resource in estimates) {\n if (estimates.hasOwnProperty(resource)) {\n var upperChars = resource.match(/([A-Z])/g) || [];\n var snakeCased = resource;\n for (var i = 0, n = upperChars.length; i < n; i++) {\n snakeCased = snakeCased.replace(new RegExp(upperChars[i]), '_' + upperChars[i].toLowerCase());\n }\n\n renderedEstimates.push(_react2.default.createElement(\n \"span\",\n null,\n \"- \",\n snakeCased + \"=\" + query.session.resourceEstimates[resource],\n \" \",\n _react2.default.createElement(\"br\", null)\n ));\n }\n }\n\n return renderedEstimates;\n }\n }, {\n key: \"renderWarningInfo\",\n value: function renderWarningInfo() {\n var query = this.state.query;\n if (query.warnings.length > 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Warnings\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\", id: \"warnings-table\" },\n query.warnings.map(function (warning) {\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n warning.warningCode.name\n ),\n _react2.default.createElement(\n \"td\",\n null,\n warning.message\n )\n );\n })\n )\n )\n );\n } else {\n return null;\n }\n }\n }, {\n key: \"renderFailureInfo\",\n value: function renderFailureInfo() {\n var query = this.state.query;\n if (query.failureInfo) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Error Information\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Error Type\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.errorType\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Error Code\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.errorCode.name + \" (\" + this.state.query.errorCode.code + \")\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Stack Trace\",\n _react2.default.createElement(\n \"a\",\n { className: \"btn copy-button\", \"data-clipboard-target\": \"#stack-trace\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n _react2.default.createElement(\n \"pre\",\n { id: \"stack-trace\" },\n QueryDetail.formatStackTrace(query.failureInfo)\n )\n )\n )\n )\n )\n )\n );\n } else {\n return \"\";\n }\n }\n }, {\n key: \"render\",\n value: function render() {\n var query = this.state.query;\n\n if (query === null || this.state.initialized === false) {\n var label = _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n );\n if (this.state.initialized) {\n label = \"Query not found\";\n }\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n label\n )\n )\n );\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(_QueryHeader.QueryHeader, { query: query }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Session\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"User\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"query-user\" },\n query.session.user\n ),\n \"\\xA0\\xA0\",\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: \"copy-button\", \"data-clipboard-target\": \"#query-user\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Principal\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n query.session.principal\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Source\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n query.session.source\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Catalog\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.catalog\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Schema\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.schema\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Client Address\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.remoteUserAddress\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Client Tags\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.session.clientTags.join(\", \")\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Session Properties\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n this.renderSessionProperties()\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Resource Estimates\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n this.renderResourceEstimates()\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Execution\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Resource Group\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n query.resourceGroupId ? query.resourceGroupId.join(\".\") : \"n/a\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Submission Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatShortDateTime)(new Date(query.queryStats.createTime))\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Completion Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.endTime ? (0, _utils.formatShortDateTime)(new Date(query.queryStats.endTime)) : \"\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Elapsed Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.elapsedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Queued Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.queuedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Planning Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.totalPlanningTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Execution Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.executionTime\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Resource Utilization Summary\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"CPU Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.totalCpuTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Scheduled Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.totalScheduledTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Blocked Time\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.totalBlockedTime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.processedInputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.processedInputDataSize\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Raw Input Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.rawInputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Raw Input Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.rawInputDataSize\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Peak User Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.peakUserMemoryReservation\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Peak Total Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.peakTotalMemoryReservation\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Memory Pool\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.memoryPool\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Cumulative User Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatDataSizeBytes)(query.queryStats.cumulativeUserMemory / 1000.0) + \" seconds\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Output Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.outputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Output Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.outputDataSize\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Written Output Rows\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n (0, _utils.formatCount)(query.queryStats.writtenOutputPositions)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Written Output Logical Data Size\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.writtenOutputLogicalDataSize\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Written Output Physical Data Size\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.writtenOutputPhysicalDataSize\n )\n ),\n (0, _utils.parseDataSize)(query.queryStats.spilledDataSize) > 0 && _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Spilled Data\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text\" },\n query.queryStats.spilledDataSize\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Timeline\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Parallelism\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"cpu-time-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.cpuTimeRate[this.state.cpuTimeRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Scheduled Time/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"scheduled-time-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.scheduledTimeRate[this.state.scheduledTimeRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Rows/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"row-input-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.rowInputRate[this.state.rowInputRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Input Bytes/s\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"byte-input-rate-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatDataSize)(this.state.byteInputRate[this.state.byteInputRate.length - 1])\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Memory Utilization\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"reserved-memory-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatDataSize)(this.state.reservedMemory[this.state.reservedMemory.length - 1])\n )\n )\n )\n )\n )\n )\n )\n ),\n this.renderWarningInfo(),\n this.renderFailureInfo(),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Query\",\n _react2.default.createElement(\n \"a\",\n { className: \"btn copy-button\", \"data-clipboard-target\": \"#query-text\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\", title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", \"aria-hidden\": \"true\", alt: \"Copy to clipboard\" })\n )\n ),\n _react2.default.createElement(\n \"pre\",\n { id: \"query\" },\n _react2.default.createElement(\n \"code\",\n { className: \"lang-sql\", id: \"query-text\" },\n query.query\n )\n )\n )\n ),\n this.renderStages()\n );\n }\n }], [{\n key: \"formatStackTrace\",\n value: function formatStackTrace(info) {\n return QueryDetail.formatStackTraceHelper(info, [], \"\", \"\");\n }\n }, {\n key: \"formatStackTraceHelper\",\n value: function formatStackTraceHelper(info, parentStack, prefix, linePrefix) {\n var s = linePrefix + prefix + QueryDetail.failureInfoToString(info) + \"\\n\";\n\n if (info.stack) {\n var sharedStackFrames = 0;\n if (parentStack !== null) {\n sharedStackFrames = QueryDetail.countSharedStackFrames(info.stack, parentStack);\n }\n\n for (var i = 0; i < info.stack.length - sharedStackFrames; i++) {\n s += linePrefix + \"\\tat \" + info.stack[i] + \"\\n\";\n }\n if (sharedStackFrames !== 0) {\n s += linePrefix + \"\\t... \" + sharedStackFrames + \" more\" + \"\\n\";\n }\n }\n\n if (info.suppressed) {\n for (var _i3 = 0; _i3 < info.suppressed.length; _i3++) {\n s += QueryDetail.formatStackTraceHelper(info.suppressed[_i3], info.stack, \"Suppressed: \", linePrefix + \"\\t\");\n }\n }\n\n if (info.cause) {\n s += QueryDetail.formatStackTraceHelper(info.cause, info.stack, \"Caused by: \", linePrefix);\n }\n\n return s;\n }\n }, {\n key: \"countSharedStackFrames\",\n value: function countSharedStackFrames(stack, parentStack) {\n var n = 0;\n var minStackLength = Math.min(stack.length, parentStack.length);\n while (n < minStackLength && stack[stack.length - 1 - n] === parentStack[parentStack.length - 1 - n]) {\n n++;\n }\n return n;\n }\n }, {\n key: \"failureInfoToString\",\n value: function failureInfoToString(t) {\n return t.message !== null ? t.type + \": \" + t.message : t.type;\n }\n }]);\n\n return QueryDetail;\n}(_react2.default.Component);\n\n//# sourceURL=webpack:///./components/QueryDetail.jsx?"); /***/ }), diff --git a/presto-main/src/main/resources/webapp/dist/stage.js b/presto-main/src/main/resources/webapp/dist/stage.js index bee49490d6827..6db65c843d387 100644 --- a/presto-main/src/main/resources/webapp/dist/stage.js +++ b/presto-main/src/main/resources/webapp/dist/stage.js @@ -118,7 +118,7 @@ eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n}); /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.StageDetail = undefined;\n\nvar _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/index.js\");\n\nvar _react2 = _interopRequireDefault(_react);\n\nvar _reactDom = __webpack_require__(/*! react-dom */ \"./node_modules/react-dom/index.js\");\n\nvar _reactDom2 = _interopRequireDefault(_reactDom);\n\nvar _server = __webpack_require__(/*! react-dom/server */ \"./node_modules/react-dom/server.browser.js\");\n\nvar _server2 = _interopRequireDefault(_server);\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nvar _utils = __webpack_require__(/*! ../utils */ \"./utils.js\");\n\nvar _QueryHeader = __webpack_require__(/*! ./QueryHeader */ \"./components/QueryHeader.jsx\");\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return call && (typeof call === \"object\" || typeof call === \"function\") ? call : self; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function, not \" + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } /*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nfunction getTotalWallTime(operator) {\n return (0, _utils.parseDuration)(operator.addInputWall) + (0, _utils.parseDuration)(operator.getOutputWall) + (0, _utils.parseDuration)(operator.finishWall) + (0, _utils.parseDuration)(operator.blockedWall);\n}\n\nvar OperatorSummary = function (_React$Component) {\n _inherits(OperatorSummary, _React$Component);\n\n function OperatorSummary() {\n _classCallCheck(this, OperatorSummary);\n\n return _possibleConstructorReturn(this, (OperatorSummary.__proto__ || Object.getPrototypeOf(OperatorSummary)).apply(this, arguments));\n }\n\n _createClass(OperatorSummary, [{\n key: \"render\",\n value: function render() {\n var operator = this.props.operator;\n\n var totalWallTime = (0, _utils.parseDuration)(operator.addInputWall) + (0, _utils.parseDuration)(operator.getOutputWall) + (0, _utils.parseDuration)(operator.finishWall) + (0, _utils.parseDuration)(operator.blockedWall);\n\n var rowInputRate = totalWallTime === 0 ? 0 : 1.0 * operator.inputPositions / (totalWallTime / 1000.0);\n var byteInputRate = totalWallTime === 0 ? 0 : 1.0 * (0, _utils.parseDataSize)(operator.inputDataSize) / (totalWallTime / 1000.0);\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"highlight-row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"header-row\" },\n operator.operatorType\n ),\n _react2.default.createElement(\n \"div\",\n null,\n (0, _utils.formatCount)(rowInputRate) + \" rows/s (\" + (0, _utils.formatDataSize)(byteInputRate) + \"/s)\"\n )\n ),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Output\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatCount)(operator.outputPositions) + \" rows (\" + operator.outputDataSize + \")\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Drivers\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n operator.totalDrivers\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Wall Time\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatDuration)(totalWallTime)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Blocked\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatDuration)((0, _utils.parseDuration)(operator.blockedWall))\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Input\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatCount)(operator.inputPositions) + \" rows (\" + operator.inputDataSize + \")\"\n )\n )\n )\n )\n );\n }\n }]);\n\n return OperatorSummary;\n}(_react2.default.Component);\n\nvar BAR_CHART_PROPERTIES = {\n type: 'bar',\n barSpacing: '0',\n height: '80px',\n barColor: '#747F96',\n zeroColor: '#8997B3',\n tooltipClassname: 'sparkline-tooltip',\n tooltipFormat: 'Task {{offset:offset}} - {{value}}',\n disableHiddenCheck: true\n};\n\nvar OperatorStatistic = function (_React$Component2) {\n _inherits(OperatorStatistic, _React$Component2);\n\n function OperatorStatistic() {\n _classCallCheck(this, OperatorStatistic);\n\n return _possibleConstructorReturn(this, (OperatorStatistic.__proto__ || Object.getPrototypeOf(OperatorStatistic)).apply(this, arguments));\n }\n\n _createClass(OperatorStatistic, [{\n key: \"componentDidMount\",\n value: function componentDidMount() {\n var operators = this.props.operators;\n var statistic = operators.map(this.props.supplier);\n var numTasks = operators.length;\n\n var tooltipValueLookups = { 'offset': {} };\n for (var i = 0; i < numTasks; i++) {\n tooltipValueLookups['offset'][i] = \"\" + i;\n }\n\n var stageBarChartProperties = $.extend({}, BAR_CHART_PROPERTIES, { barWidth: 800 / numTasks, tooltipValueLookups: tooltipValueLookups });\n $('#' + this.props.id).sparkline(statistic, $.extend({}, stageBarChartProperties, { numberFormatter: this.props.renderer }));\n }\n }, {\n key: \"render\",\n value: function render() {\n return _react2.default.createElement(\n \"div\",\n { className: \"row operator-statistic\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-2 italic-uppercase operator-statistic-title\" },\n this.props.name\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-10\" },\n _react2.default.createElement(\"span\", { className: \"bar-chart\", id: this.props.id })\n )\n );\n }\n }]);\n\n return OperatorStatistic;\n}(_react2.default.Component);\n\nvar OperatorDetail = function (_React$Component3) {\n _inherits(OperatorDetail, _React$Component3);\n\n function OperatorDetail(props) {\n _classCallCheck(this, OperatorDetail);\n\n var _this3 = _possibleConstructorReturn(this, (OperatorDetail.__proto__ || Object.getPrototypeOf(OperatorDetail)).call(this, props));\n\n _this3.state = {\n selectedStatistics: _this3.getInitialStatistics()\n };\n return _this3;\n }\n\n _createClass(OperatorDetail, [{\n key: \"getInitialStatistics\",\n value: function getInitialStatistics() {\n return [{\n name: \"Total Wall Time\",\n id: \"totalWallTime\",\n supplier: getTotalWallTime,\n renderer: _utils.formatDuration\n }, {\n name: \"Input Rows\",\n id: \"inputPositions\",\n supplier: function supplier(operator) {\n return operator.inputPositions;\n },\n renderer: _utils.formatCount\n }, {\n name: \"Input Data Size\",\n id: \"inputDataSize\",\n supplier: function supplier(operator) {\n return (0, _utils.parseDataSize)(operator.inputDataSize);\n },\n renderer: _utils.formatDataSize\n }, {\n name: \"Output Rows\",\n id: \"outputPositions\",\n supplier: function supplier(operator) {\n return operator.outputPositions;\n },\n renderer: _utils.formatCount\n }, {\n name: \"Output Data Size\",\n id: \"outputDataSize\",\n supplier: function supplier(operator) {\n return (0, _utils.parseDataSize)(operator.outputDataSize);\n },\n renderer: _utils.formatDataSize\n }];\n }\n }, {\n key: \"getOperatorTasks\",\n value: function getOperatorTasks() {\n // sort the x-axis\n var tasks = this.props.tasks.sort(function (taskA, taskB) {\n return (0, _utils.getTaskNumber)(taskA.taskStatus.taskId) - (0, _utils.getTaskNumber)(taskB.taskStatus.taskId);\n });\n\n var operatorSummary = this.props.operator;\n\n var operatorTasks = [];\n tasks.forEach(function (task) {\n task.stats.pipelines.forEach(function (pipeline) {\n if (pipeline.pipelineId === operatorSummary.pipelineId) {\n pipeline.operatorSummaries.forEach(function (operator) {\n if (operatorSummary.operatorId === operator.operatorId) {\n operatorTasks.push(operator);\n }\n });\n }\n });\n });\n\n return operatorTasks;\n }\n }, {\n key: \"render\",\n value: function render() {\n var operator = this.props.operator;\n var operatorTasks = this.getOperatorTasks();\n var totalWallTime = getTotalWallTime(operator);\n\n var rowInputRate = totalWallTime === 0 ? 0 : 1.0 * operator.inputPositions / totalWallTime;\n var byteInputRate = totalWallTime === 0 ? 0 : 1.0 * (0, _utils.parseDataSize)(operator.inputDataSize) / (totalWallTime / 1000.0);\n\n var rowOutputRate = totalWallTime === 0 ? 0 : 1.0 * operator.outputPositions / totalWallTime;\n var byteOutputRate = totalWallTime === 0 ? 0 : 1.0 * (0, _utils.parseDataSize)(operator.outputDataSize) / (totalWallTime / 1000.0);\n\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"div\",\n { className: \"modal-header\" },\n _react2.default.createElement(\n \"button\",\n { type: \"button\", className: \"close\", \"data-dismiss\": \"modal\", \"aria-label\": \"Close\" },\n _react2.default.createElement(\n \"span\",\n { \"aria-hidden\": \"true\" },\n \"\\xD7\"\n )\n ),\n _react2.default.createElement(\n \"h3\",\n null,\n _react2.default.createElement(\n \"small\",\n null,\n \"Pipeline \",\n operator.pipelineId\n ),\n _react2.default.createElement(\"br\", null),\n operator.operatorType\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Input\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatCount)(operator.inputPositions) + \" rows (\" + operator.inputDataSize + \")\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Input Rate\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatCount)(rowInputRate) + \" rows/s (\" + (0, _utils.formatDataSize)(byteInputRate) + \"/s)\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Output\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatCount)(operator.outputPositions) + \" rows (\" + operator.outputDataSize + \")\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Output Rate\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatCount)(rowOutputRate) + \" rows/s (\" + (0, _utils.formatDataSize)(byteOutputRate) + \"/s)\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Wall Time\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatDuration)(totalWallTime)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Blocked\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatDuration)((0, _utils.parseDuration)(operator.blockedWall))\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Drivers\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n operator.totalDrivers\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Tasks\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n operatorTasks.length\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row font-white\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-2 italic-uppercase\" },\n _react2.default.createElement(\n \"strong\",\n null,\n \"Statistic\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-10 italic-uppercase\" },\n _react2.default.createElement(\n \"strong\",\n null,\n \"Tasks\"\n )\n )\n ),\n this.state.selectedStatistics.map(function (statistic) {\n return _react2.default.createElement(OperatorStatistic, {\n key: statistic.id,\n id: statistic.id,\n name: statistic.name,\n supplier: statistic.supplier,\n renderer: statistic.renderer,\n operators: operatorTasks });\n }.bind(this)),\n _react2.default.createElement(\"p\", null),\n _react2.default.createElement(\"p\", null)\n )\n );\n }\n }]);\n\n return OperatorDetail;\n}(_react2.default.Component);\n\nvar StageOperatorGraph = function (_React$Component4) {\n _inherits(StageOperatorGraph, _React$Component4);\n\n function StageOperatorGraph() {\n _classCallCheck(this, StageOperatorGraph);\n\n return _possibleConstructorReturn(this, (StageOperatorGraph.__proto__ || Object.getPrototypeOf(StageOperatorGraph)).apply(this, arguments));\n }\n\n _createClass(StageOperatorGraph, [{\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.updateD3Graph();\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n this.updateD3Graph();\n }\n }, {\n key: \"handleOperatorClick\",\n value: function handleOperatorClick(operatorCssId) {\n $('#operator-detail-modal').modal();\n\n var pipelineId = parseInt(operatorCssId.split('-')[1]);\n var operatorId = parseInt(operatorCssId.split('-')[2]);\n var stage = this.props.stage;\n\n var operatorStageSummary = null;\n var operatorSummaries = stage.stageStats.operatorSummaries;\n for (var i = 0; i < operatorSummaries.length; i++) {\n if (operatorSummaries[i].pipelineId === pipelineId && operatorSummaries[i].operatorId === operatorId) {\n operatorStageSummary = operatorSummaries[i];\n }\n }\n\n _reactDom2.default.render(_react2.default.createElement(OperatorDetail, { key: operatorCssId, operator: operatorStageSummary, tasks: stage.tasks }), document.getElementById('operator-detail'));\n }\n }, {\n key: \"computeOperatorGraphs\",\n value: function computeOperatorGraphs(planNode, operatorMap) {\n var _this5 = this;\n\n var sources = (0, _utils.getChildren)(planNode);\n\n var sourceResults = new Map();\n sources.forEach(function (source) {\n var sourceResult = _this5.computeOperatorGraphs(source, operatorMap);\n sourceResult.forEach(function (operator, pipelineId) {\n if (sourceResults.has(pipelineId)) {\n console.error(\"Multiple sources for \", planNode['@type'], \" had the same pipeline ID\");\n return sourceResults;\n }\n sourceResults.set(pipelineId, operator);\n });\n });\n\n var nodeOperators = operatorMap.get(planNode.id);\n if (!nodeOperators || nodeOperators.length === 0) {\n return sourceResults;\n }\n\n var pipelineOperators = new Map();\n nodeOperators.forEach(function (operator) {\n if (!pipelineOperators.has(operator.pipelineId)) {\n pipelineOperators.set(operator.pipelineId, []);\n }\n pipelineOperators.get(operator.pipelineId).push(operator);\n });\n\n var result = new Map();\n pipelineOperators.forEach(function (pipelineOperators, pipelineId) {\n // sort deep-copied operators in this pipeline from source to sink\n var linkedOperators = pipelineOperators.map(function (a) {\n return Object.assign({}, a);\n }).sort(function (a, b) {\n return a.operatorId - b.operatorId;\n });\n var sinkOperator = linkedOperators[linkedOperators.length - 1];\n var sourceOperator = linkedOperators[0];\n\n if (sourceResults.has(pipelineId)) {\n var pipelineChildResult = sourceResults.get(pipelineId);\n if (pipelineChildResult) {\n sourceOperator.child = pipelineChildResult;\n }\n }\n\n // chain operators at this level\n var currentOperator = sourceOperator;\n linkedOperators.slice(1).forEach(function (source) {\n source.child = currentOperator;\n currentOperator = source;\n });\n\n result.set(pipelineId, sinkOperator);\n });\n\n sourceResults.forEach(function (operator, pipelineId) {\n if (!result.has(pipelineId)) {\n result.set(pipelineId, operator);\n }\n });\n\n return result;\n }\n }, {\n key: \"computeOperatorMap\",\n value: function computeOperatorMap() {\n var operatorMap = new Map();\n this.props.stage.stageStats.operatorSummaries.forEach(function (operator) {\n if (!operatorMap.has(operator.planNodeId)) {\n operatorMap.set(operator.planNodeId, []);\n }\n\n operatorMap.get(operator.planNodeId).push(operator);\n });\n\n return operatorMap;\n }\n }, {\n key: \"computeD3StageOperatorGraph\",\n value: function computeD3StageOperatorGraph(graph, operator, sink, pipelineNode) {\n var operatorNodeId = \"operator-\" + operator.pipelineId + \"-\" + operator.operatorId;\n\n // this is a non-standard use of ReactDOMServer, but it's the cleanest way to unify DagreD3 with React\n var html = _server2.default.renderToString(_react2.default.createElement(OperatorSummary, { key: operator.pipelineId + \"-\" + operator.operatorId, operator: operator }));\n graph.setNode(operatorNodeId, { class: \"operator-stats\", label: html, labelType: \"html\" });\n\n if (operator.hasOwnProperty(\"child\")) {\n this.computeD3StageOperatorGraph(graph, operator.child, operatorNodeId, pipelineNode);\n }\n\n if (sink !== null) {\n graph.setEdge(operatorNodeId, sink, { class: \"plan-edge\", arrowheadClass: \"plan-arrowhead\" });\n }\n\n graph.setParent(operatorNodeId, pipelineNode);\n }\n }, {\n key: \"updateD3Graph\",\n value: function updateD3Graph() {\n var _this6 = this;\n\n if (!this.props.stage) {\n return;\n }\n\n var stage = this.props.stage;\n var operatorMap = this.computeOperatorMap();\n var operatorGraphs = this.computeOperatorGraphs(stage.plan.root, operatorMap);\n\n var graph = (0, _utils.initializeGraph)();\n operatorGraphs.forEach(function (operator, pipelineId) {\n var pipelineNodeId = \"pipeline-\" + pipelineId;\n graph.setNode(pipelineNodeId, { label: \"Pipeline \" + pipelineId + \" \", clusterLabelPos: 'top', style: 'fill: #2b2b2b', labelStyle: 'fill: #fff' });\n _this6.computeD3StageOperatorGraph(graph, operator, null, pipelineNodeId);\n });\n\n $(\"#operator-canvas\").html(\"\");\n\n if (operatorGraphs.size > 0) {\n $(\".graph-container\").css(\"display\", \"block\");\n var svg = (0, _utils.initializeSvg)(\"#operator-canvas\");\n var render = new dagreD3.render();\n render(d3.select(\"#operator-canvas g\"), graph);\n\n svg.selectAll(\"g.operator-stats\").on(\"click\", this.handleOperatorClick.bind(this));\n svg.attr(\"height\", graph.graph().height);\n svg.attr(\"width\", graph.graph().width);\n } else {\n $(\".graph-container\").css(\"display\", \"none\");\n }\n }\n }, {\n key: \"render\",\n value: function render() {\n var stage = this.props.stage;\n\n if (!stage.hasOwnProperty('plan')) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Stage does not have a plan\"\n )\n )\n );\n }\n\n if (!stage.hasOwnProperty('stageStats') || !stage.stageStats.hasOwnProperty(\"operatorSummaries\") || stage.stageStats.operatorSummaries.length === 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Operator data not available for \",\n stage.stageId\n )\n )\n );\n }\n\n return null;\n }\n }]);\n\n return StageOperatorGraph;\n}(_react2.default.Component);\n\nvar StageDetail = exports.StageDetail = function (_React$Component5) {\n _inherits(StageDetail, _React$Component5);\n\n function StageDetail(props) {\n _classCallCheck(this, StageDetail);\n\n var _this7 = _possibleConstructorReturn(this, (StageDetail.__proto__ || Object.getPrototypeOf(StageDetail)).call(this, props));\n\n _this7.state = {\n initialized: false,\n ended: false,\n\n selectedStageId: null,\n query: null,\n\n lastRefresh: null,\n lastRender: null\n };\n\n _this7.refreshLoop = _this7.refreshLoop.bind(_this7);\n return _this7;\n }\n\n _createClass(StageDetail, [{\n key: \"resetTimer\",\n value: function resetTimer() {\n clearTimeout(this.timeoutId);\n // stop refreshing when query finishes or fails\n if (this.state.query === null || !this.state.ended) {\n this.timeoutId = setTimeout(this.refreshLoop, 1000);\n }\n }\n }, {\n key: \"refreshLoop\",\n value: function refreshLoop() {\n var _this8 = this;\n\n clearTimeout(this.timeoutId); // to stop multiple series of refreshLoop from going on simultaneously\n var queryString = (0, _utils.getFirstParameter)(window.location.search).split('.');\n var queryId = queryString[0];\n\n var selectedStageId = this.state.selectedStageId;\n if (selectedStageId === null) {\n selectedStageId = 0;\n if (queryString.length > 1) {\n selectedStageId = parseInt(queryString[1]);\n }\n }\n\n $.get('/v1/query/' + queryId, function (query) {\n _this8.setState({\n initialized: true,\n ended: query.finalQueryInfo,\n\n selectedStageId: selectedStageId,\n query: query\n });\n _this8.resetTimer();\n }).error(function () {\n _this8.setState({\n initialized: true\n });\n _this8.resetTimer();\n });\n }\n }, {\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.refreshLoop();\n }\n }, {\n key: \"findStage\",\n value: function findStage(stageId, currentStage) {\n if (stageId === null) {\n return null;\n }\n\n if (currentStage.stageId === stageId) {\n return currentStage;\n }\n\n for (var i = 0; i < currentStage.subStages.length; i++) {\n var stage = this.findStage(stageId, currentStage.subStages[i]);\n if (stage !== null) {\n return stage;\n }\n }\n\n return null;\n }\n }, {\n key: \"getAllStageIds\",\n value: function getAllStageIds(result, currentStage) {\n var _this9 = this;\n\n result.push(currentStage.plan.id);\n currentStage.subStages.forEach(function (stage) {\n _this9.getAllStageIds(result, stage);\n });\n }\n }, {\n key: \"render\",\n value: function render() {\n var _this10 = this;\n\n if (!this.state.query) {\n var label = _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n );\n if (this.state.initialized) {\n label = \"Query not found\";\n }\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n label\n )\n )\n );\n }\n\n if (!this.state.query.outputStage) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Query does not have an output stage\"\n )\n )\n );\n }\n\n var query = this.state.query;\n var allStages = [];\n this.getAllStageIds(allStages, query.outputStage);\n\n var stage = this.findStage(query.queryId + \".\" + this.state.selectedStageId, query.outputStage);\n if (stage === null) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Stage not found\"\n )\n )\n );\n }\n\n var stageOperatorGraph = null;\n if (!(0, _utils.isQueryEnded)(query)) {\n stageOperatorGraph = _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Operator graph will appear automatically when query completes.\"\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n )\n )\n );\n } else {\n stageOperatorGraph = _react2.default.createElement(StageOperatorGraph, { id: stage.stageId, stage: stage });\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(_QueryHeader.QueryHeader, { query: query }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-2\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Stage \",\n stage.plan.id\n )\n ),\n _react2.default.createElement(\"div\", { className: \"col-xs-8\" }),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-2 stage-dropdown\" },\n _react2.default.createElement(\n \"div\",\n { className: \"input-group-btn\" },\n _react2.default.createElement(\n \"button\",\n { type: \"button\", className: \"btn btn-default dropdown-toggle\", \"data-toggle\": \"dropdown\", \"aria-haspopup\": \"true\", \"aria-expanded\": \"false\" },\n \"Select Stage \",\n _react2.default.createElement(\"span\", { className: \"caret\" })\n ),\n _react2.default.createElement(\n \"ul\",\n { className: \"dropdown-menu\" },\n allStages.map(function (stageId) {\n return _react2.default.createElement(\n \"li\",\n { key: stageId },\n _react2.default.createElement(\n \"a\",\n { onClick: function onClick() {\n return _this10.setState({ selectedStageId: stageId });\n } },\n stageId\n )\n );\n })\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n stageOperatorGraph\n )\n )\n );\n }\n }]);\n\n return StageDetail;\n}(_react2.default.Component);\n\n//# sourceURL=webpack:///./components/StageDetail.jsx?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.StageDetail = undefined;\n\nvar _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/index.js\");\n\nvar _react2 = _interopRequireDefault(_react);\n\nvar _reactDom = __webpack_require__(/*! react-dom */ \"./node_modules/react-dom/index.js\");\n\nvar _reactDom2 = _interopRequireDefault(_reactDom);\n\nvar _server = __webpack_require__(/*! react-dom/server */ \"./node_modules/react-dom/server.browser.js\");\n\nvar _server2 = _interopRequireDefault(_server);\n\nvar _dagreD = __webpack_require__(/*! dagre-d3 */ \"./node_modules/dagre-d3/index.js\");\n\nvar dagreD3 = _interopRequireWildcard(_dagreD);\n\nvar _d = __webpack_require__(/*! d3 */ \"./node_modules/d3/index.js\");\n\nvar d3 = _interopRequireWildcard(_d);\n\nvar _utils = __webpack_require__(/*! ../utils */ \"./utils.js\");\n\nvar _QueryHeader = __webpack_require__(/*! ./QueryHeader */ \"./components/QueryHeader.jsx\");\n\nfunction _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return call && (typeof call === \"object\" || typeof call === \"function\") ? call : self; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function, not \" + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } /*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nfunction getTotalWallTime(operator) {\n return (0, _utils.parseDuration)(operator.addInputWall) + (0, _utils.parseDuration)(operator.getOutputWall) + (0, _utils.parseDuration)(operator.finishWall) + (0, _utils.parseDuration)(operator.blockedWall);\n}\n\nvar OperatorSummary = function (_React$Component) {\n _inherits(OperatorSummary, _React$Component);\n\n function OperatorSummary() {\n _classCallCheck(this, OperatorSummary);\n\n return _possibleConstructorReturn(this, (OperatorSummary.__proto__ || Object.getPrototypeOf(OperatorSummary)).apply(this, arguments));\n }\n\n _createClass(OperatorSummary, [{\n key: \"render\",\n value: function render() {\n var operator = this.props.operator;\n\n var totalWallTime = (0, _utils.parseDuration)(operator.addInputWall) + (0, _utils.parseDuration)(operator.getOutputWall) + (0, _utils.parseDuration)(operator.finishWall) + (0, _utils.parseDuration)(operator.blockedWall);\n\n var rowInputRate = totalWallTime === 0 ? 0 : 1.0 * operator.inputPositions / (totalWallTime / 1000.0);\n var byteInputRate = totalWallTime === 0 ? 0 : 1.0 * (0, _utils.parseDataSize)(operator.inputDataSize) / (totalWallTime / 1000.0);\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"highlight-row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"header-row\" },\n operator.operatorType\n ),\n _react2.default.createElement(\n \"div\",\n null,\n (0, _utils.formatCount)(rowInputRate) + \" rows/s (\" + (0, _utils.formatDataSize)(byteInputRate) + \"/s)\"\n )\n ),\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Output\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatCount)(operator.outputPositions) + \" rows (\" + operator.outputDataSize + \")\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Drivers\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n operator.totalDrivers\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Wall Time\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatDuration)(totalWallTime)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Blocked\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatDuration)((0, _utils.parseDuration)(operator.blockedWall))\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Input\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatCount)(operator.inputPositions) + \" rows (\" + operator.inputDataSize + \")\"\n )\n )\n )\n )\n );\n }\n }]);\n\n return OperatorSummary;\n}(_react2.default.Component);\n\nvar BAR_CHART_PROPERTIES = {\n type: 'bar',\n barSpacing: '0',\n height: '80px',\n barColor: '#747F96',\n zeroColor: '#8997B3',\n tooltipClassname: 'sparkline-tooltip',\n tooltipFormat: 'Task {{offset:offset}} - {{value}}',\n disableHiddenCheck: true\n};\n\nvar OperatorStatistic = function (_React$Component2) {\n _inherits(OperatorStatistic, _React$Component2);\n\n function OperatorStatistic() {\n _classCallCheck(this, OperatorStatistic);\n\n return _possibleConstructorReturn(this, (OperatorStatistic.__proto__ || Object.getPrototypeOf(OperatorStatistic)).apply(this, arguments));\n }\n\n _createClass(OperatorStatistic, [{\n key: \"componentDidMount\",\n value: function componentDidMount() {\n var operators = this.props.operators;\n var statistic = operators.map(this.props.supplier);\n var numTasks = operators.length;\n\n var tooltipValueLookups = { 'offset': {} };\n for (var i = 0; i < numTasks; i++) {\n tooltipValueLookups['offset'][i] = \"\" + i;\n }\n\n var stageBarChartProperties = $.extend({}, BAR_CHART_PROPERTIES, { barWidth: 800 / numTasks, tooltipValueLookups: tooltipValueLookups });\n $('#' + this.props.id).sparkline(statistic, $.extend({}, stageBarChartProperties, { numberFormatter: this.props.renderer }));\n }\n }, {\n key: \"render\",\n value: function render() {\n return _react2.default.createElement(\n \"div\",\n { className: \"row operator-statistic\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-2 italic-uppercase operator-statistic-title\" },\n this.props.name\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-10\" },\n _react2.default.createElement(\"span\", { className: \"bar-chart\", id: this.props.id })\n )\n );\n }\n }]);\n\n return OperatorStatistic;\n}(_react2.default.Component);\n\nvar OperatorDetail = function (_React$Component3) {\n _inherits(OperatorDetail, _React$Component3);\n\n function OperatorDetail(props) {\n _classCallCheck(this, OperatorDetail);\n\n var _this3 = _possibleConstructorReturn(this, (OperatorDetail.__proto__ || Object.getPrototypeOf(OperatorDetail)).call(this, props));\n\n _this3.state = {\n selectedStatistics: _this3.getInitialStatistics()\n };\n return _this3;\n }\n\n _createClass(OperatorDetail, [{\n key: \"getInitialStatistics\",\n value: function getInitialStatistics() {\n return [{\n name: \"Total Wall Time\",\n id: \"totalWallTime\",\n supplier: getTotalWallTime,\n renderer: _utils.formatDuration\n }, {\n name: \"Input Rows\",\n id: \"inputPositions\",\n supplier: function supplier(operator) {\n return operator.inputPositions;\n },\n renderer: _utils.formatCount\n }, {\n name: \"Input Data Size\",\n id: \"inputDataSize\",\n supplier: function supplier(operator) {\n return (0, _utils.parseDataSize)(operator.inputDataSize);\n },\n renderer: _utils.formatDataSize\n }, {\n name: \"Output Rows\",\n id: \"outputPositions\",\n supplier: function supplier(operator) {\n return operator.outputPositions;\n },\n renderer: _utils.formatCount\n }, {\n name: \"Output Data Size\",\n id: \"outputDataSize\",\n supplier: function supplier(operator) {\n return (0, _utils.parseDataSize)(operator.outputDataSize);\n },\n renderer: _utils.formatDataSize\n }];\n }\n }, {\n key: \"getOperatorTasks\",\n value: function getOperatorTasks() {\n // sort the x-axis\n var tasks = this.props.tasks.sort(function (taskA, taskB) {\n return (0, _utils.getTaskNumber)(taskA.taskStatus.taskId) - (0, _utils.getTaskNumber)(taskB.taskStatus.taskId);\n });\n\n var operatorSummary = this.props.operator;\n\n var operatorTasks = [];\n tasks.forEach(function (task) {\n task.stats.pipelines.forEach(function (pipeline) {\n if (pipeline.pipelineId === operatorSummary.pipelineId) {\n pipeline.operatorSummaries.forEach(function (operator) {\n if (operatorSummary.operatorId === operator.operatorId) {\n operatorTasks.push(operator);\n }\n });\n }\n });\n });\n\n return operatorTasks;\n }\n }, {\n key: \"render\",\n value: function render() {\n var operator = this.props.operator;\n var operatorTasks = this.getOperatorTasks();\n var totalWallTime = getTotalWallTime(operator);\n\n var rowInputRate = totalWallTime === 0 ? 0 : 1.0 * operator.inputPositions / totalWallTime;\n var byteInputRate = totalWallTime === 0 ? 0 : 1.0 * (0, _utils.parseDataSize)(operator.inputDataSize) / (totalWallTime / 1000.0);\n\n var rowOutputRate = totalWallTime === 0 ? 0 : 1.0 * operator.outputPositions / totalWallTime;\n var byteOutputRate = totalWallTime === 0 ? 0 : 1.0 * (0, _utils.parseDataSize)(operator.outputDataSize) / (totalWallTime / 1000.0);\n\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"div\",\n { className: \"modal-header\" },\n _react2.default.createElement(\n \"button\",\n { type: \"button\", className: \"close\", \"data-dismiss\": \"modal\", \"aria-label\": \"Close\" },\n _react2.default.createElement(\n \"span\",\n { \"aria-hidden\": \"true\" },\n \"\\xD7\"\n )\n ),\n _react2.default.createElement(\n \"h3\",\n null,\n _react2.default.createElement(\n \"small\",\n null,\n \"Pipeline \",\n operator.pipelineId\n ),\n _react2.default.createElement(\"br\", null),\n operator.operatorType\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Input\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatCount)(operator.inputPositions) + \" rows (\" + operator.inputDataSize + \")\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Input Rate\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatCount)(rowInputRate) + \" rows/s (\" + (0, _utils.formatDataSize)(byteInputRate) + \"/s)\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Output\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatCount)(operator.outputPositions) + \" rows (\" + operator.outputDataSize + \")\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Output Rate\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatCount)(rowOutputRate) + \" rows/s (\" + (0, _utils.formatDataSize)(byteOutputRate) + \"/s)\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Wall Time\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatDuration)(totalWallTime)\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Blocked\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n (0, _utils.formatDuration)((0, _utils.parseDuration)(operator.blockedWall))\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Drivers\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n operator.totalDrivers\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"Tasks\"\n ),\n _react2.default.createElement(\n \"td\",\n null,\n operatorTasks.length\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row font-white\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-2 italic-uppercase\" },\n _react2.default.createElement(\n \"strong\",\n null,\n \"Statistic\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-10 italic-uppercase\" },\n _react2.default.createElement(\n \"strong\",\n null,\n \"Tasks\"\n )\n )\n ),\n this.state.selectedStatistics.map(function (statistic) {\n return _react2.default.createElement(OperatorStatistic, {\n key: statistic.id,\n id: statistic.id,\n name: statistic.name,\n supplier: statistic.supplier,\n renderer: statistic.renderer,\n operators: operatorTasks });\n }.bind(this)),\n _react2.default.createElement(\"p\", null),\n _react2.default.createElement(\"p\", null)\n )\n );\n }\n }]);\n\n return OperatorDetail;\n}(_react2.default.Component);\n\nvar StageOperatorGraph = function (_React$Component4) {\n _inherits(StageOperatorGraph, _React$Component4);\n\n function StageOperatorGraph() {\n _classCallCheck(this, StageOperatorGraph);\n\n return _possibleConstructorReturn(this, (StageOperatorGraph.__proto__ || Object.getPrototypeOf(StageOperatorGraph)).apply(this, arguments));\n }\n\n _createClass(StageOperatorGraph, [{\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.updateD3Graph();\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n this.updateD3Graph();\n }\n }, {\n key: \"handleOperatorClick\",\n value: function handleOperatorClick(operatorCssId) {\n $('#operator-detail-modal').modal();\n\n var pipelineId = parseInt(operatorCssId.split('-')[1]);\n var operatorId = parseInt(operatorCssId.split('-')[2]);\n var stage = this.props.stage;\n\n var operatorStageSummary = null;\n var operatorSummaries = stage.latestAttemptExecutionInfo.stats.operatorSummaries;\n for (var i = 0; i < operatorSummaries.length; i++) {\n if (operatorSummaries[i].pipelineId === pipelineId && operatorSummaries[i].operatorId === operatorId) {\n operatorStageSummary = operatorSummaries[i];\n }\n }\n\n _reactDom2.default.render(_react2.default.createElement(OperatorDetail, { key: operatorCssId, operator: operatorStageSummary, tasks: stage.tasks }), document.getElementById('operator-detail'));\n }\n }, {\n key: \"computeOperatorGraphs\",\n value: function computeOperatorGraphs(planNode, operatorMap) {\n var _this5 = this;\n\n var sources = (0, _utils.getChildren)(planNode);\n\n var sourceResults = new Map();\n sources.forEach(function (source) {\n var sourceResult = _this5.computeOperatorGraphs(source, operatorMap);\n sourceResult.forEach(function (operator, pipelineId) {\n if (sourceResults.has(pipelineId)) {\n console.error(\"Multiple sources for \", planNode['@type'], \" had the same pipeline ID\");\n return sourceResults;\n }\n sourceResults.set(pipelineId, operator);\n });\n });\n\n var nodeOperators = operatorMap.get(planNode.id);\n if (!nodeOperators || nodeOperators.length === 0) {\n return sourceResults;\n }\n\n var pipelineOperators = new Map();\n nodeOperators.forEach(function (operator) {\n if (!pipelineOperators.has(operator.pipelineId)) {\n pipelineOperators.set(operator.pipelineId, []);\n }\n pipelineOperators.get(operator.pipelineId).push(operator);\n });\n\n var result = new Map();\n pipelineOperators.forEach(function (pipelineOperators, pipelineId) {\n // sort deep-copied operators in this pipeline from source to sink\n var linkedOperators = pipelineOperators.map(function (a) {\n return Object.assign({}, a);\n }).sort(function (a, b) {\n return a.operatorId - b.operatorId;\n });\n var sinkOperator = linkedOperators[linkedOperators.length - 1];\n var sourceOperator = linkedOperators[0];\n\n if (sourceResults.has(pipelineId)) {\n var pipelineChildResult = sourceResults.get(pipelineId);\n if (pipelineChildResult) {\n sourceOperator.child = pipelineChildResult;\n }\n }\n\n // chain operators at this level\n var currentOperator = sourceOperator;\n linkedOperators.slice(1).forEach(function (source) {\n source.child = currentOperator;\n currentOperator = source;\n });\n\n result.set(pipelineId, sinkOperator);\n });\n\n sourceResults.forEach(function (operator, pipelineId) {\n if (!result.has(pipelineId)) {\n result.set(pipelineId, operator);\n }\n });\n\n return result;\n }\n }, {\n key: \"computeOperatorMap\",\n value: function computeOperatorMap() {\n var operatorMap = new Map();\n this.props.stage.latestAttemptExecutionInfo.stats.operatorSummaries.forEach(function (operator) {\n if (!operatorMap.has(operator.planNodeId)) {\n operatorMap.set(operator.planNodeId, []);\n }\n\n operatorMap.get(operator.planNodeId).push(operator);\n });\n\n return operatorMap;\n }\n }, {\n key: \"computeD3StageOperatorGraph\",\n value: function computeD3StageOperatorGraph(graph, operator, sink, pipelineNode) {\n var operatorNodeId = \"operator-\" + operator.pipelineId + \"-\" + operator.operatorId;\n\n // this is a non-standard use of ReactDOMServer, but it's the cleanest way to unify DagreD3 with React\n var html = _server2.default.renderToString(_react2.default.createElement(OperatorSummary, { key: operator.pipelineId + \"-\" + operator.operatorId, operator: operator }));\n graph.setNode(operatorNodeId, { class: \"operator-stats\", label: html, labelType: \"html\" });\n\n if (operator.hasOwnProperty(\"child\")) {\n this.computeD3StageOperatorGraph(graph, operator.child, operatorNodeId, pipelineNode);\n }\n\n if (sink !== null) {\n graph.setEdge(operatorNodeId, sink, { class: \"plan-edge\", arrowheadClass: \"plan-arrowhead\" });\n }\n\n graph.setParent(operatorNodeId, pipelineNode);\n }\n }, {\n key: \"updateD3Graph\",\n value: function updateD3Graph() {\n var _this6 = this;\n\n if (!this.props.stage) {\n return;\n }\n\n var stage = this.props.stage;\n var operatorMap = this.computeOperatorMap();\n var operatorGraphs = this.computeOperatorGraphs(stage.plan.root, operatorMap);\n\n var graph = (0, _utils.initializeGraph)();\n operatorGraphs.forEach(function (operator, pipelineId) {\n var pipelineNodeId = \"pipeline-\" + pipelineId;\n graph.setNode(pipelineNodeId, { label: \"Pipeline \" + pipelineId + \" \", clusterLabelPos: 'top', style: 'fill: #2b2b2b', labelStyle: 'fill: #fff' });\n _this6.computeD3StageOperatorGraph(graph, operator, null, pipelineNodeId);\n });\n\n $(\"#operator-canvas\").html(\"\");\n\n if (operatorGraphs.size > 0) {\n $(\".graph-container\").css(\"display\", \"block\");\n var svg = (0, _utils.initializeSvg)(\"#operator-canvas\");\n var render = new dagreD3.render();\n render(d3.select(\"#operator-canvas g\"), graph);\n\n svg.selectAll(\"g.operator-stats\").on(\"click\", this.handleOperatorClick.bind(this));\n svg.attr(\"height\", graph.graph().height);\n svg.attr(\"width\", graph.graph().width);\n } else {\n $(\".graph-container\").css(\"display\", \"none\");\n }\n }\n }, {\n key: \"render\",\n value: function render() {\n var stage = this.props.stage;\n\n if (!stage.hasOwnProperty('plan')) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Stage does not have a plan\"\n )\n )\n );\n }\n\n var latestAttemptExecutionInfo = stage.latestAttemptExecutionInfo;\n if (!latestAttemptExecutionInfo.hasOwnProperty('stats') || !latestAttemptExecutionInfo.stats.hasOwnProperty(\"operatorSummaries\") || latestAttemptExecutionInfo.stats.operatorSummaries.length === 0) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Operator data not available for \",\n stage.stageId\n )\n )\n );\n }\n\n return null;\n }\n }]);\n\n return StageOperatorGraph;\n}(_react2.default.Component);\n\nvar StageDetail = exports.StageDetail = function (_React$Component5) {\n _inherits(StageDetail, _React$Component5);\n\n function StageDetail(props) {\n _classCallCheck(this, StageDetail);\n\n var _this7 = _possibleConstructorReturn(this, (StageDetail.__proto__ || Object.getPrototypeOf(StageDetail)).call(this, props));\n\n _this7.state = {\n initialized: false,\n ended: false,\n\n selectedStageId: null,\n query: null,\n\n lastRefresh: null,\n lastRender: null\n };\n\n _this7.refreshLoop = _this7.refreshLoop.bind(_this7);\n return _this7;\n }\n\n _createClass(StageDetail, [{\n key: \"resetTimer\",\n value: function resetTimer() {\n clearTimeout(this.timeoutId);\n // stop refreshing when query finishes or fails\n if (this.state.query === null || !this.state.ended) {\n this.timeoutId = setTimeout(this.refreshLoop, 1000);\n }\n }\n }, {\n key: \"refreshLoop\",\n value: function refreshLoop() {\n var _this8 = this;\n\n clearTimeout(this.timeoutId); // to stop multiple series of refreshLoop from going on simultaneously\n var queryString = (0, _utils.getFirstParameter)(window.location.search).split('.');\n var queryId = queryString[0];\n\n var selectedStageId = this.state.selectedStageId;\n if (selectedStageId === null) {\n selectedStageId = 0;\n if (queryString.length > 1) {\n selectedStageId = parseInt(queryString[1]);\n }\n }\n\n $.get('/v1/query/' + queryId, function (query) {\n _this8.setState({\n initialized: true,\n ended: query.finalQueryInfo,\n\n selectedStageId: selectedStageId,\n query: query\n });\n _this8.resetTimer();\n }).error(function () {\n _this8.setState({\n initialized: true\n });\n _this8.resetTimer();\n });\n }\n }, {\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.refreshLoop();\n }\n }, {\n key: \"findStage\",\n value: function findStage(stageId, currentStage) {\n if (stageId === null) {\n return null;\n }\n\n if (currentStage.stageId === stageId) {\n return currentStage;\n }\n\n for (var i = 0; i < currentStage.subStages.length; i++) {\n var stage = this.findStage(stageId, currentStage.subStages[i]);\n if (stage !== null) {\n return stage;\n }\n }\n\n return null;\n }\n }, {\n key: \"getAllStageIds\",\n value: function getAllStageIds(result, currentStage) {\n var _this9 = this;\n\n result.push(currentStage.plan.id);\n currentStage.subStages.forEach(function (stage) {\n _this9.getAllStageIds(result, stage);\n });\n }\n }, {\n key: \"render\",\n value: function render() {\n var _this10 = this;\n\n if (!this.state.query) {\n var label = _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n );\n if (this.state.initialized) {\n label = \"Query not found\";\n }\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n label\n )\n )\n );\n }\n\n if (!this.state.query.outputStage) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Query does not have an output stage\"\n )\n )\n );\n }\n\n var query = this.state.query;\n var allStages = [];\n this.getAllStageIds(allStages, query.outputStage);\n\n var stage = this.findStage(query.queryId + \".\" + this.state.selectedStageId, query.outputStage);\n if (stage === null) {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Stage not found\"\n )\n )\n );\n }\n\n var stageOperatorGraph = null;\n if (!(0, _utils.isQueryEnded)(query)) {\n stageOperatorGraph = _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Operator graph will appear automatically when query completes.\"\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n )\n )\n );\n } else {\n stageOperatorGraph = _react2.default.createElement(StageOperatorGraph, { id: stage.stageId, stage: stage });\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(_QueryHeader.QueryHeader, { query: query }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-2\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Stage \",\n stage.plan.id\n )\n ),\n _react2.default.createElement(\"div\", { className: \"col-xs-8\" }),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-2 stage-dropdown\" },\n _react2.default.createElement(\n \"div\",\n { className: \"input-group-btn\" },\n _react2.default.createElement(\n \"button\",\n { type: \"button\", className: \"btn btn-default dropdown-toggle\", \"data-toggle\": \"dropdown\", \"aria-haspopup\": \"true\", \"aria-expanded\": \"false\" },\n \"Select Stage \",\n _react2.default.createElement(\"span\", { className: \"caret\" })\n ),\n _react2.default.createElement(\n \"ul\",\n { className: \"dropdown-menu\" },\n allStages.map(function (stageId) {\n return _react2.default.createElement(\n \"li\",\n { key: stageId },\n _react2.default.createElement(\n \"a\",\n { onClick: function onClick() {\n return _this10.setState({ selectedStageId: stageId });\n } },\n stageId\n )\n );\n })\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n stageOperatorGraph\n )\n )\n );\n }\n }]);\n\n return StageDetail;\n}(_react2.default.Component);\n\n//# sourceURL=webpack:///./components/StageDetail.jsx?"); /***/ }), diff --git a/presto-main/src/main/resources/webapp/dist/worker.js b/presto-main/src/main/resources/webapp/dist/worker.js index f63d7c1856f68..7e1b3731b150f 100644 --- a/presto-main/src/main/resources/webapp/dist/worker.js +++ b/presto-main/src/main/resources/webapp/dist/worker.js @@ -106,7 +106,7 @@ eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n}); /***/ (function(module, exports, __webpack_require__) { "use strict"; -eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.WorkerStatus = undefined;\n\nvar _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/index.js\");\n\nvar _react2 = _interopRequireDefault(_react);\n\nvar _utils = __webpack_require__(/*! ../utils */ \"./utils.js\");\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return call && (typeof call === \"object\" || typeof call === \"function\") ? call : self; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function, not \" + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } /*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nvar SMALL_SPARKLINE_PROPERTIES = {\n width: '100%',\n height: '57px',\n fillColor: '#3F4552',\n lineColor: '#747F96',\n spotColor: '#1EDCFF',\n tooltipClassname: 'sparkline-tooltip',\n disableHiddenCheck: true\n};\n\nvar WorkerStatus = exports.WorkerStatus = function (_React$Component) {\n _inherits(WorkerStatus, _React$Component);\n\n function WorkerStatus(props) {\n _classCallCheck(this, WorkerStatus);\n\n var _this = _possibleConstructorReturn(this, (WorkerStatus.__proto__ || Object.getPrototypeOf(WorkerStatus)).call(this, props));\n\n _this.state = {\n serverInfo: null,\n initialized: false,\n ended: false,\n\n processCpuLoad: [],\n systemCpuLoad: [],\n heapPercentUsed: [],\n nonHeapUsed: []\n };\n\n _this.refreshLoop = _this.refreshLoop.bind(_this);\n return _this;\n }\n\n _createClass(WorkerStatus, [{\n key: \"resetTimer\",\n value: function resetTimer() {\n clearTimeout(this.timeoutId);\n // stop refreshing when query finishes or fails\n if (this.state.query === null || !this.state.ended) {\n this.timeoutId = setTimeout(this.refreshLoop, 1000);\n }\n }\n }, {\n key: \"refreshLoop\",\n value: function refreshLoop() {\n clearTimeout(this.timeoutId); // to stop multiple series of refreshLoop from going on simultaneously\n var nodeId = (0, _utils.getFirstParameter)(window.location.search);\n $.get('/v1/worker/' + nodeId + '/status', function (serverInfo) {\n this.setState({\n serverInfo: serverInfo,\n initialized: true,\n\n processCpuLoad: (0, _utils.addToHistory)(serverInfo.processCpuLoad * 100.0, this.state.processCpuLoad),\n systemCpuLoad: (0, _utils.addToHistory)(serverInfo.systemCpuLoad * 100.0, this.state.systemCpuLoad),\n heapPercentUsed: (0, _utils.addToHistory)(serverInfo.heapUsed * 100.0 / serverInfo.heapAvailable, this.state.heapPercentUsed),\n nonHeapUsed: (0, _utils.addToHistory)(serverInfo.nonHeapUsed * 100.0, this.state.nonHeapUsed)\n });\n\n this.resetTimer();\n }.bind(this)).error(function () {\n this.setState({\n initialized: true\n });\n this.resetTimer();\n }.bind(this));\n }\n }, {\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.refreshLoop();\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n $('#process-cpu-load-sparkline').sparkline(this.state.processCpuLoad, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { chartRangeMin: 0, numberFormatter: _utils.precisionRound }));\n $('#system-cpu-load-sparkline').sparkline(this.state.systemCpuLoad, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { chartRangeMin: 0, numberFormatter: _utils.precisionRound }));\n $('#heap-percent-used-sparkline').sparkline(this.state.heapPercentUsed, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { chartRangeMin: 0, numberFormatter: _utils.precisionRound }));\n $('#nonheap-used-sparkline').sparkline(this.state.nonHeapUsed, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { chartRangeMin: 0, numberFormatter: _utils.formatDataSize }));\n\n $('[data-toggle=\"tooltip\"]').tooltip();\n new Clipboard('.copy-button');\n }\n }, {\n key: \"renderPoolQueries\",\n value: function renderPoolQueries(pool) {\n if (!pool) {\n return;\n }\n\n var queries = {};\n var reservations = pool.queryMemoryReservations;\n var revocableReservations = pool.queryMemoryRevocableReservations;\n\n for (var query in reservations) {\n queries[query] = [reservations[query], 0];\n }\n\n for (var _query in revocableReservations) {\n if (queries.hasOwnProperty(_query)) {\n queries[_query][1] = revocableReservations[_query];\n } else {\n queries[_query] = [0, revocableReservations[_query]];\n }\n }\n\n var size = pool.maxBytes;\n\n if (Object.keys(queries).length === 0) {\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"table table-condensed\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"No queries using pool\"\n )\n )\n )\n )\n );\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n Object.keys(queries).map(function (key) {\n return WorkerStatus.renderPoolQuery(key, queries[key][0], queries[key][1], size);\n })\n )\n )\n );\n }\n }, {\n key: \"render\",\n value: function render() {\n var serverInfo = this.state.serverInfo;\n\n if (serverInfo === null) {\n if (this.state.initialized === false) {\n return _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n );\n } else {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Node information could not be loaded\"\n )\n )\n );\n }\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Overview\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Node ID\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"node-id\" },\n serverInfo.nodeId\n ),\n \"\\xA0\\xA0\",\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: \"copy-button\", \"data-clipboard-target\": \"#node-id\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\",\n title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", alt: \"Copy to clipboard\" })\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Heap Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"internal-address\" },\n (0, _utils.formatDataSize)(serverInfo.heapAvailable)\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Processors\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"internal-address\" },\n serverInfo.processors\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Uptime\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n serverInfo.uptime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"External Address\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"external-address\" },\n serverInfo.externalAddress\n ),\n \"\\xA0\\xA0\",\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: \"copy-button\", \"data-clipboard-target\": \"#external-address\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\",\n title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", alt: \"Copy to clipboard\" })\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Internal Address\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"internal-address\" },\n serverInfo.internalAddress\n ),\n \"\\xA0\\xA0\",\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: \"copy-button\", \"data-clipboard-target\": \"#internal-address\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\",\n title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", alt: \"Copy to clipboard\" })\n )\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Resource Utilization\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Process CPU Utilization\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"process-cpu-load-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.processCpuLoad[this.state.processCpuLoad.length - 1]),\n \"%\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"System CPU Utilization\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"system-cpu-load-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.systemCpuLoad[this.state.systemCpuLoad.length - 1]),\n \"%\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Heap Utilization\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"heap-percent-used-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.heapPercentUsed[this.state.heapPercentUsed.length - 1]),\n \"%\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Non-Heap Memory Used\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"nonheap-used-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatDataSize)(this.state.nonHeapUsed[this.state.nonHeapUsed.length - 1])\n )\n )\n )\n )\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Memory Pools\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n WorkerStatus.renderPoolBar(\"General\", serverInfo.memoryInfo.pools.general),\n this.renderPoolQueries(serverInfo.memoryInfo.pools.general)\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n WorkerStatus.renderPoolBar(\"Reserved\", serverInfo.memoryInfo.pools.reserved),\n this.renderPoolQueries(serverInfo.memoryInfo.pools.reserved)\n )\n )\n )\n )\n );\n }\n }], [{\n key: \"renderPoolBar\",\n value: function renderPoolBar(name, pool) {\n if (!pool) {\n return;\n }\n\n var size = pool.maxBytes;\n var reserved = pool.reservedBytes;\n var revocable = pool.reservedRevocableBytes;\n\n var percentageReservedNonRevocable = reserved - revocable === 0 ? 0 : Math.max(Math.round((reserved - revocable) * 100.0 / size), 15);\n var percentageRevocable = revocable === 0 ? 0 : Math.max(Math.round(revocable * 100.0 / size), 15);\n var percentageFree = 100 - (percentageRevocable + percentageReservedNonRevocable);\n\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-8\" },\n _react2.default.createElement(\n \"h4\",\n null,\n name,\n \" Pool\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-4\" },\n _react2.default.createElement(\n \"div\",\n { className: \"progress\", style: { marginTop: \"6px\" } },\n _react2.default.createElement(\n \"div\",\n { className: \"progress-bar memory-progress-bar memory-progress-bar-info\", role: \"progressbar\", style: { width: \"100%\" } },\n (0, _utils.formatDataSize)(size),\n \" total\"\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\"hr\", { className: \"h4-hr\" }),\n _react2.default.createElement(\n \"div\",\n { className: \"progress\" },\n _react2.default.createElement(\n \"div\",\n { className: \"progress-bar memory-progress-bar progress-bar-warning progress-bar-striped active\", role: \"progressbar\",\n style: { width: percentageReservedNonRevocable + \"%\" } },\n (0, _utils.formatDataSize)(reserved - revocable)\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"progress-bar memory-progress-bar progress-bar-danger progress-bar-striped active\", role: \"progressbar\",\n style: { width: percentageRevocable + \"%\" } },\n (0, _utils.formatDataSize)(revocable)\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"progress-bar memory-progress-bar progress-bar-success\", role: \"progressbar\", style: { width: percentageFree + \"%\" } },\n (0, _utils.formatDataSize)(size - reserved)\n )\n )\n )\n )\n )\n );\n }\n }, {\n key: \"renderPoolQuery\",\n value: function renderPoolQuery(query, reserved, revocable, total) {\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"row query-memory-list-header\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-7\" },\n _react2.default.createElement(\n \"a\",\n { href: \"query.html?\" + query, target: \"_blank\" },\n query\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-5\" },\n _react2.default.createElement(\n \"div\",\n { className: \"row text-right\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"span\",\n { \"data-toggle\": \"tooltip\", \"data-placement\": \"top\", title: \"% of pool memory reserved\" },\n Math.round(reserved * 100.0 / total),\n \"%\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"span\",\n { \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Reserved: \" + (0, _utils.formatDataSize)(reserved) + \". Revocable: \" + (0, _utils.formatDataSize)(revocable) },\n (0, _utils.formatDataSize)(reserved)\n )\n )\n )\n )\n )\n )\n );\n }\n }]);\n\n return WorkerStatus;\n}(_react2.default.Component);\n\n//# sourceURL=webpack:///./components/WorkerStatus.jsx?"); +eval("\n\nObject.defineProperty(exports, \"__esModule\", {\n value: true\n});\nexports.WorkerStatus = undefined;\n\nvar _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();\n\nvar _react = __webpack_require__(/*! react */ \"./node_modules/react/index.js\");\n\nvar _react2 = _interopRequireDefault(_react);\n\nvar _utils = __webpack_require__(/*! ../utils */ \"./utils.js\");\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return call && (typeof call === \"object\" || typeof call === \"function\") ? call : self; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function, not \" + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } /*\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nvar SMALL_SPARKLINE_PROPERTIES = {\n width: '100%',\n height: '57px',\n fillColor: '#3F4552',\n lineColor: '#747F96',\n spotColor: '#1EDCFF',\n tooltipClassname: 'sparkline-tooltip',\n disableHiddenCheck: true\n};\n\nvar WorkerStatus = exports.WorkerStatus = function (_React$Component) {\n _inherits(WorkerStatus, _React$Component);\n\n function WorkerStatus(props) {\n _classCallCheck(this, WorkerStatus);\n\n var _this = _possibleConstructorReturn(this, (WorkerStatus.__proto__ || Object.getPrototypeOf(WorkerStatus)).call(this, props));\n\n _this.state = {\n serverInfo: null,\n initialized: false,\n ended: false,\n\n processCpuLoad: [],\n systemCpuLoad: [],\n heapPercentUsed: [],\n nonHeapUsed: []\n };\n\n _this.refreshLoop = _this.refreshLoop.bind(_this);\n return _this;\n }\n\n _createClass(WorkerStatus, [{\n key: \"resetTimer\",\n value: function resetTimer() {\n clearTimeout(this.timeoutId);\n // stop refreshing when query finishes or fails\n if (this.state.query === null || !this.state.ended) {\n this.timeoutId = setTimeout(this.refreshLoop, 1000);\n }\n }\n }, {\n key: \"refreshLoop\",\n value: function refreshLoop() {\n clearTimeout(this.timeoutId); // to stop multiple series of refreshLoop from going on simultaneously\n var nodeId = (0, _utils.getFirstParameter)(window.location.search);\n $.get('/v1/worker/' + nodeId + '/status', function (serverInfo) {\n this.setState({\n serverInfo: serverInfo,\n initialized: true,\n\n processCpuLoad: (0, _utils.addToHistory)(serverInfo.processCpuLoad * 100.0, this.state.processCpuLoad),\n systemCpuLoad: (0, _utils.addToHistory)(serverInfo.systemCpuLoad * 100.0, this.state.systemCpuLoad),\n heapPercentUsed: (0, _utils.addToHistory)(serverInfo.heapUsed * 100.0 / serverInfo.heapAvailable, this.state.heapPercentUsed),\n nonHeapUsed: (0, _utils.addToHistory)(serverInfo.nonHeapUsed, this.state.nonHeapUsed)\n });\n\n this.resetTimer();\n }.bind(this)).error(function () {\n this.setState({\n initialized: true\n });\n this.resetTimer();\n }.bind(this));\n }\n }, {\n key: \"componentDidMount\",\n value: function componentDidMount() {\n this.refreshLoop();\n }\n }, {\n key: \"componentDidUpdate\",\n value: function componentDidUpdate() {\n $('#process-cpu-load-sparkline').sparkline(this.state.processCpuLoad, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { chartRangeMin: 0, numberFormatter: _utils.precisionRound }));\n $('#system-cpu-load-sparkline').sparkline(this.state.systemCpuLoad, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { chartRangeMin: 0, numberFormatter: _utils.precisionRound }));\n $('#heap-percent-used-sparkline').sparkline(this.state.heapPercentUsed, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { chartRangeMin: 0, numberFormatter: _utils.precisionRound }));\n $('#nonheap-used-sparkline').sparkline(this.state.nonHeapUsed, $.extend({}, SMALL_SPARKLINE_PROPERTIES, { chartRangeMin: 0, numberFormatter: _utils.formatDataSize }));\n\n $('[data-toggle=\"tooltip\"]').tooltip();\n new Clipboard('.copy-button');\n }\n }, {\n key: \"renderPoolQueries\",\n value: function renderPoolQueries(pool) {\n if (!pool) {\n return;\n }\n\n var queries = {};\n var reservations = pool.queryMemoryReservations;\n var revocableReservations = pool.queryMemoryRevocableReservations;\n\n for (var query in reservations) {\n queries[query] = [reservations[query], 0];\n }\n\n for (var _query in revocableReservations) {\n if (queries.hasOwnProperty(_query)) {\n queries[_query][1] = revocableReservations[_query];\n } else {\n queries[_query] = [0, revocableReservations[_query]];\n }\n }\n\n var size = pool.maxBytes;\n\n if (Object.keys(queries).length === 0) {\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"table table-condensed\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n \"No queries using pool\"\n )\n )\n )\n )\n );\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n Object.keys(queries).map(function (key) {\n return WorkerStatus.renderPoolQuery(key, queries[key][0], queries[key][1], size);\n })\n )\n )\n );\n }\n }, {\n key: \"render\",\n value: function render() {\n var serverInfo = this.state.serverInfo;\n\n if (serverInfo === null) {\n if (this.state.initialized === false) {\n return _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading...\"\n );\n } else {\n return _react2.default.createElement(\n \"div\",\n { className: \"row error-message\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h4\",\n null,\n \"Node information could not be loaded\"\n )\n )\n );\n }\n }\n\n return _react2.default.createElement(\n \"div\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Overview\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Node ID\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"node-id\" },\n serverInfo.nodeId\n ),\n \"\\xA0\\xA0\",\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: \"copy-button\", \"data-clipboard-target\": \"#node-id\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\",\n title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", alt: \"Copy to clipboard\" })\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Heap Memory\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"internal-address\" },\n (0, _utils.formatDataSize)(serverInfo.heapAvailable)\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Processors\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"internal-address\" },\n serverInfo.processors\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Uptime\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n serverInfo.uptime\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"External Address\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"external-address\" },\n serverInfo.externalAddress\n ),\n \"\\xA0\\xA0\",\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: \"copy-button\", \"data-clipboard-target\": \"#external-address\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\",\n title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", alt: \"Copy to clipboard\" })\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Internal Address\"\n ),\n _react2.default.createElement(\n \"td\",\n { className: \"info-text wrap-text\" },\n _react2.default.createElement(\n \"span\",\n { id: \"internal-address\" },\n serverInfo.internalAddress\n ),\n \"\\xA0\\xA0\",\n _react2.default.createElement(\n \"a\",\n { href: \"#\", className: \"copy-button\", \"data-clipboard-target\": \"#internal-address\", \"data-toggle\": \"tooltip\", \"data-placement\": \"right\",\n title: \"Copy to clipboard\" },\n _react2.default.createElement(\"span\", { className: \"glyphicon glyphicon-copy\", alt: \"Copy to clipboard\" })\n )\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Resource Utilization\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Process CPU Utilization\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"process-cpu-load-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.processCpuLoad[this.state.processCpuLoad.length - 1]),\n \"%\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"System CPU Utilization\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"system-cpu-load-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.systemCpuLoad[this.state.systemCpuLoad.length - 1]),\n \"%\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"table\",\n { className: \"table\" },\n _react2.default.createElement(\n \"tbody\",\n null,\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Heap Utilization\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"heap-percent-used-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatCount)(this.state.heapPercentUsed[this.state.heapPercentUsed.length - 1]),\n \"%\"\n )\n ),\n _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n { className: \"info-title\" },\n \"Non-Heap Memory Used\"\n ),\n _react2.default.createElement(\n \"td\",\n { rowSpan: \"2\" },\n _react2.default.createElement(\n \"div\",\n { className: \"query-stats-sparkline-container\" },\n _react2.default.createElement(\n \"span\",\n { className: \"sparkline\", id: \"nonheap-used-sparkline\" },\n _react2.default.createElement(\n \"div\",\n { className: \"loader\" },\n \"Loading ...\"\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"tr\",\n { className: \"tr-noborder\" },\n _react2.default.createElement(\n \"td\",\n { className: \"info-sparkline-text\" },\n (0, _utils.formatDataSize)(this.state.nonHeapUsed[this.state.nonHeapUsed.length - 1])\n )\n )\n )\n )\n )\n )\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"h3\",\n null,\n \"Memory Pools\"\n ),\n _react2.default.createElement(\"hr\", { className: \"h3-hr\" }),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n WorkerStatus.renderPoolBar(\"General\", serverInfo.memoryInfo.pools.general),\n this.renderPoolQueries(serverInfo.memoryInfo.pools.general)\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n WorkerStatus.renderPoolBar(\"Reserved\", serverInfo.memoryInfo.pools.reserved),\n this.renderPoolQueries(serverInfo.memoryInfo.pools.reserved)\n )\n )\n )\n )\n );\n }\n }], [{\n key: \"renderPoolBar\",\n value: function renderPoolBar(name, pool) {\n if (!pool) {\n return;\n }\n\n var size = pool.maxBytes;\n var reserved = pool.reservedBytes;\n var revocable = pool.reservedRevocableBytes;\n\n var percentageReservedNonRevocable = reserved - revocable === 0 ? 0 : Math.max(Math.round((reserved - revocable) * 100.0 / size), 15);\n var percentageRevocable = revocable === 0 ? 0 : Math.max(Math.round(revocable * 100.0 / size), 15);\n var percentageFree = 100 - (percentageRevocable + percentageReservedNonRevocable);\n\n return _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-8\" },\n _react2.default.createElement(\n \"h4\",\n null,\n name,\n \" Pool\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-4\" },\n _react2.default.createElement(\n \"div\",\n { className: \"progress\", style: { marginTop: \"6px\" } },\n _react2.default.createElement(\n \"div\",\n { className: \"progress-bar memory-progress-bar memory-progress-bar-info\", role: \"progressbar\", style: { width: \"100%\" } },\n (0, _utils.formatDataSize)(size),\n \" total\"\n )\n )\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"row\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-12\" },\n _react2.default.createElement(\"hr\", { className: \"h4-hr\" }),\n _react2.default.createElement(\n \"div\",\n { className: \"progress\" },\n _react2.default.createElement(\n \"div\",\n { className: \"progress-bar memory-progress-bar progress-bar-warning progress-bar-striped active\", role: \"progressbar\",\n style: { width: percentageReservedNonRevocable + \"%\" } },\n (0, _utils.formatDataSize)(reserved - revocable)\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"progress-bar memory-progress-bar progress-bar-danger progress-bar-striped active\", role: \"progressbar\",\n style: { width: percentageRevocable + \"%\" } },\n (0, _utils.formatDataSize)(revocable)\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"progress-bar memory-progress-bar progress-bar-success\", role: \"progressbar\", style: { width: percentageFree + \"%\" } },\n (0, _utils.formatDataSize)(size - reserved)\n )\n )\n )\n )\n )\n );\n }\n }, {\n key: \"renderPoolQuery\",\n value: function renderPoolQuery(query, reserved, revocable, total) {\n return _react2.default.createElement(\n \"tr\",\n null,\n _react2.default.createElement(\n \"td\",\n null,\n _react2.default.createElement(\n \"div\",\n { className: \"row query-memory-list-header\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-7\" },\n _react2.default.createElement(\n \"a\",\n { href: \"query.html?\" + query, target: \"_blank\" },\n query\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-5\" },\n _react2.default.createElement(\n \"div\",\n { className: \"row text-right\" },\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"span\",\n { \"data-toggle\": \"tooltip\", \"data-placement\": \"top\", title: \"% of pool memory reserved\" },\n Math.round(reserved * 100.0 / total),\n \"%\"\n )\n ),\n _react2.default.createElement(\n \"div\",\n { className: \"col-xs-6\" },\n _react2.default.createElement(\n \"span\",\n { \"data-toggle\": \"tooltip\", \"data-placement\": \"top\",\n title: \"Reserved: \" + (0, _utils.formatDataSize)(reserved) + \". Revocable: \" + (0, _utils.formatDataSize)(revocable) },\n (0, _utils.formatDataSize)(reserved)\n )\n )\n )\n )\n )\n )\n );\n }\n }]);\n\n return WorkerStatus;\n}(_react2.default.Component);\n\n//# sourceURL=webpack:///./components/WorkerStatus.jsx?"); /***/ }), diff --git a/presto-main/src/main/resources/webapp/src/components/ClusterHUD.jsx b/presto-main/src/main/resources/webapp/src/components/ClusterHUD.jsx index 78ae5784a4c64..80e9768a520d5 100644 --- a/presto-main/src/main/resources/webapp/src/components/ClusterHUD.jsx +++ b/presto-main/src/main/resources/webapp/src/components/ClusterHUD.jsx @@ -14,13 +14,7 @@ import React from "react"; -import { - addExponentiallyWeightedToHistory, - addToHistory, - formatCount, - formatDataSizeBytes, - precisionRound -} from "../utils"; +import {addExponentiallyWeightedToHistory, addToHistory, formatCount, formatDataSizeBytes, precisionRound} from "../utils"; const SPARKLINE_PROPERTIES = { width: '100%', diff --git a/presto-main/src/main/resources/webapp/src/components/LivePlan.jsx b/presto-main/src/main/resources/webapp/src/components/LivePlan.jsx index d0da6367f4dc5..114b16d0d821d 100644 --- a/presto-main/src/main/resources/webapp/src/components/LivePlan.jsx +++ b/presto-main/src/main/resources/webapp/src/components/LivePlan.jsx @@ -18,13 +18,7 @@ import ReactDOMServer from "react-dom/server"; import * as dagreD3 from "dagre-d3"; import * as d3 from "d3"; -import { - formatRows, - getStageStateColor, - initializeGraph, - initializeSvg, - truncateString -} from "../utils"; +import {formatRows, getStageStateColor, initializeGraph, initializeSvg, truncateString} from "../utils"; import {QueryHeader} from "./QueryHeader"; type StageStatisticsProps = { @@ -61,8 +55,8 @@ class StageStatistics extends React.Component

- - - - - - - - - - - - - - + + + + + + + + + + + + + + {renderedTasks}
IDHostStateRowsRows/sBytesBytes/sElapsedCPU TimeBufferedIDHostStateRowsRows/sBytesBytes/sElapsedCPU TimeBuffered
@@ -248,7 +253,8 @@ class StageSummary extends React.Component { super(props); this.state = { expanded: false, - lastRender: null + lastRender: null, + taskFilter: TASK_FILTER.ALL }; } @@ -299,13 +305,13 @@ class StageSummary extends React.Component { componentDidUpdate() { const stage = this.props.stage; - const numTasks = stage.tasks.length; + const numTasks = stage.latestAttemptExecutionInfo.tasks.length; // sort the x-axis - stage.tasks.sort((taskA, taskB) => getTaskNumber(taskA.taskStatus.taskId) - getTaskNumber(taskB.taskStatus.taskId)); + stage.latestAttemptExecutionInfo.tasks.sort((taskA, taskB) => getTaskNumber(taskA.taskStatus.taskId) - getTaskNumber(taskB.taskStatus.taskId)); - const scheduledTimes = stage.tasks.map(task => parseDuration(task.stats.totalScheduledTime)); - const cpuTimes = stage.tasks.map(task => parseDuration(task.stats.totalCpuTime)); + const scheduledTimes = stage.latestAttemptExecutionInfo.tasks.map(task => parseDuration(task.stats.totalScheduledTime)); + const cpuTimes = stage.latestAttemptExecutionInfo.tasks.map(task => parseDuration(task.stats.totalCpuTime)); // prevent multiple calls to componentDidUpdate (resulting from calls to setState or otherwise) within the refresh interval from re-rendering sparklines/charts if (this.state.lastRender === null || (Date.now() - this.state.lastRender) >= 1000) { @@ -334,6 +340,71 @@ class StageSummary extends React.Component { } } + renderStageExecutionAttemptsTasks(attempts) { + return attempts.map(attempt => { + return this.renderTaskList(attempt.tasks) + }); + } + + renderTaskList(tasks) { + tasks = this.state.expanded ? tasks : []; + tasks = tasks.filter(task => this.state.taskFilter.predicate(task.taskStatus.state), this); + return ( + + + + + + ); + } + + renderTaskFilterListItem(taskFilter) { + return ( +

  • {taskFilter.text}
    • + ); + } + + handleTaskFilterClick(filter, event) { + this.setState({ + taskFilter: filter + }); + event.preventDefault(); + } + + renderTaskFilter() { + return (
    +
    +

    Tasks

    +
    +
    + + + + + + +
    +
    + +
      + {this.renderTaskFilterListItem(TASK_FILTER.ALL)} + {this.renderTaskFilterListItem(TASK_FILTER.PLANNED)} + {this.renderTaskFilterListItem(TASK_FILTER.RUNNING)} + {this.renderTaskFilterListItem(TASK_FILTER.FINISHED)} + {this.renderTaskFilterListItem(TASK_FILTER.FAILED)} +
    +
    +
    +
    +
    ); + + } + render() { const stage = this.props.stage; if (stage === undefined || !stage.hasOwnProperty('plan')) { @@ -343,7 +414,7 @@ class StageSummary extends React.Component { ); } - const totalBufferedBytes = stage.tasks + const totalBufferedBytes = stage.latestAttemptExecutionInfo.tasks .map(task => task.outputBuffers.totalBufferedBytes) .reduce((a, b) => a + b, 0); @@ -374,7 +445,7 @@ class StageSummary extends React.Component { Scheduled - {stage.stageStats.totalScheduledTime} + {stage.latestAttemptExecutionInfo.stats.totalScheduledTime} @@ -382,7 +453,7 @@ class StageSummary extends React.Component { Blocked - {stage.stageStats.totalBlockedTime} + {stage.latestAttemptExecutionInfo.stats.totalBlockedTime} @@ -390,7 +461,7 @@ class StageSummary extends React.Component { CPU - {stage.stageStats.totalCpuTime} + {stage.latestAttemptExecutionInfo.stats.totalCpuTime} @@ -412,7 +483,7 @@ class StageSummary extends React.Component { Cumulative - {formatDataSizeBytes(stage.stageStats.cumulativeUserMemory / 1000)} + {formatDataSizeBytes(stage.latestAttemptExecutionInfo.stats.cumulativeUserMemory / 1000)} @@ -420,7 +491,7 @@ class StageSummary extends React.Component { Current - {stage.stageStats.userMemoryReservation} + {stage.latestAttemptExecutionInfo.stats.userMemoryReservation} @@ -436,7 +507,7 @@ class StageSummary extends React.Component { Peak - {stage.stageStats.peakUserMemoryReservation} + {stage.latestAttemptExecutionInfo.stats.peakUserMemoryReservation} @@ -458,7 +529,7 @@ class StageSummary extends React.Component { Pending - {stage.tasks.filter(task => task.taskStatus.state === "PLANNED").length} + {stage.latestAttemptExecutionInfo.tasks.filter(task => task.taskStatus.state === "PLANNED").length} @@ -466,7 +537,7 @@ class StageSummary extends React.Component { Running - {stage.tasks.filter(task => task.taskStatus.state === "RUNNING").length} + {stage.latestAttemptExecutionInfo.tasks.filter(task => task.taskStatus.state === "RUNNING").length} @@ -474,7 +545,7 @@ class StageSummary extends React.Component { Blocked - {stage.tasks.filter(task => task.stats.fullyBlocked).length} + {stage.latestAttemptExecutionInfo.tasks.filter(task => task.stats.fullyBlocked).length} @@ -482,7 +553,7 @@ class StageSummary extends React.Component { Total - {stage.tasks.length} + {stage.latestAttemptExecutionInfo.tasks.length} @@ -562,6 +633,14 @@ class StageSummary extends React.Component { + + + {this.renderTaskFilter()} + + + {this.renderStageExecutionAttemptsTasks([stage.latestAttemptExecutionInfo])} + + {this.renderStageExecutionAttemptsTasks(stage.previousAttemptsExecutionInfos)} @@ -618,10 +697,6 @@ const SMALL_SPARKLINE_PROPERTIES = { }; const TASK_FILTER = { - NONE: { - text: "None", - predicate: function () { return false } - }, ALL: { text: "All", predicate: function () { return true } @@ -651,7 +726,6 @@ export class QueryDetail extends React.Component { this.state = { query: null, lastSnapshotStages: null, - lastSnapshotTasks: null, lastScheduledTime: 0, lastCpuTime: 0, @@ -672,9 +746,6 @@ export class QueryDetail extends React.Component { lastRender: null, stageRefresh: true, - taskRefresh: true, - - taskFilter: TASK_FILTER.NONE, }; this.refreshLoop = this.refreshLoop.bind(this); @@ -744,10 +815,6 @@ export class QueryDetail extends React.Component { if (this.state.stageRefresh) { lastSnapshotStages = query.outputStage; } - let lastSnapshotTasks = this.state.lastSnapshotTasks; - if (this.state.taskRefresh) { - lastSnapshotTasks = query.outputStage; - } let lastRefresh = this.state.lastRefresh; const lastScheduledTime = this.state.lastScheduledTime; @@ -760,7 +827,6 @@ export class QueryDetail extends React.Component { this.setState({ query: query, lastSnapshotStage: lastSnapshotStages, - lastSnapshotTasks: lastSnapshotTasks, lastScheduledTime: parseDuration(query.queryStats.totalScheduledTime), lastCpuTime: parseDuration(query.queryStats.totalCpuTime), @@ -807,29 +873,6 @@ export class QueryDetail extends React.Component { }); } - handleTaskRefreshClick() { - if (this.state.taskRefresh) { - this.setState({ - taskRefresh: false, - lastSnapshotTasks: this.state.query.outputStage, - }); - } - else { - this.setState({ - taskRefresh: true, - }); - } - } - - renderTaskRefreshButton() { - if (this.state.taskRefresh) { - return - } - else { - return - } - } - handleStageRefreshClick() { if (this.state.stageRefresh) { this.setState({ @@ -853,27 +896,6 @@ export class QueryDetail extends React.Component { } } - renderTaskFilterListItem(taskFilter) { - return ( -
  • {taskFilter.text}
    • - ); - } - - handleTaskFilterClick(filter, event) { - this.setState({ - taskFilter: filter - }); - event.preventDefault(); - } - - getTasksFromStage(stage) { - if (stage === undefined || !stage.hasOwnProperty('subStages') || !stage.hasOwnProperty('tasks')) { - return [] - } - - return [].concat.apply(stage.tasks, stage.subStages.map(this.getTasksFromStage, this)); - } - componentDidMount() { this.refreshLoop(); } @@ -906,57 +928,6 @@ export class QueryDetail extends React.Component { new Clipboard('.copy-button'); } - renderTasks() { - if (this.state.lastSnapshotTasks === null) { - return; - } - - let tasks = []; - if (this.state.taskFilter !== TASK_FILTER.NONE) { - tasks = this.getTasksFromStage(this.state.lastSnapshotTasks).filter(task => this.state.taskFilter.predicate(task.taskStatus.state), this); - } - - return ( -
    -
    -
    -

    Tasks

    -
    -
    - - - - - - - -
    -
    - -
      - {this.renderTaskFilterListItem(TASK_FILTER.NONE)} - {this.renderTaskFilterListItem(TASK_FILTER.ALL)} - {this.renderTaskFilterListItem(TASK_FILTER.PLANNED)} - {this.renderTaskFilterListItem(TASK_FILTER.RUNNING)} - {this.renderTaskFilterListItem(TASK_FILTER.FINISHED)} - {this.renderTaskFilterListItem(TASK_FILTER.FAILED)} -
    -
    -     		  {this.renderTaskRefreshButton()}
    -
    -
    -
    -
    - -
    -
    -
    - ); - } - renderStages() { if (this.state.lastSnapshotStage === null) { return; @@ -1538,7 +1509,6 @@ export class QueryDetail extends React.Component { {this.renderStages()} - {this.renderTasks()} ); } diff --git a/presto-main/src/main/resources/webapp/src/components/QueryHeader.jsx b/presto-main/src/main/resources/webapp/src/components/QueryHeader.jsx index 20d2295d8f957..660bcd6080963 100644 --- a/presto-main/src/main/resources/webapp/src/components/QueryHeader.jsx +++ b/presto-main/src/main/resources/webapp/src/components/QueryHeader.jsx @@ -14,12 +14,7 @@ import React from "react"; -import { - getProgressBarPercentage, - getProgressBarTitle, - getQueryStateColor, - isQueryEnded -} from "../utils"; +import {getProgressBarPercentage, getProgressBarTitle, getQueryStateColor, isQueryEnded} from "../utils"; export class QueryHeader extends React.Component { constructor(props) { diff --git a/presto-main/src/main/resources/webapp/src/components/StageDetail.jsx b/presto-main/src/main/resources/webapp/src/components/StageDetail.jsx index a0f33ada3581d..55defb9711b4e 100644 --- a/presto-main/src/main/resources/webapp/src/components/StageDetail.jsx +++ b/presto-main/src/main/resources/webapp/src/components/StageDetail.jsx @@ -19,10 +19,10 @@ import * as dagreD3 from "dagre-d3"; import * as d3 from "d3"; import { - getChildren, formatCount, formatDataSize, formatDuration, + getChildren, getFirstParameter, getTaskNumber, initializeGraph, @@ -362,7 +362,7 @@ class StageOperatorGraph extends React.Component { const stage = this.props.stage; let operatorStageSummary = null; - const operatorSummaries = stage.stageStats.operatorSummaries; + const operatorSummaries = stage.latestAttemptExecutionInfo.stats.operatorSummaries; for (let i = 0; i < operatorSummaries.length; i++) { if (operatorSummaries[i].pipelineId === pipelineId && operatorSummaries[i].operatorId === operatorId) { operatorStageSummary = operatorSummaries[i]; @@ -436,7 +436,7 @@ class StageOperatorGraph extends React.Component { computeOperatorMap() { const operatorMap = new Map(); - this.props.stage.stageStats.operatorSummaries.forEach(operator => { + this.props.stage.latestAttemptExecutionInfo.stats.operatorSummaries.forEach(operator => { if (!operatorMap.has(operator.planNodeId)) { operatorMap.set(operator.planNodeId, []) } @@ -509,7 +509,8 @@ class StageOperatorGraph extends React.Component { ); } - if (!stage.hasOwnProperty('stageStats') || !stage.stageStats.hasOwnProperty("operatorSummaries") || stage.stageStats.operatorSummaries.length === 0) { + const latestAttemptExecutionInfo = stage.latestAttemptExecutionInfo; + if (!latestAttemptExecutionInfo.hasOwnProperty('stats') || !latestAttemptExecutionInfo.stats.hasOwnProperty("operatorSummaries") || latestAttemptExecutionInfo.stats.operatorSummaries.length === 0) { return (
    diff --git a/presto-main/src/main/resources/webapp/src/components/WorkerStatus.jsx b/presto-main/src/main/resources/webapp/src/components/WorkerStatus.jsx index d5e6bfc935d51..d31759ef57461 100644 --- a/presto-main/src/main/resources/webapp/src/components/WorkerStatus.jsx +++ b/presto-main/src/main/resources/webapp/src/components/WorkerStatus.jsx @@ -14,13 +14,7 @@ import React from "react"; -import { - addToHistory, - formatCount, - formatDataSize, - getFirstParameter, - precisionRound -} from "../utils"; +import {addToHistory, formatCount, formatDataSize, getFirstParameter, precisionRound} from "../utils"; const SMALL_SPARKLINE_PROPERTIES = { width: '100%', @@ -68,7 +62,7 @@ export class WorkerStatus extends React.Component { processCpuLoad: addToHistory(serverInfo.processCpuLoad * 100.0, this.state.processCpuLoad), systemCpuLoad: addToHistory(serverInfo.systemCpuLoad * 100.0, this.state.systemCpuLoad), heapPercentUsed: addToHistory(serverInfo.heapUsed * 100.0 / serverInfo.heapAvailable, this.state.heapPercentUsed), - nonHeapUsed: addToHistory(serverInfo.nonHeapUsed * 100.0, this.state.nonHeapUsed), + nonHeapUsed: addToHistory(serverInfo.nonHeapUsed, this.state.nonHeapUsed), }); this.resetTimer(); diff --git a/presto-main/src/main/resources/webapp/src/components/WorkerThreadList.jsx b/presto-main/src/main/resources/webapp/src/components/WorkerThreadList.jsx index 615bb5f65a87c..606b7f5fe91d0 100644 --- a/presto-main/src/main/resources/webapp/src/components/WorkerThreadList.jsx +++ b/presto-main/src/main/resources/webapp/src/components/WorkerThreadList.jsx @@ -14,9 +14,7 @@ import React from "react"; -import { - getFirstParameter -} from "../utils"; +import {getFirstParameter} from "../utils"; const ALL_THREADS = "All Threads"; const QUERY_THREADS = "Running Queries"; diff --git a/presto-main/src/main/resources/webapp/timeline.html b/presto-main/src/main/resources/webapp/timeline.html index f29b12f58041e..0a3203030daf9 100644 --- a/presto-main/src/main/resources/webapp/timeline.html +++ b/presto-main/src/main/resources/webapp/timeline.html @@ -115,7 +115,7 @@

    Timeline

    function renderTimeline(data) { function getTasks(stage) { return [].concat.apply( - stage.tasks, + stage.latestAttemptExecutionInfo.tasks, stage.subStages.map(getTasks)); } tasks = getTasks(data.outputStage); diff --git a/presto-main/src/test/java/com/facebook/presto/SessionTestUtils.java b/presto-main/src/test/java/com/facebook/presto/SessionTestUtils.java index dfb9cb75007a7..74b200ab6547e 100644 --- a/presto-main/src/test/java/com/facebook/presto/SessionTestUtils.java +++ b/presto-main/src/test/java/com/facebook/presto/SessionTestUtils.java @@ -13,21 +13,14 @@ */ package com.facebook.presto; -import com.facebook.presto.client.ClientCapabilities; - import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static java.util.Arrays.stream; public final class SessionTestUtils { public static final Session TEST_SESSION = testSessionBuilder() .setCatalog("tpch") .setSchema(TINY_SCHEMA_NAME) - .setClientCapabilities(stream(ClientCapabilities.values()) - .map(ClientCapabilities::toString) - .collect(toImmutableSet())) .build(); private SessionTestUtils() diff --git a/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java b/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java index fb109080aa31e..a547a0a33635b 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java +++ b/presto-main/src/test/java/com/facebook/presto/block/AbstractTestBlock.java @@ -322,7 +322,7 @@ protected void assertCheckedPositionValue(Block block, int position, T expec } } else if (expectedValue instanceof long[]) { - Block actual = block.getObject(position, Block.class); + Block actual = block.getBlock(position); long[] expected = (long[]) expectedValue; assertEquals(actual.getPositionCount(), expected.length); for (int i = 0; i < expected.length; i++) { @@ -330,7 +330,7 @@ else if (expectedValue instanceof long[]) { } } else if (expectedValue instanceof Slice[]) { - Block actual = block.getObject(position, Block.class); + Block actual = block.getBlock(position); Slice[] expected = (Slice[]) expectedValue; assertEquals(actual.getPositionCount(), expected.length); for (int i = 0; i < expected.length; i++) { @@ -338,7 +338,7 @@ else if (expectedValue instanceof Slice[]) { } } else if (expectedValue instanceof long[][]) { - Block actual = block.getObject(position, Block.class); + Block actual = block.getBlock(position); long[][] expected = (long[][]) expectedValue; assertEquals(actual.getPositionCount(), expected.length); for (int i = 0; i < expected.length; i++) { @@ -534,7 +534,7 @@ private static Block copyBlockViaWriteStructure(Block block, Supplier values) return builder.build(); } + public static Block createRandomStringBlock(int positionCount, boolean allowNulls, int maxStringLength) + { + return createStringsBlock( + IntStream.range(0, positionCount) + .mapToObj(i -> allowNulls && i % 7 == 1 ? null : generateRandomStringWithLength(maxStringLength)) + .collect(Collectors.toList())); + } + public static Block createSlicesBlock(Slice... values) { requireNonNull(values, "varargs 'values' is null"); @@ -177,6 +215,18 @@ public static Block createStringArraysBlock(Iterable> return builder.build(); } + public static Block createMapBlock(MapType type, Map map) + { + BlockBuilder blockBuilder = type.createBlockBuilder(null, map.size()); + for (Map.Entry entry : map.entrySet()) { + BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); + appendToBlockBuilder(BIGINT, entry.getKey(), entryBuilder); + appendToBlockBuilder(BIGINT, entry.getValue(), entryBuilder); + blockBuilder.closeEntry(); + } + return blockBuilder.build(); + } + public static Block createBooleansBlock(Boolean... values) { requireNonNull(values, "varargs 'values' is null"); @@ -205,6 +255,14 @@ public static Block createBooleansBlock(Iterable values) return builder.build(); } + public static Block createRandomBooleansBlock(int positionCount, boolean allowNulls) + { + return createBooleansBlock( + IntStream.range(0, positionCount) + .mapToObj(i -> allowNulls && i % 7 == 1 ? null : ThreadLocalRandom.current().nextBoolean()) + .collect(Collectors.toList())); + } + public static Block createShortDecimalsBlock(String... values) { requireNonNull(values, "varargs 'values' is null"); @@ -229,6 +287,14 @@ public static Block createShortDecimalsBlock(Iterable values) return builder.build(); } + public static Block createRandomShortDecimalsBlock(int positionCount, boolean allowNulls) + { + return createShortDecimalsBlock( + IntStream.range(0, positionCount) + .mapToObj(i -> allowNulls && i % 7 == 1 ? null : Double.toString(ThreadLocalRandom.current().nextDouble() * ThreadLocalRandom.current().nextInt())) + .collect(Collectors.toList())); + } + public static Block createLongDecimalsBlock(String... values) { requireNonNull(values, "varargs 'values' is null"); @@ -253,6 +319,15 @@ public static Block createLongDecimalsBlock(Iterable values) return builder.build(); } + public static Block createRandomLongDecimalsBlock(int positionCount, boolean allowNulls) + { + checkArgument(positionCount >= 10, "positionCount is less than 10"); + return createLongDecimalsBlock( + IntStream.range(0, positionCount) + .mapToObj(i -> allowNulls && i % 7 == 1 ? null : Double.toString(ThreadLocalRandom.current().nextDouble() * ThreadLocalRandom.current().nextInt())) + .collect(Collectors.toList())); + } + public static Block createIntsBlock(Integer... values) { requireNonNull(values, "varargs 'values' is null"); @@ -276,6 +351,60 @@ public static Block createIntsBlock(Iterable values) return builder.build(); } + public static Block createRowBlock(List fieldTypes, Object[]... rows) + { + BlockBuilder rowBlockBuilder = new RowBlockBuilder(fieldTypes, null, 1); + for (Object[] row : rows) { + if (row == null) { + rowBlockBuilder.appendNull(); + continue; + } + BlockBuilder singleRowBlockWriter = rowBlockBuilder.beginBlockEntry(); + for (Object fieldValue : row) { + if (fieldValue == null) { + singleRowBlockWriter.appendNull(); + continue; + } + + if (fieldValue instanceof String) { + VARCHAR.writeSlice(singleRowBlockWriter, utf8Slice((String) fieldValue)); + } + else if (fieldValue instanceof Slice) { + VARBINARY.writeSlice(singleRowBlockWriter, (Slice) fieldValue); + } + else if (fieldValue instanceof Double) { + DOUBLE.writeDouble(singleRowBlockWriter, ((Double) fieldValue).doubleValue()); + } + else if (fieldValue instanceof Long) { + BIGINT.writeLong(singleRowBlockWriter, ((Long) fieldValue).longValue()); + } + else if (fieldValue instanceof Boolean) { + BOOLEAN.writeBoolean(singleRowBlockWriter, ((Boolean) fieldValue).booleanValue()); + } + else if (fieldValue instanceof Block) { + singleRowBlockWriter.appendStructure((Block) fieldValue); + } + else if (fieldValue instanceof Integer) { + INTEGER.writeLong(singleRowBlockWriter, ((Integer) fieldValue).intValue()); + } + else { + throw new IllegalArgumentException(); + } + } + rowBlockBuilder.closeEntry(); + } + + return rowBlockBuilder.build(); + } + + public static Block createRandomIntsBlock(int positionCount, boolean allowNulls) + { + return createIntsBlock( + IntStream.range(0, positionCount) + .mapToObj(i -> allowNulls && i % 7 == 1 ? null : ThreadLocalRandom.current().nextInt()) + .collect(Collectors.toList())); + } + public static Block createEmptyLongsBlock() { return BIGINT.createFixedSizeBlockBuilder(0).build(); @@ -305,6 +434,14 @@ public static Block createLongsBlock(Iterable values) return createTypedLongsBlock(BIGINT, values); } + public static Block createRandomLongsBlock(int positionCount, boolean allowNulls) + { + return createLongsBlock( + IntStream.range(0, positionCount) + .mapToObj(i -> allowNulls && i % 7 == 1 ? null : ThreadLocalRandom.current().nextLong()) + .collect(Collectors.toList())); + } + public static Block createTypedLongsBlock(Type type, Iterable values) { BlockBuilder builder = type.createBlockBuilder(null, 100); @@ -321,6 +458,15 @@ public static Block createTypedLongsBlock(Type type, Iterable values) return builder.build(); } + public static Block createRandomSmallintsBlock(int positionCount, boolean allowNulls) + { + return createTypedLongsBlock( + SMALLINT, + IntStream.range(0, positionCount) + .mapToObj(i -> allowNulls && i % 7 == 1 ? null : ThreadLocalRandom.current().nextLong() % Short.MIN_VALUE) + .collect(Collectors.toList())); + } + public static Block createLongSequenceBlock(int start, int end) { BlockBuilder builder = BIGINT.createFixedSizeBlockBuilder(end - start); @@ -519,4 +665,160 @@ public static RunLengthEncodedBlock createRLEBlock(long value, int positionCount BIGINT.writeLong(blockBuilder, value); return new RunLengthEncodedBlock(blockBuilder.build(), positionCount); } + + public static RunLengthEncodedBlock createRLEBlock(String value, int positionCount) + { + BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 1); + VARCHAR.writeSlice(blockBuilder, wrappedBuffer(value.getBytes())); + return new RunLengthEncodedBlock(blockBuilder.build(), positionCount); + } + + public static RunLengthEncodedBlock createRleBlockWithRandomValue(Block block, int positionCount) + { + checkArgument(block.getPositionCount() > 0, format("block positions %d is less than or equal to 0", block.getPositionCount())); + return new RunLengthEncodedBlock(block.getRegion(block.getPositionCount() / 2, 1), positionCount); + } + + public static DictionaryBlock createRandomDictionaryBlock(Block dictionary, int positionCount) + { + int[] ids = IntStream.range(0, positionCount).map(i -> ThreadLocalRandom.current().nextInt(dictionary.getPositionCount() / 10)).toArray(); + return new DictionaryBlock(dictionary, ids); + } + + public static Block createRandomBlockForType(Type type, int positionCount, boolean allowNulls, boolean createView, List wrappings) + { + Block block = null; + + if (createView) { + positionCount *= 2; + } + + if (type == BOOLEAN) { + block = createRandomBooleansBlock(positionCount, allowNulls); + } + else if (type == BIGINT) { + block = createRandomLongsBlock(positionCount, allowNulls); + } + else if (type == INTEGER || type == REAL) { + block = createRandomIntsBlock(positionCount, allowNulls); + } + else if (type == SMALLINT) { + block = createRandomSmallintsBlock(positionCount, allowNulls); + } + else if (type instanceof DecimalType) { + DecimalType decimalType = (DecimalType) type; + if (decimalType.isShort()) { + block = createRandomLongsBlock(positionCount, allowNulls); + } + else { + block = createRandomLongDecimalsBlock(positionCount, allowNulls); + } + } + else if (type == VARCHAR) { + block = createRandomStringBlock(positionCount, allowNulls, MAX_STRING_SIZE); + } + else { + // Nested types + // Build isNull and offsets of size positionCount + boolean[] isNull = new boolean[positionCount]; + int[] offsets = new int[positionCount + 1]; + for (int position = 0; position < positionCount; position++) { + if (allowNulls && position % 7 == 1) { + isNull[position] = true; + offsets[position + 1] = offsets[position]; + } + else { + offsets[position + 1] = offsets[position] + (type instanceof RowType ? 1 : ThreadLocalRandom.current().nextInt(ENTRY_SIZE) + 1); + } + } + + // Build the nested block of size offsets[positionCount]. + if (type instanceof ArrayType) { + Block valuesBlock = createRandomBlockForType(((ArrayType) type).getElementType(), offsets[positionCount], allowNulls, createView, wrappings); + block = fromElementBlock(positionCount, Optional.of(isNull), offsets, valuesBlock); + } + else if (type instanceof MapType) { + MapType mapType = (MapType) type; + Block keyBlock = createRandomBlockForType(mapType.getKeyType(), offsets[positionCount], false, createView, wrappings); + Block valueBlock = createRandomBlockForType(mapType.getValueType(), offsets[positionCount], allowNulls, createView, wrappings); + + block = mapType.createBlockFromKeyValue(positionCount, Optional.of(isNull), offsets, keyBlock, valueBlock); + } + else if (type instanceof RowType) { + List fieldTypes = type.getTypeParameters(); + Block[] fieldBlocks = new Block[fieldTypes.size()]; + + for (int i = 0; i < fieldBlocks.length; i++) { + fieldBlocks[i] = createRandomBlockForType(fieldTypes.get(i), positionCount, allowNulls, createView, wrappings); + } + + block = fromFieldBlocks(positionCount, Optional.of(isNull), fieldBlocks); + } + else { + throw new IllegalArgumentException(format("type %s is not supported.", type)); + } + } + + if (createView) { + positionCount /= 2; + int offset = positionCount / 2; + block = block.getRegion(offset, positionCount); + } + + if (!wrappings.isEmpty()) { + block = wrapBlock(block, positionCount, wrappings); + } + + return block; + } + + public static Block wrapBlock(Block block, int positionCount, List wrappings) + { + checkArgument(!wrappings.isEmpty(), "wrappings is empty"); + + Block wrappedBlock = block; + + for (int i = wrappings.size() - 1; i >= 0; i--) { + switch (wrappings.get(i)) { + case DICTIONARY: + wrappedBlock = createRandomDictionaryBlock(wrappedBlock, positionCount); + break; + case RUN_LENGTH: + wrappedBlock = createRleBlockWithRandomValue(wrappedBlock, positionCount); + break; + default: + throw new IllegalArgumentException(format("wrappings %s is incorrect", wrappings)); + } + } + return wrappedBlock; + } + + public static MapType createMapType(Type keyType, Type valueType) + { + MethodHandle keyNativeEquals = TYPE_MANAGER.resolveOperator(OperatorType.EQUAL, ImmutableList.of(keyType, keyType)); + MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); + MethodHandle keyBlockEquals = compose(keyNativeEquals, nativeValueGetter(keyType), nativeValueGetter(keyType)); + MethodHandle keyNativeHashCode = TYPE_MANAGER.resolveOperator(OperatorType.HASH_CODE, ImmutableList.of(keyType)); + MethodHandle keyBlockHashCode = compose(keyNativeHashCode, nativeValueGetter(keyType)); + return new MapType( + keyType, + valueType, + keyBlockNativeEquals, + keyBlockEquals, + keyNativeHashCode, + keyBlockHashCode); + } + + public enum Encoding + { + DICTIONARY, + RUN_LENGTH + } + + private static String generateRandomStringWithLength(int length) + { + byte[] array = new byte[length]; + ThreadLocalRandom.current().nextBytes(array); + return new String(array, UTF_8); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/block/ColumnarTestUtils.java b/presto-main/src/test/java/com/facebook/presto/block/ColumnarTestUtils.java index 25d8ff501798c..b9ed3026f0da5 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/ColumnarTestUtils.java +++ b/presto-main/src/test/java/com/facebook/presto/block/ColumnarTestUtils.java @@ -31,7 +31,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; -final class ColumnarTestUtils +public final class ColumnarTestUtils { private static final TypeManager TYPE_MANAGER = new TypeRegistry(); private static final BlockEncodingSerde BLOCK_ENCODING_SERDE = new BlockEncodingManager(TYPE_MANAGER); @@ -81,12 +81,12 @@ private static void assertPositionValue(Block block, int position, T expecte } else if (expectedValue instanceof Slice[]) { // array or row - Block actual = block.getObject(position, Block.class); + Block actual = block.getBlock(position); assertBlock(actual, (Slice[]) expectedValue); } else if (expectedValue instanceof Slice[][]) { // map - Block actual = block.getObject(position, Block.class); + Block actual = block.getBlock(position); // a map is exposed as a block alternating key and value entries, so we need to flatten the expected values array assertBlock(actual, flattenMapEntries((Slice[][]) expectedValue)); } diff --git a/presto-main/src/test/java/com/facebook/presto/block/TestColumnarArray.java b/presto-main/src/test/java/com/facebook/presto/block/TestColumnarArray.java index b96be7ff2a37c..0641cb9e87c06 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/TestColumnarArray.java +++ b/presto-main/src/test/java/com/facebook/presto/block/TestColumnarArray.java @@ -115,6 +115,7 @@ private static void assertColumnarArray(Block block, T[] expectedValues) T expectedArray = expectedValues[position]; assertEquals(columnarArray.isNull(position), expectedArray == null); assertEquals(columnarArray.getLength(position), expectedArray == null ? 0 : Array.getLength(expectedArray)); + assertEquals(elementsPosition, columnarArray.getOffset(position)); for (int i = 0; i < columnarArray.getLength(position); i++) { Object expectedElement = Array.get(expectedArray, i); diff --git a/presto-main/src/test/java/com/facebook/presto/block/TestColumnarMap.java b/presto-main/src/test/java/com/facebook/presto/block/TestColumnarMap.java index 70eb7c304a4b3..71438836fc9ae 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/TestColumnarMap.java +++ b/presto-main/src/test/java/com/facebook/presto/block/TestColumnarMap.java @@ -119,11 +119,13 @@ private static void assertColumnarMap(Block block, Slice[][][] expectedValues) Block keysBlock = columnarMap.getKeysBlock(); Block valuesBlock = columnarMap.getValuesBlock(); - int keysPosition = 0; - int valuesPosition = 0; + int elementsPosition = 0; + for (int position = 0; position < expectedValues.length; position++) { Slice[][] expectedMap = expectedValues[position]; assertEquals(columnarMap.isNull(position), expectedMap == null); + assertEquals(columnarMap.getOffset(position), elementsPosition); + if (expectedMap == null) { assertEquals(columnarMap.getEntryCount(position), 0); continue; @@ -134,12 +136,12 @@ private static void assertColumnarMap(Block block, Slice[][][] expectedValues) Slice[] expectedEntry = expectedMap[i]; Slice expectedKey = expectedEntry[0]; - assertBlockPosition(keysBlock, keysPosition, expectedKey); - keysPosition++; + assertBlockPosition(keysBlock, elementsPosition, expectedKey); Slice expectedValue = expectedEntry[1]; - assertBlockPosition(valuesBlock, valuesPosition, expectedValue); - valuesPosition++; + assertBlockPosition(valuesBlock, elementsPosition, expectedValue); + + elementsPosition++; } } } diff --git a/presto-main/src/test/java/com/facebook/presto/block/TestMapBlock.java b/presto-main/src/test/java/com/facebook/presto/block/TestMapBlock.java index 3b51104e5891d..03b7922d631d1 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/TestMapBlock.java +++ b/presto-main/src/test/java/com/facebook/presto/block/TestMapBlock.java @@ -74,52 +74,95 @@ public void testCompactBlock() int[] offsets = {0, 1, 1, 2, 4, 8, 16}; boolean[] mapIsNull = {false, true, false, false, false, false}; - testCompactBlock(mapType(TINYINT, TINYINT).createBlockFromKeyValue(Optional.empty(), new int[1], emptyBlock, emptyBlock)); - testCompactBlock(mapType(TINYINT, TINYINT).createBlockFromKeyValue(Optional.of(mapIsNull), offsets, compactKeyBlock, compactValueBlock)); + testCompactBlock(mapType(TINYINT, TINYINT).createBlockFromKeyValue(0, Optional.empty(), new int[1], emptyBlock, emptyBlock)); + testCompactBlock(mapType(TINYINT, TINYINT).createBlockFromKeyValue(mapIsNull.length, Optional.of(mapIsNull), offsets, compactKeyBlock, compactValueBlock)); // TODO: Add test case for a sliced MapBlock // underlying key/value block is not compact - testIncompactBlock(mapType(TINYINT, TINYINT).createBlockFromKeyValue(Optional.of(mapIsNull), offsets, inCompactKeyBlock, inCompactValueBlock)); + testIncompactBlock(mapType(TINYINT, TINYINT).createBlockFromKeyValue(mapIsNull.length, Optional.of(mapIsNull), offsets, inCompactKeyBlock, inCompactValueBlock)); } // TODO: remove this test when we have a more unified testWith() using assertBlock() @Test public void testLazyHashTableBuildOverBlockRegion() { - Map[] values = createTestMap(9, 3, 4, 0, 8, 0, 6, 5); - Block block = createBlockWithValuesFromKeyValueBlock(values); - BlockBuilder blockBuilder = createBlockBuilderWithValues(values); + assertLazyHashTableBuildOverBlockRegion(createTestMap(9, 3, 4, 0, 8, 0, 6, 5)); + assertLazyHashTableBuildOverBlockRegion(alternatingNullValues(createTestMap(9, 3, 4, 0, 8, 0, 6, 5))); + } + + private void assertLazyHashTableBuildOverBlockRegion(Map[] testValues) + { + // use prefix block to build the hash table + { + MapBlock block = createBlockWithValuesFromKeyValueBlock(testValues); + BlockBuilder blockBuilder = createBlockBuilderWithValues(testValues); - // Create a MapBlock that is a region of another MapBlock. It doesn't have hashtables built at the time of creation. - int offset = block.getPositionCount() / 2; - Block blockRegion = block.getRegion(offset, block.getPositionCount() - offset); + MapBlock prefix = (MapBlock) block.getRegion(0, 4); - assertFalse(((MapBlock) blockRegion).isHashTablesPresent()); - assertFalse(((MapBlock) block).isHashTablesPresent()); + assertFalse(block.isHashTablesPresent()); + assertFalse(prefix.isHashTablesPresent()); - // Lazily build the hashtables for the block region and use them to do position/value check. - Map[] expectedValues = Arrays.copyOfRange(values, values.length / 2, values.length); - assertBlock(blockRegion, () -> blockBuilder.newBlockBuilderLike(null), expectedValues); + assertBlock(prefix, () -> blockBuilder.newBlockBuilderLike(null), Arrays.copyOfRange(testValues, 0, 4)); - assertTrue(((MapBlock) blockRegion).isHashTablesPresent()); - assertTrue(((MapBlock) block).isHashTablesPresent()); + assertTrue(block.isHashTablesPresent()); + assertTrue(prefix.isHashTablesPresent()); - Map[] valuesWithNull = alternatingNullValues(values); - Block blockWithNull = createBlockWithValuesFromKeyValueBlock(valuesWithNull); + MapBlock midSection = (MapBlock) block.getRegion(2, 4); + assertTrue(midSection.isHashTablesPresent()); + assertBlock(midSection, () -> blockBuilder.newBlockBuilderLike(null), Arrays.copyOfRange(testValues, 2, 6)); + + MapBlock suffix = (MapBlock) block.getRegion(4, 4); + assertTrue(suffix.isHashTablesPresent()); + assertBlock(suffix, () -> blockBuilder.newBlockBuilderLike(null), Arrays.copyOfRange(testValues, 4, 8)); + } - // Create a MapBlock that is a region of another MapBlock with null values. It doesn't have hashtables built at the time of creation. - offset = blockWithNull.getPositionCount() / 2; - Block blockRegionWithNull = blockWithNull.getRegion(offset, blockWithNull.getPositionCount() - offset); + // use mid-section block to build the hash table + { + MapBlock block = createBlockWithValuesFromKeyValueBlock(testValues); + BlockBuilder blockBuilder = createBlockBuilderWithValues(testValues); - assertFalse(((MapBlock) blockRegionWithNull).isHashTablesPresent()); - assertFalse(((MapBlock) blockWithNull).isHashTablesPresent()); + MapBlock midSection = (MapBlock) block.getRegion(2, 4); - // Lazily build the hashtables for the block region and use them to do position/value check. - Map[] expectedValuesWithNull = Arrays.copyOfRange(valuesWithNull, valuesWithNull.length / 2, valuesWithNull.length); - assertBlock(blockRegionWithNull, () -> blockBuilder.newBlockBuilderLike(null), expectedValuesWithNull); + assertFalse(block.isHashTablesPresent()); + assertFalse(midSection.isHashTablesPresent()); - assertTrue(((MapBlock) blockRegionWithNull).isHashTablesPresent()); - assertTrue(((MapBlock) blockWithNull).isHashTablesPresent()); + assertBlock(midSection, () -> blockBuilder.newBlockBuilderLike(null), Arrays.copyOfRange(testValues, 2, 6)); + + assertTrue(block.isHashTablesPresent()); + assertTrue(midSection.isHashTablesPresent()); + + MapBlock prefix = (MapBlock) block.getRegion(0, 4); + assertTrue(prefix.isHashTablesPresent()); + assertBlock(prefix, () -> blockBuilder.newBlockBuilderLike(null), Arrays.copyOfRange(testValues, 0, 4)); + + MapBlock suffix = (MapBlock) block.getRegion(4, 4); + assertTrue(suffix.isHashTablesPresent()); + assertBlock(suffix, () -> blockBuilder.newBlockBuilderLike(null), Arrays.copyOfRange(testValues, 4, 8)); + } + + // use suffix block to build the hash table + { + MapBlock block = createBlockWithValuesFromKeyValueBlock(testValues); + BlockBuilder blockBuilder = createBlockBuilderWithValues(testValues); + + MapBlock suffix = (MapBlock) block.getRegion(4, 4); + + assertFalse(block.isHashTablesPresent()); + assertFalse(suffix.isHashTablesPresent()); + + assertBlock(suffix, () -> blockBuilder.newBlockBuilderLike(null), Arrays.copyOfRange(testValues, 4, 8)); + + assertTrue(block.isHashTablesPresent()); + assertTrue(suffix.isHashTablesPresent()); + + MapBlock prefix = (MapBlock) block.getRegion(0, 4); + assertTrue(prefix.isHashTablesPresent()); + assertBlock(prefix, () -> blockBuilder.newBlockBuilderLike(null), Arrays.copyOfRange(testValues, 0, 4)); + + MapBlock midSection = (MapBlock) block.getRegion(2, 4); + assertTrue(midSection.isHashTablesPresent()); + assertBlock(midSection, () -> blockBuilder.newBlockBuilderLike(null), Arrays.copyOfRange(testValues, 2, 6)); + } } private Map[] createTestMap(int... entryCounts) @@ -180,13 +223,14 @@ private BlockBuilder createBlockBuilderWithValues(Map[] maps) return mapBlockBuilder; } - private Block createBlockWithValuesFromKeyValueBlock(Map[] maps) + private MapBlock createBlockWithValuesFromKeyValueBlock(Map[] maps) { List keys = new ArrayList<>(); List values = new ArrayList<>(); - int[] offsets = new int[maps.length + 1]; - boolean[] mapIsNull = new boolean[maps.length]; - for (int i = 0; i < maps.length; i++) { + int positionCount = maps.length; + int[] offsets = new int[positionCount + 1]; + boolean[] mapIsNull = new boolean[positionCount]; + for (int i = 0; i < positionCount; i++) { Map map = maps[i]; mapIsNull[i] = map == null; if (map == null) { @@ -200,7 +244,7 @@ private Block createBlockWithValuesFromKeyValueBlock(Map[] maps) offsets[i + 1] = offsets[i] + map.size(); } } - return mapType(VARCHAR, BIGINT).createBlockFromKeyValue(Optional.of(mapIsNull), offsets, createStringsBlock(keys), createLongsBlock(values)); + return (MapBlock) mapType(VARCHAR, BIGINT).createBlockFromKeyValue(positionCount, Optional.of(mapIsNull), offsets, createStringsBlock(keys), createLongsBlock(values)); } private void createBlockBuilderWithValues(Map map, BlockBuilder mapBlockBuilder) diff --git a/presto-main/src/test/java/com/facebook/presto/block/TestRowBlock.java b/presto-main/src/test/java/com/facebook/presto/block/TestRowBlock.java index 51b2fe974d0d3..bb23980dc445c 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/TestRowBlock.java +++ b/presto-main/src/test/java/com/facebook/presto/block/TestRowBlock.java @@ -168,7 +168,7 @@ private void assertValue(Block rowBlock, int position, List row) requireNonNull(row, "row is null"); assertFalse(rowBlock.isNull(position)); - SingleRowBlock singleRowBlock = (SingleRowBlock) rowBlock.getObject(position, Block.class); + SingleRowBlock singleRowBlock = (SingleRowBlock) rowBlock.getBlock(position); assertEquals(singleRowBlock.getPositionCount(), row.size()); for (int i = 0; i < row.size(); i++) { diff --git a/presto-main/src/test/java/com/facebook/presto/connector/system/TestSystemSplit.java b/presto-main/src/test/java/com/facebook/presto/connector/system/TestSystemSplit.java index 8940a792c6783..a972963ed507b 100644 --- a/presto-main/src/test/java/com/facebook/presto/connector/system/TestSystemSplit.java +++ b/presto-main/src/test/java/com/facebook/presto/connector/system/TestSystemSplit.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.connector.system; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.predicate.TupleDomain; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; public class TestSystemSplit diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java index 4d5d72f8d7c40..c2a62ed9f904e 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestCostCalculator.java @@ -27,16 +27,20 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.NodePartitioningManager; +import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.PlanFragmenter; import com.facebook.presto.sql.planner.PlanVariableAllocator; @@ -44,12 +48,9 @@ import com.facebook.presto.sql.planner.SubPlan; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.IterativeOptimizer; -import com.facebook.presto.sql.planner.optimizations.TranslateExpressions; -import com.facebook.presto.sql.planner.plan.AggregationNode; +import com.facebook.presto.sql.planner.iterative.rule.TranslateExpressions; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.tree.Cast; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.SymbolReference; @@ -60,7 +61,6 @@ import com.facebook.presto.transaction.TransactionManager; import com.facebook.presto.util.FinalizerService; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.testng.annotations.AfterClass; @@ -75,11 +75,11 @@ import java.util.function.Function; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; import static com.facebook.presto.sql.planner.optimizations.AggregationNodeUtils.count; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.LOCAL; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.replicatedExchange; @@ -133,7 +133,8 @@ public void setUp() new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)); - nodePartitioningManager = new NodePartitioningManager(nodeScheduler); + PartitioningProviderManager partitioningProviderManager = new PartitioningProviderManager(); + nodePartitioningManager = new NodePartitioningManager(nodeScheduler, partitioningProviderManager); planFragmenter = new PlanFragmenter(metadata, nodePartitioningManager, new QueryManagerConfig(), new SqlParser()); } @@ -159,12 +160,12 @@ public void testTableScan() TableScanNode tableScan = tableScan("ts", "orderkey"); Map types = ImmutableMap.of("orderkey", BIGINT); - assertCost(tableScan, ImmutableMap.of(), ImmutableMap.of("ts", statsEstimate(tableScan, 1000)), types) + assertCost(tableScan, ImmutableMap.of(), ImmutableMap.of("ts", statsEstimate(tableScan, 1000))) .cpu(1000 * IS_NULL_OVERHEAD) .memory(0) .network(0); - assertCostEstimatedExchanges(tableScan, ImmutableMap.of(), ImmutableMap.of("ts", statsEstimate(tableScan, 1000)), types) + assertCostEstimatedExchanges(tableScan, ImmutableMap.of(), ImmutableMap.of("ts", statsEstimate(tableScan, 1000))) .cpu(1000 * IS_NULL_OVERHEAD) .memory(0) .network(0); @@ -174,7 +175,7 @@ public void testTableScan() .memory(0) .network(0); - assertCostHasUnknownComponentsForUnknownStats(tableScan, types); + assertCostHasUnknownComponentsForUnknownStats(tableScan); } @Test @@ -190,12 +191,12 @@ public void testProject() "orderkey", BIGINT, "string", VARCHAR); - assertCost(project, costs, stats, types) + assertCost(project, costs, stats) .cpu(1000 + 4000 * OFFSET_AND_IS_NULL_OVERHEAD) .memory(0) .network(0); - assertCostEstimatedExchanges(project, costs, stats, types) + assertCostEstimatedExchanges(project, costs, stats) .cpu(1000 + 4000 * OFFSET_AND_IS_NULL_OVERHEAD) .memory(0) .network(0); @@ -205,7 +206,7 @@ public void testProject() .memory(0) .network(0); - assertCostHasUnknownComponentsForUnknownStats(project, types); + assertCostHasUnknownComponentsForUnknownStats(project); } @Test @@ -232,12 +233,12 @@ public void testRepartitionedJoin() "orderkey", BIGINT, "orderkey_0", BIGINT); - assertCost(join, costs, stats, types) + assertCost(join, costs, stats) .cpu(6000 + 1000 + (12000 + 6000 + 1000) * IS_NULL_OVERHEAD) .memory(1000 * IS_NULL_OVERHEAD) .network(0); - assertCostEstimatedExchanges(join, costs, stats, types) + assertCostEstimatedExchanges(join, costs, stats) .cpu(6000 + 1000 + (12000 + 6000 + 1000 + 6000 + 1000 + 1000) * IS_NULL_OVERHEAD) .memory(1000 * IS_NULL_OVERHEAD) .network((6000 + 1000) * IS_NULL_OVERHEAD); @@ -247,7 +248,7 @@ public void testRepartitionedJoin() .memory(1000 * IS_NULL_OVERHEAD) .network(0); - assertCostHasUnknownComponentsForUnknownStats(join, types); + assertCostHasUnknownComponentsForUnknownStats(join); } @Test @@ -275,12 +276,12 @@ public void testReplicatedJoin() "orderkey", BIGINT, "orderkey_0", BIGINT); - assertCost(join, costs, stats, types) + assertCost(join, costs, stats) .cpu(1000 + 6000 + (12000 + 6000 + 10000 + 1000 * (NUMBER_OF_NODES - 1)) * IS_NULL_OVERHEAD) .memory(1000 * NUMBER_OF_NODES * IS_NULL_OVERHEAD) .network(0); - assertCostEstimatedExchanges(join, costs, stats, types) + assertCostEstimatedExchanges(join, costs, stats) .cpu(1000 + 6000 + (12000 + 6000 + 10000 + 1000 * NUMBER_OF_NODES) * IS_NULL_OVERHEAD) .memory(1000 * NUMBER_OF_NODES * IS_NULL_OVERHEAD) .network(1000 * NUMBER_OF_NODES * IS_NULL_OVERHEAD); @@ -290,7 +291,7 @@ public void testReplicatedJoin() .memory(1000 * NUMBER_OF_NODES * IS_NULL_OVERHEAD) .network(0); - assertCostHasUnknownComponentsForUnknownStats(join, types); + assertCostHasUnknownComponentsForUnknownStats(join); } @Test @@ -334,7 +335,7 @@ public void testMemoryCostJoinAboveJoin() Map types = ImmutableMap.of("key1", BIGINT, "key2", BIGINT, "key3", BIGINT); - assertCost(join23, costs, stats, types) + assertCost(join23, costs, stats) .memory( 100 * IS_NULL_OVERHEAD // join23 memory footprint + 64 + 32) // ts2, ts3 memory footprint @@ -342,7 +343,7 @@ public void testMemoryCostJoinAboveJoin() 100 * IS_NULL_OVERHEAD // join23 memory footprint + 64); // ts2 memory footprint - assertCost(join, costs, stats, types) + assertCost(join, costs, stats) .memory( 2000 * IS_NULL_OVERHEAD // join memory footprint + 100 * IS_NULL_OVERHEAD + 64 // join23 total memory when outputting @@ -351,7 +352,7 @@ public void testMemoryCostJoinAboveJoin() 2000 * IS_NULL_OVERHEAD // join memory footprint + 128); // ts1 memory footprint - assertCostEstimatedExchanges(join23, costs, stats, types) + assertCostEstimatedExchanges(join23, costs, stats) .memory( 100 * IS_NULL_OVERHEAD // join23 memory footprint + 64 + 32) // ts2, ts3 memory footprint @@ -359,7 +360,7 @@ public void testMemoryCostJoinAboveJoin() 100 * IS_NULL_OVERHEAD // join23 memory footprint + 64); // ts2 memory footprint - assertCostEstimatedExchanges(join, costs, stats, types) + assertCostEstimatedExchanges(join, costs, stats) .memory( 2000 * IS_NULL_OVERHEAD // join memory footprint + 100 * IS_NULL_OVERHEAD + 64 // join23 total memory when outputting @@ -400,12 +401,12 @@ public void testAggregation() "orderkey", BIGINT, "count", BIGINT); - assertCost(aggregation, costs, stats, types) + assertCost(aggregation, costs, stats) .cpu(6000 * IS_NULL_OVERHEAD + 6000) .memory(13 * IS_NULL_OVERHEAD) .network(0); - assertCostEstimatedExchanges(aggregation, costs, stats, types) + assertCostEstimatedExchanges(aggregation, costs, stats) .cpu((6000 + 6000 + 6000) * IS_NULL_OVERHEAD + 6000) .memory(13 * IS_NULL_OVERHEAD) .network(6000 * IS_NULL_OVERHEAD); @@ -415,7 +416,7 @@ public void testAggregation() .memory(13 * IS_NULL_OVERHEAD) .network(0 * IS_NULL_OVERHEAD); - assertCostHasUnknownComponentsForUnknownStats(aggregation, types); + assertCostHasUnknownComponentsForUnknownStats(aggregation); } @Test @@ -506,10 +507,13 @@ public void testUnion() { TableScanNode ts1 = tableScan("ts1", "orderkey"); TableScanNode ts2 = tableScan("ts2", "orderkey_0"); - ImmutableListMultimap.Builder outputMappings = ImmutableListMultimap.builder(); - outputMappings.put(new VariableReferenceExpression("orderkey_1", BIGINT), new VariableReferenceExpression("orderkey", BIGINT)); - outputMappings.put(new VariableReferenceExpression("orderkey_1", BIGINT), new VariableReferenceExpression("orderkey_0", BIGINT)); - UnionNode union = new UnionNode(new PlanNodeId("union"), ImmutableList.of(ts1, ts2), outputMappings.build()); + UnionNode union = new UnionNode( + new PlanNodeId("union"), + ImmutableList.of(ts1, ts2), + ImmutableList.of(new VariableReferenceExpression("orderkey_1", BIGINT)), + ImmutableMap.of( + new VariableReferenceExpression("orderkey_1", BIGINT), + ImmutableList.of(new VariableReferenceExpression("orderkey", BIGINT), new VariableReferenceExpression("orderkey_0", BIGINT)))); Map stats = ImmutableMap.of( "ts1", statsEstimate(ts1, 4000), "ts2", statsEstimate(ts2, 1000), @@ -521,11 +525,11 @@ public void testUnion() "orderkey", BIGINT, "orderkey_0", BIGINT, "orderkey_1", BIGINT); - assertCost(union, costs, stats, types) + assertCost(union, costs, stats) .cpu(2000) .memory(0) .network(0); - assertCostEstimatedExchanges(union, costs, stats, types) + assertCostEstimatedExchanges(union, costs, stats) .cpu(2000) .memory(0) .network(5000 * IS_NULL_OVERHEAD); @@ -534,19 +538,17 @@ public void testUnion() private CostAssertionBuilder assertCost( PlanNode node, Map costs, - Map stats, - Map types) + Map stats) { - return assertCost(costCalculatorUsingExchanges, node, costs, stats, types); + return assertCost(costCalculatorUsingExchanges, node, costs, stats); } private CostAssertionBuilder assertCostEstimatedExchanges( PlanNode node, Map costs, - Map stats, - Map types) + Map stats) { - return assertCost(costCalculatorWithEstimatedExchanges, node, costs, stats, types); + return assertCost(costCalculatorWithEstimatedExchanges, node, costs, stats); } private CostAssertionBuilder assertCostFragmentedPlan( @@ -557,7 +559,7 @@ private CostAssertionBuilder assertCostFragmentedPlan( { TypeProvider typeProvider = TypeProvider.copyOf(types); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator(stats), session, typeProvider); - CostProvider costProvider = new TestingCostProvider(costs, costCalculatorUsingExchanges, statsProvider, session, typeProvider); + CostProvider costProvider = new TestingCostProvider(costs, costCalculatorUsingExchanges, statsProvider, session); PlanNode plan = translateExpression(node, statsCalculator(stats), typeProvider); SubPlan subPlan = fragment(new Plan(plan, typeProvider, StatsAndCosts.create(node, statsProvider, costProvider))); return new CostAssertionBuilder(subPlan.getFragment().getStatsAndCosts().getCosts().getOrDefault(node.getId(), PlanCostEstimate.unknown())); @@ -576,15 +578,13 @@ private static class TestingCostProvider private final CostCalculator costCalculator; private final StatsProvider statsProvider; private final Session session; - private final TypeProvider types; - private TestingCostProvider(Map costs, CostCalculator costCalculator, StatsProvider statsProvider, Session session, TypeProvider types) + private TestingCostProvider(Map costs, CostCalculator costCalculator, StatsProvider statsProvider, Session session) { this.costs = ImmutableMap.copyOf(requireNonNull(costs, "costs is null")); this.costCalculator = requireNonNull(costCalculator, "costCalculator is null"); this.statsProvider = requireNonNull(statsProvider, "statsProvider is null"); this.session = requireNonNull(session, "session is null"); - this.types = requireNonNull(types, "types is null"); } @Override @@ -593,7 +593,7 @@ public PlanCostEstimate getCost(PlanNode node) if (costs.containsKey(node.getId().toString())) { return costs.get(node.getId().toString()); } - return costCalculator.calculateCost(node, statsProvider, this, session, types); + return costCalculator.calculateCost(node, statsProvider, this, session); } } @@ -601,24 +601,21 @@ private CostAssertionBuilder assertCost( CostCalculator costCalculator, PlanNode node, Map costs, - Map stats, - Map types) + Map stats) { Function statsProvider = planNode -> stats.get(planNode.getId().toString()); PlanCostEstimate cost = calculateCost( costCalculator, node, - sourceCostProvider(costCalculator, costs, statsProvider, types), - statsProvider, - types); + sourceCostProvider(costCalculator, costs, statsProvider), + statsProvider); return new CostAssertionBuilder(cost); } private Function sourceCostProvider( CostCalculator costCalculator, Map costs, - Function statsProvider, - Map types) + Function statsProvider) { return node -> { PlanCostEstimate providedCost = costs.get(node.getId().toString()); @@ -628,27 +625,24 @@ private Function sourceCostProvider( return calculateCost( costCalculator, node, - sourceCostProvider(costCalculator, costs, statsProvider, types), - statsProvider, - types); + sourceCostProvider(costCalculator, costs, statsProvider), + statsProvider); }; } - private void assertCostHasUnknownComponentsForUnknownStats(PlanNode node, Map types) + private void assertCostHasUnknownComponentsForUnknownStats(PlanNode node) { new CostAssertionBuilder(calculateCost( costCalculatorUsingExchanges, node, planNode -> PlanCostEstimate.unknown(), - planNode -> PlanNodeStatsEstimate.unknown(), - types)) + planNode -> PlanNodeStatsEstimate.unknown())) .hasUnknownComponents(); new CostAssertionBuilder(calculateCost( costCalculatorWithEstimatedExchanges, node, planNode -> PlanCostEstimate.unknown(), - planNode -> PlanNodeStatsEstimate.unknown(), - types)) + planNode -> PlanNodeStatsEstimate.unknown())) .hasUnknownComponents(); } @@ -670,22 +664,20 @@ private PlanCostEstimate calculateCost( CostCalculator costCalculator, PlanNode node, Function costs, - Function stats, - Map types) + Function stats) { return costCalculator.calculateCost( node, planNode -> requireNonNull(stats.apply(planNode), "no stats for node"), source -> requireNonNull(costs.apply(source), format("no cost for source: %s", source.getId())), - session, - TypeProvider.copyOf(types)); + session); } private PlanCostEstimate calculateCost(PlanNode node, CostCalculator costCalculator, StatsCalculator statsCalculator, Map types) { TypeProvider typeProvider = TypeProvider.copyOf(types); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, typeProvider); - CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.empty(), session, typeProvider); + CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.empty(), session); return costProvider.getCost(node); } @@ -693,7 +685,7 @@ private PlanCostEstimate calculateCostFragmentedPlan(PlanNode node, StatsCalcula { TypeProvider typeProvider = TypeProvider.copyOf(types); StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, typeProvider); - CostProvider costProvider = new CachingCostProvider(costCalculatorUsingExchanges, statsProvider, Optional.empty(), session, typeProvider); + CostProvider costProvider = new CachingCostProvider(costCalculatorUsingExchanges, statsProvider, Optional.empty(), session); node = translateExpression(node, statsCalculator, typeProvider); SubPlan subPlan = fragment(new Plan(node, typeProvider, StatsAndCosts.create(node, statsProvider, costProvider))); return subPlan.getFragment().getStatsAndCosts().getCosts().getOrDefault(node.getId(), PlanCostEstimate.unknown()); diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestFilterStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/TestFilterStatsCalculator.java index 68d6c8d5d7fed..173f7bbda0633 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestFilterStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestFilterStatsCalculator.java @@ -175,7 +175,7 @@ public void testBooleanLiteralStats() // For ExpressionInterpreter, cast will return unknown; but RowExpressionInterpreter can actually evaluate cast. Expression expression = expression("cast(null as boolean) AND sin(x) > x"); RowExpression rowExpression = translator.translateAndOptimize(expression, standardTypes); - PlanNodeStatsEstimate rowExpressionStatsEstimate = statsCalculator.filterStats(standardInputStatistics, rowExpression, session, standardTypes); + PlanNodeStatsEstimate rowExpressionStatsEstimate = statsCalculator.filterStats(standardInputStatistics, rowExpression, session); PlanNodeStatsAssertion.assertThat(rowExpressionStatsEstimate).outputRowsCount(0.0); } @@ -596,7 +596,7 @@ private PlanNodeStatsAssertion assertExpression(Expression expression) // assert both visitors yield the same result RowExpression rowExpression = translator.translateAndOptimize(expression, standardTypes); PlanNodeStatsEstimate expressionStatsEstimate = statsCalculator.filterStats(standardInputStatistics, expression, session, standardTypes); - PlanNodeStatsEstimate rowExpressionStatsEstimate = statsCalculator.filterStats(standardInputStatistics, rowExpression, session, standardTypes); + PlanNodeStatsEstimate rowExpressionStatsEstimate = statsCalculator.filterStats(standardInputStatistics, rowExpression, session); assertEquals(expressionStatsEstimate, rowExpressionStatsEstimate); return PlanNodeStatsAssertion.assertThat(expressionStatsEstimate); diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java index 39241010716e5..6fb3918b6c209 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java @@ -287,7 +287,7 @@ private VariableStatsAssertion assertCalculate(Expression scalarExpression, Plan private VariableStatsAssertion assertCalculate(Expression scalarExpression, PlanNodeStatsEstimate inputStatistics, TypeProvider types) { // assert both visitors yield the same result - RowExpression scalarRowExpression = translator.translateAndOptimize(scalarExpression, types); + RowExpression scalarRowExpression = translator.translate(scalarExpression, types); VariableStatsEstimate expressionVariableStatsEstimate = calculator.calculate(scalarExpression, inputStatistics, session, types); VariableStatsEstimate rowExpressionVariableStatsEstimate = calculator.calculate(scalarRowExpression, inputStatistics, session); assertEquals(expressionVariableStatsEstimate, rowExpressionVariableStatsEstimate); diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestSemiJoinStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/TestSemiJoinStatsCalculator.java index a2c7414ff9716..21fc1d7535603 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestSemiJoinStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestSemiJoinStatsCalculator.java @@ -15,7 +15,7 @@ package com.facebook.presto.cost; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import static com.facebook.presto.cost.PlanNodeStatsAssertion.assertThat; @@ -52,7 +52,7 @@ public class TestSemiJoinStatsCalculator private VariableReferenceExpression unknown = new VariableReferenceExpression("unknown", BIGINT); private VariableReferenceExpression fractionalNdv = new VariableReferenceExpression("fractionalNdv", BIGINT); - @BeforeMethod + @BeforeClass public void setUp() throws Exception { diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestValuesNodeStats.java b/presto-main/src/test/java/com/facebook/presto/cost/TestValuesNodeStats.java index 3a62874be75f3..5a3da3b72c62f 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestValuesNodeStats.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestValuesNodeStats.java @@ -28,6 +28,7 @@ import static com.facebook.presto.sql.relational.Expressions.constantNull; import static com.facebook.presto.sql.tree.ArithmeticBinaryExpression.Operator.ADD; import static com.facebook.presto.type.UnknownType.UNKNOWN; +import static io.airlift.slice.Slices.utf8Slice; public class TestValuesNodeStats extends BaseStatsCalculatorTest @@ -41,7 +42,7 @@ public void testStatsForValuesNode() ImmutableList.of(pb.variable("a", BIGINT), pb.variable("b", DOUBLE)), ImmutableList.of( ImmutableList.of(call(ADD.name(), resolution.arithmeticFunction(ADD, BIGINT, BIGINT), BIGINT, constantExpressions(BIGINT, 3L, 3L)), constant(13.5, DOUBLE)), - ImmutableList.of(constant(55, BIGINT), constantNull(DOUBLE)), + ImmutableList.of(constant(55L, BIGINT), constantNull(DOUBLE)), ImmutableList.of(constant(6L, BIGINT), constant(13.5, DOUBLE))))) .check(outputStats -> outputStats.equalTo( PlanNodeStatsEstimate.builder() @@ -68,9 +69,9 @@ public void testStatsForValuesNode() .values( ImmutableList.of(pb.variable("v", createVarcharType(30))), ImmutableList.of( - constantExpressions(VARCHAR, "Alice"), - constantExpressions(VARCHAR, "has"), - constantExpressions(VARCHAR, "a cat"), + constantExpressions(VARCHAR, utf8Slice("Alice")), + constantExpressions(VARCHAR, utf8Slice("has")), + constantExpressions(VARCHAR, utf8Slice("a cat")), ImmutableList.of(constantNull(VARCHAR))))) .check(outputStats -> outputStats.equalTo( PlanNodeStatsEstimate.builder() @@ -98,7 +99,7 @@ public void testStatsForValuesNodeWithJustNulls() .values( ImmutableList.of(pb.variable("a", BIGINT)), ImmutableList.of( - ImmutableList.of(call(ADD.name(), resolution.arithmeticFunction(ADD, BIGINT, BIGINT), BIGINT, constant(3, BIGINT), constantNull(BIGINT)))))) + ImmutableList.of(call(ADD.name(), resolution.arithmeticFunction(ADD, BIGINT, BIGINT), BIGINT, constant(3L, BIGINT), constantNull(BIGINT)))))) .check(outputStats -> outputStats.equalTo(bigintNullAStats)); tester().assertStatsFor(pb -> pb diff --git a/presto-main/src/test/java/com/facebook/presto/event/TestQueryMonitorConfig.java b/presto-main/src/test/java/com/facebook/presto/event/TestQueryMonitorConfig.java index 963b191c52128..4f9987cce3ec3 100644 --- a/presto-main/src/test/java/com/facebook/presto/event/TestQueryMonitorConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/event/TestQueryMonitorConfig.java @@ -19,9 +19,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static io.airlift.units.DataSize.Unit; public class TestQueryMonitorConfig diff --git a/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java index b2d91960c02d7..1651a0cdbf5b5 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/BenchmarkNodeScheduler.java @@ -65,10 +65,10 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.execution.scheduler.NodeSchedulerConfig.NetworkTopologyType.BENCHMARK; import static com.facebook.presto.execution.scheduler.NodeSchedulerConfig.NetworkTopologyType.FLAT; import static com.facebook.presto.execution.scheduler.NodeSchedulerConfig.NetworkTopologyType.LEGACY; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; @@ -158,7 +158,7 @@ public void setup() for (int j = 0; j < MAX_SPLITS_PER_NODE + MAX_PENDING_SPLITS_PER_TASK_PER_NODE; j++) { initialSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(i))); } - TaskId taskId = new TaskId("test", 1, i); + TaskId taskId = new TaskId("test", 1, 0, i); MockRemoteTaskFactory.MockRemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, initialSplits.build(), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId)); nodeTaskMap.addTask(node, remoteTask); taskMap.put(node, remoteTask); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/MockQueryExecution.java b/presto-main/src/test/java/com/facebook/presto/execution/MockQueryExecution.java index 0138fa10d7c7c..a40447982a3c2 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/MockQueryExecution.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/MockQueryExecution.java @@ -122,6 +122,7 @@ public QueryInfo getQueryInfo() 9, 10, 11, + 11, 12, 13, @@ -160,7 +161,6 @@ public QueryInfo getQueryInfo() ImmutableList.of()), Optional.empty(), Optional.empty(), - Optional.empty(), ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of(), @@ -264,6 +264,12 @@ public BasicQueryInfo getBasicQueryInfo() return new BasicQueryInfo(getQueryInfo()); } + @Override + public int getRunningTaskCount() + { + return getQueryInfo().getQueryStats().getRunningTasks(); + } + @Override public DataSize getUserMemoryReservation() { diff --git a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java index d62ca5dca5224..2e19f72863be7 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java @@ -13,12 +13,14 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.stats.TestingGcMonitor; import com.facebook.presto.Session; import com.facebook.presto.cost.StatsAndCosts; import com.facebook.presto.execution.NodeTaskMap.PartitionedSplitCountTracker; import com.facebook.presto.execution.buffer.LazyOutputBuffer; import com.facebook.presto.execution.buffer.OutputBuffer; import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.memory.MemoryPool; import com.facebook.presto.memory.QueryContext; import com.facebook.presto.memory.context.SimpleLocalMemoryContext; @@ -52,7 +54,6 @@ import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import io.airlift.stats.TestingGcMonitor; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -130,7 +131,17 @@ public MockRemoteTask createTableScanTask(TaskId taskId, InternalNode newNode, L for (Split sourceSplit : splits) { initialSplits.put(sourceId, sourceSplit); } - return createRemoteTask(TEST_SESSION, taskId, newNode, testFragment, initialSplits.build(), OptionalInt.empty(), createInitialEmptyOutputBuffers(BROADCAST), partitionedSplitCountTracker, true); + return createRemoteTask( + TEST_SESSION, + taskId, + newNode, + testFragment, + initialSplits.build(), + OptionalInt.empty(), + createInitialEmptyOutputBuffers(BROADCAST), + partitionedSplitCountTracker, + true, + new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())); } @Override @@ -143,7 +154,8 @@ public MockRemoteTask createRemoteTask( OptionalInt totalPartitions, OutputBuffers outputBuffers, PartitionedSplitCountTracker partitionedSplitCountTracker, - boolean summarizeTaskInfo) + boolean summarizeTaskInfo, + TableWriteInfo tableWriteInfo) { return new MockRemoteTask(taskId, fragment, node.getNodeIdentifier(), executor, scheduledExecutor, initialSplits, totalPartitions, partitionedSplitCountTracker); } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TaskTestUtils.java b/presto-main/src/test/java/com/facebook/presto/execution/TaskTestUtils.java index 562c76e3381b3..176a306e21a69 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TaskTestUtils.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TaskTestUtils.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.cost.StatsAndCosts; import com.facebook.presto.event.SplitMonitor; @@ -21,6 +22,7 @@ import com.facebook.presto.execution.scheduler.LegacyNetworkTopology; import com.facebook.presto.execution.scheduler.NodeScheduler; import com.facebook.presto.execution.scheduler.NodeSchedulerConfig; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.index.IndexManager; import com.facebook.presto.metadata.InMemoryNodeManager; import com.facebook.presto.metadata.MetadataManager; @@ -47,10 +49,10 @@ import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; import com.facebook.presto.sql.gen.OrderingCompiler; import com.facebook.presto.sql.gen.PageFunctionCompiler; -import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.LocalExecutionPlanner; import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.sql.planner.Partitioning; +import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.sql.planner.PartitioningScheme; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.PlanFragmentId; @@ -62,17 +64,16 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.json.ObjectMapperProvider; import java.util.List; import java.util.Optional; import java.util.OptionalInt; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION; -import static io.airlift.json.JsonCodec.jsonCodec; public final class TaskTestUtils { @@ -126,15 +127,16 @@ public static LocalExecutionPlanner createTestingPlanner() new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)); - NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler); + PartitioningProviderManager partitioningProviderManager = new PartitioningProviderManager(); + NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler, partitioningProviderManager); PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0); return new LocalExecutionPlanner( metadata, - new SqlParser(), Optional.empty(), pageSourceManager, new IndexManager(), + partitioningProviderManager, nodePartitioningManager, new PageSinkManager(), new ExpressionCompiler(metadata, pageFunctionCompiler), @@ -161,7 +163,7 @@ public static LocalExecutionPlanner createTestingPlanner() public static TaskInfo updateTask(SqlTask sqlTask, List taskSources, OutputBuffers outputBuffers) { - return sqlTask.updateTask(TEST_SESSION, Optional.of(PLAN_FRAGMENT), taskSources, outputBuffers, OptionalInt.empty()); + return sqlTask.updateTask(TEST_SESSION, Optional.of(PLAN_FRAGMENT), taskSources, outputBuffers, OptionalInt.empty(), Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); } public static SplitMonitor createTestSplitMonitor() diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestColumn.java b/presto-main/src/test/java/com/facebook/presto/execution/TestColumn.java index 26d56c0c7e5dc..6d25b33bf8449 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestColumn.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestColumn.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.execution; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestCommitTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestCommitTask.java index 2cbea2cf3afd2..9b15025e97382 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestCommitTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestCommitTask.java @@ -32,13 +32,13 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.spi.StandardErrorCode.NOT_IN_TRANSACTION; import static com.facebook.presto.spi.StandardErrorCode.UNKNOWN_TRANSACTION; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Collections.emptyList; import static java.util.concurrent.Executors.newCachedThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestCreateTableTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestCreateTableTask.java index df26aaa82a234..d22224a9effb4 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestCreateTableTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestCreateTableTask.java @@ -48,6 +48,7 @@ import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS; import static com.facebook.presto.spi.connector.ConnectorCapabilities.NOT_NULL_COLUMN_CONSTRAINT; import static com.facebook.presto.spi.session.PropertyMetadata.stringProperty; @@ -56,7 +57,6 @@ import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; import static com.google.common.collect.Sets.immutableEnumSet; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static java.util.Collections.emptyList; import static java.util.Collections.emptySet; import static java.util.Locale.ENGLISH; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestDeallocateTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestDeallocateTask.java index c44f15dcbc76a..9c22c3f43601a 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestDeallocateTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestDeallocateTask.java @@ -33,12 +33,12 @@ import java.util.Set; import java.util.concurrent.ExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Collections.emptyList; import static java.util.concurrent.Executors.newCachedThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestInput.java b/presto-main/src/test/java/com/facebook/presto/execution/TestInput.java index 4dd446859ef79..51a8782212751 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestInput.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestInput.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.ConnectorId; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; import java.util.Optional; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestMemoryRevokingScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/TestMemoryRevokingScheduler.java index f67565d3ed7e7..c06fba295372a 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestMemoryRevokingScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestMemoryRevokingScheduler.java @@ -14,6 +14,8 @@ package com.facebook.presto.execution; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.TestingGcMonitor; import com.facebook.presto.Session; import com.facebook.presto.execution.TestSqlTaskManager.MockExchangeClientSupplier; import com.facebook.presto.execution.executor.TaskExecutor; @@ -35,8 +37,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import io.airlift.stats.CounterStat; -import io.airlift.stats.TestingGcMonitor; import io.airlift.units.DataSize; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -51,11 +51,11 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicInteger; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static com.facebook.presto.execution.SqlTask.createSqlTask; import static com.facebook.presto.execution.TaskTestUtils.createTestSplitMonitor; import static com.facebook.presto.execution.TaskTestUtils.createTestingPlanner; import static com.facebook.presto.memory.LocalMemoryManager.GENERAL_POOL; -import static io.airlift.concurrent.Threads.threadsNamed; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; @@ -119,7 +119,7 @@ public void testScheduleMemoryRevoking() SqlTask sqlTask1 = newSqlTask(); SqlTask sqlTask2 = newSqlTask(); - TaskContext taskContext1 = sqlTask1.getQueryContext().addTaskContext(new TaskStateMachine(new TaskId("q1", 1, 1), executor), session, false, false, OptionalInt.empty(), false); + TaskContext taskContext1 = sqlTask1.getQueryContext().addTaskContext(new TaskStateMachine(new TaskId("q1", 1, 0, 1), executor), session, false, false, OptionalInt.empty(), false); PipelineContext pipelineContext11 = taskContext1.addPipelineContext(0, false, false, false); DriverContext driverContext111 = pipelineContext11.addDriverContext(); OperatorContext operatorContext1 = driverContext111.addOperatorContext(1, new PlanNodeId("na"), "na"); @@ -127,7 +127,7 @@ public void testScheduleMemoryRevoking() DriverContext driverContext112 = pipelineContext11.addDriverContext(); OperatorContext operatorContext3 = driverContext112.addOperatorContext(3, new PlanNodeId("na"), "na"); - TaskContext taskContext2 = sqlTask2.getQueryContext().addTaskContext(new TaskStateMachine(new TaskId("q2", 1, 1), executor), session, false, false, OptionalInt.empty(), false); + TaskContext taskContext2 = sqlTask2.getQueryContext().addTaskContext(new TaskStateMachine(new TaskId("q2", 1, 0, 1), executor), session, false, false, OptionalInt.empty(), false); PipelineContext pipelineContext21 = taskContext2.addPipelineContext(1, false, false, false); DriverContext driverContext211 = pipelineContext21.addDriverContext(); OperatorContext operatorContext4 = driverContext211.addOperatorContext(4, new PlanNodeId("na"), "na"); @@ -249,7 +249,7 @@ public void testImmediateMemoryRevoking() private OperatorContext createContexts(SqlTask sqlTask) { - TaskContext taskContext = sqlTask.getQueryContext().addTaskContext(new TaskStateMachine(new TaskId("q", 1, 1), executor), session, false, false, OptionalInt.empty(), false); + TaskContext taskContext = sqlTask.getQueryContext().addTaskContext(new TaskStateMachine(new TaskId("q", 1, 0, 1), executor), session, false, false, OptionalInt.empty(), false); PipelineContext pipelineContext = taskContext.addPipelineContext(0, false, false, false); DriverContext driverContext = pipelineContext.addDriverContext(); OperatorContext operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("na"), "na"); @@ -287,7 +287,7 @@ private void assertMemoryRevokingNotRequested() private SqlTask newSqlTask() { - TaskId taskId = new TaskId("query", 0, idGeneator.incrementAndGet()); + TaskId taskId = new TaskId("query", 0, 0, idGeneator.incrementAndGet()); URI location = URI.create("fake://task/" + taskId); return createSqlTask( diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java index a589d8f53c3ee..a5016fbbb02bf 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java @@ -21,6 +21,7 @@ import com.facebook.presto.execution.scheduler.NodeScheduler; import com.facebook.presto.execution.scheduler.NodeSchedulerConfig; import com.facebook.presto.execution.scheduler.NodeSelector; +import com.facebook.presto.execution.scheduler.SplitPlacementResult; import com.facebook.presto.metadata.InMemoryNodeManager; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; @@ -37,6 +38,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import io.airlift.units.Duration; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -53,12 +55,13 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadLocalRandom; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.execution.scheduler.NetworkLocation.ROOT_LOCATION; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -161,7 +164,7 @@ public NetworkLocation get(HostAddress host) } } }; - NodeScheduler nodeScheduler = new NodeScheduler(locationCache, topology, nodeManager, nodeSchedulerConfig, nodeTaskMap); + NodeScheduler nodeScheduler = new NodeScheduler(locationCache, topology, nodeManager, nodeSchedulerConfig, nodeTaskMap, new Duration(5, SECONDS)); NodeSelector nodeSelector = nodeScheduler.createNodeSelector(CONNECTOR_ID); // Fill up the nodes with non-local data @@ -174,7 +177,7 @@ public NetworkLocation get(HostAddress host) MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); int task = 0; for (InternalNode node : assignments.keySet()) { - TaskId taskId = new TaskId("test", 1, task); + TaskId taskId = new TaskId("test", 1, 0, task); task++; MockRemoteTaskFactory.MockRemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, ImmutableList.copyOf(assignments.get(node)), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId)); remoteTask.startSplits(25); @@ -301,11 +304,11 @@ public void testMaxSplitsPerNode() MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); // Max out number of splits on node - TaskId taskId1 = new TaskId("test", 1, 1); + TaskId taskId1 = new TaskId("test", 1, 0, 1); RemoteTask remoteTask1 = remoteTaskFactory.createTableScanTask(taskId1, newNode, initialSplits.build(), nodeTaskMap.createPartitionedSplitCountTracker(newNode, taskId1)); nodeTaskMap.addTask(newNode, remoteTask1); - TaskId taskId2 = new TaskId("test", 1, 2); + TaskId taskId2 = new TaskId("test", 1, 0, 2); RemoteTask remoteTask2 = remoteTaskFactory.createTableScanTask(taskId2, newNode, initialSplits.build(), nodeTaskMap.createPartitionedSplitCountTracker(newNode, taskId2)); nodeTaskMap.addTask(newNode, remoteTask2); @@ -341,13 +344,13 @@ public void testMaxSplitsPerNodePerTask() MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) { // Max out number of splits on node - TaskId taskId = new TaskId("test", 1, 1); + TaskId taskId = new TaskId("test", 1, 0, 1); RemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, initialSplits.build(), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId)); nodeTaskMap.addTask(node, remoteTask); tasks.add(remoteTask); } - TaskId taskId = new TaskId("test", 1, 2); + TaskId taskId = new TaskId("test", 1, 0, 2); RemoteTask newRemoteTask = remoteTaskFactory.createTableScanTask(taskId, newNode, initialSplits.build(), nodeTaskMap.createPartitionedSplitCountTracker(newNode, taskId)); // Max out pending splits on new node taskMap.put(newNode, newRemoteTask); @@ -377,7 +380,7 @@ public void testTaskCompletion() { MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); InternalNode chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0); - TaskId taskId = new TaskId("test", 1, 1); + TaskId taskId = new TaskId("test", 1, 0, 1); RemoteTask remoteTask = remoteTaskFactory.createTableScanTask( taskId, chosenNode, @@ -399,7 +402,7 @@ public void testSplitCount() MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); InternalNode chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0); - TaskId taskId1 = new TaskId("test", 1, 1); + TaskId taskId1 = new TaskId("test", 1, 0, 1); RemoteTask remoteTask1 = remoteTaskFactory.createTableScanTask(taskId1, chosenNode, ImmutableList.of( @@ -407,7 +410,7 @@ public void testSplitCount() new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())), nodeTaskMap.createPartitionedSplitCountTracker(chosenNode, taskId1)); - TaskId taskId2 = new TaskId("test", 1, 2); + TaskId taskId2 = new TaskId("test", 1, 0, 2); RemoteTask remoteTask2 = remoteTaskFactory.createTableScanTask( taskId2, chosenNode, @@ -424,6 +427,97 @@ public void testSplitCount() assertEquals(nodeTaskMap.getPartitionedSplitsOnNode(chosenNode), 0); } + @Test + public void testMaxTasksPerStageWittLimit() + { + NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); + TestingTransactionHandle transactionHandle = TestingTransactionHandle.create(); + NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig() + .setMaxSplitsPerNode(20) + .setIncludeCoordinator(false) + .setMaxPendingSplitsPerTask(10); + + NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, nodeSchedulerConfig, nodeTaskMap); + NodeSelector nodeSelector = nodeScheduler.createNodeSelector(CONNECTOR_ID, 2); + + Set splits = new HashSet<>(); + + splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); + SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of()); + Set internalNodes = splitPlacementResult.getAssignments().keySet(); + assertEquals(internalNodes.size(), 1); + + // adding one more split. Total 2 + splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); + splitPlacementResult = nodeSelector.computeAssignments(splits, getRemoteTableScanTask(splitPlacementResult)); + Set internalNodesSecondCall = splitPlacementResult.getAssignments().keySet(); + assertEquals(internalNodesSecondCall.size(), 2); + assertTrue(internalNodesSecondCall.containsAll(internalNodes)); + + // adding one more split. Total 3 + splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); + splitPlacementResult = nodeSelector.computeAssignments(splits, getRemoteTableScanTask(splitPlacementResult)); + assertEquals(splitPlacementResult.getAssignments().keySet().size(), 2); + assertEquals(splitPlacementResult.getAssignments().keySet(), internalNodesSecondCall); + } + + @Test + public void testMaxTasksPerStageAddingNewNodes() + { + InMemoryNodeManager nodeManager = new InMemoryNodeManager(); + + NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService); + TestingTransactionHandle transactionHandle = TestingTransactionHandle.create(); + NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig() + .setMaxSplitsPerNode(20) + .setIncludeCoordinator(false) + .setMaxPendingSplitsPerTask(10); + + LegacyNetworkTopology networkTopology = new LegacyNetworkTopology(); + // refresh interval is 1 nanosecond + NodeScheduler nodeScheduler = new NodeScheduler(new NetworkLocationCache(networkTopology), networkTopology, nodeManager, nodeSchedulerConfig, nodeTaskMap, Duration.valueOf("0s")); + NodeSelector nodeSelector = nodeScheduler.createNodeSelector(CONNECTOR_ID, 2); + + Set splits = new HashSet<>(); + splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); + splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); + splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())); + + nodeManager.addNode(CONNECTOR_ID, ImmutableList.of(new InternalNode("node1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false))); + SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of()); + Set internalNodes = splitPlacementResult.getAssignments().keySet(); + assertEquals(internalNodes.size(), 1); + + nodeManager.addNode(CONNECTOR_ID, ImmutableList.of(new InternalNode("node2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false))); + + splitPlacementResult = nodeSelector.computeAssignments(splits, getRemoteTableScanTask(splitPlacementResult)); + Set internalNodesSecondCall = splitPlacementResult.getAssignments().keySet(); + assertEquals(internalNodesSecondCall.size(), 2); + assertTrue(internalNodesSecondCall.containsAll(internalNodes)); + + nodeManager.addNode(CONNECTOR_ID, ImmutableList.of(new InternalNode("node2", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false))); + internalNodes = splitPlacementResult.getAssignments().keySet(); + assertEquals(internalNodes.size(), 2); + assertTrue(internalNodesSecondCall.containsAll(internalNodes)); + } + + private List getRemoteTableScanTask(SplitPlacementResult splitPlacementResult) + { + Map taskMap = new HashMap<>(); + Multimap assignments = splitPlacementResult.getAssignments(); + MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor); + int task = 0; + for (InternalNode node : assignments.keySet()) { + TaskId taskId = new TaskId("test", 1, 1, task); + task++; + MockRemoteTaskFactory.MockRemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, ImmutableList.copyOf(assignments.get(node)), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId)); + remoteTask.startSplits(25); + nodeTaskMap.addTask(node, remoteTask); + taskMap.put(node, remoteTask); + } + return ImmutableList.copyOf(taskMap.values()); + } + private static class TestSplitLocal implements ConnectorSplit { diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeSchedulerConfig.java b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeSchedulerConfig.java index 6dd1aa6311b86..d70aa77bef60d 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestNodeSchedulerConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestNodeSchedulerConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.facebook.presto.execution.scheduler.NodeSchedulerConfig; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import org.testng.annotations.Test; import java.util.Map; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestOutput.java b/presto-main/src/test/java/com/facebook/presto/execution/TestOutput.java index 3b1b9a1076548..2b61aec21e793 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestOutput.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestOutput.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.ConnectorId; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestPageSplitterUtil.java b/presto-main/src/test/java/com/facebook/presto/execution/TestPageSplitterUtil.java index 901eb9b5185df..3c1110fc6f18c 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestPageSplitterUtil.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestPageSplitterUtil.java @@ -25,6 +25,8 @@ import java.util.List; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.airlift.testing.Assertions.assertLessThanOrEqual; import static com.facebook.presto.SequencePageBuilder.createSequencePage; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.execution.buffer.PageSplitterUtil.splitPage; @@ -33,8 +35,6 @@ import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.testing.assertions.Assert.assertEquals; import static io.airlift.slice.Slices.wrappedBuffer; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertLessThanOrEqual; public class TestPageSplitterUtil { diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestPlannerWarnings.java b/presto-main/src/test/java/com/facebook/presto/execution/TestPlannerWarnings.java index 8d70422149495..c502ead27b8d6 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestPlannerWarnings.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestPlannerWarnings.java @@ -21,14 +21,15 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.PrestoWarning; import com.facebook.presto.spi.WarningCode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.sql.analyzer.SemanticException; import com.facebook.presto.sql.planner.LogicalPlanner; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.iterative.IterativeOptimizer; import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.iterative.rule.TranslateExpressions; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.tpch.TpchConnectorFactory; import com.google.common.collect.ImmutableList; @@ -127,7 +128,14 @@ private static Plan createPlan(LocalQueryRunner queryRunner, Session session, St queryRunner.getCostCalculator(), ImmutableSet.copyOf(rules)); - return queryRunner.createPlan(session, sql, ImmutableList.of(optimizer, queryRunner.translateExpressions()), LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, warningCollector); + // Translate all OriginalExpression in planNodes to RowExpression so that we can do plan pattern asserting and printing on RowExpression only. + PlanOptimizer expressionTranslator = new IterativeOptimizer( + new RuleStatsRecorder(), + queryRunner.getStatsCalculator(), + queryRunner.getCostCalculator(), + ImmutableSet.copyOf(new TranslateExpressions(queryRunner.getMetadata(), queryRunner.getSqlParser()).rules())); + + return queryRunner.createPlan(session, sql, ImmutableList.of(optimizer, expressionTranslator), LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, warningCollector); } public static List createTestWarnings(int numberOfWarnings) diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestPrepareTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestPrepareTask.java index 90afab1e5ef32..04c81dc0b4c9f 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestPrepareTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestPrepareTask.java @@ -37,6 +37,7 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; @@ -46,7 +47,6 @@ import static com.facebook.presto.sql.QueryUtil.table; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Collections.emptyList; import static java.util.concurrent.Executors.newCachedThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryManagerConfig.java b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryManagerConfig.java index 04d67750d51bf..d4c1d844841b2 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryManagerConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryManagerConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.facebook.presto.execution.QueryManagerConfig.ExchangeMaterializationStrategy; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.Duration; import org.testng.annotations.Test; @@ -33,6 +33,8 @@ public void testDefaults() .setMaxQueryLength(1_000_000) .setMaxStageCount(100) .setStageCountWarningThreshold(50) + .setMaxTotalRunningTaskCount(Integer.MAX_VALUE) + .setMaxQueryRunningTaskCount(Integer.MAX_VALUE) .setClientTimeout(new Duration(5, TimeUnit.MINUTES)) .setScheduleSplitBatchSize(1000) .setMinScheduleSplitBatchSize(100) @@ -52,7 +54,8 @@ public void testDefaults() .setInitializationRequiredWorkers(1) .setInitializationTimeout(new Duration(5, TimeUnit.MINUTES)) .setRequiredWorkers(1) - .setRequiredWorkersMaxWait(new Duration(5, TimeUnit.MINUTES))); + .setRequiredWorkersMaxWait(new Duration(5, TimeUnit.MINUTES)) + .setQuerySubmissionMaxThreads(Runtime.getRuntime().availableProcessors() * 2)); } @Test @@ -65,6 +68,8 @@ public void testExplicitPropertyMappings() .put("query.max-length", "10000") .put("query.max-stage-count", "12345") .put("query.stage-count-warning-threshold", "12300") + .put("experimental.max-total-running-task-count", "60000") + .put("experimental.max-query-running-task-count", "10000") .put("query.schedule-split-batch-size", "99") .put("query.min-schedule-split-batch-size", "9") .put("query.max-concurrent-queries", "10") @@ -84,6 +89,7 @@ public void testExplicitPropertyMappings() .put("query-manager.initialization-timeout", "1m") .put("query-manager.required-workers", "333") .put("query-manager.required-workers-max-wait", "33m") + .put("query-manager.query-submission-max-threads", "5") .build(); QueryManagerConfig expected = new QueryManagerConfig() @@ -92,6 +98,8 @@ public void testExplicitPropertyMappings() .setMaxQueryLength(10000) .setMaxStageCount(12345) .setStageCountWarningThreshold(12300) + .setMaxTotalRunningTaskCount(60000) + .setMaxQueryRunningTaskCount(10000) .setClientTimeout(new Duration(10, TimeUnit.SECONDS)) .setScheduleSplitBatchSize(99) .setMinScheduleSplitBatchSize(9) @@ -111,7 +119,8 @@ public void testExplicitPropertyMappings() .setInitializationRequiredWorkers(200) .setInitializationTimeout(new Duration(1, TimeUnit.MINUTES)) .setRequiredWorkers(333) - .setRequiredWorkersMaxWait(new Duration(33, TimeUnit.MINUTES)); + .setRequiredWorkersMaxWait(new Duration(33, TimeUnit.MINUTES)) + .setQuerySubmissionMaxThreads(5); ConfigAssertions.assertFullMapping(properties, expected); } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStateMachine.java b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStateMachine.java index f373f8d7c3bc9..141942ff3886e 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStateMachine.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStateMachine.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.testing.TestingTicker; import com.facebook.presto.Session; import com.facebook.presto.client.FailureInfo; import com.facebook.presto.execution.warnings.WarningCollector; @@ -27,11 +28,14 @@ import com.facebook.presto.spi.resourceGroups.QueryType; import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import com.facebook.presto.spi.type.Type; +import com.facebook.presto.transaction.DelegatingTransactionManager; +import com.facebook.presto.transaction.TransactionId; import com.facebook.presto.transaction.TransactionManager; import com.google.common.base.Ticker; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.testing.TestingTicker; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; import io.airlift.units.Duration; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -46,6 +50,7 @@ import java.util.concurrent.ExecutorService; import java.util.function.Consumer; +import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.execution.QueryState.FAILED; import static com.facebook.presto.execution.QueryState.FINISHED; @@ -59,7 +64,7 @@ import static com.facebook.presto.spi.StandardErrorCode.USER_CANCELED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; -import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; +import static com.google.common.util.concurrent.Futures.allAsList; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; @@ -352,6 +357,77 @@ public void testUpdateMemoryUsage() assertEquals(stateMachine.getPeakTaskTotalMemory(), 5); } + @Test + public void testTransitionToFailedAfterTransitionToFinishing() + { + SettableFuture commitFuture = SettableFuture.create(); + TransactionManager transactionManager = new DelegatingTransactionManager(createTestTransactionManager()) + { + @Override + public ListenableFuture asyncCommit(TransactionId transactionId) + { + return allAsList(commitFuture, super.asyncCommit(transactionId)); + } + }; + + QueryStateMachine stateMachine = createQueryStateMachine(transactionManager); + stateMachine.transitionToFinishing(); + assertEquals(stateMachine.getQueryState(), FINISHING); + assertFalse(stateMachine.transitionToFailed(new RuntimeException("failed"))); + assertEquals(stateMachine.getQueryState(), FINISHING); + commitFuture.set(null); + tryGetFutureValue(stateMachine.getStateChange(FINISHED), 2, SECONDS); + assertEquals(stateMachine.getQueryState(), FINISHED); + } + + @Test + public void testTransitionToCanceledAfterTransitionToFinishing() + { + SettableFuture commitFuture = SettableFuture.create(); + TransactionManager transactionManager = new DelegatingTransactionManager(createTestTransactionManager()) + { + @Override + public ListenableFuture asyncCommit(TransactionId transactionId) + { + return allAsList(commitFuture, super.asyncCommit(transactionId)); + } + }; + + QueryStateMachine stateMachine = createQueryStateMachine(transactionManager); + stateMachine.transitionToFinishing(); + assertEquals(stateMachine.getQueryState(), FINISHING); + assertTrue(stateMachine.transitionToCanceled()); + assertEquals(stateMachine.getQueryState(), FAILED); + commitFuture.set(null); + assertEquals(stateMachine.getQueryState(), FAILED); + assertEquals(stateMachine.getFailureInfo().get().getMessage(), "Query was canceled"); + } + + @Test + public void testCommitFailure() + { + SettableFuture commitFuture = SettableFuture.create(); + TransactionManager transactionManager = new DelegatingTransactionManager(createTestTransactionManager()) + { + @Override + public ListenableFuture asyncCommit(TransactionId transactionId) + { + return allAsList(commitFuture, super.asyncCommit(transactionId)); + } + }; + + QueryStateMachine stateMachine = createQueryStateMachine(transactionManager); + stateMachine.transitionToFinishing(); + // after transitioning to finishing, the transaction is gone + assertEquals(stateMachine.getQueryState(), FINISHING); + assertFalse(stateMachine.transitionToFailed(new RuntimeException("failed"))); + assertEquals(stateMachine.getQueryState(), FINISHING); + commitFuture.setException(new RuntimeException("transaction failed")); + tryGetFutureValue(stateMachine.getStateChange(FAILED), 2, SECONDS); + assertEquals(stateMachine.getQueryState(), FAILED); + assertEquals(stateMachine.getFailureInfo().get().getMessage(), "transaction failed"); + } + private static void assertFinalState(QueryStateMachine stateMachine, QueryState expectedState) { assertFinalState(stateMachine, expectedState, null); @@ -455,10 +531,19 @@ private QueryStateMachine createQueryStateMachine() return createQueryStateMachineWithTicker(Ticker.systemTicker()); } + private QueryStateMachine createQueryStateMachine(TransactionManager transactionManager) + { + return createQueryStateMachineWithTicker(Ticker.systemTicker(), transactionManager); + } + private QueryStateMachine createQueryStateMachineWithTicker(Ticker ticker) + { + return createQueryStateMachineWithTicker(ticker, createTestTransactionManager()); + } + + private QueryStateMachine createQueryStateMachineWithTicker(Ticker ticker, TransactionManager transactionManager) { Metadata metadata = MetadataManager.createTestMetadataManager(); - TransactionManager transactionManager = createTestTransactionManager(); AccessControl accessControl = new AccessControlManager(transactionManager); QueryStateMachine stateMachine = QueryStateMachine.beginWithTicker( QUERY, diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStats.java b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStats.java index e0a85018cfd32..2706963667d4d 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStats.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestQueryStats.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.operator.FilterAndProjectOperator; import com.facebook.presto.operator.OperatorStats; import com.facebook.presto.operator.TableWriterOperator; @@ -20,7 +21,6 @@ import com.facebook.presto.spi.plan.PlanNodeId; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -40,6 +40,7 @@ public class TestQueryStats public static final List operatorSummaries = ImmutableList.of( new OperatorStats( 10, + 101, 11, 12, new PlanNodeId("13"), @@ -49,6 +50,7 @@ public class TestQueryStats new Duration(16, NANOSECONDS), new Duration(17, NANOSECONDS), succinctBytes(18L), + 200, succinctBytes(19L), 110L, 111.0, @@ -73,6 +75,7 @@ public class TestQueryStats null), new OperatorStats( 20, + 201, 21, 22, new PlanNodeId("23"), @@ -82,6 +85,7 @@ public class TestQueryStats new Duration(26, NANOSECONDS), new Duration(27, NANOSECONDS), succinctBytes(28L), + 250, succinctBytes(29L), 210L, 211.0, @@ -106,6 +110,7 @@ public class TestQueryStats null), new OperatorStats( 30, + 301, 31, 32, new PlanNodeId("33"), @@ -115,6 +120,7 @@ public class TestQueryStats new Duration(36, NANOSECONDS), new Duration(37, NANOSECONDS), succinctBytes(38L), + 350, succinctBytes(39L), 310L, 311.0, @@ -155,6 +161,7 @@ public class TestQueryStats 9, 10, 11, + 11, 12, 13, @@ -194,6 +201,7 @@ public class TestQueryStats ImmutableList.of(new StageGcStatistics( 101, + 1001, 102, 103, 104, @@ -231,6 +239,7 @@ public static void assertExpectedQueryStats(QueryStats actual) assertEquals(actual.getTotalTasks(), 9); assertEquals(actual.getRunningTasks(), 10); + assertEquals(actual.getPeakRunningTasks(), 11); assertEquals(actual.getCompletedTasks(), 11); assertEquals(actual.getTotalDrivers(), 12); @@ -271,6 +280,7 @@ public static void assertExpectedQueryStats(QueryStats actual) assertEquals(actual.getStageGcStatistics().size(), 1); StageGcStatistics gcStatistics = actual.getStageGcStatistics().get(0); assertEquals(gcStatistics.getStageId(), 101); + assertEquals(gcStatistics.getStageExecutionId(), 1001); assertEquals(gcStatistics.getTasks(), 102); assertEquals(gcStatistics.getFullGcTasks(), 103); assertEquals(gcStatistics.getMinFullGcSec(), 104); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestResetSessionTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestResetSessionTask.java index adccc2041aef6..e79650896f922 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestResetSessionTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestResetSessionTask.java @@ -42,12 +42,12 @@ import java.util.Set; import java.util.concurrent.ExecutorService; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.spi.session.PropertyMetadata.stringProperty; import static com.facebook.presto.testing.TestingSession.createBogusTestingCatalog; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Collections.emptyList; import static java.util.concurrent.Executors.newCachedThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestRollbackTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestRollbackTask.java index e8b2a7d740d15..03ae953a61b37 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestRollbackTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestRollbackTask.java @@ -31,12 +31,12 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.spi.StandardErrorCode.NOT_IN_TRANSACTION; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Collections.emptyList; import static java.util.concurrent.Executors.newCachedThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSetPathTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSetPathTask.java deleted file mode 100644 index aa760e993b925..0000000000000 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSetPathTask.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.execution; - -import com.facebook.presto.block.BlockEncodingManager; -import com.facebook.presto.execution.warnings.WarningCollector; -import com.facebook.presto.metadata.AnalyzePropertyManager; -import com.facebook.presto.metadata.CatalogManager; -import com.facebook.presto.metadata.ColumnPropertyManager; -import com.facebook.presto.metadata.MetadataManager; -import com.facebook.presto.metadata.SchemaPropertyManager; -import com.facebook.presto.metadata.SessionPropertyManager; -import com.facebook.presto.metadata.TablePropertyManager; -import com.facebook.presto.security.AccessControl; -import com.facebook.presto.security.AllowAllAccessControl; -import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.resourceGroups.ResourceGroupId; -import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.tree.Identifier; -import com.facebook.presto.sql.tree.PathElement; -import com.facebook.presto.sql.tree.PathSpecification; -import com.facebook.presto.sql.tree.SetPath; -import com.facebook.presto.transaction.TransactionManager; -import com.facebook.presto.type.TypeRegistry; -import com.google.common.collect.ImmutableList; -import org.testng.annotations.AfterClass; -import org.testng.annotations.Test; - -import java.net.URI; -import java.util.Optional; -import java.util.concurrent.ExecutorService; - -import static com.facebook.presto.SessionTestUtils.TEST_SESSION; -import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static java.util.Collections.emptyList; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static org.testng.Assert.assertEquals; - -public class TestSetPathTask -{ - private final TransactionManager transactionManager; - private final AccessControl accessControl; - private final MetadataManager metadata; - - private final ExecutorService executor = newCachedThreadPool(daemonThreadsNamed("stage-executor-%s")); - - public TestSetPathTask() - { - CatalogManager catalogManager = new CatalogManager(); - transactionManager = createTestTransactionManager(catalogManager); - accessControl = new AllowAllAccessControl(); - - metadata = new MetadataManager( - new FeaturesConfig(), - new TypeRegistry(), - new BlockEncodingManager(new TypeRegistry()), - new SessionPropertyManager(), - new SchemaPropertyManager(), - new TablePropertyManager(), - new ColumnPropertyManager(), - new AnalyzePropertyManager(), - transactionManager); - } - - @AfterClass(alwaysRun = true) - public void tearDown() - { - executor.shutdownNow(); - } - - @Test - public void testSetPath() - { - PathSpecification pathSpecification = new PathSpecification(Optional.empty(), ImmutableList.of( - new PathElement(Optional.empty(), new Identifier("foo")))); - - QueryStateMachine stateMachine = createQueryStateMachine("SET PATH foo"); - executeSetPathTask(pathSpecification, stateMachine); - - assertEquals(stateMachine.getSetPath(), "foo"); - } - - @Test(expectedExceptions = PrestoException.class, expectedExceptionsMessageRegExp = "Catalog does not exist: .*") - public void testSetPathInvalidCatalog() - { - PathSpecification invalidPathSpecification = new PathSpecification(Optional.empty(), ImmutableList.of( - new PathElement(Optional.of(new Identifier("invalidCatalog")), new Identifier("thisDoesNotMatter")))); - - QueryStateMachine stateMachine = createQueryStateMachine("SET PATH invalidCatalog.thisDoesNotMatter"); - executeSetPathTask(invalidPathSpecification, stateMachine); - } - - private QueryStateMachine createQueryStateMachine(String query) - { - return QueryStateMachine.begin( - query, - TEST_SESSION, - URI.create("fake://uri"), - new ResourceGroupId("test"), - Optional.empty(), - false, - transactionManager, - accessControl, - executor, - metadata, - WarningCollector.NOOP); - } - - private void executeSetPathTask(PathSpecification pathSpecification, QueryStateMachine stateMachine) - { - getFutureValue(new SetPathTask().execute( - new SetPath(pathSpecification), - transactionManager, - metadata, - accessControl, - stateMachine, - emptyList())); - } -} diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSetRoleTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSetRoleTask.java index 07afa175d06cd..bd980b3e102cb 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSetRoleTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSetRoleTask.java @@ -46,10 +46,10 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.testing.TestingSession.createBogusTestingCatalog; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSetSessionTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSetSessionTask.java index 8a207cdbc2761..e01fe2c2c4e96 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSetSessionTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSetSessionTask.java @@ -50,14 +50,14 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.spi.StandardErrorCode.INVALID_SESSION_PROPERTY; import static com.facebook.presto.spi.session.PropertyMetadata.stringProperty; import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.testing.TestingSession.createBogusTestingCatalog; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.concurrent.Executors.newCachedThreadPool; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java index cbbc5f6ba212f..32ee70177a39e 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java @@ -15,8 +15,8 @@ import com.facebook.presto.client.NodeVersion; import com.facebook.presto.cost.StatsAndCosts; -import com.facebook.presto.execution.TestSqlTaskManager.MockLocationFactory; import com.facebook.presto.execution.scheduler.SplitSchedulerStats; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.failureDetector.NoOpFailureDetector; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.operator.StageExecutionDescriptor; @@ -45,6 +45,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.execution.SqlStageExecution.createSqlStageExecution; import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.ARBITRARY; @@ -53,7 +54,6 @@ import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static java.util.concurrent.TimeUnit.MINUTES; @@ -100,8 +100,7 @@ private void testFinalStageInfoInternal() StageId stageId = new StageId(new QueryId("query"), 0); SqlStageExecution stage = createSqlStageExecution( - stageId, - new MockLocationFactory().createStageLocation(stageId), + new StageExecutionId(stageId, 0), createExchangePlanFragment(), new MockRemoteTaskFactory(executor, scheduledExecutor), TEST_SESSION, @@ -109,11 +108,12 @@ private void testFinalStageInfoInternal() nodeTaskMap, executor, new NoOpFailureDetector(), - new SplitSchedulerStats()); + new SplitSchedulerStats(), + new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())); stage.setOutputBuffers(createInitialEmptyOutputBuffers(ARBITRARY)); // add listener that fetches stage info when the final status is available - SettableFuture finalStageInfo = SettableFuture.create(); + SettableFuture finalStageInfo = SettableFuture.create(); stage.addFinalStageInfoListener(finalStageInfo::set); // in a background thread add a ton of tasks @@ -142,14 +142,14 @@ private void testFinalStageInfoInternal() // wait for some tasks to be created, and then abort the query latch.await(1, MINUTES); - assertFalse(stage.getStageInfo().getTasks().isEmpty()); + assertFalse(stage.getStageExecutionInfo().getTasks().isEmpty()); stage.abort(); // once the final stage info is available, verify that it is complete - StageInfo stageInfo = finalStageInfo.get(1, MINUTES); + StageExecutionInfo stageInfo = finalStageInfo.get(1, MINUTES); assertFalse(stageInfo.getTasks().isEmpty()); - assertTrue(stageInfo.isFinalStageInfo()); - assertSame(stage.getStageInfo(), stageInfo); + assertTrue(stageInfo.isFinal()); + assertSame(stage.getStageExecutionInfo(), stageInfo); // cancel the background thread adding tasks addTasksTask.cancel(true); @@ -161,6 +161,7 @@ private static PlanFragment createExchangePlanFragment() new PlanNodeId("exchange"), ImmutableList.of(new PlanFragmentId(0)), ImmutableList.of(new VariableReferenceExpression("column", VARCHAR)), + false, Optional.empty(), REPARTITION); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTask.java index 075d7956ef86d..c719aa726bfa1 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTask.java @@ -13,12 +13,15 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.TestingGcMonitor; import com.facebook.presto.execution.TestSqlTaskManager.MockExchangeClientSupplier; import com.facebook.presto.execution.buffer.BufferResult; import com.facebook.presto.execution.buffer.BufferState; import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId; import com.facebook.presto.execution.executor.TaskExecutor; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.memory.MemoryPool; import com.facebook.presto.memory.QueryContext; import com.facebook.presto.spi.QueryId; @@ -30,8 +33,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.stats.CounterStat; -import io.airlift.stats.TestingGcMonitor; import io.airlift.units.DataSize; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -43,6 +44,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.execution.SqlTask.createSqlTask; import static com.facebook.presto.execution.TaskTestUtils.EMPTY_SOURCES; @@ -55,7 +57,6 @@ import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.PARTITIONED; import static com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.concurrent.Threads.threadsNamed; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.util.concurrent.Executors.newScheduledThreadPool; @@ -114,7 +115,8 @@ public void testEmptyQuery() ImmutableList.of(), createInitialEmptyOutputBuffers(PARTITIONED) .withNoMoreBufferIds(), - OptionalInt.empty()); + OptionalInt.empty(), + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); taskInfo = sqlTask.getTaskInfo(); @@ -125,7 +127,8 @@ public void testEmptyQuery() ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(), true)), createInitialEmptyOutputBuffers(PARTITIONED) .withNoMoreBufferIds(), - OptionalInt.empty()); + OptionalInt.empty(), + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.FINISHED); taskInfo = sqlTask.getTaskInfo(); @@ -142,7 +145,8 @@ public void testSimpleQuery() Optional.of(PLAN_FRAGMENT), ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(SPLIT), true)), createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds(), - OptionalInt.empty()); + OptionalInt.empty(), + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); taskInfo = sqlTask.getTaskInfo(); @@ -180,7 +184,8 @@ public void testCancel() createInitialEmptyOutputBuffers(PARTITIONED) .withBuffer(OUT, 0) .withNoMoreBufferIds(), - OptionalInt.empty()); + OptionalInt.empty(), + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); assertNull(taskInfo.getStats().getEndTime()); @@ -207,7 +212,8 @@ public void testAbort() Optional.of(PLAN_FRAGMENT), ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(SPLIT), true)), createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds(), - OptionalInt.empty()); + OptionalInt.empty(), + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); taskInfo = sqlTask.getTaskInfo(); @@ -299,7 +305,7 @@ public void testBufferNotCloseOnFail() public SqlTask createInitialTask() { - TaskId taskId = new TaskId("query", 0, nextTaskId.incrementAndGet()); + TaskId taskId = new TaskId("query", 0, 0, nextTaskId.incrementAndGet()); URI location = URI.create("fake://task/" + taskId); QueryContext queryContext = new QueryContext(new QueryId("query"), diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskExecution.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskExecution.java index 6d8ea123817f2..e374a2a3c1be6 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskExecution.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskExecution.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.stats.TestingGcMonitor; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.execution.buffer.BufferResult; import com.facebook.presto.execution.buffer.BufferState; @@ -60,7 +61,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import io.airlift.stats.TestingGcMonitor; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.testng.annotations.DataProvider; @@ -82,6 +82,8 @@ import java.util.function.Function; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.block.BlockAssertions.createStringSequenceBlock; import static com.facebook.presto.block.BlockAssertions.createStringsBlock; @@ -98,8 +100,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.threadsNamed; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.util.Objects.requireNonNull; @@ -117,7 +117,7 @@ public class TestSqlTaskExecution private static final ConnectorId CONNECTOR_ID = new ConnectorId("test"); private static final ConnectorTransactionHandle TRANSACTION_HANDLE = TestingTransactionHandle.create(); private static final Duration ASSERT_WAIT_TIMEOUT = new Duration(1, HOURS); - private static final TaskId TASK_ID = TaskId.valueOf("queryid.0.0"); + private static final TaskId TASK_ID = TaskId.valueOf("queryid.0.0.0"); @DataProvider public static Object[][] executionStrategies() diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java index 0c4745f3216c7..e71f0e719b4a8 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java @@ -13,11 +13,14 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.node.NodeInfo; +import com.facebook.airlift.stats.TestingGcMonitor; import com.facebook.presto.execution.buffer.BufferResult; import com.facebook.presto.execution.buffer.BufferState; import com.facebook.presto.execution.buffer.OutputBuffers; import com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId; import com.facebook.presto.execution.executor.TaskExecutor; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.memory.LocalMemoryManager; import com.facebook.presto.memory.NodeMemoryConfig; import com.facebook.presto.memory.context.LocalMemoryContext; @@ -30,8 +33,6 @@ import com.google.common.base.Ticker; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.node.NodeInfo; -import io.airlift.stats.TestingGcMonitor; import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import io.airlift.units.Duration; @@ -61,7 +62,7 @@ @Test public class TestSqlTaskManager { - private static final TaskId TASK_ID = new TaskId("query", 0, 1); + private static final TaskId TASK_ID = new TaskId("query", 0, 0, 1); public static final OutputBufferId OUT = new OutputBufferId(0); private final TaskExecutor taskExecutor; @@ -256,7 +257,8 @@ private TaskInfo createTask(SqlTaskManager sqlTaskManager, TaskId taskId, Immuta Optional.of(PLAN_FRAGMENT), ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, splits, true)), outputBuffers, - OptionalInt.empty()); + OptionalInt.empty(), + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); } private TaskInfo createTask(SqlTaskManager sqlTaskManager, TaskId taskId, OutputBuffers outputBuffers) @@ -275,7 +277,8 @@ private TaskInfo createTask(SqlTaskManager sqlTaskManager, TaskId taskId, Output Optional.of(PLAN_FRAGMENT), ImmutableList.of(), outputBuffers, - OptionalInt.empty()); + OptionalInt.empty(), + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); } public static class MockExchangeClientSupplier diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestStageStateMachine.java b/presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStateMachine.java similarity index 65% rename from presto-main/src/test/java/com/facebook/presto/execution/TestStageStateMachine.java rename to presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStateMachine.java index 7ecf9b7930df4..be8a5c11f90d0 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestStageStateMachine.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStateMachine.java @@ -26,16 +26,15 @@ import com.facebook.presto.sql.planner.plan.PlanFragmentId; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import io.airlift.slice.Slices; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; import java.io.IOException; -import java.net.URI; import java.sql.SQLException; import java.util.Optional; import java.util.concurrent.ExecutorService; -import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION; @@ -43,14 +42,11 @@ import static java.util.concurrent.Executors.newCachedThreadPool; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; -public class TestStageStateMachine +public class TestStageExecutionStateMachine { - private static final StageId STAGE_ID = new StageId(new QueryId("query"), 0); - private static final URI LOCATION = URI.create("fake://fake-stage"); - private static final PlanFragment PLAN_FRAGMENT = createValuesPlan(); + private static final StageExecutionId STAGE_ID = new StageExecutionId(new StageId(new QueryId("query"), 0), 0); private static final SQLException FAILED_CAUSE = new SQLException("FAILED"); private final ExecutorService executor = newCachedThreadPool(); @@ -64,201 +60,201 @@ public void tearDown() @Test public void testBasicStateChanges() { - StageStateMachine stateMachine = createStageStateMachine(); - assertState(stateMachine, StageState.PLANNED); + StageExecutionStateMachine stateMachine = createStageStateMachine(); + assertState(stateMachine, StageExecutionState.PLANNED); assertTrue(stateMachine.transitionToScheduling()); - assertState(stateMachine, StageState.SCHEDULING); + assertState(stateMachine, StageExecutionState.SCHEDULING); assertTrue(stateMachine.transitionToScheduled()); - assertState(stateMachine, StageState.SCHEDULED); + assertState(stateMachine, StageExecutionState.SCHEDULED); assertTrue(stateMachine.transitionToRunning()); - assertState(stateMachine, StageState.RUNNING); + assertState(stateMachine, StageExecutionState.RUNNING); assertTrue(stateMachine.transitionToFinished()); - assertState(stateMachine, StageState.FINISHED); + assertState(stateMachine, StageExecutionState.FINISHED); } @Test public void testPlanned() { - StageStateMachine stateMachine = createStageStateMachine(); - assertState(stateMachine, StageState.PLANNED); + StageExecutionStateMachine stateMachine = createStageStateMachine(); + assertState(stateMachine, StageExecutionState.PLANNED); stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToScheduling()); - assertState(stateMachine, StageState.SCHEDULING); + assertState(stateMachine, StageExecutionState.SCHEDULING); stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToRunning()); - assertState(stateMachine, StageState.RUNNING); + assertState(stateMachine, StageExecutionState.RUNNING); stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToFinished()); - assertState(stateMachine, StageState.FINISHED); + assertState(stateMachine, StageExecutionState.FINISHED); stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToFailed(FAILED_CAUSE)); - assertState(stateMachine, StageState.FAILED); + assertState(stateMachine, StageExecutionState.FAILED); stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToAborted()); - assertState(stateMachine, StageState.ABORTED); + assertState(stateMachine, StageExecutionState.ABORTED); stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToCanceled()); - assertState(stateMachine, StageState.CANCELED); + assertState(stateMachine, StageExecutionState.CANCELED); } @Test public void testScheduling() { - StageStateMachine stateMachine = createStageStateMachine(); + StageExecutionStateMachine stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToScheduling()); - assertState(stateMachine, StageState.SCHEDULING); + assertState(stateMachine, StageExecutionState.SCHEDULING); assertFalse(stateMachine.transitionToScheduling()); - assertState(stateMachine, StageState.SCHEDULING); + assertState(stateMachine, StageExecutionState.SCHEDULING); assertTrue(stateMachine.transitionToScheduled()); - assertState(stateMachine, StageState.SCHEDULED); + assertState(stateMachine, StageExecutionState.SCHEDULED); stateMachine = createStageStateMachine(); stateMachine.transitionToScheduling(); assertTrue(stateMachine.transitionToRunning()); - assertState(stateMachine, StageState.RUNNING); + assertState(stateMachine, StageExecutionState.RUNNING); stateMachine = createStageStateMachine(); stateMachine.transitionToScheduling(); assertTrue(stateMachine.transitionToFinished()); - assertState(stateMachine, StageState.FINISHED); + assertState(stateMachine, StageExecutionState.FINISHED); stateMachine = createStageStateMachine(); stateMachine.transitionToScheduling(); assertTrue(stateMachine.transitionToFailed(FAILED_CAUSE)); - assertState(stateMachine, StageState.FAILED); + assertState(stateMachine, StageExecutionState.FAILED); stateMachine = createStageStateMachine(); stateMachine.transitionToScheduling(); assertTrue(stateMachine.transitionToAborted()); - assertState(stateMachine, StageState.ABORTED); + assertState(stateMachine, StageExecutionState.ABORTED); stateMachine = createStageStateMachine(); stateMachine.transitionToScheduling(); assertTrue(stateMachine.transitionToCanceled()); - assertState(stateMachine, StageState.CANCELED); + assertState(stateMachine, StageExecutionState.CANCELED); } @Test public void testScheduled() { - StageStateMachine stateMachine = createStageStateMachine(); + StageExecutionStateMachine stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToScheduled()); - assertState(stateMachine, StageState.SCHEDULED); + assertState(stateMachine, StageExecutionState.SCHEDULED); assertFalse(stateMachine.transitionToScheduling()); - assertState(stateMachine, StageState.SCHEDULED); + assertState(stateMachine, StageExecutionState.SCHEDULED); assertFalse(stateMachine.transitionToScheduled()); - assertState(stateMachine, StageState.SCHEDULED); + assertState(stateMachine, StageExecutionState.SCHEDULED); assertTrue(stateMachine.transitionToRunning()); - assertState(stateMachine, StageState.RUNNING); + assertState(stateMachine, StageExecutionState.RUNNING); stateMachine = createStageStateMachine(); stateMachine.transitionToScheduled(); assertTrue(stateMachine.transitionToFinished()); - assertState(stateMachine, StageState.FINISHED); + assertState(stateMachine, StageExecutionState.FINISHED); stateMachine = createStageStateMachine(); stateMachine.transitionToScheduled(); assertTrue(stateMachine.transitionToFailed(FAILED_CAUSE)); - assertState(stateMachine, StageState.FAILED); + assertState(stateMachine, StageExecutionState.FAILED); stateMachine = createStageStateMachine(); stateMachine.transitionToScheduled(); assertTrue(stateMachine.transitionToAborted()); - assertState(stateMachine, StageState.ABORTED); + assertState(stateMachine, StageExecutionState.ABORTED); stateMachine = createStageStateMachine(); stateMachine.transitionToScheduled(); assertTrue(stateMachine.transitionToCanceled()); - assertState(stateMachine, StageState.CANCELED); + assertState(stateMachine, StageExecutionState.CANCELED); } @Test public void testRunning() { - StageStateMachine stateMachine = createStageStateMachine(); + StageExecutionStateMachine stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToRunning()); - assertState(stateMachine, StageState.RUNNING); + assertState(stateMachine, StageExecutionState.RUNNING); assertFalse(stateMachine.transitionToScheduling()); - assertState(stateMachine, StageState.RUNNING); + assertState(stateMachine, StageExecutionState.RUNNING); assertFalse(stateMachine.transitionToScheduled()); - assertState(stateMachine, StageState.RUNNING); + assertState(stateMachine, StageExecutionState.RUNNING); assertFalse(stateMachine.transitionToRunning()); - assertState(stateMachine, StageState.RUNNING); + assertState(stateMachine, StageExecutionState.RUNNING); assertTrue(stateMachine.transitionToFinished()); - assertState(stateMachine, StageState.FINISHED); + assertState(stateMachine, StageExecutionState.FINISHED); stateMachine = createStageStateMachine(); stateMachine.transitionToRunning(); assertTrue(stateMachine.transitionToFailed(FAILED_CAUSE)); - assertState(stateMachine, StageState.FAILED); + assertState(stateMachine, StageExecutionState.FAILED); stateMachine = createStageStateMachine(); stateMachine.transitionToRunning(); assertTrue(stateMachine.transitionToAborted()); - assertState(stateMachine, StageState.ABORTED); + assertState(stateMachine, StageExecutionState.ABORTED); stateMachine = createStageStateMachine(); stateMachine.transitionToRunning(); assertTrue(stateMachine.transitionToCanceled()); - assertState(stateMachine, StageState.CANCELED); + assertState(stateMachine, StageExecutionState.CANCELED); } @Test public void testFinished() { - StageStateMachine stateMachine = createStageStateMachine(); + StageExecutionStateMachine stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToFinished()); - assertFinalState(stateMachine, StageState.FINISHED); + assertFinalState(stateMachine, StageExecutionState.FINISHED); } @Test public void testFailed() { - StageStateMachine stateMachine = createStageStateMachine(); + StageExecutionStateMachine stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToFailed(FAILED_CAUSE)); - assertFinalState(stateMachine, StageState.FAILED); + assertFinalState(stateMachine, StageExecutionState.FAILED); } @Test public void testAborted() { - StageStateMachine stateMachine = createStageStateMachine(); + StageExecutionStateMachine stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToAborted()); - assertFinalState(stateMachine, StageState.ABORTED); + assertFinalState(stateMachine, StageExecutionState.ABORTED); } @Test public void testCanceled() { - StageStateMachine stateMachine = createStageStateMachine(); + StageExecutionStateMachine stateMachine = createStageStateMachine(); assertTrue(stateMachine.transitionToCanceled()); - assertFinalState(stateMachine, StageState.CANCELED); + assertFinalState(stateMachine, StageExecutionState.CANCELED); } - private static void assertFinalState(StageStateMachine stateMachine, StageState expectedState) + private static void assertFinalState(StageExecutionStateMachine stateMachine, StageExecutionState expectedState) { assertTrue(expectedState.isDone()); @@ -287,36 +283,30 @@ private static void assertFinalState(StageStateMachine stateMachine, StageState assertState(stateMachine, expectedState); } - private static void assertState(StageStateMachine stateMachine, StageState expectedState) + private static void assertState(StageExecutionStateMachine stateMachine, StageExecutionState expectedState) { - assertEquals(stateMachine.getStageId(), STAGE_ID); - assertEquals(stateMachine.getLocation(), LOCATION); - assertSame(stateMachine.getSession(), TEST_SESSION); - - StageInfo stageInfo = stateMachine.getStageInfo(ImmutableList::of, 0, 0); - assertEquals(stageInfo.getStageId(), STAGE_ID); - assertEquals(stageInfo.getSelf(), LOCATION); - assertEquals(stageInfo.getSubStages(), ImmutableList.of()); - assertEquals(stageInfo.getTasks(), ImmutableList.of()); - assertEquals(stageInfo.getTypes(), ImmutableList.of(VARCHAR)); - assertSame(stageInfo.getPlan().get(), PLAN_FRAGMENT); + assertEquals(stateMachine.getStageExecutionId(), STAGE_ID); + + StageExecutionInfo stageExecutionInfo = stateMachine.getStageExecutionInfo(ImmutableList::of, 0, 0); + assertEquals(stageExecutionInfo.getStageExecutionId(), STAGE_ID); + assertEquals(stageExecutionInfo.getTasks(), ImmutableList.of()); assertEquals(stateMachine.getState(), expectedState); - assertEquals(stageInfo.getState(), expectedState); + assertEquals(stageExecutionInfo.getState(), expectedState); - if (expectedState == StageState.FAILED) { - ExecutionFailureInfo failure = stageInfo.getFailureCause().get(); + if (expectedState == StageExecutionState.FAILED) { + ExecutionFailureInfo failure = stageExecutionInfo.getFailureCause().get(); assertEquals(failure.getMessage(), FAILED_CAUSE.getMessage()); assertEquals(failure.getType(), FAILED_CAUSE.getClass().getName()); } else { - assertFalse(stageInfo.getFailureCause().isPresent()); + assertFalse(stageExecutionInfo.getFailureCause().isPresent()); } } - private StageStateMachine createStageStateMachine() + private StageExecutionStateMachine createStageStateMachine() { - return new StageStateMachine(STAGE_ID, LOCATION, TEST_SESSION, PLAN_FRAGMENT, executor, new SplitSchedulerStats()); + return new StageExecutionStateMachine(STAGE_ID, executor, new SplitSchedulerStats(), false); } private static PlanFragment createValuesPlan() @@ -327,7 +317,7 @@ private static PlanFragment createValuesPlan() new PlanFragmentId(0), new ValuesNode(valuesNodeId, ImmutableList.of(variable), - ImmutableList.of(ImmutableList.of(constant("foo", VARCHAR)))), + ImmutableList.of(ImmutableList.of(constant(Slices.utf8Slice("foo"), VARCHAR)))), ImmutableSet.of(variable), SOURCE_DISTRIBUTION, ImmutableList.of(valuesNodeId), diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestStageStats.java b/presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStats.java similarity index 88% rename from presto-main/src/test/java/com/facebook/presto/execution/TestStageStats.java rename to presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStats.java index 1c4c6bd1ba5d9..01ab23ba97496 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestStageStats.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStats.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.stats.Distribution; +import com.facebook.airlift.stats.Distribution.DistributionSnapshot; import com.facebook.presto.spi.eventlistener.StageGcStatistics; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; -import io.airlift.stats.Distribution; -import io.airlift.stats.Distribution.DistributionSnapshot; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -28,9 +28,9 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.testng.Assert.assertEquals; -public class TestStageStats +public class TestStageExecutionStats { - private static final StageStats EXPECTED = new StageStats( + private static final StageExecutionStats EXPECTED = new StageExecutionStats( new DateTime(0), getTestDistribution(1), @@ -73,6 +73,7 @@ public class TestStageStats new StageGcStatistics( 101, + 1001, 102, 103, 104, @@ -85,15 +86,15 @@ public class TestStageStats @Test public void testJson() { - JsonCodec codec = JsonCodec.jsonCodec(StageStats.class); + JsonCodec codec = JsonCodec.jsonCodec(StageExecutionStats.class); String json = codec.toJson(EXPECTED); - StageStats actual = codec.fromJson(json); + StageExecutionStats actual = codec.fromJson(json); assertExpectedStageStats(actual); } - private static void assertExpectedStageStats(StageStats actual) + private static void assertExpectedStageStats(StageExecutionStats actual) { assertEquals(actual.getSchedulingComplete().getMillis(), 0); @@ -134,6 +135,7 @@ private static void assertExpectedStageStats(StageStats actual) assertEquals(actual.getPhysicalWrittenDataSize(), new DataSize(26, BYTE)); assertEquals(actual.getGcInfo().getStageId(), 101); + assertEquals(actual.getGcInfo().getStageExecutionId(), 1001); assertEquals(actual.getGcInfo().getTasks(), 102); assertEquals(actual.getGcInfo().getFullGcTasks(), 103); assertEquals(actual.getGcInfo().getMinFullGcSec(), 104); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestStartTransactionTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestStartTransactionTask.java index c540726e3a51b..e194d2527875f 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestStartTransactionTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestStartTransactionTask.java @@ -43,14 +43,14 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.spi.StandardErrorCode.INCOMPATIBLE_CLIENT; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_TRANSACTION_MODE; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Collections.emptyList; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestStateMachine.java b/presto-main/src/test/java/com/facebook/presto/execution/TestStateMachine.java index eb753a9a51f90..3e415888d35f1 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestStateMachine.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestStateMachine.java @@ -21,8 +21,8 @@ import java.util.concurrent.ExecutorService; -import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.TimeUnit.SECONDS; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestTaskManagerConfig.java b/presto-main/src/test/java/com/facebook/presto/execution/TestTaskManagerConfig.java index 58df2f058e63a..742201218f522 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestTaskManagerConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestTaskManagerConfig.java @@ -22,9 +22,9 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static io.airlift.units.DataSize.Unit; public class TestTaskManagerConfig @@ -54,6 +54,7 @@ public void testDefaults() .setSinkMaxBufferSize(new DataSize(32, Unit.MEGABYTE)) .setMaxPagePartitioningBufferSize(new DataSize(32, Unit.MEGABYTE)) .setWriterCount(1) + .setPartitionedWriterCount(null) .setTaskConcurrency(16) .setHttpResponseThreads(100) .setHttpTimeoutThreads(3) @@ -89,6 +90,7 @@ public void testExplicitPropertyMappings() .put("sink.max-buffer-size", "42MB") .put("driver.max-page-partitioning-buffer-size", "40MB") .put("task.writer-count", "4") + .put("task.partitioned-writer-count", "8") .put("task.concurrency", "8") .put("task.http-response-threads", "4") .put("task.http-timeout-threads", "10") @@ -121,6 +123,7 @@ public void testExplicitPropertyMappings() .setSinkMaxBufferSize(new DataSize(42, Unit.MEGABYTE)) .setMaxPagePartitioningBufferSize(new DataSize(40, Unit.MEGABYTE)) .setWriterCount(4) + .setPartitionedWriterCount(8) .setTaskConcurrency(8) .setHttpResponseThreads(4) .setHttpTimeoutThreads(10) diff --git a/presto-main/src/test/java/com/facebook/presto/execution/buffer/BufferTestUtils.java b/presto-main/src/test/java/com/facebook/presto/execution/buffer/BufferTestUtils.java index 011e85a1ba256..251288ec51639 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/buffer/BufferTestUtils.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/buffer/BufferTestUtils.java @@ -29,9 +29,9 @@ import java.util.concurrent.Future; import java.util.stream.Collectors; +import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; import static com.facebook.presto.execution.buffer.TestingPagesSerdeFactory.testingPagesSerde; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; import static io.airlift.units.DataSize.Unit.BYTE; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/buffer/TestArbitraryOutputBuffer.java b/presto-main/src/test/java/com/facebook/presto/execution/buffer/TestArbitraryOutputBuffer.java index 195c668854109..98006df0749db 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/buffer/TestArbitraryOutputBuffer.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/buffer/TestArbitraryOutputBuffer.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.execution.buffer.BufferResult.emptyResults; import static com.facebook.presto.execution.buffer.BufferState.OPEN; import static com.facebook.presto.execution.buffer.BufferState.TERMINAL_BUFFER_STATES; @@ -50,7 +51,6 @@ import static com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.BYTE; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/buffer/TestBroadcastOutputBuffer.java b/presto-main/src/test/java/com/facebook/presto/execution/buffer/TestBroadcastOutputBuffer.java index 19c00266db359..5fdc11d1cf02c 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/buffer/TestBroadcastOutputBuffer.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/buffer/TestBroadcastOutputBuffer.java @@ -33,6 +33,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.execution.buffer.BufferResult.emptyResults; import static com.facebook.presto.execution.buffer.BufferState.OPEN; import static com.facebook.presto.execution.buffer.BufferState.TERMINAL_BUFFER_STATES; @@ -59,7 +60,6 @@ import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.BYTE; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newScheduledThreadPool; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/buffer/TestPartitionedOutputBuffer.java b/presto-main/src/test/java/com/facebook/presto/execution/buffer/TestPartitionedOutputBuffer.java index 216d88f0df17c..f1669e6ce7adb 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/buffer/TestPartitionedOutputBuffer.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/buffer/TestPartitionedOutputBuffer.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.execution.buffer.BufferResult.emptyResults; import static com.facebook.presto.execution.buffer.BufferState.OPEN; import static com.facebook.presto.execution.buffer.BufferState.TERMINAL_BUFFER_STATES; @@ -52,7 +53,6 @@ import static com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.BYTE; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/executor/SimulationController.java b/presto-main/src/test/java/com/facebook/presto/execution/executor/SimulationController.java index 10db510432d91..476c20a3e413b 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/executor/SimulationController.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/executor/SimulationController.java @@ -182,13 +182,13 @@ private void createTask(TaskSpecification specification) runningTasks.put(specification, new LeafTask( taskExecutor, specification, - new TaskId(specification.getName(), 0, runningTasks.get(specification).size() + completedTasks.get(specification).size()))); + new TaskId(specification.getName(), 0, 0, runningTasks.get(specification).size() + completedTasks.get(specification).size()))); } else { runningTasks.put(specification, new IntermediateTask( taskExecutor, specification, - new TaskId(specification.getName(), 0, runningTasks.get(specification).size() + completedTasks.get(specification).size()))); + new TaskId(specification.getName(), 0, 0, runningTasks.get(specification).size() + completedTasks.get(specification).size()))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/executor/TaskExecutorSimulator.java b/presto-main/src/test/java/com/facebook/presto/execution/executor/TaskExecutorSimulator.java index 418074adadc87..d1d0902ed7d60 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/executor/TaskExecutorSimulator.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/executor/TaskExecutorSimulator.java @@ -38,12 +38,12 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.stream.Collectors; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static com.facebook.presto.execution.executor.Histogram.fromContinuous; import static com.facebook.presto.execution.executor.Histogram.fromDiscrete; import static com.facebook.presto.execution.executor.SimulationController.TaskSpecification.Type.INTERMEDIATE; import static com.facebook.presto.execution.executor.SimulationController.TaskSpecification.Type.LEAF; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; -import static io.airlift.concurrent.Threads.threadsNamed; import static io.airlift.units.Duration.nanosSince; import static io.airlift.units.Duration.succinctNanos; import static java.util.concurrent.Executors.newCachedThreadPool; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/executor/TestTaskExecutor.java b/presto-main/src/test/java/com/facebook/presto/execution/executor/TestTaskExecutor.java index 3b964657cc411..73d5fa542fbcc 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/executor/TestTaskExecutor.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/executor/TestTaskExecutor.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.execution.executor; +import com.facebook.airlift.testing.TestingTicker; import com.facebook.presto.execution.SplitRunner; import com.facebook.presto.execution.TaskId; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import io.airlift.testing.TestingTicker; import io.airlift.units.Duration; import org.testng.annotations.Test; @@ -31,11 +31,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static com.facebook.presto.execution.executor.MultilevelSplitQueue.LEVEL_CONTRIBUTION_CAP; import static com.facebook.presto.execution.executor.MultilevelSplitQueue.LEVEL_THRESHOLD_SECONDS; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertLessThan; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; @@ -55,7 +55,7 @@ public void testTasksComplete() ticker.increment(20, MILLISECONDS); try { - TaskId taskId = new TaskId("test", 0, 0); + TaskId taskId = new TaskId("test", 0, 0, 0); TaskHandle taskHandle = taskExecutor.addTask(taskId, () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); Phaser beginPhase = new Phaser(); @@ -149,8 +149,8 @@ public void testQuantaFairness() ticker.increment(20, MILLISECONDS); try { - TaskHandle shortQuantaTaskHandle = taskExecutor.addTask(new TaskId("shortQuanta", 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); - TaskHandle longQuantaTaskHandle = taskExecutor.addTask(new TaskId("longQuanta", 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); + TaskHandle shortQuantaTaskHandle = taskExecutor.addTask(new TaskId("short_quanta", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); + TaskHandle longQuantaTaskHandle = taskExecutor.addTask(new TaskId("long_quanta", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); Phaser globalPhaser = new Phaser(); @@ -183,7 +183,7 @@ public void testLevelMovement() ticker.increment(20, MILLISECONDS); try { - TaskHandle testTaskHandle = taskExecutor.addTask(new TaskId("test", 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); + TaskHandle testTaskHandle = taskExecutor.addTask(new TaskId("test", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); Phaser globalPhaser = new Phaser(); globalPhaser.bulkRegister(3); @@ -224,9 +224,9 @@ public void testLevelMultipliers() try { for (int i = 0; i < (LEVEL_THRESHOLD_SECONDS.length - 1); i++) { TaskHandle[] taskHandles = { - taskExecutor.addTask(new TaskId("test1", 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()), - taskExecutor.addTask(new TaskId("test2", 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()), - taskExecutor.addTask(new TaskId("test3", 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()) + taskExecutor.addTask(new TaskId("test1", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()), + taskExecutor.addTask(new TaskId("test2", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()), + taskExecutor.addTask(new TaskId("test3", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()) }; // move task 0 to next level @@ -305,7 +305,7 @@ public void testTaskHandle() taskExecutor.start(); try { - TaskId taskId = new TaskId("test", 0, 0); + TaskId taskId = new TaskId("test", 0, 0, 0); TaskHandle taskHandle = taskExecutor.addTask(taskId, () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); Phaser beginPhase = new Phaser(); @@ -337,8 +337,8 @@ public void testTaskHandle() public void testLevelContributionCap() { MultilevelSplitQueue splitQueue = new MultilevelSplitQueue(2); - TaskHandle handle0 = new TaskHandle(new TaskId("test0", 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); - TaskHandle handle1 = new TaskHandle(new TaskId("test1", 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); + TaskHandle handle0 = new TaskHandle(new TaskId("test0", 0, 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); + TaskHandle handle1 = new TaskHandle(new TaskId("test1", 0, 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); for (int i = 0; i < (LEVEL_THRESHOLD_SECONDS.length - 1); i++) { long levelAdvanceTime = SECONDS.toNanos(LEVEL_THRESHOLD_SECONDS[i + 1] - LEVEL_THRESHOLD_SECONDS[i]); @@ -357,7 +357,7 @@ public void testLevelContributionCap() public void testUpdateLevelWithCap() { MultilevelSplitQueue splitQueue = new MultilevelSplitQueue(2); - TaskHandle handle0 = new TaskHandle(new TaskId("test0", 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); + TaskHandle handle0 = new TaskHandle(new TaskId("test0", 0, 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); long quantaNanos = MINUTES.toNanos(10); handle0.addScheduledNanos(quantaNanos); @@ -379,7 +379,7 @@ public void testMinMaxDriversPerTask() TaskExecutor taskExecutor = new TaskExecutor(4, 16, 1, maxDriversPerTask, splitQueue, ticker); taskExecutor.start(); try { - TaskHandle testTaskHandle = taskExecutor.addTask(new TaskId("test", 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); + TaskHandle testTaskHandle = taskExecutor.addTask(new TaskId("test", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); // enqueue all batches of splits int batchCount = 4; @@ -420,7 +420,7 @@ public void testUserSpecifiedMaxDriversPerTask() taskExecutor.start(); try { // overwrite the max drivers per task to be 1 - TaskHandle testTaskHandle = taskExecutor.addTask(new TaskId("test", 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.of(1)); + TaskHandle testTaskHandle = taskExecutor.addTask(new TaskId("test", 0, 0, 0), () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.of(1)); // enqueue all batches of splits int batchCount = 4; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/resourceGroups/TestResourceGroups.java b/presto-main/src/test/java/com/facebook/presto/execution/resourceGroups/TestResourceGroups.java index 2a5c5466a5a34..651399c618252 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/resourceGroups/TestResourceGroups.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/resourceGroups/TestResourceGroups.java @@ -34,6 +34,9 @@ import java.util.SortedMap; import java.util.TreeMap; +import static com.facebook.airlift.testing.Assertions.assertBetweenInclusive; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static com.facebook.presto.execution.QueryState.FAILED; import static com.facebook.presto.execution.QueryState.QUEUED; import static com.facebook.presto.execution.QueryState.RUNNING; @@ -44,9 +47,6 @@ import static com.facebook.presto.spi.resourceGroups.SchedulingPolicy.WEIGHTED; import static com.facebook.presto.spi.resourceGroups.SchedulingPolicy.WEIGHTED_FAIR; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.testing.Assertions.assertBetweenInclusive; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertLessThan; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/resourceGroups/TestStochasticPriorityQueue.java b/presto-main/src/test/java/com/facebook/presto/execution/resourceGroups/TestStochasticPriorityQueue.java index 9acd5d8309685..ab4007e2a7644 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/resourceGroups/TestStochasticPriorityQueue.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/resourceGroups/TestStochasticPriorityQueue.java @@ -16,8 +16,8 @@ import org.apache.commons.math3.distribution.BinomialDistribution; import org.testng.annotations.Test; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertLessThan; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java index 8fcc8a69253ec..36976088e7363 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestFixedCountScheduler.java @@ -30,9 +30,9 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.stream.IntStream; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; @@ -61,7 +61,7 @@ public void testSingleNode() { FixedCountScheduler nodeScheduler = new FixedCountScheduler( (node, partition, totalPartitions) -> Optional.of(taskFactory.createTableScanTask( - new TaskId("test", 1, 1), + new TaskId("test", 1, 0, 1), node, ImmutableList.of(), new PartitionedSplitCountTracker(delta -> {}))), generateRandomNodes(1)); @@ -78,7 +78,7 @@ public void testMultipleNodes() { FixedCountScheduler nodeScheduler = new FixedCountScheduler( (node, partition, totalPartitions) -> Optional.of(taskFactory.createTableScanTask( - new TaskId("test", 1, 1), + new TaskId("test", 1, 0, 1), node, ImmutableList.of(), new PartitionedSplitCountTracker(delta -> {}))), generateRandomNodes(5)); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestPhasedExecutionSchedule.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestPhasedExecutionSchedule.java index d2fac43693cd6..91be673c6c926 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestPhasedExecutionSchedule.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestPhasedExecutionSchedule.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.Partitioning; @@ -28,12 +29,10 @@ import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.PlanFragmentId; import com.facebook.presto.sql.planner.plan.RemoteSourceNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.testing.TestingMetadata.TestingColumnHandle; import com.facebook.presto.testing.TestingMetadata.TestingTableHandle; import com.facebook.presto.testing.TestingTransactionHandle; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.testng.annotations.Test; @@ -161,6 +160,7 @@ private static PlanFragment createExchangePlanFragment(String name, PlanFragment .map(PlanFragment::getId) .collect(toImmutableList()), fragments[0].getPartitioningScheme().getOutputLayout(), + false, Optional.empty(), REPARTITION); @@ -176,10 +176,12 @@ private static PlanFragment createUnionPlanFragment(String name, PlanFragment... new PlanNodeId(fragment.getId().toString()), fragment.getId(), fragment.getPartitioningScheme().getOutputLayout(), + false, Optional.empty(), REPARTITION)) .collect(toImmutableList()), - ImmutableListMultimap.of()); + ImmutableList.of(), + ImmutableMap.of()); return createFragment(planNode); } @@ -199,7 +201,7 @@ private static PlanFragment createBroadcastJoinPlanFragment(String name, PlanFra TupleDomain.all(), TupleDomain.all()); - RemoteSourceNode remote = new RemoteSourceNode(new PlanNodeId("build_id"), buildFragment.getId(), ImmutableList.of(), Optional.empty(), REPLICATE); + RemoteSourceNode remote = new RemoteSourceNode(new PlanNodeId("build_id"), buildFragment.getId(), ImmutableList.of(), false, Optional.empty(), REPLICATE); PlanNode join = new JoinNode( new PlanNodeId(name + "_id"), INNER, @@ -220,8 +222,8 @@ private static PlanFragment createBroadcastJoinPlanFragment(String name, PlanFra private static PlanFragment createJoinPlanFragment(JoinNode.Type joinType, String name, PlanFragment buildFragment, PlanFragment probeFragment) { - RemoteSourceNode probe = new RemoteSourceNode(new PlanNodeId("probe_id"), probeFragment.getId(), ImmutableList.of(), Optional.empty(), REPARTITION); - RemoteSourceNode build = new RemoteSourceNode(new PlanNodeId("build_id"), buildFragment.getId(), ImmutableList.of(), Optional.empty(), REPARTITION); + RemoteSourceNode probe = new RemoteSourceNode(new PlanNodeId("probe_id"), probeFragment.getId(), ImmutableList.of(), false, Optional.empty(), REPARTITION); + RemoteSourceNode build = new RemoteSourceNode(new PlanNodeId("build_id"), buildFragment.getId(), ImmutableList.of(), false, Optional.empty(), REPARTITION); PlanNode planNode = new JoinNode( new PlanNodeId(name + "_id"), joinType, diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java index dd87e199e2664..75686f938cd23 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java @@ -21,6 +21,7 @@ import com.facebook.presto.execution.NodeTaskMap; import com.facebook.presto.execution.RemoteTask; import com.facebook.presto.execution.SqlStageExecution; +import com.facebook.presto.execution.StageExecutionId; import com.facebook.presto.execution.StageId; import com.facebook.presto.execution.TestSqlTaskManager.MockLocationFactory; import com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId; @@ -39,6 +40,7 @@ import com.facebook.presto.spi.connector.ConnectorPartitionHandle; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.split.ConnectorAwareSplitSource; import com.facebook.presto.split.SplitSource; @@ -71,6 +73,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.PARTITIONED; import static com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers; @@ -83,7 +86,6 @@ import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.Integer.min; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; @@ -458,9 +460,11 @@ private static SubPlan createPlan() TABLE_SCAN_NODE_ID, new TableHandle(CONNECTOR_ID, new TestingTableHandle(), TestingTransactionHandle.create(), Optional.empty()), ImmutableList.of(variable), - ImmutableMap.of(variable, new TestingColumnHandle("column"))); + ImmutableMap.of(variable, new TestingColumnHandle("column")), + TupleDomain.all(), + TupleDomain.all()); - RemoteSourceNode remote = new RemoteSourceNode(new PlanNodeId("remote_id"), new PlanFragmentId(0), ImmutableList.of(), Optional.empty(), GATHER); + RemoteSourceNode remote = new RemoteSourceNode(new PlanNodeId("remote_id"), new PlanFragmentId(0), ImmutableList.of(), false, Optional.empty(), GATHER); PlanFragment testFragment = new PlanFragment( new PlanFragmentId(0), new JoinNode(new PlanNodeId("join_id"), @@ -501,8 +505,8 @@ private static ConnectorSplitSource createFixedSplitSource(int splitCount, Suppl private SqlStageExecution createSqlStageExecution(SubPlan tableScanPlan, NodeTaskMap nodeTaskMap) { StageId stageId = new StageId(new QueryId("query"), 0); - SqlStageExecution stage = SqlStageExecution.createSqlStageExecution(stageId, - locationFactory.createStageLocation(stageId), + SqlStageExecution stage = SqlStageExecution.createSqlStageExecution( + new StageExecutionId(stageId, 0), tableScanPlan.getFragment(), new MockRemoteTaskFactory(queryExecutor, scheduledExecutor), TEST_SESSION, @@ -510,7 +514,8 @@ private SqlStageExecution createSqlStageExecution(SubPlan tableScanPlan, NodeTas nodeTaskMap, queryExecutor, new NoOpFailureDetector(), - new SplitSchedulerStats()); + new SplitSchedulerStats(), + new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())); stage.setOutputBuffers(createInitialEmptyOutputBuffers(PARTITIONED) .withBuffer(OUT, 0) diff --git a/presto-main/src/test/java/com/facebook/presto/failureDetector/TestHeartbeatFailureDetector.java b/presto-main/src/test/java/com/facebook/presto/failureDetector/TestHeartbeatFailureDetector.java index 6fe7479845134..ac314294bdb1f 100644 --- a/presto-main/src/test/java/com/facebook/presto/failureDetector/TestHeartbeatFailureDetector.java +++ b/presto-main/src/test/java/com/facebook/presto/failureDetector/TestHeartbeatFailureDetector.java @@ -13,6 +13,16 @@ */ package com.facebook.presto.failureDetector; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.discovery.client.ServiceSelector; +import com.facebook.airlift.discovery.client.testing.TestingDiscoveryModule; +import com.facebook.airlift.http.server.testing.TestingHttpServerModule; +import com.facebook.airlift.jaxrs.JaxrsModule; +import com.facebook.airlift.jmx.testing.TestingJmxModule; +import com.facebook.airlift.json.JsonModule; +import com.facebook.airlift.json.ObjectMapperProvider; +import com.facebook.airlift.node.testing.TestingNodeModule; +import com.facebook.airlift.tracetoken.TraceTokenModule; import com.facebook.presto.execution.QueryManagerConfig; import com.facebook.presto.failureDetector.HeartbeatFailureDetector.Stats; import com.facebook.presto.server.InternalCommunicationConfig; @@ -22,16 +32,6 @@ import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.discovery.client.ServiceSelector; -import io.airlift.discovery.client.testing.TestingDiscoveryModule; -import io.airlift.http.server.testing.TestingHttpServerModule; -import io.airlift.jaxrs.JaxrsModule; -import io.airlift.jmx.testing.TestingJmxModule; -import io.airlift.json.JsonModule; -import io.airlift.json.ObjectMapperProvider; -import io.airlift.node.testing.TestingNodeModule; -import io.airlift.tracetoken.TraceTokenModule; import org.testng.annotations.Test; import javax.ws.rs.GET; @@ -40,10 +40,10 @@ import java.net.SocketTimeoutException; import java.net.URI; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.discovery.client.DiscoveryBinder.discoveryBinder; -import static io.airlift.discovery.client.ServiceTypes.serviceType; -import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.discovery.client.DiscoveryBinder.discoveryBinder; +import static com.facebook.airlift.discovery.client.ServiceTypes.serviceType; +import static com.facebook.airlift.jaxrs.JaxrsBinder.jaxrsBinder; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; diff --git a/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryManagerConfig.java b/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryManagerConfig.java index 7b28131ffcaec..61846a5692a66 100644 --- a/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryManagerConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryManagerConfig.java @@ -13,18 +13,18 @@ */ package com.facebook.presto.memory; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.testng.annotations.Test; import java.util.Map; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static com.facebook.presto.memory.MemoryManagerConfig.LowMemoryKillerPolicy.NONE; import static com.facebook.presto.memory.MemoryManagerConfig.LowMemoryKillerPolicy.TOTAL_RESERVATION_ON_BLOCKED_NODES; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryPools.java b/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryPools.java index ecb2b4b607862..4da5a3f53aaa0 100644 --- a/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryPools.java +++ b/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryPools.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.memory; +import com.facebook.airlift.stats.TestingGcMonitor; import com.facebook.presto.Session; import com.facebook.presto.execution.buffer.TestingPagesSerdeFactory; import com.facebook.presto.memory.context.LocalMemoryContext; @@ -37,7 +38,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.stats.TestingGcMonitor; import io.airlift.units.DataSize; import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; @@ -236,18 +236,21 @@ public void testTaggedAllocations() testPool.reserve(testQuery, "test_tag", 10); - Map allocations = testPool.getTaggedMemoryAllocations().get(testQuery); + Map allocations = testPool.getTaggedMemoryAllocations(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); // free 5 bytes for test_tag testPool.free(testQuery, "test_tag", 5); + allocations = testPool.getTaggedMemoryAllocations(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 5L)); testPool.reserve(testQuery, "test_tag2", 20); + allocations = testPool.getTaggedMemoryAllocations(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 5L, "test_tag2", 20L)); // free the remaining 5 bytes for test_tag testPool.free(testQuery, "test_tag", 5); + allocations = testPool.getTaggedMemoryAllocations(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag2", 20L)); // free all for test_tag2 @@ -263,12 +266,12 @@ public void testMoveQuery() MemoryPool pool2 = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE)); pool1.reserve(testQuery, "test_tag", 10); - Map allocations = pool1.getTaggedMemoryAllocations().get(testQuery); + Map allocations = pool1.getTaggedMemoryAllocations(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); pool1.moveQuery(testQuery, pool2); - assertNull(pool1.getTaggedMemoryAllocations().get(testQuery)); - allocations = pool2.getTaggedMemoryAllocations().get(testQuery); + assertNull(pool1.getTaggedMemoryAllocations(testQuery)); + allocations = pool2.getTaggedMemoryAllocations(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); assertEquals(pool1.getFreeBytes(), 1000); diff --git a/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryTracking.java b/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryTracking.java index c2af5001c012e..1d32522de9d2d 100644 --- a/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryTracking.java +++ b/presto-main/src/test/java/com/facebook/presto/memory/TestMemoryTracking.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.memory; +import com.facebook.airlift.stats.TestingGcMonitor; import com.facebook.presto.ExceededMemoryLimitException; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.TaskStateMachine; @@ -30,7 +31,6 @@ import com.facebook.presto.spi.memory.MemoryPoolId; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spiller.SpillSpaceTracker; -import io.airlift.stats.TestingGcMonitor; import io.airlift.units.DataSize; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -43,8 +43,8 @@ import java.util.function.Consumer; import java.util.regex.Pattern; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static java.lang.String.format; import static java.util.concurrent.Executors.newCachedThreadPool; @@ -108,7 +108,7 @@ public void setUpTest() queryMaxSpillSize, spillSpaceTracker); taskContext = queryContext.addTaskContext( - new TaskStateMachine(new TaskId("query", 0, 0), notificationExecutor), + new TaskStateMachine(new TaskId("query", 0, 0, 0), notificationExecutor), testSessionBuilder().build(), true, true, diff --git a/presto-main/src/test/java/com/facebook/presto/memory/TestNodeMemoryConfig.java b/presto-main/src/test/java/com/facebook/presto/memory/TestNodeMemoryConfig.java index f86c08d19aeee..3c027b8fe1bbd 100644 --- a/presto-main/src/test/java/com/facebook/presto/memory/TestNodeMemoryConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/memory/TestNodeMemoryConfig.java @@ -13,16 +13,16 @@ */ package com.facebook.presto.memory; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.DataSize; import org.testng.annotations.Test; import java.util.Map; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static com.facebook.presto.memory.LocalMemoryManager.validateHeapHeadroom; import static com.facebook.presto.memory.NodeMemoryConfig.AVAILABLE_HEAP_MEMORY; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.GIGABYTE; diff --git a/presto-main/src/test/java/com/facebook/presto/memory/TestQueryContext.java b/presto-main/src/test/java/com/facebook/presto/memory/TestQueryContext.java index 835a9d16520fe..1612498150d66 100644 --- a/presto-main/src/test/java/com/facebook/presto/memory/TestQueryContext.java +++ b/presto-main/src/test/java/com/facebook/presto/memory/TestQueryContext.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.memory; +import com.facebook.airlift.stats.TestingGcMonitor; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.TaskStateMachine; import com.facebook.presto.memory.context.LocalMemoryContext; @@ -24,7 +25,6 @@ import com.facebook.presto.spiller.SpillSpaceTracker; import com.facebook.presto.testing.LocalQueryRunner; import com.google.common.collect.ImmutableMap; -import io.airlift.stats.TestingGcMonitor; import io.airlift.units.DataSize; import org.testng.annotations.AfterClass; import org.testng.annotations.DataProvider; @@ -34,10 +34,10 @@ import java.util.OptionalInt; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.memory.LocalMemoryManager.GENERAL_POOL; import static com.facebook.presto.memory.LocalMemoryManager.RESERVED_POOL; -import static io.airlift.concurrent.Threads.threadsNamed; import static io.airlift.units.DataSize.Unit.BYTE; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; @@ -111,7 +111,7 @@ public void testMoveTaggedAllocations() MemoryPool reservedPool = new MemoryPool(RESERVED_POOL, new DataSize(10_000, BYTE)); QueryId queryId = new QueryId("query"); QueryContext queryContext = createQueryContext(queryId, generalPool); - TaskStateMachine taskStateMachine = new TaskStateMachine(TaskId.valueOf("task-id"), TEST_EXECUTOR); + TaskStateMachine taskStateMachine = new TaskStateMachine(TaskId.valueOf("queryid.0.0.0"), TEST_EXECUTOR); TaskContext taskContext = queryContext.addTaskContext(taskStateMachine, TEST_SESSION, false, false, OptionalInt.empty(), false); DriverContext driverContext = taskContext.addPipelineContext(0, false, false, false).addDriverContext(); OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), "test"); @@ -120,13 +120,13 @@ public void testMoveTaggedAllocations() LocalMemoryContext memoryContext = operatorContext.aggregateUserMemoryContext().newLocalMemoryContext("test_context"); memoryContext.setBytes(1_000); - Map allocations = generalPool.getTaggedMemoryAllocations().get(queryId); + Map allocations = generalPool.getTaggedMemoryAllocations(queryId); assertEquals(allocations, ImmutableMap.of("test_context", 1_000L)); queryContext.setMemoryPool(reservedPool); - assertNull(generalPool.getTaggedMemoryAllocations().get(queryId)); - allocations = reservedPool.getTaggedMemoryAllocations().get(queryId); + assertNull(generalPool.getTaggedMemoryAllocations(queryId)); + allocations = reservedPool.getTaggedMemoryAllocations(queryId); assertEquals(allocations, ImmutableMap.of("test_context", 1_000L)); assertEquals(generalPool.getFreeBytes(), 10_000); diff --git a/presto-main/src/test/java/com/facebook/presto/memory/TestReservedSystemMemoryConfig.java b/presto-main/src/test/java/com/facebook/presto/memory/TestReservedSystemMemoryConfig.java index dc48dcde65e7c..ef9a0ed40bb86 100644 --- a/presto-main/src/test/java/com/facebook/presto/memory/TestReservedSystemMemoryConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/memory/TestReservedSystemMemoryConfig.java @@ -19,7 +19,7 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static org.testng.Assert.fail; diff --git a/presto-main/src/test/java/com/facebook/presto/memory/TestSystemMemoryBlocking.java b/presto-main/src/test/java/com/facebook/presto/memory/TestSystemMemoryBlocking.java index 62ab40f2ffd38..a09fea561f56a 100644 --- a/presto-main/src/test/java/com/facebook/presto/memory/TestSystemMemoryBlocking.java +++ b/presto-main/src/test/java/com/facebook/presto/memory/TestSystemMemoryBlocking.java @@ -22,9 +22,12 @@ import com.facebook.presto.operator.TaskContext; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableHandle; import com.facebook.presto.spi.FixedPageSource; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.QueryId; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.Type; import com.facebook.presto.testing.MaterializedResult; @@ -41,14 +44,15 @@ import org.testng.annotations.Test; import java.util.List; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.function.Function; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -96,13 +100,18 @@ public void testTableScanSystemMemoryBlocking() final List types = ImmutableList.of(VARCHAR); TableScanOperator source = new TableScanOperator(driverContext.addOperatorContext(1, new PlanNodeId("test"), "values"), sourceId, - (session, split, columns) -> new FixedPageSource(rowPagesBuilder(types) + (session, split, table, columns) -> new FixedPageSource(rowPagesBuilder(types) .addSequencePage(10, 1) .addSequencePage(10, 1) .addSequencePage(10, 1) .addSequencePage(10, 1) .addSequencePage(10, 1) .build()), + new TableHandle( + new ConnectorId("test"), + new ConnectorTableHandle() {}, + new ConnectorTransactionHandle() {}, + Optional.empty()), ImmutableList.of()); PageConsumerOperator sink = createSinkOperator(types); Driver driver = Driver.createDriver(driverContext, source, sink); diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java b/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java index e4b1329078adb..58efad33b52d5 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java @@ -25,6 +25,7 @@ import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.connector.ConnectorCapabilities; import com.facebook.presto.spi.connector.ConnectorOutputMetadata; +import com.facebook.presto.spi.function.SqlFunction; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.security.GrantInfo; @@ -38,6 +39,7 @@ import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.sql.planner.PartitioningHandle; +import com.google.common.util.concurrent.ListenableFuture; import io.airlift.slice.Slice; import java.util.Collection; @@ -74,7 +76,7 @@ public List listFunctions() } @Override - public void addFunctions(List functions) + public void registerBuiltInFunctions(List functions) { throw new UnsupportedOperationException(); } @@ -170,7 +172,7 @@ public TableMetadata getTableMetadata(Session session, TableHandle tableHandle) } @Override - public TableStatistics getTableStatistics(Session session, TableHandle tableHandle, Constraint constraint) + public TableStatistics getTableStatistics(Session session, TableHandle tableHandle, List columnHandles, Constraint constraint) { throw new UnsupportedOperationException(); } @@ -470,13 +472,13 @@ public List listTablePrivileges(Session session, QualifiedTablePrefix } @Override - public void commitPartition(Session session, OutputTableHandle tableHandle, Collection fragments) + public ListenableFuture commitPartitionAsync(Session session, OutputTableHandle tableHandle, Collection fragments) { throw new UnsupportedOperationException(); } @Override - public void commitPartition(Session session, InsertTableHandle tableHandle, Collection fragments) + public ListenableFuture commitPartitionAsync(Session session, InsertTableHandle tableHandle, Collection fragments) { throw new UnsupportedOperationException(); } diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java index 785170957bad2..afd3a10bc793a 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestDiscoveryNodeManager.java @@ -13,6 +13,13 @@ */ package com.facebook.presto.metadata; +import com.facebook.airlift.discovery.client.ServiceDescriptor; +import com.facebook.airlift.discovery.client.ServiceSelector; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.testing.TestingHttpClient; +import com.facebook.airlift.http.client.testing.TestingResponse; +import com.facebook.airlift.node.NodeConfig; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.client.NodeVersion; import com.facebook.presto.failureDetector.NoOpFailureDetector; import com.facebook.presto.server.InternalCommunicationConfig; @@ -21,13 +28,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.discovery.client.ServiceDescriptor; -import io.airlift.discovery.client.ServiceSelector; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.testing.TestingHttpClient; -import io.airlift.http.client.testing.TestingResponse; -import io.airlift.node.NodeConfig; -import io.airlift.node.NodeInfo; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -40,12 +40,12 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; +import static com.facebook.airlift.discovery.client.ServiceDescriptor.serviceDescriptor; +import static com.facebook.airlift.discovery.client.ServiceSelectorConfig.DEFAULT_POOL; +import static com.facebook.airlift.http.client.HttpStatus.OK; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static com.facebook.presto.spi.NodeState.ACTIVE; import static com.facebook.presto.spi.NodeState.INACTIVE; -import static io.airlift.discovery.client.ServiceDescriptor.serviceDescriptor; -import static io.airlift.discovery.client.ServiceSelectorConfig.DEFAULT_POOL; -import static io.airlift.http.client.HttpStatus.OK; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotSame; diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionInvokerProvider.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionInvokerProvider.java index 0b517e6f76719..4d2729d9800a9 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionInvokerProvider.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionInvokerProvider.java @@ -14,7 +14,7 @@ package com.facebook.presto.metadata; import com.facebook.presto.operator.scalar.AbstractTestFunctions; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty; import com.facebook.presto.spi.function.InvocationConvention; import com.facebook.presto.spi.function.InvocationConvention.InvocationReturnConvention; import com.google.common.collect.ImmutableList; @@ -23,11 +23,11 @@ import java.util.Optional; import static com.facebook.presto.metadata.FunctionInvokerProvider.checkChoice; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentType.VALUE_TYPE; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentType.VALUE_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; import static com.facebook.presto.spi.function.InvocationConvention.InvocationArgumentConvention.BLOCK_POSITION; import static com.facebook.presto.spi.function.InvocationConvention.InvocationArgumentConvention.BOXED_NULLABLE; import static com.facebook.presto.spi.function.InvocationConvention.InvocationArgumentConvention.NEVER_NULL; diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionManager.java similarity index 79% rename from presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java rename to presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionManager.java index b016e5c134fe1..acd0bd62fd7be 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespace.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionManager.java @@ -14,13 +14,15 @@ package com.facebook.presto.metadata; import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.operator.scalar.CustomFunctions; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.BlockEncodingSerde; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunction; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.function.TypeVariableConstraint; import com.facebook.presto.spi.type.StandardTypes; @@ -37,13 +39,13 @@ import java.lang.invoke.MethodHandles; import java.util.List; -import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; -import static com.facebook.presto.metadata.OperatorSignatureUtils.unmangleOperator; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.OperatorType.CAST; import static com.facebook.presto.spi.function.OperatorType.SATURATED_FLOOR_CAST; +import static com.facebook.presto.spi.function.OperatorType.tryGetOperatorType; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.HyperLogLogType.HYPER_LOG_LOG; @@ -63,25 +65,25 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -public class TestStaticFunctionNamespace +public class TestFunctionManager { @Test public void testIdentityCast() { TypeRegistry typeManager = new TypeRegistry(); - StaticFunctionNamespace staticFunctionNamespace = createStaticFunctionNamespace(typeManager); - FunctionHandle exactOperator = staticFunctionNamespace.lookupCast(CastType.CAST, HYPER_LOG_LOG.getTypeSignature(), HYPER_LOG_LOG.getTypeSignature()); - assertEquals(exactOperator, new StaticFunctionHandle(new Signature(mangleOperatorName(CAST.name()), SCALAR, HYPER_LOG_LOG.getTypeSignature(), HYPER_LOG_LOG.getTypeSignature()))); + FunctionManager functionManager = createFunctionManager(typeManager); + FunctionHandle exactOperator = functionManager.lookupCast(CastType.CAST, HYPER_LOG_LOG.getTypeSignature(), HYPER_LOG_LOG.getTypeSignature()); + assertEquals(exactOperator, new BuiltInFunctionHandle(new Signature(CAST.getFunctionName(), SCALAR, HYPER_LOG_LOG.getTypeSignature(), HYPER_LOG_LOG.getTypeSignature()))); } @Test public void testExactMatchBeforeCoercion() { TypeRegistry typeManager = new TypeRegistry(); - StaticFunctionNamespace staticFunctionNamespace = createStaticFunctionNamespace(typeManager); + FunctionManager functionManager = createFunctionManager(typeManager); boolean foundOperator = false; - for (SqlFunction function : staticFunctionNamespace.listOperators()) { - OperatorType operatorType = unmangleOperator(function.getSignature().getName()); + for (SqlFunction function : functionManager.listOperators()) { + OperatorType operatorType = tryGetOperatorType(function.getSignature().getName()).get(); if (operatorType == CAST || operatorType == SATURATED_FLOOR_CAST) { continue; } @@ -91,7 +93,7 @@ public void testExactMatchBeforeCoercion() if (function.getSignature().getArgumentTypes().stream().anyMatch(TypeSignature::isCalculated)) { continue; } - StaticFunctionHandle exactOperator = (StaticFunctionHandle) staticFunctionNamespace.resolveOperator(operatorType, fromTypeSignatures(function.getSignature().getArgumentTypes())); + BuiltInFunctionHandle exactOperator = (BuiltInFunctionHandle) functionManager.resolveOperator(operatorType, fromTypeSignatures(function.getSignature().getArgumentTypes())); assertEquals(exactOperator.getSignature(), function.getSignature()); foundOperator = true; } @@ -102,52 +104,52 @@ public void testExactMatchBeforeCoercion() public void testMagicLiteralFunction() { Signature signature = getMagicLiteralFunctionSignature(TIMESTAMP_WITH_TIME_ZONE); - assertEquals(signature.getName(), "$literal$timestamp with time zone"); + assertEquals(signature.getNameSuffix(), "$literal$timestamp with time zone"); assertEquals(signature.getArgumentTypes(), ImmutableList.of(parseTypeSignature(StandardTypes.BIGINT))); assertEquals(signature.getReturnType().getBase(), StandardTypes.TIMESTAMP_WITH_TIME_ZONE); TypeRegistry typeManager = new TypeRegistry(); - StaticFunctionNamespace staticFunctionNamespace = createStaticFunctionNamespace(typeManager); - StaticFunctionHandle functionHandle = (StaticFunctionHandle) staticFunctionNamespace.resolveFunction(QualifiedName.of(signature.getName()), fromTypeSignatures(signature.getArgumentTypes())); - assertEquals(staticFunctionNamespace.getFunctionMetadata(functionHandle).getArgumentTypes(), ImmutableList.of(parseTypeSignature(StandardTypes.BIGINT))); + FunctionManager functionManager = createFunctionManager(typeManager); + BuiltInFunctionHandle functionHandle = (BuiltInFunctionHandle) functionManager.resolveFunction(TEST_SESSION.getTransactionId(), signature.getName(), fromTypeSignatures(signature.getArgumentTypes())); + assertEquals(functionManager.getFunctionMetadata(functionHandle).getArgumentTypes(), ImmutableList.of(parseTypeSignature(StandardTypes.BIGINT))); assertEquals(signature.getReturnType().getBase(), StandardTypes.TIMESTAMP_WITH_TIME_ZONE); } - @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "\\QFunction already registered: custom_add(bigint,bigint):bigint\\E") + @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "\\QFunction already registered: presto.default.custom_add(bigint,bigint):bigint\\E") public void testDuplicateFunctions() { - List functions = new FunctionListBuilder() + List functions = new FunctionListBuilder() .scalars(CustomFunctions.class) .getFunctions() .stream() - .filter(input -> input.getSignature().getName().equals("custom_add")) + .filter(input -> input.getSignature().getNameSuffix().equals("custom_add")) .collect(toImmutableList()); TypeRegistry typeManager = new TypeRegistry(); - StaticFunctionNamespace staticFunctionNamespace = createStaticFunctionNamespace(typeManager); - staticFunctionNamespace.addFunctions(functions); - staticFunctionNamespace.addFunctions(functions); + FunctionManager functionManager = createFunctionManager(typeManager); + functionManager.registerBuiltInFunctions(functions); + functionManager.registerBuiltInFunctions(functions); } - @Test(expectedExceptions = IllegalStateException.class, expectedExceptionsMessageRegExp = "'sum' is both an aggregation and a scalar function") + @Test(expectedExceptions = IllegalStateException.class, expectedExceptionsMessageRegExp = "'presto.default.sum' is both an aggregation and a scalar function") public void testConflictingScalarAggregation() { - List functions = new FunctionListBuilder() + List functions = new FunctionListBuilder() .scalars(ScalarSum.class) .getFunctions(); TypeRegistry typeManager = new TypeRegistry(); - StaticFunctionNamespace staticFunctionNamespace = createStaticFunctionNamespace(typeManager); - staticFunctionNamespace.addFunctions(functions); + FunctionManager functionManager = createFunctionManager(typeManager); + functionManager.registerBuiltInFunctions(functions); } @Test public void testListingHiddenFunctions() { TypeRegistry typeManager = new TypeRegistry(); - StaticFunctionNamespace staticFunctionNamespace = createStaticFunctionNamespace(typeManager); - List functions = staticFunctionNamespace.listFunctions(); - List names = transform(functions, input -> input.getSignature().getName()); + FunctionManager functionManager = createFunctionManager(typeManager); + List functions = functionManager.listFunctions(); + List names = transform(functions, input -> input.getSignature().getNameSuffix()); assertTrue(names.contains("length"), "Expected function names " + names + " to contain 'length'"); assertTrue(names.contains("stddev"), "Expected function names " + names + " to contain 'stddev'"); @@ -171,6 +173,22 @@ public void testOperatorTypes() assertFalse(functionManager.getFunctionMetadata(functionResolution.notFunction()).getOperatorType().isPresent()); } + @Test(expectedExceptions = PrestoException.class, expectedExceptionsMessageRegExp = ".*Non-builtin functions must be reference by three parts: catalog\\.schema\\.function_name, found: a\\.b") + public void testSqlFunctionReferenceTooShort() + { + TypeRegistry typeManager = new TypeRegistry(); + FunctionManager functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + functionManager.resolveFunction(TEST_SESSION.getTransactionId(), QualifiedName.of("a", "b"), ImmutableList.of()); + } + + @Test(expectedExceptions = PrestoException.class, expectedExceptionsMessageRegExp = ".*Non-builtin functions must be reference by three parts: catalog\\.schema\\.function_name, found: a\\.b\\.c\\.d") + public void testSqlFunctionReferenceTooLong() + { + TypeRegistry typeManager = new TypeRegistry(); + FunctionManager functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + functionManager.resolveFunction(TEST_SESSION.getTransactionId(), QualifiedName.of("a", "b", "c", "d"), ImmutableList.of()); + } + @Test public void testResolveFunctionByExactMatch() { @@ -312,12 +330,11 @@ public void testResolveFunctionForUnknown() .failsWithMessage("Could not choose a best candidate operator. Explicit type casts must be added."); } - private StaticFunctionNamespace createStaticFunctionNamespace(TypeRegistry typeManager) + private FunctionManager createFunctionManager(TypeRegistry typeManager) { BlockEncodingManager blockEncodingManager = new BlockEncodingManager(typeManager); FeaturesConfig featuresConfig = new FeaturesConfig(); - FunctionManager functionManager = new FunctionManager(typeManager, blockEncodingManager, featuresConfig); - return new StaticFunctionNamespace(typeManager, blockEncodingManager, featuresConfig, functionManager); + return new FunctionManager(typeManager, blockEncodingManager, featuresConfig); } private SignatureBuilder functionSignature(String... argumentTypes) @@ -372,7 +389,7 @@ public ResolveFunctionAssertion forParameters(String... parameters) public ResolveFunctionAssertion returns(SignatureBuilder functionSignature) { - FunctionHandle expectedFunction = new StaticFunctionHandle(functionSignature.name(TEST_FUNCTION_NAME).build()); + FunctionHandle expectedFunction = new BuiltInFunctionHandle(functionSignature.name(TEST_FUNCTION_NAME).build()); FunctionHandle actualFunction = resolveFunctionHandle(); assertEquals(expectedFunction, actualFunction); return this; @@ -399,26 +416,25 @@ private FunctionHandle resolveFunctionHandle() { FeaturesConfig featuresConfig = new FeaturesConfig(); FunctionManager functionManager = new FunctionManager(typeRegistry, blockEncoding, featuresConfig); - StaticFunctionNamespace staticFunctionNamespace = new StaticFunctionNamespace(typeRegistry, blockEncoding, featuresConfig, functionManager); - staticFunctionNamespace.addFunctions(createFunctionsFromSignatures()); - return staticFunctionNamespace.resolveFunction(QualifiedName.of(TEST_FUNCTION_NAME), fromTypeSignatures(parameterTypes)); + functionManager.registerBuiltInFunctions(createFunctionsFromSignatures()); + return functionManager.resolveFunction(TEST_SESSION.getTransactionId(), QualifiedName.of("presto", "default", TEST_FUNCTION_NAME), fromTypeSignatures(parameterTypes)); } - private List createFunctionsFromSignatures() + private List createFunctionsFromSignatures() { - ImmutableList.Builder functions = ImmutableList.builder(); + ImmutableList.Builder functions = ImmutableList.builder(); for (SignatureBuilder functionSignature : functionSignatures) { Signature signature = functionSignature.name(TEST_FUNCTION_NAME).build(); functions.add(new SqlScalarFunction(signature) { @Override - public ScalarFunctionImplementation specialize( + public BuiltInScalarFunctionImplementation specialize( BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, nCopies(arity, valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), MethodHandles.identity(Void.class)); diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionNamespaceManager.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionNamespaceManager.java new file mode 100644 index 0000000000000..60f9438c0e290 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionNamespaceManager.java @@ -0,0 +1,163 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionNamespaceTransactionHandle; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.facebook.presto.sqlfunction.SqlInvokedFunctionNamespaceManagerConfig; +import com.facebook.presto.testing.InMemoryFunctionNamespaceManager; +import com.google.common.collect.ImmutableSet; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.Collection; +import java.util.Optional; + +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR; +import static com.facebook.presto.sqlfunction.testing.SqlInvokedFunctionTestUtils.FUNCTION_POWER_TOWER_DOUBLE; +import static com.facebook.presto.sqlfunction.testing.SqlInvokedFunctionTestUtils.FUNCTION_POWER_TOWER_DOUBLE_UPDATED; +import static com.facebook.presto.sqlfunction.testing.SqlInvokedFunctionTestUtils.FUNCTION_POWER_TOWER_INT; +import static com.facebook.presto.sqlfunction.testing.SqlInvokedFunctionTestUtils.POWER_TOWER; +import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static com.google.common.collect.Iterables.getOnlyElement; +import static java.lang.String.format; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.testng.Assert.assertNotSame; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +public class TestFunctionNamespaceManager +{ + @Test + public void testCreateFunction() + { + InMemoryFunctionNamespaceManager functionNamespaceManager = createFunctionNamespaceManager(); + functionNamespaceManager.createFunction(FUNCTION_POWER_TOWER_DOUBLE, false); + assertEquals(functionNamespaceManager.listFunctions(), ImmutableSet.of(FUNCTION_POWER_TOWER_DOUBLE.withVersion(1))); + + functionNamespaceManager.createFunction(FUNCTION_POWER_TOWER_INT, false); + assertEquals( + ImmutableSet.copyOf(functionNamespaceManager.listFunctions()), + ImmutableSet.of(FUNCTION_POWER_TOWER_DOUBLE.withVersion(1), FUNCTION_POWER_TOWER_INT.withVersion(1))); + + functionNamespaceManager.createFunction(FUNCTION_POWER_TOWER_DOUBLE_UPDATED, true); + assertEquals( + ImmutableSet.copyOf(functionNamespaceManager.listFunctions()), + ImmutableSet.of(FUNCTION_POWER_TOWER_DOUBLE_UPDATED.withVersion(2), FUNCTION_POWER_TOWER_INT.withVersion(1))); + } + + @Test + public void testCreateFunctionFailed() + { + InMemoryFunctionNamespaceManager functionNamespaceManager = createFunctionNamespaceManager(); + functionNamespaceManager.createFunction(FUNCTION_POWER_TOWER_DOUBLE, false); + assertPrestoException( + () -> functionNamespaceManager.createFunction(FUNCTION_POWER_TOWER_DOUBLE_UPDATED, false), + GENERIC_USER_ERROR, + ".*Function 'unittest.memory.power_tower\\(double\\)' already exists"); + } + + @Test + public void testTransactionalGetFunction() + { + InMemoryFunctionNamespaceManager functionNamespaceManager = new InMemoryFunctionNamespaceManager( + new SqlInvokedFunctionNamespaceManagerConfig() + .setFunctionCacheExpiration(new Duration(0, MILLISECONDS)) + .setFunctionInstanceCacheExpiration(new Duration(0, MILLISECONDS))); + + // begin first transaction + FunctionNamespaceTransactionHandle transaction1 = functionNamespaceManager.beginTransaction(); + assertEquals(functionNamespaceManager.getFunctions(Optional.of(transaction1), POWER_TOWER).size(), 0); + + // create function, first transaction still sees no functions + functionNamespaceManager.createFunction(FUNCTION_POWER_TOWER_DOUBLE, false); + assertEquals(functionNamespaceManager.getFunctions(Optional.of(transaction1), POWER_TOWER).size(), 0); + + // second transaction sees newly created function + FunctionNamespaceTransactionHandle transaction2 = functionNamespaceManager.beginTransaction(); + Collection functions2 = functionNamespaceManager.getFunctions(Optional.of(transaction2), POWER_TOWER); + assertEquals(functions2.size(), 1); + assertEquals(getOnlyElement(functions2), FUNCTION_POWER_TOWER_DOUBLE.withVersion(1)); + + // update the function, second transaction still sees the old functions + functionNamespaceManager.createFunction(FUNCTION_POWER_TOWER_DOUBLE_UPDATED, true); + functions2 = functionNamespaceManager.getFunctions(Optional.of(transaction2), POWER_TOWER); + assertEquals(functions2.size(), 1); + assertEquals(getOnlyElement(functions2), FUNCTION_POWER_TOWER_DOUBLE.withVersion(1)); + + // third transaction sees the updated function + FunctionNamespaceTransactionHandle transaction3 = functionNamespaceManager.beginTransaction(); + Collection functions3 = functionNamespaceManager.getFunctions(Optional.of(transaction3), POWER_TOWER); + assertEquals(functions3.size(), 1); + assertEquals(getOnlyElement(functions3), FUNCTION_POWER_TOWER_DOUBLE_UPDATED.withVersion(2)); + + functionNamespaceManager.commit(transaction1); + functionNamespaceManager.commit(transaction2); + functionNamespaceManager.commit(transaction3); + } + + @Test + public void testCaching() + { + InMemoryFunctionNamespaceManager functionNamespaceManager = createFunctionNamespaceManager(); + functionNamespaceManager.createFunction(FUNCTION_POWER_TOWER_DOUBLE, false); + + // fetchFunctionsDirect does not produce the same function reference + SqlInvokedFunction function1 = getOnlyElement(functionNamespaceManager.fetchFunctionsDirect(POWER_TOWER)); + SqlInvokedFunction function2 = getOnlyElement(functionNamespaceManager.fetchFunctionsDirect(POWER_TOWER)); + assertEquals(function1, function2); + assertNotSame(function1, function2); + + // fetchFunctionMetadataDirect does not produce the same metdata reference + FunctionMetadata metadata1 = functionNamespaceManager.fetchFunctionMetadataDirect(function1.getRequiredFunctionHandle()); + FunctionMetadata metadata2 = functionNamespaceManager.fetchFunctionMetadataDirect(function2.getRequiredFunctionHandle()); + assertEquals(metadata1, metadata2); + assertNotSame(metadata1, metadata2); + + // getFunctionMetadata produces the same metadata reference + metadata1 = functionNamespaceManager.getFunctionMetadata(function1.getRequiredFunctionHandle()); + metadata2 = functionNamespaceManager.getFunctionMetadata(function2.getRequiredFunctionHandle()); + assertSame(metadata1, metadata2); + + // getFunctions produces the same function collection reference + functionNamespaceManager.createFunction(FUNCTION_POWER_TOWER_INT, false); + FunctionNamespaceTransactionHandle transaction1 = functionNamespaceManager.beginTransaction(); + FunctionNamespaceTransactionHandle transaction2 = functionNamespaceManager.beginTransaction(); + Collection functions1 = functionNamespaceManager.getFunctions(Optional.of(transaction1), POWER_TOWER); + Collection functions2 = functionNamespaceManager.getFunctions(Optional.of(transaction2), POWER_TOWER); + assertEquals(functions1.size(), 2); + assertSame(functions1, functions2); + } + + private static InMemoryFunctionNamespaceManager createFunctionNamespaceManager() + { + return new InMemoryFunctionNamespaceManager(new SqlInvokedFunctionNamespaceManagerConfig()); + } + + private static void assertPrestoException(Runnable runnable, ErrorCodeSupplier expectedErrorCode, String expectedMessageRegex) + { + try { + runnable.run(); + fail(format("Expected PrestoException with error code '%s', but not Exception is thrown", expectedErrorCode.toErrorCode().getName())); + } + catch (PrestoException e) { + assertEquals(e.getErrorCode(), expectedErrorCode.toErrorCode()); + assertTrue(e.getMessage().matches(expectedMessageRegex)); + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestInformationSchemaMetadata.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestInformationSchemaMetadata.java index dc4777a348f8a..6e5971e40efd3 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestInformationSchemaMetadata.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestInformationSchemaMetadata.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.metadata; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.block.BlockEncodingManager; -import com.facebook.presto.client.ClientCapabilities; import com.facebook.presto.connector.MockConnectorFactory; import com.facebook.presto.connector.informationSchema.InformationSchemaColumnHandle; import com.facebook.presto.connector.informationSchema.InformationSchemaMetadata; @@ -40,7 +40,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import org.testng.annotations.Test; @@ -53,8 +52,6 @@ import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static java.util.Arrays.stream; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -173,9 +170,6 @@ private ConnectorSession createNewSession(TransactionId transactionId) return testSessionBuilder() .setCatalog("test_catalog") .setSchema("information_schema") - .setClientCapabilities(stream(ClientCapabilities.values()) - .map(ClientCapabilities::toString) - .collect(toImmutableSet())) .setTransactionId(transactionId) .build() .toConnectorSession(); diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestInformationSchemaTableHandle.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestInformationSchemaTableHandle.java index b0a4fccc60ae9..8b072926db5c1 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestInformationSchemaTableHandle.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestInformationSchemaTableHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.connector.informationSchema.InformationSchemaTableHandle; import com.facebook.presto.spi.ConnectorTableHandle; import com.fasterxml.jackson.core.type.TypeReference; @@ -20,13 +21,12 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Guice; import com.google.inject.Injector; -import io.airlift.json.JsonModule; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.util.Map; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestPolymorphicScalarFunction.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestPolymorphicScalarFunction.java index 68634044b92a7..13d9c92436387 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestPolymorphicScalarFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestPolymorphicScalarFunction.java @@ -14,7 +14,7 @@ package com.facebook.presto.metadata; import com.facebook.presto.block.BlockEncodingManager; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.LongArrayBlock; import com.facebook.presto.spi.function.Signature; @@ -34,9 +34,9 @@ import static com.facebook.presto.metadata.TestPolymorphicScalarFunction.TestMethods.VARCHAR_TO_BIGINT_RETURN_VALUE; import static com.facebook.presto.metadata.TestPolymorphicScalarFunction.TestMethods.VARCHAR_TO_VARCHAR_RETURN_VALUE; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.OperatorType.ADD; import static com.facebook.presto.spi.function.OperatorType.IS_DISTINCT_FROM; @@ -108,7 +108,7 @@ public void testSelectsMultipleChoiceWithBlockPosition() asList(Optional.of(long.class), Optional.of(long.class))))) .build(); - ScalarFunctionImplementation functionImplementation = function.specialize(SHORT_DECIMAL_BOUND_VARIABLES, 2, TYPE_REGISTRY, FUNCTION_MANAGER); + BuiltInScalarFunctionImplementation functionImplementation = function.specialize(SHORT_DECIMAL_BOUND_VARIABLES, 2, TYPE_REGISTRY, FUNCTION_MANAGER); assertEquals(functionImplementation.getAllChoices().size(), 2); assertEquals(functionImplementation.getAllChoices().get(0).getArgumentProperties(), Collections.nCopies(2, valueTypeArgumentProperty(USE_NULL_FLAG))); @@ -135,7 +135,7 @@ public void testSelectsMethodBasedOnArgumentTypes() .withExtraParameters(context -> ImmutableList.of(context.getLiteral("x"))))) .build(); - ScalarFunctionImplementation functionImplementation = function.specialize(BOUND_VARIABLES, 1, TYPE_REGISTRY, FUNCTION_MANAGER); + BuiltInScalarFunctionImplementation functionImplementation = function.specialize(BOUND_VARIABLES, 1, TYPE_REGISTRY, FUNCTION_MANAGER); assertEquals(functionImplementation.getMethodHandle().invoke(INPUT_SLICE), INPUT_VARCHAR_LENGTH); } @@ -154,7 +154,7 @@ public void testSelectsMethodBasedOnReturnType() .withExtraParameters(context -> ImmutableList.of(42)))) .build(); - ScalarFunctionImplementation functionImplementation = function.specialize(BOUND_VARIABLES, 1, TYPE_REGISTRY, FUNCTION_MANAGER); + BuiltInScalarFunctionImplementation functionImplementation = function.specialize(BOUND_VARIABLES, 1, TYPE_REGISTRY, FUNCTION_MANAGER); assertEquals(functionImplementation.getMethodHandle().invoke(INPUT_SLICE), VARCHAR_TO_BIGINT_RETURN_VALUE); } @@ -178,7 +178,7 @@ public void testSameLiteralInArgumentsAndReturnValue() .implementation(methodsGroup -> methodsGroup.methods("varcharToVarchar"))) .build(); - ScalarFunctionImplementation functionImplementation = function.specialize(BOUND_VARIABLES, 1, TYPE_REGISTRY, FUNCTION_MANAGER); + BuiltInScalarFunctionImplementation functionImplementation = function.specialize(BOUND_VARIABLES, 1, TYPE_REGISTRY, FUNCTION_MANAGER); Slice slice = (Slice) functionImplementation.getMethodHandle().invoke(INPUT_SLICE); assertEquals(slice, VARCHAR_TO_VARCHAR_RETURN_VALUE); } @@ -203,7 +203,7 @@ public void testTypeParameters() .implementation(methodsGroup -> methodsGroup.methods("varcharToVarchar"))) .build(); - ScalarFunctionImplementation functionImplementation = function.specialize(BOUND_VARIABLES, 1, TYPE_REGISTRY, FUNCTION_MANAGER); + BuiltInScalarFunctionImplementation functionImplementation = function.specialize(BOUND_VARIABLES, 1, TYPE_REGISTRY, FUNCTION_MANAGER); Slice slice = (Slice) functionImplementation.getMethodHandle().invoke(INPUT_SLICE); assertEquals(slice, VARCHAR_TO_VARCHAR_RETURN_VALUE); } @@ -225,7 +225,7 @@ public void testSetsHiddenToTrueForOperators() .implementation(methodsGroup -> methodsGroup.methods("varcharToVarchar"))) .build(); - ScalarFunctionImplementation functionImplementation = function.specialize(BOUND_VARIABLES, 1, TYPE_REGISTRY, FUNCTION_MANAGER); + BuiltInScalarFunctionImplementation functionImplementation = function.specialize(BOUND_VARIABLES, 1, TYPE_REGISTRY, FUNCTION_MANAGER); } @Test(expectedExceptions = {IllegalStateException.class}, diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestQualifiedTablePrefix.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestQualifiedTablePrefix.java index f198815adaf2f..73f52d23c5352 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestQualifiedTablePrefix.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestQualifiedTablePrefix.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.metadata; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import org.testng.annotations.Test; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestSignature.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestSignature.java index 8fb95345113b7..2117e81753c2e 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestSignature.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestSignature.java @@ -13,6 +13,10 @@ */ package com.facebook.presto.metadata; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; @@ -20,11 +24,9 @@ import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; import org.testng.annotations.Test; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static org.testng.Assert.assertEquals; @@ -39,7 +41,7 @@ public void testSerializationRoundTrip() JsonCodec codec = new JsonCodecFactory(objectMapperProvider, true).jsonCodec(Signature.class); Signature expected = new Signature( - "function", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "function"), SCALAR, parseTypeSignature(StandardTypes.BIGINT), ImmutableList.of(parseTypeSignature(StandardTypes.BOOLEAN), parseTypeSignature(StandardTypes.DOUBLE), parseTypeSignature(StandardTypes.VARCHAR))); @@ -47,7 +49,7 @@ public void testSerializationRoundTrip() String json = codec.toJson(expected); Signature actual = codec.fromJson(json); - assertEquals(actual.getName(), expected.getName()); + assertEquals(actual.getNameSuffix(), expected.getNameSuffix()); assertEquals(actual.getKind(), expected.getKind()); assertEquals(actual.getReturnType(), expected.getReturnType()); assertEquals(actual.getArgumentTypes(), expected.getArgumentTypes()); diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticCatalogStoreConfig.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticCatalogStoreConfig.java index b00ba918434fe..289fb1f0a0651 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticCatalogStoreConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticCatalogStoreConfig.java @@ -20,9 +20,9 @@ import java.io.File; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestStaticCatalogStoreConfig { diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespaceStoreConfig.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespaceStoreConfig.java new file mode 100644 index 0000000000000..4aee022471ec1 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestStaticFunctionNamespaceStoreConfig.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.io.File; +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestStaticFunctionNamespaceStoreConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(StaticFunctionNamespaceStoreConfig.class) + .setFunctionNamespaceConfigurationDir(new File("etc/function-namespace"))); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("function-namespace.config-dir", "/foo") + .build(); + + StaticFunctionNamespaceStoreConfig expected = new StaticFunctionNamespaceStoreConfig() + .setFunctionNamespaceConfigurationDir(new File("/foo")); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestSystemTableHandle.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestSystemTableHandle.java index e386b8be8f7e5..0e291c6a8fb59 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestSystemTableHandle.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestSystemTableHandle.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.connector.system.SystemTableHandle; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorTableHandle; @@ -22,13 +23,12 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Guice; import com.google.inject.Injector; -import io.airlift.json.JsonModule; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.util.Map; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkCompressToByteBuffer.java b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkCompressToByteBuffer.java new file mode 100644 index 0000000000000..b93f521470107 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkCompressToByteBuffer.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import io.airlift.compress.Compressor; +import io.airlift.compress.lz4.Lz4Compressor; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.VerboseMode; + +import java.nio.ByteBuffer; +import java.util.Random; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +@State(Scope.Thread) +@OutputTimeUnit(MICROSECONDS) +@Fork(3) +@Warmup(iterations = 20, time = 500, timeUnit = MILLISECONDS) +@Measurement(iterations = 20, time = 500, timeUnit = MILLISECONDS) +@BenchmarkMode(Mode.AverageTime) +public class BenchmarkCompressToByteBuffer +{ + @Benchmark + public void compressToByteBuffer(BenchmarkData data) + { + data.byteBuffer.mark(); + data.COMPRESSOR.compress(data.slice.toByteBuffer(), data.byteBuffer); + data.byteBuffer.reset(); + } + + @Benchmark + public void compressToByteArray(BenchmarkData data) + { + data.COMPRESSOR.compress((byte[]) data.slice.getBase(), 0, data.slice.length(), data.bytes, 0, data.MAX_COMPRESSED_SIZE); + } + + @State(Scope.Thread) + public static class BenchmarkData + { + private static final Random RANDOM = new Random(0); + private static final Compressor COMPRESSOR = new Lz4Compressor(); + private static final int UNCOMPRESSED_SIZE = 1_000_000; + private static final int MAX_COMPRESSED_SIZE = COMPRESSOR.maxCompressedLength(UNCOMPRESSED_SIZE); + + private final byte[] byteValues = new byte[UNCOMPRESSED_SIZE]; + private final Slice slice = Slices.wrappedBuffer(byteValues); + + private final byte[] bytes = new byte[MAX_COMPRESSED_SIZE]; + private final ByteBuffer byteBuffer = ByteBuffer.allocate(MAX_COMPRESSED_SIZE); + + @Setup + public void setup() + { + // Generate discontinuous runs of random values and 0's to avoid LZ4 enters uncompressible fast-path + int runLength = UNCOMPRESSED_SIZE / 10; + byte[] randomBytes = new byte[runLength]; + for (int i = 0; i < 10; i += 2) { + RANDOM.nextBytes(randomBytes); + System.arraycopy(randomBytes, 0, byteValues, i * runLength, runLength); + } + } + } + + public static void main(String[] args) + throws RunnerException + { + Options options = new OptionsBuilder() + .verbosity(VerboseMode.NORMAL) + .include(".*" + BenchmarkCompressToByteBuffer.class.getSimpleName() + ".*") + .build(); + new Runner(options).run(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashAndStreamingAggregationOperators.java b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashAndStreamingAggregationOperators.java index 1b0752bcb9440..2d51dd00d4363 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashAndStreamingAggregationOperators.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashAndStreamingAggregationOperators.java @@ -21,11 +21,11 @@ import com.facebook.presto.operator.aggregation.InternalAggregationFunction; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spiller.SpillerFactory; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.JoinCompiler; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.testing.TestingTaskContext; import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; @@ -51,6 +51,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.block.BlockAssertions.createLongSequenceBlock; import static com.facebook.presto.operator.BenchmarkHashAndStreamingAggregationOperators.Context.ROWS_PER_PAGE; @@ -58,7 +59,6 @@ import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static io.airlift.units.DataSize.succinctBytes; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashBuildAndJoinOperators.java b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashBuildAndJoinOperators.java index 09a460b44b908..05f43ce1c38c2 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashBuildAndJoinOperators.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkHashBuildAndJoinOperators.java @@ -49,14 +49,14 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.spiller.PartitioningSpillerFactory.unsupportedPartitioningSpillerFactory; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static java.lang.String.format; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkMoreByteArrays.java b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkMoreByteArrays.java new file mode 100644 index 0000000000000..410b0009f34a6 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkMoreByteArrays.java @@ -0,0 +1,154 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.operator; + +import io.airlift.slice.DynamicSliceOutput; +import io.airlift.slice.Slice; +import io.airlift.slice.SliceOutput; +import io.airlift.slice.Slices; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.VerboseMode; + +import java.util.Random; +import java.util.stream.IntStream; + +import static com.facebook.presto.operator.MoreByteArrays.fill; +import static com.facebook.presto.operator.MoreByteArrays.setBytes; +import static com.facebook.presto.operator.MoreByteArrays.setInts; +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static sun.misc.Unsafe.ARRAY_LONG_INDEX_SCALE; + +@State(Scope.Thread) +@OutputTimeUnit(MICROSECONDS) +@Fork(3) +@Warmup(iterations = 10, time = 500, timeUnit = MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = MILLISECONDS) +@BenchmarkMode(Mode.AverageTime) +public class BenchmarkMoreByteArrays +{ + @Benchmark + public void fillToByteArray(BenchmarkData data) + { + fill(data.bytes, 0, data.bytes.length, (byte) 5); + } + + @Benchmark + public void fillToBasicSliceOutput(BenchmarkData data) + { + data.basicSliceOutput.reset(); + for (int i = 0; i < data.bytes.length; i++) { + data.basicSliceOutput.writeByte((byte) 5); + } + } + + @Benchmark + public void fillToDynamicSliceOutput(BenchmarkData data) + { + data.dynamicSliceOutput.reset(); + for (int i = 0; i < data.bytes.length; i++) { + data.dynamicSliceOutput.writeByte((byte) 5); + } + } + + @Benchmark + public void setBytesToByteArray(BenchmarkData data) + { + setBytes(data.bytes, 0, data.byteValues, 0, data.bytes.length); + } + + @Benchmark + public void setBytesToBasicSliceOutput(BenchmarkData data) + { + data.basicSliceOutput.reset(); + data.basicSliceOutput.writeBytes(data.slice, 0, data.slice.length()); + } + + @Benchmark + public void setBytesToDynamicSliceOutput(BenchmarkData data) + { + data.dynamicSliceOutput.reset(); + data.dynamicSliceOutput.writeBytes(data.slice, 0, data.slice.length()); + } + + @Benchmark + public void setIntsToByteArray(BenchmarkData data) + { + setInts(data.bytes, 0, data.intValues, 0, data.POSITIONS_PER_PAGE); + } + + @Benchmark + public void setIntsToBasicSliceOutput(BenchmarkData data) + { + data.basicSliceOutput.reset(); + for (int i = 0; i < data.POSITIONS_PER_PAGE; i++) { + data.basicSliceOutput.writeInt(data.intValues[i]); + } + } + + @Benchmark + public void setIntsToDynamicSliceOutput(BenchmarkData data) + { + data.dynamicSliceOutput.reset(); + for (int i = 0; i < data.POSITIONS_PER_PAGE; i++) { + data.dynamicSliceOutput.writeInt(data.intValues[i]); + } + } + + @State(Scope.Thread) + public static class BenchmarkData + { + private static final int POSITIONS_PER_PAGE = 10000; + + private final Random random = new Random(0); + + private final int[] intValues = IntStream.range(0, POSITIONS_PER_PAGE).map(i -> random.nextInt()).toArray(); + private final byte[] byteValues = new byte[POSITIONS_PER_PAGE * ARRAY_LONG_INDEX_SCALE]; + private final Slice slice = Slices.wrappedBuffer(byteValues); + + private final byte[] bytes = new byte[POSITIONS_PER_PAGE * ARRAY_LONG_INDEX_SCALE]; + private final SliceOutput basicSliceOutput = Slices.wrappedBuffer(new byte[POSITIONS_PER_PAGE * ARRAY_LONG_INDEX_SCALE]).getOutput(); + private final SliceOutput dynamicSliceOutput = new DynamicSliceOutput(POSITIONS_PER_PAGE * ARRAY_LONG_INDEX_SCALE); + + @Setup + public void setup() + { + random.nextBytes(byteValues); + } + } + + public static void main(String[] args) + throws RunnerException + { + Options options = new OptionsBuilder() + .verbosity(VerboseMode.NORMAL) + .include(".*" + BenchmarkMoreByteArrays.class.getSimpleName() + ".*") + .build(); + new Runner(options).run(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkPartitionedOutputOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkPartitionedOutputOperator.java deleted file mode 100644 index 7eec1e79f9f13..0000000000000 --- a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkPartitionedOutputOperator.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.operator; - -import com.facebook.presto.block.BlockEncodingManager; -import com.facebook.presto.execution.StateMachine; -import com.facebook.presto.execution.buffer.OutputBuffers; -import com.facebook.presto.execution.buffer.PagesSerdeFactory; -import com.facebook.presto.execution.buffer.PartitionedOutputBuffer; -import com.facebook.presto.memory.context.SimpleLocalMemoryContext; -import com.facebook.presto.operator.PartitionedOutputOperator.PartitionedOutputFactory; -import com.facebook.presto.operator.exchange.LocalPartitionGenerator; -import com.facebook.presto.spi.Page; -import com.facebook.presto.spi.PageBuilder; -import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.spi.plan.PlanNodeId; -import com.facebook.presto.spi.type.RowType; -import com.facebook.presto.spi.type.Type; -import com.facebook.presto.testing.TestingTaskContext; -import com.facebook.presto.type.TypeRegistry; -import com.google.common.collect.ImmutableList; -import io.airlift.units.DataSize; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.runner.Runner; -import org.openjdk.jmh.runner.RunnerException; -import org.openjdk.jmh.runner.options.Options; -import org.openjdk.jmh.runner.options.OptionsBuilder; -import org.openjdk.jmh.runner.options.VerboseMode; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.function.Function; - -import static com.facebook.presto.SessionTestUtils.TEST_SESSION; -import static com.facebook.presto.execution.buffer.BufferState.OPEN; -import static com.facebook.presto.execution.buffer.BufferState.TERMINAL_BUFFER_STATES; -import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.PARTITIONED; -import static com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers; -import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; -import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static com.facebook.presto.spi.type.VarcharType.VARCHAR; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.slice.Slices.utf8Slice; -import static io.airlift.units.DataSize.Unit.BYTE; -import static io.airlift.units.DataSize.Unit.GIGABYTE; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -@State(Scope.Thread) -@OutputTimeUnit(MILLISECONDS) -@Fork(2) -@Warmup(iterations = 20, time = 500, timeUnit = MILLISECONDS) -@Measurement(iterations = 20, time = 500, timeUnit = MILLISECONDS) -@BenchmarkMode(Mode.AverageTime) -public class BenchmarkPartitionedOutputOperator -{ - @Benchmark - public void addPage(BenchmarkData data) - { - PartitionedOutputOperator operator = data.createPartitionedOutputOperator(); - for (int i = 0; i < data.getPageCount(); i++) { - operator.addInput(data.getDataPage()); - } - operator.finish(); - } - - @State(Scope.Thread) - public static class BenchmarkData - { - private static final int PAGE_COUNT = 5000; - private static final int PARTITION_COUNT = 512; - private static final int ENTRIES_PER_PAGE = 256; - private static final DataSize MAX_MEMORY = new DataSize(1, GIGABYTE); - private static final RowType rowType = RowType.anonymous(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR, VARCHAR)); - private static final List TYPES = ImmutableList.of(BIGINT, rowType, rowType, rowType); - private static final ExecutorService EXECUTOR = newCachedThreadPool(daemonThreadsNamed("test-EXECUTOR-%s")); - private static final ScheduledExecutorService SCHEDULER = newScheduledThreadPool(1, daemonThreadsNamed("test-%s")); - - private final Page dataPage = createPage(); - - private int getPageCount() - { - return PAGE_COUNT; - } - - public Page getDataPage() - { - return dataPage; - } - - private PartitionedOutputOperator createPartitionedOutputOperator() - { - PartitionFunction partitionFunction = new LocalPartitionGenerator(new InterpretedHashGenerator(ImmutableList.of(BIGINT), new int[] {0}), PARTITION_COUNT); - PagesSerdeFactory serdeFactory = new PagesSerdeFactory(new BlockEncodingManager(new TypeRegistry()), false); - OutputBuffers buffers = createInitialEmptyOutputBuffers(PARTITIONED); - for (int partition = 0; partition < PARTITION_COUNT; partition++) { - buffers = buffers.withBuffer(new OutputBuffers.OutputBufferId(partition), partition); - } - PartitionedOutputBuffer buffer = createPartitionedBuffer( - buffers.withNoMoreBufferIds(), - new DataSize(Long.MAX_VALUE, BYTE)); // don't let output buffer block - buffer.registerLifespanCompletionCallback(ignore -> {}); - PartitionedOutputFactory operatorFactory = new PartitionedOutputFactory( - partitionFunction, - ImmutableList.of(0), - ImmutableList.of(Optional.empty()), - false, - OptionalInt.empty(), - buffer, - new DataSize(1, GIGABYTE)); - return (PartitionedOutputOperator) operatorFactory - .createOutputOperator(0, new PlanNodeId("plan-node-0"), TYPES, Function.identity(), serdeFactory) - .createOperator(createDriverContext()); - } - - private Page createPage() - { - List[] testRows = generateTestRows(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR, VARCHAR), ENTRIES_PER_PAGE); - PageBuilder pageBuilder = new PageBuilder(TYPES); - BlockBuilder bigintBlockBuilder = pageBuilder.getBlockBuilder(0); - BlockBuilder rowBlockBuilder = pageBuilder.getBlockBuilder(1); - BlockBuilder rowBlockBuilder2 = pageBuilder.getBlockBuilder(2); - BlockBuilder rowBlockBuilder3 = pageBuilder.getBlockBuilder(3); - for (int i = 0; i < ENTRIES_PER_PAGE; i++) { - BIGINT.writeLong(bigintBlockBuilder, i); - writeRow(testRows[i], rowBlockBuilder); - writeRow(testRows[i], rowBlockBuilder2); - writeRow(testRows[i], rowBlockBuilder3); - } - pageBuilder.declarePositions(ENTRIES_PER_PAGE); - return pageBuilder.build(); - } - - private void writeRow(List testRow, BlockBuilder rowBlockBuilder) - { - BlockBuilder singleRowBlockWriter = rowBlockBuilder.beginBlockEntry(); - for (Object fieldValue : testRow) { - if (fieldValue instanceof String) { - VARCHAR.writeSlice(singleRowBlockWriter, utf8Slice((String) fieldValue)); - } - else { - throw new UnsupportedOperationException(); - } - } - rowBlockBuilder.closeEntry(); - } - - // copied & modifed from TestRowBlock - private List[] generateTestRows(List fieldTypes, int numRows) - { - List[] testRows = new List[numRows]; - for (int i = 0; i < numRows; i++) { - List testRow = new ArrayList<>(fieldTypes.size()); - for (int j = 0; j < fieldTypes.size(); j++) { - if (fieldTypes.get(j) == VARCHAR) { - byte[] data = new byte[ThreadLocalRandom.current().nextInt(128)]; - ThreadLocalRandom.current().nextBytes(data); - testRow.add(new String(data)); - } - else { - throw new UnsupportedOperationException(); - } - } - testRows[i] = testRow; - } - return testRows; - } - - private DriverContext createDriverContext() - { - return TestingTaskContext.builder(EXECUTOR, SCHEDULER, TEST_SESSION) - .setMemoryPoolSize(MAX_MEMORY) - .setQueryMaxTotalMemory(MAX_MEMORY) - .build() - .addPipelineContext(0, true, true, false) - .addDriverContext(); - } - - private PartitionedOutputBuffer createPartitionedBuffer(OutputBuffers buffers, DataSize dataSize) - { - return new PartitionedOutputBuffer( - "task-instance-id", - new StateMachine<>("bufferState", SCHEDULER, OPEN, TERMINAL_BUFFER_STATES), - buffers, - dataSize, - () -> new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), - SCHEDULER); - } - } - - public static void main(String[] args) - throws RunnerException - { - // assure the benchmarks are valid before running - BenchmarkData data = new BenchmarkData(); - new BenchmarkPartitionedOutputOperator().addPage(data); - Options options = new OptionsBuilder() - .verbosity(VerboseMode.NORMAL) - .jvmArgs("-Xmx10g") - .include(".*" + BenchmarkPartitionedOutputOperator.class.getSimpleName() + ".*") - .build(); - new Runner(options).run(); - } -} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkReadBlock.java b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkReadBlock.java new file mode 100644 index 0000000000000..4c05469759119 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkReadBlock.java @@ -0,0 +1,297 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import com.facebook.presto.block.BlockAssertions; +import com.facebook.presto.spi.block.Block; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.VerboseMode; + +import java.util.Random; + +import static com.facebook.presto.block.BlockAssertions.createRandomDictionaryBlock; +import static com.facebook.presto.operator.UncheckedByteArrays.setLongUnchecked; +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static sun.misc.Unsafe.ARRAY_LONG_INDEX_SCALE; + +@State(Scope.Thread) +@OutputTimeUnit(MICROSECONDS) +@Fork(0) +@Warmup(iterations = 10, time = 500, timeUnit = MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = MILLISECONDS) +@BenchmarkMode(Mode.AverageTime) +public class BenchmarkReadBlock +{ + @Benchmark + public int sequentialCopyLongValues(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + index = setLongUnchecked(data.bytes, index, data.longValues[i]); + } + return index; + } + + @Benchmark + public int sequentialCopyLongArrayBlock(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + index = setLongUnchecked(data.bytes, index, data.blockNoNulls.getLong(i)); + } + return index; + } + + @Benchmark + public int sequentialCopyUncheckedLongArrayBlock(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + index = setLongUnchecked(data.bytes, index, data.blockNoNulls.getLongUnchecked(i)); + } + return index; + } + + @Benchmark + public int randomCopyLongValues(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + index = setLongUnchecked(data.bytes, index, data.longValues[data.positions[i]]); + } + return index; + } + + @Benchmark + public int randomCopyLongArrayBlock(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + index = setLongUnchecked(data.bytes, index, data.blockNoNulls.getLong(data.positions[i])); + } + return index; + } + + @Benchmark + public int randomCopyUncheckedLongArrayBlock(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + index = setLongUnchecked(data.bytes, index, data.blockNoNulls.getLongUnchecked(data.positions[i])); + } + return index; + } + + @Benchmark + public int sequentialCopyLongValuesWithNulls(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + int newIndex = setLongUnchecked(data.bytes, index, data.longValues[i]); + if (!data.nulls[i]) { + index = newIndex; + } + } + return index; + } + + @Benchmark + public int sequentialCopyLongArrayBlockWithNulls(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + int newIndex = setLongUnchecked(data.bytes, index, data.blockWithNulls.getLong(i)); + if (!data.blockWithNulls.isNull(i)) { + index = newIndex; + } + } + return index; + } + + @Benchmark + public int sequentialCopyUncheckedLongArrayBlockWithNulls(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + int newIndex = setLongUnchecked(data.bytes, index, data.blockWithNulls.getLongUnchecked(i)); + if (!data.blockWithNulls.isNullUnchecked(i)) { + index = newIndex; + } + } + return index; + } + + @Benchmark + public int randomCopyLongValuesWithNulls(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + int newIndex = setLongUnchecked(data.bytes, index, data.longValues[data.positions[i]]); + if (!data.nulls[data.positions[i]]) { + index = newIndex; + } + } + return index; + } + + @Benchmark + public int randomCopyLongArrayBlockWithNulls(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + int newIndex = setLongUnchecked(data.bytes, index, data.blockNoNulls.getLong(data.positions[i])); + if (!data.blockWithNulls.isNull(data.positions[i])) { + index = newIndex; + } + } + return index; + } + + @Benchmark + public int randomCopyUncheckedLongArrayBlockWithNulls(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + int newIndex = setLongUnchecked(data.bytes, index, data.blockNoNulls.getLongUnchecked(data.positions[i])); + if (!data.blockWithNulls.isNullUnchecked(data.positions[i])) { + index = newIndex; + } + } + return index; + } + + @Benchmark + public int randomCopyLongValuesWithDictionary(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + index = setLongUnchecked(data.bytes, index, data.longValues[data.ids[data.positions[i]]]); + } + return index; + } + + @Benchmark + public int randomCopyDictionaryBlock(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + index = setLongUnchecked(data.bytes, index, data.dictionaryBlockNoNulls.getLong(data.positions[i])); + } + return index; + } + + @Benchmark + public int randomCopyUncheckedDictionaryBlock(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + index = setLongUnchecked(data.bytes, index, data.dictionaryBlockNoNulls.getLongUnchecked(data.positions[i])); + } + return index; + } + + @Benchmark + public int randomCopyLongValuesWithDictionaryWithNulls(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + int newIndex = setLongUnchecked(data.bytes, index, data.longValues[data.positions[i]]); + if (!data.nulls[data.ids[data.positions[i]]]) { + index = newIndex; + } + } + return index; + } + + @Benchmark + public int randomCopyDictionaryBlockWithNulls(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + int newIndex = setLongUnchecked(data.bytes, index, data.dictionaryBlockWithNulls.getLong(data.positions[i])); + if (!data.dictionaryBlockWithNulls.isNull(data.positions[i])) { + index = newIndex; + } + } + return index; + } + + @Benchmark + public int randomCopyUncheckedDictionaryBlockWithNulls(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + int newIndex = setLongUnchecked(data.bytes, index, data.dictionaryBlockWithNulls.getLongUnchecked(data.positions[i])); + if (!data.dictionaryBlockWithNulls.isNullUnchecked(data.positions[i])) { + index = newIndex; + } + } + return index; + } + + @State(Scope.Thread) + public static class BenchmarkData + { + private static final int POSITIONS_PER_PAGE = 10000; + + private final Random random = new Random(0); + + private final long[] longValues = new long[POSITIONS_PER_PAGE]; + private final boolean[] nulls = new boolean[POSITIONS_PER_PAGE]; + private final int[] ids = new int[POSITIONS_PER_PAGE]; + private final int[] positions = new int[POSITIONS_PER_PAGE]; + + private final Block blockNoNulls = BlockAssertions.createRandomLongsBlock(POSITIONS_PER_PAGE, false); + private final Block blockWithNulls = BlockAssertions.createRandomLongsBlock(POSITIONS_PER_PAGE, true); + private final Block dictionaryBlockNoNulls = createRandomDictionaryBlock(blockNoNulls, POSITIONS_PER_PAGE); + private final Block dictionaryBlockWithNulls = createRandomDictionaryBlock(blockWithNulls, POSITIONS_PER_PAGE); + + private final byte[] bytes = new byte[POSITIONS_PER_PAGE * ARRAY_LONG_INDEX_SCALE]; + + @Setup + public void setup() + { + for (int i = 0; i < POSITIONS_PER_PAGE; i++) { + longValues[i] = random.nextLong(); + ids[i] = random.nextInt(POSITIONS_PER_PAGE / 10); + positions[i] = i; + nulls[i] = i % 7 == 0; + } + } + } + + public static void main(String[] args) + throws RunnerException + { + Options options = new OptionsBuilder() + .verbosity(VerboseMode.NORMAL) + .include(".*" + BenchmarkReadBlock.class.getSimpleName() + ".*") + .build(); + new Runner(options).run(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkUncheckedByteArrays.java b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkUncheckedByteArrays.java new file mode 100644 index 0000000000000..2ee4ae92a4c0d --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkUncheckedByteArrays.java @@ -0,0 +1,287 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import com.google.common.primitives.Booleans; +import io.airlift.slice.DynamicSliceOutput; +import io.airlift.slice.SliceOutput; +import io.airlift.slice.Slices; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.VerboseMode; +import sun.misc.Unsafe; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.LongStream; +import java.util.stream.Stream; + +import static com.facebook.presto.operator.UncheckedByteArrays.setLongUnchecked; +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET; +import static sun.misc.Unsafe.ARRAY_LONG_INDEX_SCALE; + +@State(Scope.Thread) +@OutputTimeUnit(MICROSECONDS) +@Fork(3) +@Warmup(iterations = 10, time = 500, timeUnit = MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = MILLISECONDS) +@BenchmarkMode(Mode.AverageTime) +public class BenchmarkUncheckedByteArrays +{ + public static void main(String[] args) + throws RunnerException + { + Options options = new OptionsBuilder() + .verbosity(VerboseMode.NORMAL) + .include(".*" + BenchmarkUncheckedByteArrays.class.getSimpleName() + ".*") + .build(); + new Runner(options).run(); + } + + @Benchmark + public int sequentialCopyToByteArray(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + index = setLongUnchecked(data.bytes, index, data.longValues[i]); + } + return index; + } + + @Benchmark + public int sequentialCopyToBasicSliceOutput(BenchmarkData data) + { + return sequentialCopyToSliceOutput(data.basicSliceOutput, data.longValues); + } + + @Benchmark + public int sequentialCopyToDynamicSliceOutput(BenchmarkData data) + { + return sequentialCopyToSliceOutput(data.dynamicSliceOutput, data.longValues); + } + + @Benchmark + public int sequentialCopyToTestingSliceOutput(BenchmarkData data) + { + TestingSliceOutput sliceOutput = data.testingSliceOutput; + sliceOutput.reset(); + for (int i = 0; i < data.longValues.length; i++) { + sliceOutput.writeLong(data.longValues[i]); + } + return sliceOutput.size(); + } + + @Benchmark + public int sequentialCopyWithNullsToByteArray(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + int newIndex = setLongUnchecked(data.bytes, index, data.longValues[i]); + // Only update the index when it's not null. This will be compiled to conditional move instruction instead of compare and jumps. + if (data.nulls[i]) { + index = newIndex; + } + } + return index; + } + + @Benchmark + public int sequentialCopyWithNullsToByteArrayNaive(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + // This will be compiled to compare and jumps. + if (!data.nulls[i]) { + index = setLongUnchecked(data.bytes, index, data.longValues[i]); + } + } + return index; + } + + @Benchmark + public int sequentialCopyWithNullsToBasicSliceOutput(BenchmarkData data) + { + return sequentialCopyWithNullsToSliceOutput(data.basicSliceOutput, data.longValues, data.nulls); + } + + @Benchmark + public int sequentialCopyWithNullsToDynamicSliceOutput(BenchmarkData data) + { + return sequentialCopyWithNullsToSliceOutput(data.dynamicSliceOutput, data.longValues, data.nulls); + } + + @Benchmark + public int sequentialCopyWithNullsToTestingSliceOutputNaive(BenchmarkData data) + { + TestingSliceOutput sliceOutput = data.testingSliceOutput; + sliceOutput.reset(); + for (int i = 0; i < data.longValues.length; i++) { + if (data.nulls[i]) { + sliceOutput.writeLong(data.longValues[i]); + } + } + return sliceOutput.size(); + } + + @Benchmark + public int sequentialCopyWithNullsToTestingSliceOutput(BenchmarkData data) + { + TestingSliceOutput sliceOutput = data.testingSliceOutput; + sliceOutput.reset(); + for (int i = 0; i < data.longValues.length; i++) { + sliceOutput.writeLong(data.longValues[i], data.nulls[i]); + } + return sliceOutput.size(); + } + + @Benchmark + public int randomCopyToByteArray(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + index = setLongUnchecked(data.bytes, data.positions[i] * ARRAY_LONG_INDEX_SCALE, data.longValues[i]); + } + return index; + } + + // We can't apply the trick on null check when copying to random locations in the byte array + @Benchmark + public int randomCopyWithNullsToByteArray(BenchmarkData data) + { + int index = 0; + for (int i = 0; i < data.longValues.length; i++) { + if (data.nulls[i]) { + index = setLongUnchecked(data.bytes, data.positions[i] * ARRAY_LONG_INDEX_SCALE, data.longValues[i]); + } + } + return index; + } + + private int sequentialCopyToSliceOutput(SliceOutput sliceOutput, long[] values) + { + sliceOutput.reset(); + for (int i = 0; i < values.length; i++) { + sliceOutput.writeLong(values[i]); + } + return sliceOutput.size(); + } + + private int sequentialCopyWithNullsToSliceOutput(SliceOutput sliceOutput, long[] values, boolean[] nulls) + { + sliceOutput.reset(); + for (int i = 0; i < values.length; i++) { + if (!nulls[i]) { + sliceOutput.writeLong(values[i]); + } + } + return sliceOutput.size(); + } + + private static class TestingSliceOutput + { + private static Unsafe unsafe; + public int size; + private byte[] buffer; + + TestingSliceOutput(int initialSize) + { + buffer = new byte[initialSize]; + } + + public void writeLong(long value) + { + unsafe.putLong(buffer, (long) size + ARRAY_BYTE_BASE_OFFSET, value); + size += ARRAY_LONG_INDEX_SCALE; + } + + public void writeLong(long value, int address) + { + unsafe.putLong(buffer, (long) address + ARRAY_BYTE_BASE_OFFSET, value); + } + + public void writeLong(long value, boolean isNull) + { + unsafe.putLong(buffer, (long) size + ARRAY_BYTE_BASE_OFFSET, value); + if (!isNull) { + size += ARRAY_LONG_INDEX_SCALE; + } + } + + public int size() + { + return size; + } + + public void reset() + { + size = 0; + } + + static { + try { + Field field = Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); + unsafe = (Unsafe) field.get(null); + if (unsafe == null) { + throw new RuntimeException("Unsafe access not available"); + } + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + @State(Scope.Thread) + public static class BenchmarkData + { + private static final int POSITIONS_PER_PAGE = 10000; + + private final Random random = new Random(0); + + private final long[] longValues = LongStream.range(0, POSITIONS_PER_PAGE).map(i -> random.nextLong()).toArray(); + private final boolean[] nulls = Booleans.toArray(Stream.generate(() -> random.nextBoolean()).limit(POSITIONS_PER_PAGE).collect(Collectors.toCollection(ArrayList::new))); + private final int[] positions = IntStream.range(0, POSITIONS_PER_PAGE).toArray(); + + private final byte[] byteValues = new byte[POSITIONS_PER_PAGE * ARRAY_LONG_INDEX_SCALE]; + + private final byte[] bytes = new byte[POSITIONS_PER_PAGE * ARRAY_LONG_INDEX_SCALE]; + private final SliceOutput basicSliceOutput = Slices.wrappedBuffer(new byte[POSITIONS_PER_PAGE * ARRAY_LONG_INDEX_SCALE]).getOutput(); + private final SliceOutput dynamicSliceOutput = new DynamicSliceOutput(POSITIONS_PER_PAGE * ARRAY_LONG_INDEX_SCALE); + private final TestingSliceOutput testingSliceOutput = new TestingSliceOutput(POSITIONS_PER_PAGE * ARRAY_LONG_INDEX_SCALE); + + @Setup + public void setup() + { + random.nextBytes(byteValues); + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkUnnestOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkUnnestOperator.java new file mode 100644 index 0000000000000..c4aa2b3059e8c --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkUnnestOperator.java @@ -0,0 +1,424 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.operator.unnest.UnnestOperator.UnnestOperatorFactory; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.block.RowBlock; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.MapType; +import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeSignature; +import com.facebook.presto.testing.TestingTaskContext; +import com.google.common.collect.ImmutableList; +import io.airlift.units.DataSize; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.VerboseMode; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.util.StructuralTestUtil.mapType; +import static com.google.common.base.Preconditions.checkArgument; +import static io.airlift.units.DataSize.Unit.GIGABYTE; +import static java.util.concurrent.Executors.newCachedThreadPool; +import static java.util.concurrent.Executors.newScheduledThreadPool; +import static org.testng.Assert.assertEquals; + +@State(Scope.Thread) +@OutputTimeUnit(TimeUnit.SECONDS) +@Fork(3) +@Warmup(iterations = 20, time = 1000, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 20, time = 1000, timeUnit = TimeUnit.MILLISECONDS) +public class BenchmarkUnnestOperator +{ + private static final int TOTAL_POSITIONS = 10000; + + private static final String ARRAY_OF_VARCHAR = "array(varchar)"; + private static final String MAP_STRING_TO_STRING = "map(varchar,varchar)"; + private static final String ARRAY_OF_ROW_THREE_STRINGS = "array(row(varchar, varchar, varchar))"; + + @State(Scope.Thread) + public static class BenchmarkContext + { + @Param("varchar") + private String replicateType = "varchar"; + + @Param({ARRAY_OF_VARCHAR, MAP_STRING_TO_STRING, ARRAY_OF_ROW_THREE_STRINGS}) + private String nestedTypeOne = ARRAY_OF_ROW_THREE_STRINGS; + + @Param({"NONE", ARRAY_OF_VARCHAR}) + private String nestedTypeTwo = "NONE"; + + @Param({"0.0", "0.2"}) + private double primitiveNullsRatioNestedOne; // % of nulls in input primitive elements + + @Param({"0.0", "0.05"}) + private double rowNullsRatioNestedOne; // % of nulls in row type elements + + @Param("1000") + private int positionsPerPage = 1000; + + @Param("50") + private int stringLengths = 50; // max length of varchars + + @Param("300") + private int nestedLengths = 300; // max entries in one nested structure (array, map) + + private ExecutorService executor; + private ScheduledExecutorService scheduledExecutor; + private OperatorFactory operatorFactory; + private List pages; + + @Setup + public void setup() + { + executor = newCachedThreadPool(daemonThreadsNamed("test-executor-%s")); + scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s")); + Metadata metadata = createTestMetadataManager(); + + InputGenerator inputGenerator = new InputGenerator(0, nestedLengths, stringLengths); + + ImmutableList.Builder typesBuilder = ImmutableList.builder(); + ImmutableList.Builder channelsBuilder = ImmutableList.builder(); + + Type replicateType = getType(metadata, this.replicateType).get(); + typesBuilder.add(replicateType); + channelsBuilder.add(0); + + Type nestedTypeOne = getType(metadata, this.nestedTypeOne).get(); + typesBuilder.add(nestedTypeOne); + channelsBuilder.add(1); + + if (!nestedTypeTwo.equals("NONE")) { + Type nestedTypeTwo = getType(metadata, this.nestedTypeTwo).get(); + typesBuilder.add(nestedTypeTwo); + channelsBuilder.add(2); + } + + List types = typesBuilder.build(); + List channels = channelsBuilder.build(); + + this.pages = createInputPages(positionsPerPage, typesBuilder.build(), inputGenerator, + primitiveNullsRatioNestedOne, rowNullsRatioNestedOne); + + operatorFactory = new UnnestOperatorFactory( + 0, + new PlanNodeId("test"), + channels.subList(0, 1), + types.subList(0, 1), + channels.subList(1, channels.size()), + types.subList(1, types.size()), + true); + } + + public Optional getType(Metadata metadata, String typeString) + { + if (typeString.equals("NONE")) { + return Optional.empty(); + } + TypeSignature signature = TypeSignature.parseTypeSignature(typeString); + return Optional.of(metadata.getType(signature)); + } + + @TearDown + public void cleanup() + { + executor.shutdownNow(); + scheduledExecutor.shutdownNow(); + } + + public TaskContext createTaskContext() + { + return TestingTaskContext.createTaskContext(executor, scheduledExecutor, TEST_SESSION, new DataSize(2, GIGABYTE)); + } + + public OperatorFactory getOperatorFactory() + { + return operatorFactory; + } + + public List getPages() + { + return pages; + } + + private static List createInputPages(int positionsPerPage, List types, InputGenerator inputGenerator, double primitiveNullsRatioUnnest, double rowNullsRatioUnnest) + { + ImmutableList.Builder pages = ImmutableList.builder(); + int pageCount = TOTAL_POSITIONS / positionsPerPage; + + for (int i = 0; i < pageCount; i++) { + Block[] blocks = new Block[types.size()]; + blocks[0] = inputGenerator.produceBlock(types.get(0), positionsPerPage, 0.0, 0.0); + blocks[1] = inputGenerator.produceBlock(types.get(1), positionsPerPage, primitiveNullsRatioUnnest, rowNullsRatioUnnest); + if (blocks.length == 3) { + blocks[2] = inputGenerator.produceBlock(types.get(2), positionsPerPage, 0.0, 0.0); + } + pages.add(new Page(blocks)); + } + + return pages.build(); + } + } + + @Benchmark + public List unnest(BenchmarkContext context) + { + DriverContext driverContext = context.createTaskContext().addPipelineContext(0, true, true, false).addDriverContext(); + Operator operator = context.getOperatorFactory().createOperator(driverContext); + + Iterator input = context.getPages().iterator(); + ImmutableList.Builder outputPages = ImmutableList.builder(); + + boolean finishing = false; + for (int loops = 0; !operator.isFinished() && loops < 1_000_000; loops++) { + if (operator.needsInput()) { + if (input.hasNext()) { + Page inputPage = input.next(); + operator.addInput(inputPage); + } + else if (!finishing) { + operator.finish(); + finishing = true; + } + } + + Page outputPage = operator.getOutput(); + if (outputPage != null) { + outputPages.add(outputPage); + } + } + + return outputPages.build(); + } + + public static class InputGenerator + { + private static final int INT_UPPER_LIMIT = 1_000_000_000; + private static final String LETTERS = "abcdefghijklmnopqrstuvwxyz"; + + private final Random random = new Random(); + + // lower and upper bounds on nested structures in the input + private final int minNestedCardinality; + private final int maxNestedCardinality; + + // upper bound on length of strings generated for VARCHAR elements + private final int maxStringLength; + + public InputGenerator(int minNestedCardinality, int maxNestedCardinality, int maxStringLength) + { + checkArgument(minNestedCardinality >= 0, "minNestedCardinality must be >= 0"); + checkArgument(minNestedCardinality <= maxNestedCardinality, "minNestedCardinality must be <= maxNestedCardinality"); + checkArgument(maxStringLength >= 0, "maxStringLength must be >= 0"); + this.minNestedCardinality = minNestedCardinality; + this.maxNestedCardinality = maxNestedCardinality; + this.maxStringLength = maxStringLength; + } + + /** + * Generates a block with {@code entries} positions given an input {@code type}. + * + * {@code primitiveNullsRatio} indicates what percentage of primitive elements + * (VARCHAR or INT) should be null. Similarly, the value {@code rowNullsRatio} + * indicates the percentage for Row elements. + * + * The null percentages are not propagated inside a MapBlock's keyBlock. Everything + * is always non-null inside the returned keyBlock. + * + * The set S of valid input for {@code type} can be defined as: + * S = {VARCHAR, INTEGER, ROW(x), ARRAY(x), MAP(x, x)} where x belongs to S + * + */ + public Block produceBlock(Type type, int entries, double primitiveNullsRatio, double rowNullsRatio) + { + if (type instanceof ArrayType) { + return produceArrayBlock((ArrayType) type, entries, primitiveNullsRatio, rowNullsRatio); + } + else if (type instanceof MapType) { + return produceMapBlock((MapType) type, entries, primitiveNullsRatio, rowNullsRatio); + } + else if (type instanceof RowType) { + return produceRowBlock((RowType) type, entries, primitiveNullsRatio, rowNullsRatio); + } + else if (type == VARCHAR) { + return produceStringBlock(entries, primitiveNullsRatio); + } + else if (type == INTEGER) { + return produceIntBlock(entries, primitiveNullsRatio); + } + + throw new RuntimeException("not supported"); + } + + private Block produceStringBlock(int entries, double primitiveNullsRatio) + { + BlockBuilder builder = VARCHAR.createBlockBuilder(null, 100); + + for (int i = 0; i < entries; i++) { + if (random.nextDouble() <= primitiveNullsRatio) { + builder.appendNull(); + } + else { + VARCHAR.writeString(builder, generateRandomString(random, random.nextInt(maxStringLength))); + } + } + + return builder.build(); + } + + private Block produceIntBlock(int entries, double primitiveNullsRatio) + { + BlockBuilder builder = INTEGER.createBlockBuilder(null, 100); + + for (int i = 0; i < entries; i++) { + if (random.nextDouble() < primitiveNullsRatio) { + builder.appendNull(); + } + else { + INTEGER.writeLong(builder, random.nextInt(INT_UPPER_LIMIT)); + } + } + + return builder.build(); + } + + private Block produceArrayBlock(ArrayType arrayType, int entries, double primitiveNullsRatio, double rowNullsRatio) + { + BlockBuilder builder = arrayType.createBlockBuilder(null, 100); + Type elementType = arrayType.getElementType(); + + for (int i = 0; i < entries; i++) { + int arrayLength = minNestedCardinality + random.nextInt(maxNestedCardinality - minNestedCardinality); + builder.appendStructure(produceBlock(elementType, arrayLength, primitiveNullsRatio, rowNullsRatio)); + } + return builder.build(); + } + + private Block produceMapBlock(MapType mapType, int entries, double primitiveNullsRatio, double rowNullsRatio) + { + int[] mapLengths = new int[entries]; + int[] offsets = new int[entries + 1]; + + int elementCount = 0; + for (int i = 0; i < entries; i++) { + mapLengths[i] = minNestedCardinality + random.nextInt(maxNestedCardinality - minNestedCardinality); + offsets[i + 1] = elementCount; + elementCount += mapLengths[i]; + } + + // No null map objects OR null keys + // This stops propagation of null percentages, but for benchmarks this is okay + Block keyBlock = produceBlock(mapType.getKeyType(), elementCount, 0.0, 0.0); + Block valueBlock = produceBlock(mapType.getValueType(), elementCount, primitiveNullsRatio, rowNullsRatio); + + return mapType.createBlockFromKeyValue(mapLengths.length, Optional.empty(), offsets, keyBlock, valueBlock); + } + + public Block produceRowBlock(RowType rowType, int entries, double primitiveNullsRatio, double rowNullsRatio) + { + boolean[] rowIsNull = new boolean[entries]; + int nonNullCount = 0; + for (int i = 0; i < entries; i++) { + if (random.nextDouble() < rowNullsRatio) { + rowIsNull[i] = true; + } + else { + rowIsNull[i] = false; + nonNullCount++; + } + } + + int fieldCount = rowType.getTypeParameters().size(); + Block[] fieldBlocks = new Block[fieldCount]; + + for (int i = 0; i < fieldCount; i++) { + fieldBlocks[i] = produceBlock(rowType.getTypeParameters().get(i), nonNullCount, primitiveNullsRatio, rowNullsRatio); + } + + return RowBlock.fromFieldBlocks(entries, Optional.of(rowIsNull), fieldBlocks); + } + + public static String generateRandomString(Random random, int length) + { + char[] chars = new char[length]; + for (int i = 0; i < length; i++) { + chars[i] = LETTERS.charAt(random.nextInt(LETTERS.length())); + } + return new String(chars); + } + } + + @Test + public void testBlocks() + { + InputGenerator generator = new InputGenerator(0, 50, 50); + + Block block = generator.produceBlock(new ArrayType(VARCHAR), 100, 0.1, 0.1); + assertEquals(block.getPositionCount(), 100); + + block = generator.produceBlock(mapType(VARCHAR, INTEGER), 100, 0.1, 0.1); + assertEquals(block.getPositionCount(), 100); + + block = generator.produceBlock(RowType.anonymous(Arrays.asList(VARCHAR, VARCHAR)), 100, 0.1, 0.1); + assertEquals(block.getPositionCount(), 100); + + block = generator.produceBlock(new ArrayType(RowType.anonymous(Arrays.asList(VARCHAR, VARCHAR, VARCHAR))), 100, 0.1, 0.1); + assertEquals(block.getPositionCount(), 100); + } + + public static void main(String[] args) + throws RunnerException + { + Options options = new OptionsBuilder() + .verbosity(VerboseMode.NORMAL) + .include(".*" + BenchmarkUnnestOperator.class.getSimpleName() + ".*") + .addProfiler(GCProfiler.class) + .build(); + + new Runner(options).run(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkWindowOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkWindowOperator.java index 353b92ffa542f..086b782ffb493 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkWindowOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/BenchmarkWindowOperator.java @@ -44,6 +44,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.block.BlockAssertions.createLongSequenceBlock; import static com.facebook.presto.operator.BenchmarkWindowOperator.Context.ROWS_PER_PAGE; @@ -51,7 +52,6 @@ import static com.facebook.presto.operator.TestWindowOperator.ROW_NUMBER; import static com.facebook.presto.operator.TestWindowOperator.createFactoryUnbounded; import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/GenericLongFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/GenericLongFunction.java index e722a5acc650d..4bbd8f694f337 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/GenericLongFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/GenericLongFunction.java @@ -16,7 +16,8 @@ import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -24,8 +25,9 @@ import java.lang.invoke.MethodHandle; import java.util.function.LongUnaryOperator; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.type.StandardTypes.BIGINT; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -43,7 +45,7 @@ public final class GenericLongFunction GenericLongFunction(String suffix, LongUnaryOperator longUnaryOperator) { - super(new Signature("generic_long_" + requireNonNull(suffix, "suffix is null"), SCALAR, + super(new Signature(QualifiedFunctionName.of(DEFAULT_NAMESPACE, "generic_long_" + requireNonNull(suffix, "suffix is null")), SCALAR, emptyList(), emptyList(), parseTypeSignature(BIGINT), singletonList(parseTypeSignature(BIGINT)), false)); this.longUnaryOperator = longUnaryOperator; } @@ -67,10 +69,10 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { MethodHandle methodHandle = METHOD_HANDLE.bindTo(longUnaryOperator); - return new ScalarFunctionImplementation(false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); + return new BuiltInScalarFunctionImplementation(false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle); } public static long apply(LongUnaryOperator longUnaryOperator, long value) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/GroupByHashYieldAssertion.java b/presto-main/src/test/java/com/facebook/presto/operator/GroupByHashYieldAssertion.java index 5019523e6c0a8..5241850210923 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/GroupByHashYieldAssertion.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/GroupByHashYieldAssertion.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.stats.TestingGcMonitor; import com.facebook.presto.RowPagesBuilder; import com.facebook.presto.memory.MemoryPool; import com.facebook.presto.memory.QueryContext; @@ -22,7 +23,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.spiller.SpillSpaceTracker; import com.google.common.collect.ImmutableList; -import io.airlift.stats.TestingGcMonitor; import io.airlift.units.DataSize; import java.util.LinkedList; @@ -31,16 +31,16 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.function.Function; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertBetweenInclusive; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.OperatorAssertion.finishOperator; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; import static com.facebook.presto.testing.assertions.Assert.assertEquals; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertBetweenInclusive; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertLessThan; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.util.Objects.requireNonNull; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/MockExchangeRequestProcessor.java b/presto-main/src/test/java/com/facebook/presto/operator/MockExchangeRequestProcessor.java index 90e6d9e40703f..4d96581d75735 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/MockExchangeRequestProcessor.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/MockExchangeRequestProcessor.java @@ -13,6 +13,11 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.http.client.HttpStatus; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.testing.TestingHttpClient; +import com.facebook.airlift.http.client.testing.TestingResponse; import com.facebook.presto.client.PrestoHeaders; import com.facebook.presto.execution.buffer.BufferResult; import com.facebook.presto.execution.buffer.PagesSerde; @@ -23,11 +28,6 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableListMultimap; -import io.airlift.http.client.HttpStatus; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.testing.TestingHttpClient; -import io.airlift.http.client.testing.TestingResponse; import io.airlift.slice.DynamicSliceOutput; import io.airlift.units.DataSize; @@ -48,6 +48,7 @@ import static com.facebook.presto.execution.buffer.TestingPagesSerdeFactory.testingPagesSerde; import static com.google.common.base.Preconditions.checkState; import static com.google.common.net.HttpHeaders.CONTENT_TYPE; +import static java.util.Collections.synchronizedList; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -61,6 +62,8 @@ public class MockExchangeRequestProcessor private final DataSize expectedMaxSize; + private final List requestMaxSizes = synchronizedList(new ArrayList<>()); + public MockExchangeRequestProcessor(DataSize expectedMaxSize) { this.expectedMaxSize = expectedMaxSize; @@ -86,7 +89,8 @@ public Response handle(Request request) // verify we got a data size and it parses correctly assertTrue(!request.getHeaders().get(PrestoHeaders.PRESTO_MAX_SIZE).isEmpty()); DataSize maxSize = DataSize.valueOf(request.getHeader(PrestoHeaders.PRESTO_MAX_SIZE)); - assertEquals(maxSize, expectedMaxSize); + assertTrue(maxSize.compareTo(expectedMaxSize) <= 0); + requestMaxSizes.add(maxSize); RequestLocation requestLocation = new RequestLocation(request.getUri()); URI location = requestLocation.getLocation(); @@ -116,6 +120,11 @@ public Response handle(Request request) bytes); } + public List getRequestMaxSizes() + { + return requestMaxSizes; + } + private class RequestLocation { private final URI location; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/OperatorAssertion.java b/presto-main/src/test/java/com/facebook/presto/operator/OperatorAssertion.java index 3ab8ab613c70d..ff1a74a262d7b 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/OperatorAssertion.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/OperatorAssertion.java @@ -37,15 +37,15 @@ import java.util.concurrent.TimeoutException; import java.util.stream.IntStream; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static com.facebook.presto.operator.PageAssertions.assertPageEquals; import static com.facebook.presto.testing.assertions.Assert.assertEquals; import static com.facebook.presto.util.StructuralTestUtil.appendToBlockBuilder; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Throwables.throwIfUnchecked; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.testng.Assert.fail; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/PageAssertions.java b/presto-main/src/test/java/com/facebook/presto/operator/PageAssertions.java index 7dbf77349159a..12675e8566302 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/PageAssertions.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/PageAssertions.java @@ -13,12 +13,25 @@ */ package com.facebook.presto.operator; +import com.facebook.presto.block.BlockAssertions; +import com.facebook.presto.block.BlockAssertions.Encoding; import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.PageBuilder; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; import java.util.List; +import static com.facebook.presto.block.BlockAssertions.Encoding.DICTIONARY; +import static com.facebook.presto.block.BlockAssertions.Encoding.RUN_LENGTH; import static com.facebook.presto.block.BlockAssertions.assertBlockEquals; +import static com.facebook.presto.block.BlockAssertions.createAllNullsBlock; +import static com.facebook.presto.block.BlockAssertions.createRandomBlockForType; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.google.common.base.Verify.verify; +import static java.lang.String.format; import static org.testng.Assert.assertEquals; public final class PageAssertions @@ -36,4 +49,94 @@ public static void assertPageEquals(List types, Page actualPage, assertBlockEquals(types.get(i), actualPage.getBlock(i), expectedPage.getBlock(i)); } } + + public static Page createPageWithRandomData(List types, int positionCount, boolean allowNulls) + { + return createPageWithRandomData(types, positionCount, true, false, allowNulls, false, ImmutableList.of()); + } + + public static Page createDictionaryPageWithRandomData(List types, int positionCount, boolean allowNulls) + { + return createPageWithRandomData(types, positionCount, true, false, allowNulls, false, ImmutableList.of(DICTIONARY)); + } + + public static Page createRlePageWithRandomData(List types, int positionCount, boolean allowNulls) + { + return createPageWithRandomData(types, positionCount, true, false, allowNulls, false, ImmutableList.of(RUN_LENGTH)); + } + + public static Page createPageWithRandomData( + List types, + int positionCount, + boolean addPreComputedHashBlock, + boolean addNullBlock, + boolean allowNulls, + boolean useBlockView, + List wrappings) + { + int channelCount = types.size(); + int preComputedChannelCount = (addPreComputedHashBlock ? 1 : 0); + int nullChannelCount = (addNullBlock ? 1 : 0); + + Block[] blocks = new Block[channelCount + preComputedChannelCount + nullChannelCount]; + + if (addPreComputedHashBlock) { + blocks[0] = BlockAssertions.createRandomLongsBlock(positionCount, false); + } + + for (int i = 0; i < channelCount; i++) { + blocks[i + preComputedChannelCount] = createRandomBlockForType(types.get(i), positionCount, allowNulls, useBlockView, wrappings); + } + + if (addNullBlock) { + blocks[channelCount + preComputedChannelCount] = createAllNullsBlock(BIGINT, positionCount); + } + + return new Page(positionCount, blocks); + } + + public static Page mergePages(List types, List pages) + { + PageBuilder pageBuilder = new PageBuilder(types); + int totalPositionCount = 0; + for (Page page : pages) { + verify(page.getChannelCount() == types.size(), format("Number of channels in page %d is not equal to number of types %d", page.getChannelCount(), types.size())); + + for (int i = 0; i < types.size(); i++) { + BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i); + Block block = page.getBlock(i); + for (int position = 0; position < page.getPositionCount(); position++) { + if (block.isNull(position)) { + blockBuilder.appendNull(); + } + else { + block.writePositionTo(position, blockBuilder); + } + } + } + totalPositionCount += page.getPositionCount(); + } + pageBuilder.declarePositions(totalPositionCount); + return pageBuilder.build(); + } + + /** + * Create a new types list that prepends the BIGINT type in front if addPreComputedHashBlock is true, and append the BIGINT type at the end if addNullBlock is true. + */ + public static List updateBlockTypesWithHashBlockAndNullBlock(List types, boolean addPreComputedHashBlock, boolean addNullBlock) + { + ImmutableList.Builder newTypes = ImmutableList.builder(); + + if (addPreComputedHashBlock) { + newTypes.add(BIGINT); + } + + newTypes.addAll(types); + + if (addNullBlock) { + newTypes.add(BIGINT); + } + + return newTypes.build(); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java index f57c42aaee0aa..cc8ee4055bb49 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestAggregationOperator.java @@ -18,9 +18,9 @@ import com.facebook.presto.operator.AggregationOperator.AggregationOperatorFactory; import com.facebook.presto.operator.aggregation.InternalAggregationFunction; import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.testing.MaterializedResult; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterMethod; @@ -32,6 +32,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.OperatorAssertion.assertOperatorEquals; @@ -44,7 +45,6 @@ import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Collections.emptyIterator; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForAggregates.java b/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForAggregates.java index 6c81e492f0a66..2f2ca4328bbc8 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForAggregates.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForAggregates.java @@ -44,6 +44,7 @@ import com.facebook.presto.spi.function.LongVariableConstraint; import com.facebook.presto.spi.function.OperatorDependency; import com.facebook.presto.spi.function.OutputFunction; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.function.TypeParameter; @@ -66,6 +67,7 @@ import java.lang.invoke.MethodHandle; import java.util.List; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.operator.aggregation.AggregationFromAnnotationsParser.parseFunctionDefinition; import static com.facebook.presto.operator.aggregation.AggregationFromAnnotationsParser.parseFunctionDefinitions; import static com.facebook.presto.spi.function.OperatorType.LESS_THAN; @@ -109,7 +111,7 @@ public static void output(@AggregationState NullableDoubleState state, BlockBuil public void testSimpleExactAggregationParse() { Signature expectedSignature = new Signature( - "simple_exact_aggregate", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "simple_exact_aggregate"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); @@ -161,7 +163,7 @@ public static void output(BlockBuilder out, @AggregationState NullableDoubleStat public void testStateOnDifferentThanFirstPositionAggregationParse() { Signature expectedSignature = new Signature( - "simple_exact_aggregate_aggregation_state_moved", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "simple_exact_aggregate_aggregation_state_moved"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); @@ -304,7 +306,7 @@ public static AccumulatorStateSerializer createSerializer() public void testNotDecomposableAggregationParse() { Signature expectedSignature = new Signature( - "custom_decomposable_aggregate", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "custom_decomposable_aggregate"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); @@ -379,7 +381,7 @@ public static void output( public void testSimpleGenericAggregationFunctionParse() { Signature expectedSignature = new Signature( - "simple_generic_implementations", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "simple_generic_implementations"), FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -455,7 +457,7 @@ public static void output( public void testSimpleBlockInputAggregationParse() { Signature expectedSignature = new Signature( - "block_input_aggregate", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "block_input_aggregate"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); @@ -539,7 +541,7 @@ public static void output( public void testSimpleImplicitSpecializedAggregationParse() { Signature expectedSignature = new Signature( - "implicit_specialized_aggregate", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "implicit_specialized_aggregate"), FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -631,7 +633,7 @@ public static void output( public void testSimpleExplicitSpecializedAggregationParse() { Signature expectedSignature = new Signature( - "explicit_specialized_aggregate", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "explicit_specialized_aggregate"), FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -706,13 +708,13 @@ public static void output2( public void testMultiOutputAggregationParse() { Signature expectedSignature1 = new Signature( - "multi_output_aggregate_1", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "multi_output_aggregate_1"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); Signature expectedSignature2 = new Signature( - "multi_output_aggregate_2", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "multi_output_aggregate_2"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); @@ -720,11 +722,11 @@ public void testMultiOutputAggregationParse() List aggregations = parseFunctionDefinitions(MultiOutputAggregationFunction.class); assertEquals(aggregations.size(), 2); - ParametricAggregation aggregation1 = aggregations.stream().filter(aggregate -> aggregate.getSignature().getName().equals("multi_output_aggregate_1")).collect(toImmutableList()).get(0); + ParametricAggregation aggregation1 = aggregations.stream().filter(aggregate -> aggregate.getSignature().getNameSuffix().equals("multi_output_aggregate_1")).collect(toImmutableList()).get(0); assertEquals(aggregation1.getSignature(), expectedSignature1); assertEquals(aggregation1.getDescription(), "Simple multi output function aggregate specialized description"); - ParametricAggregation aggregation2 = aggregations.stream().filter(aggregate -> aggregate.getSignature().getName().equals("multi_output_aggregate_2")).collect(toImmutableList()).get(0); + ParametricAggregation aggregation2 = aggregations.stream().filter(aggregate -> aggregate.getSignature().getNameSuffix().equals("multi_output_aggregate_2")).collect(toImmutableList()).get(0); assertEquals(aggregation2.getSignature(), expectedSignature2); assertEquals(aggregation2.getDescription(), "Simple multi output function aggregate generic description"); @@ -792,7 +794,7 @@ public static CustomStateSerializerAggregationFunction.CustomSerializer createSe public void testInjectOperatorAggregateParse() { Signature expectedSignature = new Signature( - "inject_operator_aggregate", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "inject_operator_aggregate"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); @@ -872,7 +874,7 @@ public static CustomStateSerializerAggregationFunction.CustomSerializer createSe public void testInjectTypeAggregateParse() { Signature expectedSignature = new Signature( - "inject_type_aggregate", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "inject_type_aggregate"), FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -956,7 +958,7 @@ public static CustomStateSerializerAggregationFunction.CustomSerializer createSe public void testInjectLiteralAggregateParse() { Signature expectedSignature = new Signature( - "inject_literal_aggregate", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "inject_literal_aggregate"), FunctionKind.AGGREGATE, parseTypeSignature("varchar(x)", ImmutableSet.of("x")), ImmutableList.of(parseTypeSignature("varchar(x)", ImmutableSet.of("x")))); @@ -1029,7 +1031,7 @@ public static void output( public void testLongConstraintAggregateFunctionParse() { Signature expectedSignature = new Signature( - "parametric_aggregate_long_constraint", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "parametric_aggregate_long_constraint"), FunctionKind.AGGREGATE, ImmutableList.of(), ImmutableList.of(new LongVariableConstraint("z", "x + y")), @@ -1106,7 +1108,7 @@ public static void output( public void testFixedTypeParameterInjectionAggregateFunctionParse() { Signature expectedSignature = new Signature( - "fixed_type_parameter_injection", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "fixed_type_parameter_injection"), FunctionKind.AGGREGATE, ImmutableList.of(), ImmutableList.of(), @@ -1172,7 +1174,7 @@ public static void output( public void testPartiallyFixedTypeParameterInjectionAggregateFunctionParse() { Signature expectedSignature = new Signature( - "partially_fixed_type_parameter_injection", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "partially_fixed_type_parameter_injection"), FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T1"), typeVariable("T2")), ImmutableList.of(), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForScalars.java b/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForScalars.java index 8a1d1e3bc04e7..74841b28c84eb 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForScalars.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForScalars.java @@ -18,8 +18,8 @@ import com.facebook.presto.operator.annotations.ImplementationDependency; import com.facebook.presto.operator.annotations.LiteralImplementationDependency; import com.facebook.presto.operator.annotations.TypeImplementationDependency; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.operator.scalar.ParametricScalar; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; import com.facebook.presto.operator.scalar.annotations.ParametricScalarImplementation.ParametricScalarImplementationChoice; import com.facebook.presto.operator.scalar.annotations.ScalarFromAnnotationsParser; import com.facebook.presto.spi.block.Block; @@ -27,6 +27,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.IsNull; import com.facebook.presto.spi.function.LiteralParameters; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.function.SqlNullable; @@ -43,10 +44,11 @@ import java.util.List; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -76,7 +78,7 @@ public static double fun(@SqlType(StandardTypes.DOUBLE) double v) public void testSingleImplementationScalarParse() { Signature expectedSignature = new Signature( - "single_implementation_parametric_scalar", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "single_implementation_parametric_scalar"), FunctionKind.SCALAR, DOUBLE.getTypeSignature(), ImmutableList.of(DOUBLE.getTypeSignature())); @@ -92,7 +94,7 @@ public void testSingleImplementationScalarParse() assertImplementationCount(scalar, 1, 0, 0); - ScalarFunctionImplementation specialized = scalar.specialize(BoundVariables.builder().build(), 1, new TypeRegistry(), null); + BuiltInScalarFunctionImplementation specialized = scalar.specialize(BoundVariables.builder().build(), 1, new TypeRegistry(), null); assertFalse(specialized.getInstanceFactory().isPresent()); assertEquals(specialized.getArgumentProperty(0).getNullConvention(), RETURN_NULL_ON_NULL); @@ -160,7 +162,7 @@ public static double fun( public void testWithNullablePrimitiveArgScalarParse() { Signature expectedSignature = new Signature( - "scalar_with_nullable", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "scalar_with_nullable"), FunctionKind.SCALAR, DOUBLE.getTypeSignature(), ImmutableList.of(DOUBLE.getTypeSignature(), DOUBLE.getTypeSignature())); @@ -174,7 +176,7 @@ public void testWithNullablePrimitiveArgScalarParse() assertFalse(scalar.isHidden()); assertEquals(scalar.getDescription(), "Simple scalar with nullable primitive"); - ScalarFunctionImplementation specialized = scalar.specialize(BoundVariables.builder().build(), 2, new TypeRegistry(), null); + BuiltInScalarFunctionImplementation specialized = scalar.specialize(BoundVariables.builder().build(), 2, new TypeRegistry(), null); assertFalse(specialized.getInstanceFactory().isPresent()); assertEquals(specialized.getArgumentProperty(0), valueTypeArgumentProperty(RETURN_NULL_ON_NULL)); @@ -198,7 +200,7 @@ public static double fun( public void testWithNullableComplexArgScalarParse() { Signature expectedSignature = new Signature( - "scalar_with_nullable_complex", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "scalar_with_nullable_complex"), FunctionKind.SCALAR, DOUBLE.getTypeSignature(), ImmutableList.of(DOUBLE.getTypeSignature(), DOUBLE.getTypeSignature())); @@ -212,7 +214,7 @@ public void testWithNullableComplexArgScalarParse() assertFalse(scalar.isHidden()); assertEquals(scalar.getDescription(), "Simple scalar with nullable complex type"); - ScalarFunctionImplementation specialized = scalar.specialize(BoundVariables.builder().build(), 2, new TypeRegistry(), null); + BuiltInScalarFunctionImplementation specialized = scalar.specialize(BoundVariables.builder().build(), 2, new TypeRegistry(), null); assertFalse(specialized.getInstanceFactory().isPresent()); assertEquals(specialized.getArgumentProperty(0), valueTypeArgumentProperty(RETURN_NULL_ON_NULL)); @@ -234,7 +236,7 @@ public static double fun(@SqlType(StandardTypes.DOUBLE) double v) public void testStaticMethodScalarParse() { Signature expectedSignature = new Signature( - "static_method_scalar", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "static_method_scalar"), FunctionKind.SCALAR, DOUBLE.getTypeSignature(), ImmutableList.of(DOUBLE.getTypeSignature())); @@ -272,13 +274,13 @@ public static long fun2(@SqlType(StandardTypes.BIGINT) long v) public void testMultiScalarParse() { Signature expectedSignature1 = new Signature( - "static_method_scalar_1", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "static_method_scalar_1"), FunctionKind.SCALAR, DOUBLE.getTypeSignature(), ImmutableList.of(DOUBLE.getTypeSignature())); Signature expectedSignature2 = new Signature( - "static_method_scalar_2", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "static_method_scalar_2"), FunctionKind.SCALAR, BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature())); @@ -325,7 +327,7 @@ public static long fun(@SqlType("T") long v) public void testParametricScalarParse() { Signature expectedSignature = new Signature( - "parametric_scalar", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "parametric_scalar"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -366,7 +368,7 @@ public static boolean fun2(@SqlType("array(varchar(17))") Block array) public void testComplexParametricScalarParse() { Signature expectedSignature = new Signature( - "with_exact_scalar", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "with_exact_scalar"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), @@ -375,7 +377,7 @@ public void testComplexParametricScalarParse() false); Signature exactSignature = new Signature( - "with_exact_scalar", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "with_exact_scalar"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), @@ -413,7 +415,7 @@ public static long fun( public void testSimpleInjectionScalarParse() { Signature expectedSignature = new Signature( - "parametric_scalar_inject", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "parametric_scalar_inject"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), @@ -473,7 +475,7 @@ public long funDouble(@SqlType("array(double)") Block val) public void testConstructorInjectionScalarParse() { Signature expectedSignature = new Signature( - "parametric_scalar_inject_constructor", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "parametric_scalar_inject_constructor"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -516,7 +518,7 @@ public static long fun( public void testFixedTypeParameterParse() { Signature expectedSignature = new Signature( - "fixed_type_parameter_scalar_function", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "fixed_type_parameter_scalar_function"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), @@ -554,7 +556,7 @@ public static long fun( public void testPartiallyFixedTypeParameterParse() { Signature expectedSignature = new Signature( - "partially_fixed_type_parameter_scalar_function", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "partially_fixed_type_parameter_scalar_function"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T1"), typeVariable("T2")), ImmutableList.of(), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestColumnarPageProcessor.java b/presto-main/src/test/java/com/facebook/presto/operator/TestColumnarPageProcessor.java index be88b3bf64252..431b39489ae6d 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestColumnarPageProcessor.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestColumnarPageProcessor.java @@ -81,6 +81,6 @@ private static Page createPage(List types, boolean dictionary) private PageProcessor newPageProcessor() { return new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)) - .compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, types.get(0)), field(1, types.get(1))), MAX_BATCH_SIZE).get(); + .compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), ImmutableList.of(field(0, types.get(0)), field(1, types.get(1))), MAX_BATCH_SIZE).get(); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestDistinctLimitOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestDistinctLimitOperator.java index 7db5aef94215e..4d9fb7c6c96ca 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestDistinctLimitOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestDistinctLimitOperator.java @@ -33,6 +33,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.GroupByHashYieldAssertion.createPagesWithDistinctHashKeys; @@ -42,8 +44,6 @@ import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertGreaterThan; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestDriver.java b/presto-main/src/test/java/com/facebook/presto/operator/TestDriver.java index bac85c6aacb86..e4a1e7df5b496 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestDriver.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestDriver.java @@ -22,10 +22,13 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableHandle; import com.facebook.presto.spi.FixedPageSource; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.Type; import com.facebook.presto.split.PageSourceProvider; @@ -44,6 +47,7 @@ import java.io.Closeable; import java.util.List; +import java.util.Optional; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -54,6 +58,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; @@ -61,7 +66,6 @@ import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; @@ -73,6 +77,12 @@ @Test(singleThreaded = true) public class TestDriver { + private static final TableHandle TESTING_TABLE_HANDLE = new TableHandle( + new ConnectorId("test"), + new ConnectorTableHandle() {}, + new ConnectorTransactionHandle() {}, + Optional.empty()); + private ExecutorService executor; private ScheduledExecutorService scheduledExecutor; private DriverContext driverContext; @@ -172,13 +182,14 @@ public void testAddSourceFinish() new PageSourceProvider() { @Override - public ConnectorPageSource createPageSource(Session session, Split split, List columns) + public ConnectorPageSource createPageSource(Session session, Split split, TableHandle table, List columns) { return new FixedPageSource(rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); } }, + TESTING_TABLE_HANDLE, ImmutableList.of()); PageConsumerOperator sink = createSinkOperator(types); @@ -270,13 +281,14 @@ public void testMemoryRevocationRace() new PageSourceProvider() { @Override - public ConnectorPageSource createPageSource(Session session, Split split, List columns) + public ConnectorPageSource createPageSource(Session session, Split split, TableHandle table, List columns) { return new FixedPageSource(rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); } }, + TESTING_TABLE_HANDLE, ImmutableList.of()); Driver driver = Driver.createDriver(driverContext, source, createSinkOperator(types)); @@ -298,13 +310,14 @@ public void testBrokenOperatorAddSource() new PageSourceProvider() { @Override - public ConnectorPageSource createPageSource(Session session, Split split, List columns) + public ConnectorPageSource createPageSource(Session session, Split split, TableHandle table, List columns) { return new FixedPageSource(rowPagesBuilder(types) .addSequencePage(10, 20, 30, 40) .build()); } }, + TESTING_TABLE_HANDLE, ImmutableList.of()); BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source")); @@ -482,9 +495,10 @@ public AlwaysBlockedMemoryRevokingTableScanOperator( OperatorContext operatorContext, PlanNodeId planNodeId, PageSourceProvider pageSourceProvider, + TableHandle table, Iterable columns) { - super(operatorContext, planNodeId, pageSourceProvider, columns); + super(operatorContext, planNodeId, pageSourceProvider, table, columns); } @Override @@ -506,9 +520,10 @@ public NotBlockedTableScanOperator( OperatorContext operatorContext, PlanNodeId planNodeId, PageSourceProvider pageSourceProvider, + TableHandle table, Iterable columns) { - super(operatorContext, planNodeId, pageSourceProvider, columns); + super(operatorContext, planNodeId, pageSourceProvider, table, columns); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestDriverStats.java b/presto-main/src/test/java/com/facebook/presto/operator/TestDriverStats.java index 1098d5114b4e9..80ed94bd42b70 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestDriverStats.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestDriverStats.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.execution.Lifespan; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClient.java b/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClient.java index 594e49d3b8bc6..8cef09fe22ac3 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClient.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClient.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.testing.TestingHttpClient; import com.facebook.presto.block.BlockAssertions; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.buffer.PagesSerde; @@ -22,7 +25,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.http.client.testing.TestingHttpClient; +import com.google.common.util.concurrent.UncheckedTimeoutException; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.testng.annotations.AfterClass; @@ -30,20 +33,25 @@ import org.testng.annotations.Test; import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static com.facebook.presto.execution.buffer.TestingPagesSerdeFactory.testingPagesSerde; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static com.facebook.presto.spi.block.PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES; import static com.google.common.collect.Maps.uniqueIndex; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static com.google.common.util.concurrent.Uninterruptibles.awaitUninterruptibly; import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; -import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertLessThan; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.util.concurrent.Executors.newCachedThreadPool; @@ -112,12 +120,13 @@ public void testHappyPath() 1, new Duration(1, MINUTES), true, + 0.2, new TestingHttpClient(processor, scheduler), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor); - exchangeClient.addLocation(location, TaskId.valueOf("taskid")); + exchangeClient.addLocation(location, TaskId.valueOf("queryid.0.0.0")); exchangeClient.noMoreLocations(); assertFalse(exchangeClient.isClosed()); @@ -151,6 +160,7 @@ public void testAddLocation() 1, new Duration(1, MINUTES), true, + 0.2, new TestingHttpClient(processor, testingHttpClientExecutor), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), @@ -161,7 +171,7 @@ public void testAddLocation() processor.addPage(location1, createPage(2)); processor.addPage(location1, createPage(3)); processor.setComplete(location1); - exchangeClient.addLocation(location1, TaskId.valueOf("foo")); + exchangeClient.addLocation(location1, TaskId.valueOf("foo.0.0.0")); assertFalse(exchangeClient.isClosed()); assertPageEquals(getNextPage(exchangeClient), createPage(1)); @@ -178,7 +188,7 @@ public void testAddLocation() processor.addPage(location2, createPage(5)); processor.addPage(location2, createPage(6)); processor.setComplete(location2); - exchangeClient.addLocation(location2, TaskId.valueOf("bar")); + exchangeClient.addLocation(location2, TaskId.valueOf("bar.0.0.0")); assertFalse(exchangeClient.isClosed()); assertPageEquals(getNextPage(exchangeClient), createPage(4)); @@ -223,12 +233,13 @@ public void testBufferLimit() 1, new Duration(1, MINUTES), true, + 0.2, new TestingHttpClient(processor, testingHttpClientExecutor), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor); - exchangeClient.addLocation(location, TaskId.valueOf("taskid")); + exchangeClient.addLocation(location, TaskId.valueOf("taskid.0.0.0")); exchangeClient.noMoreLocations(); assertFalse(exchangeClient.isClosed()); @@ -305,11 +316,12 @@ public void testClose() 1, new Duration(1, MINUTES), true, + 0.2, new TestingHttpClient(processor, testingHttpClientExecutor), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor); - exchangeClient.addLocation(location, TaskId.valueOf("taskid")); + exchangeClient.addLocation(location, TaskId.valueOf("taskid.0.0.0")); exchangeClient.noMoreLocations(); // fetch a page @@ -329,6 +341,100 @@ public void testClose() assertStatus(clientStatus, location, "closed", "not scheduled"); } + @Test + public void testInitialRequestLimit() + { + DataSize maxResponseSize = new DataSize(DEFAULT_MAX_PAGE_SIZE_IN_BYTES, BYTE); + CountDownLatch countDownLatch = new CountDownLatch(1); + MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize) { + @Override + public Response handle(Request request) + { + if (!awaitUninterruptibly(countDownLatch, 10, SECONDS)) { + throw new UncheckedTimeoutException(); + } + return super.handle(request); + } + }; + + List locations = new ArrayList<>(); + int numLocations = 16; + List expectedMaxSizes = new ArrayList<>(); + + // add pages + for (int i = 0; i < numLocations; i++) { + URI location = URI.create("http://localhost:" + (8080 + i)); + locations.add(location); + + processor.addPage(location, createPage(DEFAULT_MAX_PAGE_SIZE_IN_BYTES)); + processor.addPage(location, createPage(DEFAULT_MAX_PAGE_SIZE_IN_BYTES)); + processor.addPage(location, createPage(DEFAULT_MAX_PAGE_SIZE_IN_BYTES)); + + processor.setComplete(location); + + expectedMaxSizes.add(maxResponseSize); + } + + try (ExchangeClient exchangeClient = new ExchangeClient( + new DataSize(16, MEGABYTE), + maxResponseSize, + 1, + new Duration(1, MINUTES), + true, + 0.2, + new TestingHttpClient(processor, testingHttpClientExecutor), + scheduler, + new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), + pageBufferClientCallbackExecutor)) { + for (int i = 0; i < numLocations; i++) { + exchangeClient.addLocation(locations.get(i), TaskId.valueOf("taskid.0.0." + i)); + } + exchangeClient.noMoreLocations(); + assertFalse(exchangeClient.isClosed()); + + long start = System.nanoTime(); + countDownLatch.countDown(); + + // wait for a page to be fetched + do { + // there is no thread coordination here, so sleep is the best we can do + assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS)); + sleepUninterruptibly(100, MILLISECONDS); + } + while (exchangeClient.getStatus().getBufferedPages() < 16); + + // Client should have sent 16 requests for a single page (0) and gotten them back + // The memory limit should be hit immediately and then it doesn't fetch the third page from each + assertEquals(exchangeClient.getStatus().getBufferedPages(), 16); + assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0); + List pageBufferClientStatuses = exchangeClient.getStatus().getPageBufferClientStatuses(); + assertEquals( + 16, + pageBufferClientStatuses.stream() + .filter(status -> status.getPagesReceived() == 1) + .mapToInt(PageBufferClientStatus::getPagesReceived) + .sum()); + assertEquals(processor.getRequestMaxSizes(), expectedMaxSizes); + + for (int i = 0; i < numLocations * 3; i++) { + assertNotNull(getNextPage(exchangeClient)); + } + + do { + // there is no thread coordination here, so sleep is the best we can do + assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS)); + sleepUninterruptibly(100, MILLISECONDS); + } + while (processor.getRequestMaxSizes().size() < 64); + + for (int i = 0; i < 48; i++) { + expectedMaxSizes.add(maxResponseSize); + } + + assertEquals(processor.getRequestMaxSizes(), expectedMaxSizes); + } + } + @Test public void testRemoveRemoteSource() throws Exception @@ -336,10 +442,10 @@ public void testRemoveRemoteSource() DataSize maxResponseSize = new DataSize(1, BYTE); MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize); - URI location1 = URI.create("http://localhost:8081/foo"); - TaskId taskId1 = TaskId.valueOf("foo"); - URI location2 = URI.create("http://localhost:8082/bar"); - TaskId taskId2 = TaskId.valueOf("bar"); + URI location1 = URI.create("http://localhost:8081/foo.0.0.0"); + TaskId taskId1 = TaskId.valueOf("foo.0.0.0"); + URI location2 = URI.create("http://localhost:8082/bar.0.0.0"); + TaskId taskId2 = TaskId.valueOf("bar.0.0.0"); processor.addPage(location1, createPage(1)); processor.addPage(location1, createPage(2)); @@ -351,6 +457,7 @@ public void testRemoveRemoteSource() 1, new Duration(1, MINUTES), true, + 0.2, new TestingHttpClient(processor, testingHttpClientExecutor), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClientConfig.java b/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClientConfig.java index 34065a0efdc1a..378fd5d918021 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClientConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeClientConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.http.client.HttpClientConfig; import com.google.common.collect.ImmutableMap; -import io.airlift.http.client.HttpClientConfig; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.testng.annotations.Test; @@ -22,9 +22,9 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static io.airlift.units.DataSize.Unit; public class TestExchangeClientConfig @@ -40,7 +40,8 @@ public void testDefaults() .setMaxResponseSize(new HttpClientConfig().getMaxContentLength()) .setPageBufferClientMaxCallbackThreads(25) .setClientThreads(25) - .setAcknowledgePages(true)); + .setAcknowledgePages(true) + .setResponseSizeExponentialMovingAverageDecayingAlpha(0.1)); } @Test @@ -55,6 +56,7 @@ public void testExplicitPropertyMappings() .put("exchange.client-threads", "2") .put("exchange.page-buffer-client.max-callback-threads", "16") .put("exchange.acknowledge-pages", "false") + .put("exchange.response-size-exponential-moving-average-decaying-alpha", "0.42") .build(); ExchangeClientConfig expected = new ExchangeClientConfig() @@ -65,7 +67,8 @@ public void testExplicitPropertyMappings() .setMaxResponseSize(new DataSize(1, Unit.MEGABYTE)) .setClientThreads(2) .setPageBufferClientMaxCallbackThreads(16) - .setAcknowledgePages(false); + .setAcknowledgePages(false) + .setResponseSizeExponentialMovingAverageDecayingAlpha(0.42); assertFullMapping(properties, expected); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeOperator.java index 3b5f8e4f74d0b..caa7a4a7f0682 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestExchangeOperator.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.testing.TestingHttpClient; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.buffer.PagesSerdeFactory; import com.facebook.presto.execution.buffer.TestingPagesSerdeFactory; @@ -27,8 +29,6 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.testing.TestingHttpClient; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.testng.annotations.AfterClass; @@ -44,13 +44,13 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.ExchangeOperator.REMOTE_CONNECTOR_ID; import static com.facebook.presto.operator.PageAssertions.assertPageEquals; import static com.facebook.presto.operator.TestingTaskBuffer.PAGE; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; @@ -63,9 +63,9 @@ public class TestExchangeOperator private static final List TYPES = ImmutableList.of(VARCHAR); private static final PagesSerdeFactory SERDE_FACTORY = new TestingPagesSerdeFactory(); - private static final String TASK_1_ID = "task1"; - private static final String TASK_2_ID = "task2"; - private static final String TASK_3_ID = "task3"; + private static final String TASK_1_ID = "task1.0.0.0"; + private static final String TASK_2_ID = "task2.0.0.0"; + private static final String TASK_3_ID = "task3.0.0.0"; private final LoadingCache taskBuffers = CacheBuilder.newBuilder().build(CacheLoader.from(TestingTaskBuffer::new)); @@ -90,6 +90,7 @@ public void setUp() 3, new Duration(1, TimeUnit.MINUTES), true, + 0.2, httpClient, scheduler, systemMemoryUsageListener, diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestFilterAndProjectOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestFilterAndProjectOperator.java index 667b185165431..f8edc70e96364 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestFilterAndProjectOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestFilterAndProjectOperator.java @@ -34,6 +34,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; @@ -49,7 +50,6 @@ import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.field; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.KILOBYTE; import static java.util.concurrent.Executors.newCachedThreadPool; @@ -106,7 +106,7 @@ public void test() constant(5L, BIGINT)); ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); - Supplier processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field0, add5)); + Supplier processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(filter), ImmutableList.of(field0, add5)); OperatorFactory operatorFactory = new FilterAndProjectOperator.FilterAndProjectOperatorFactory( 0, @@ -151,7 +151,7 @@ public void testMergeOutput() constant(10L, BIGINT)); ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); - Supplier processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field(1, BIGINT))); + Supplier processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(filter), ImmutableList.of(field(1, BIGINT))); OperatorFactory operatorFactory = new FilterAndProjectOperator.FilterAndProjectOperatorFactory( 0, diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestGroupIdOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestGroupIdOperator.java index 7d9d21474b1d0..aed98e97cd7cc 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestGroupIdOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestGroupIdOperator.java @@ -28,6 +28,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.OperatorAssertion.assertOperatorEqualsIgnoreOrder; @@ -36,7 +37,6 @@ import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestGroupedTopNBuilder.java b/presto-main/src/test/java/com/facebook/presto/operator/TestGroupedTopNBuilder.java index 30de3c3a9605a..d5e16c1b08231 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestGroupedTopNBuilder.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestGroupedTopNBuilder.java @@ -33,6 +33,7 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.operator.PageAssertions.assertPageEquals; @@ -41,7 +42,6 @@ import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static io.airlift.slice.SizeOf.sizeOf; -import static io.airlift.testing.Assertions.assertGreaterThan; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java index b397eae2f1af0..07fb3c322089d 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestHashAggregationOperator.java @@ -23,15 +23,18 @@ import com.facebook.presto.operator.aggregation.builder.HashAggregationBuilder; import com.facebook.presto.operator.aggregation.builder.InMemoryHashAggregationBuilder; import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.block.ByteArrayBlock; import com.facebook.presto.spi.block.PageBuilderStatus; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spiller.Spiller; import com.facebook.presto.spiller.SpillerFactory; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.JoinCompiler; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.TestingTaskContext; import com.google.common.collect.ImmutableList; @@ -53,6 +56,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.GroupByHashYieldAssertion.GroupByHashYieldResult; @@ -74,11 +80,8 @@ import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateFuture; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.slice.SizeOf.SIZE_OF_DOUBLE; import static io.airlift.slice.SizeOf.SIZE_OF_LONG; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; -import static io.airlift.testing.Assertions.assertGreaterThan; import static io.airlift.units.DataSize.Unit.KILOBYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static io.airlift.units.DataSize.succinctBytes; @@ -659,6 +662,44 @@ public void testSpillerFailure() } } + @Test + public void testMask() + { + int positions = 4; + Block groupingBlock = RunLengthEncodedBlock.create(BIGINT, 1L, positions); + Block countBlock = RunLengthEncodedBlock.create(BIGINT, 1L, positions); + Block maskBlock = new ByteArrayBlock(positions, Optional.of(new boolean[] {false, false, true, true}), new byte[] {(byte) 0, (byte) 1, (byte) 0, (byte) 1}); + Page page = new Page(groupingBlock, countBlock, maskBlock); + + HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory( + 0, + new PlanNodeId("test"), + ImmutableList.of(BIGINT), + ImmutableList.of(0), + ImmutableList.of(), + Step.SINGLE, + false, + ImmutableList.of(COUNT.bind(ImmutableList.of(1), Optional.of(2))), + Optional.empty(), + Optional.empty(), + 1, + Optional.of(new DataSize(16, MEGABYTE)), + false, + new DataSize(16, MEGABYTE), + new DataSize(16, MEGABYTE), + new FailingSpillerFactory(), + joinCompiler, + false); + + List outputPages = toPages(operatorFactory, createDriverContext(), ImmutableList.of(page)).stream() + .filter(p -> p.getPositionCount() > 0) + .collect(toImmutableList()); + assertEquals(outputPages.size(), 1); + Page outputPage = outputPages.get(0); + assertEquals(outputPage.getBlock(0).getLong(0), 1L); + assertEquals(outputPage.getBlock(1).getLong(0), 1L); + } + @Test public void testMemoryTracking() throws Exception diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestHashJoinOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestHashJoinOperator.java index fad6f470e0b19..374fe7c3c2a7e 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestHashJoinOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestHashJoinOperator.java @@ -15,6 +15,7 @@ import com.facebook.presto.ExceededMemoryLimitException; import com.facebook.presto.RowPagesBuilder; +import com.facebook.presto.Session; import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.TaskStateMachine; @@ -36,6 +37,7 @@ import com.facebook.presto.spiller.SingleStreamSpiller; import com.facebook.presto.spiller.SingleStreamSpillerFactory; import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler.JoinFilterFunctionFactory; +import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.TestingTaskContext; import com.google.common.collect.ImmutableList; @@ -68,6 +70,9 @@ import java.util.function.Function; import java.util.stream.IntStream; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.OperatorAssertion.assertOperatorEquals; @@ -78,6 +83,7 @@ import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.testing.assertions.Assert.assertEquals; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -85,9 +91,6 @@ import static com.google.common.collect.Iterators.unmodifiableIterator; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateFuture; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static io.airlift.units.DataSize.Unit.BYTE; import static java.lang.String.format; import static java.util.Arrays.asList; @@ -112,6 +115,8 @@ public class TestHashJoinOperator private ExecutorService executor; private ScheduledExecutorService scheduledExecutor; + private PartitioningProviderManager partitioningProviderManager; + private Session session; @BeforeMethod public void setUp() @@ -131,6 +136,8 @@ public void setUp() daemonThreadsNamed("test-executor-%s"), new ThreadPoolExecutor.DiscardPolicy()); scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s")); + partitioningProviderManager = new PartitioningProviderManager(); + session = testSessionBuilder().build(); } @AfterMethod(alwaysRun = true) @@ -138,6 +145,8 @@ public void tearDown() { executor.shutdownNow(); scheduledExecutor.shutdownNow(); + partitioningProviderManager = null; + session = null; } @DataProvider(name = "hashJoinTestValues") @@ -362,7 +371,7 @@ public void testInnerJoinWithFailingSpill(boolean probeHashEnabled, List whenSpill, SingleStreamSpillerFactory buildSpillerFactory, PartitioningSpillerFactory joinSpillerFactory) throws Exception { - TaskStateMachine taskStateMachine = new TaskStateMachine(new TaskId("query", 0, 0), executor); + TaskStateMachine taskStateMachine = new TaskStateMachine(new TaskId("query", 0, 0, 0), executor); TaskContext taskContext = TestingTaskContext.createTaskContext(executor, scheduledExecutor, TEST_SESSION, taskStateMachine); DriverContext joinDriverContext = taskContext.addPipelineContext(2, true, true, false).addDriverContext(); @@ -1294,6 +1303,8 @@ private BuildSideSetup setupBuildSide( int partitionCount = parallelBuild ? PARTITION_COUNT : 1; LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory( + partitioningProviderManager, + session, FIXED_HASH_DISTRIBUTION, partitionCount, buildPages.getTypes(), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestHashSemiJoinOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestHashSemiJoinOperator.java index 1fbf3f266e667..b31f79dfea018 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestHashSemiJoinOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestHashSemiJoinOperator.java @@ -36,6 +36,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; @@ -47,9 +50,6 @@ import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; import static com.google.common.collect.Iterables.concat; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertGreaterThanOrEqual; import static io.airlift.units.DataSize.Unit.BYTE; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestHttpPageBufferClient.java b/presto-main/src/test/java/com/facebook/presto/operator/TestHttpPageBufferClient.java index b1b2da2270964..0746e1dd6b116 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestHttpPageBufferClient.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestHttpPageBufferClient.java @@ -13,18 +13,18 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.http.client.HttpStatus; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.testing.TestingHttpClient; +import com.facebook.airlift.http.client.testing.TestingResponse; +import com.facebook.airlift.testing.TestingTicker; import com.facebook.presto.execution.buffer.PagesSerde; import com.facebook.presto.execution.buffer.SerializedPage; import com.facebook.presto.operator.HttpPageBufferClient.ClientCallback; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.Page; import com.google.common.collect.ImmutableListMultimap; -import io.airlift.http.client.HttpStatus; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.testing.TestingHttpClient; -import io.airlift.http.client.testing.TestingResponse; -import io.airlift.testing.TestingTicker; import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import io.airlift.units.Duration; @@ -47,6 +47,9 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertContains; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.PrestoMediaTypes.PRESTO_PAGES; import static com.facebook.presto.execution.buffer.TestingPagesSerdeFactory.testingPagesSerde; import static com.facebook.presto.spi.StandardErrorCode.PAGE_TOO_LARGE; @@ -54,9 +57,6 @@ import static com.facebook.presto.spi.StandardErrorCode.PAGE_TRANSPORT_TIMEOUT; import static com.facebook.presto.util.Failures.WORKER_NODE_ERROR; import static com.google.common.net.HttpHeaders.CONTENT_TYPE; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertContains; -import static io.airlift.testing.Assertions.assertInstanceOf; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; @@ -102,7 +102,6 @@ public void testHappyPath() URI location = URI.create("http://localhost:8080"); HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, scheduler), - expectedMaxSize, new Duration(1, TimeUnit.MINUTES), true, location, @@ -115,7 +114,7 @@ public void testHappyPath() // fetch a page and verify processor.addPage(location, expectedPage); callback.resetStats(); - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); requestComplete.await(10, TimeUnit.SECONDS); assertEquals(callback.getPages().size(), 1); @@ -126,7 +125,7 @@ public void testHappyPath() // fetch no data and verify callback.resetStats(); - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); requestComplete.await(10, TimeUnit.SECONDS); assertEquals(callback.getPages().size(), 0); @@ -138,7 +137,7 @@ public void testHappyPath() processor.addPage(location, expectedPage); processor.addPage(location, expectedPage); callback.resetStats(); - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); requestComplete.await(10, TimeUnit.SECONDS); assertEquals(callback.getPages().size(), 2); @@ -153,7 +152,7 @@ public void testHappyPath() // finish and verify callback.resetStats(); processor.setComplete(location); - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); requestComplete.await(10, TimeUnit.SECONDS); // get the buffer complete signal @@ -162,7 +161,7 @@ public void testHappyPath() // schedule the delete call to the buffer callback.resetStats(); - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); requestComplete.await(10, TimeUnit.SECONDS); assertEquals(callback.getFinishedBuffers(), 1); @@ -177,6 +176,7 @@ public void testHappyPath() public void testLifecycle() throws Exception { + DataSize expectedMaxSize = new DataSize(10, Unit.MEGABYTE); CyclicBarrier beforeRequest = new CyclicBarrier(2); CyclicBarrier afterRequest = new CyclicBarrier(2); StaticRequestProcessor processor = new StaticRequestProcessor(beforeRequest, afterRequest); @@ -187,7 +187,6 @@ public void testLifecycle() URI location = URI.create("http://localhost:8080"); HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, scheduler), - new DataSize(10, Unit.MEGABYTE), new Duration(1, TimeUnit.MINUTES), true, location, @@ -197,7 +196,7 @@ public void testLifecycle() assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled"); - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); beforeRequest.await(10, TimeUnit.SECONDS); assertStatus(client, location, "running", 0, 1, 0, 0, "PROCESSING_REQUEST"); assertEquals(client.isRunning(), true); @@ -218,6 +217,7 @@ public void testLifecycle() public void testInvalidResponses() throws Exception { + DataSize expectedMaxSize = new DataSize(10, Unit.MEGABYTE); CyclicBarrier beforeRequest = new CyclicBarrier(1); CyclicBarrier afterRequest = new CyclicBarrier(1); StaticRequestProcessor processor = new StaticRequestProcessor(beforeRequest, afterRequest); @@ -227,7 +227,6 @@ public void testInvalidResponses() URI location = URI.create("http://localhost:8080"); HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, scheduler), - new DataSize(10, Unit.MEGABYTE), new Duration(1, TimeUnit.MINUTES), true, location, @@ -239,7 +238,7 @@ public void testInvalidResponses() // send not found response and verify response was ignored processor.setResponse(new TestingResponse(HttpStatus.NOT_FOUND, ImmutableListMultimap.of(CONTENT_TYPE, PRESTO_PAGES), new byte[0])); - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); requestComplete.await(10, TimeUnit.SECONDS); assertEquals(callback.getPages().size(), 0); assertEquals(callback.getCompletedRequests(), 1); @@ -252,7 +251,7 @@ public void testInvalidResponses() // send invalid content type response and verify response was ignored callback.resetStats(); processor.setResponse(new TestingResponse(HttpStatus.OK, ImmutableListMultimap.of(CONTENT_TYPE, "INVALID_TYPE"), new byte[0])); - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); requestComplete.await(10, TimeUnit.SECONDS); assertEquals(callback.getPages().size(), 0); assertEquals(callback.getCompletedRequests(), 1); @@ -265,7 +264,7 @@ public void testInvalidResponses() // send unexpected content type response and verify response was ignored callback.resetStats(); processor.setResponse(new TestingResponse(HttpStatus.OK, ImmutableListMultimap.of(CONTENT_TYPE, "text/plain"), new byte[0])); - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); requestComplete.await(10, TimeUnit.SECONDS); assertEquals(callback.getPages().size(), 0); assertEquals(callback.getCompletedRequests(), 1); @@ -285,6 +284,7 @@ public void testInvalidResponses() public void testCloseDuringPendingRequest() throws Exception { + DataSize expectedMaxSize = new DataSize(10, Unit.MEGABYTE); CyclicBarrier beforeRequest = new CyclicBarrier(2); CyclicBarrier afterRequest = new CyclicBarrier(2); StaticRequestProcessor processor = new StaticRequestProcessor(beforeRequest, afterRequest); @@ -295,7 +295,6 @@ public void testCloseDuringPendingRequest() URI location = URI.create("http://localhost:8080"); HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, scheduler), - new DataSize(10, Unit.MEGABYTE), new Duration(1, TimeUnit.MINUTES), true, location, @@ -306,7 +305,7 @@ public void testCloseDuringPendingRequest() assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled"); // send request - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); beforeRequest.await(10, TimeUnit.SECONDS); assertStatus(client, location, "running", 0, 1, 0, 0, "PROCESSING_REQUEST"); assertEquals(client.isRunning(), true); @@ -335,6 +334,7 @@ public void testCloseDuringPendingRequest() public void testExceptionFromResponseHandler() throws Exception { + DataSize expectedMaxSize = new DataSize(10, Unit.MEGABYTE); TestingTicker ticker = new TestingTicker(); AtomicReference tickerIncrement = new AtomicReference<>(new Duration(0, TimeUnit.SECONDS)); @@ -349,7 +349,6 @@ public void testExceptionFromResponseHandler() URI location = URI.create("http://localhost:8080"); HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, scheduler), - new DataSize(10, Unit.MEGABYTE), new Duration(30, TimeUnit.SECONDS), true, location, @@ -362,7 +361,7 @@ public void testExceptionFromResponseHandler() // request processor will throw exception, verify the request is marked a completed // this starts the error stopwatch - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); requestComplete.await(10, TimeUnit.SECONDS); assertEquals(callback.getPages().size(), 0); assertEquals(callback.getCompletedRequests(), 1); @@ -374,7 +373,7 @@ public void testExceptionFromResponseHandler() tickerIncrement.set(new Duration(30, TimeUnit.SECONDS)); // verify that the client has not failed - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); requestComplete.await(10, TimeUnit.SECONDS); assertEquals(callback.getPages().size(), 0); assertEquals(callback.getCompletedRequests(), 2); @@ -386,7 +385,7 @@ public void testExceptionFromResponseHandler() tickerIncrement.set(new Duration(31, TimeUnit.SECONDS)); // verify that the client has failed - client.scheduleRequest(); + client.scheduleRequest(expectedMaxSize); requestComplete.await(10, TimeUnit.SECONDS); assertEquals(callback.getPages().size(), 0); assertEquals(callback.getCompletedRequests(), 3); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestLimitOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestLimitOperator.java index f7b045114ad74..13def7b6f6555 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestLimitOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestLimitOperator.java @@ -26,13 +26,13 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SequencePageBuilder.createSequencePage; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestMarkDistinctOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestMarkDistinctOperator.java index 7619a9e1a6c1c..46e4454c6e2b5 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestMarkDistinctOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestMarkDistinctOperator.java @@ -34,6 +34,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.GroupByHashYieldAssertion.createPagesWithDistinctHashKeys; @@ -43,8 +45,6 @@ import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertGreaterThan; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestMergeOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestMergeOperator.java index b33568d3110f3..4dddbeebadd0e 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestMergeOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestMergeOperator.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.testing.TestingHttpClient; import com.facebook.presto.execution.TaskId; import com.facebook.presto.execution.buffer.PagesSerdeFactory; import com.facebook.presto.execution.buffer.TestingPagesSerdeFactory; @@ -28,8 +30,6 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.testing.TestingHttpClient; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.OperatorAssertion.assertOperatorIsBlocked; @@ -52,7 +53,6 @@ import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; @@ -61,9 +61,9 @@ @Test(singleThreaded = true) public class TestMergeOperator { - private static final String TASK_1_ID = "task1"; - private static final String TASK_2_ID = "task2"; - private static final String TASK_3_ID = "task3"; + private static final String TASK_1_ID = "task1.0.0.0"; + private static final String TASK_2_ID = "task2.0.0.0"; + private static final String TASK_3_ID = "task3.0.0.0"; private AtomicInteger operatorId = new AtomicInteger(); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestMoreByteArrays.java b/presto-main/src/test/java/com/facebook/presto/operator/TestMoreByteArrays.java new file mode 100644 index 0000000000000..80d586bb673a4 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestMoreByteArrays.java @@ -0,0 +1,91 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.operator; + +import com.google.common.primitives.Bytes; +import io.airlift.slice.ByteArrays; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.BiFunction; +import java.util.stream.IntStream; + +import static com.facebook.presto.operator.MoreByteArrays.fill; +import static com.facebook.presto.operator.MoreByteArrays.setBytes; +import static com.facebook.presto.operator.MoreByteArrays.setInts; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Collections.nCopies; +import static org.testng.Assert.assertEquals; +import static sun.misc.Unsafe.ARRAY_BYTE_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_INT_INDEX_SCALE; + +public class TestMoreByteArrays +{ + private static final int POSITIONS_PER_PAGE = 10_000; + + private TestMoreByteArrays() + {} + + @Test + public static void testFill() + { + byte[] destination = new byte[POSITIONS_PER_PAGE]; + int filledBytes = fill(destination, 0, POSITIONS_PER_PAGE, (byte) 5); + + assertEquals(filledBytes, POSITIONS_PER_PAGE); + assertCopied( + nCopies(POSITIONS_PER_PAGE, (byte) 5), + destination, + ARRAY_BYTE_INDEX_SCALE, + MoreByteArrays::getByte); + } + + @Test + public static void testSetBytes() + { + byte[] destination = new byte[POSITIONS_PER_PAGE]; + + byte[] source = new byte[POSITIONS_PER_PAGE]; + ThreadLocalRandom.current().nextBytes(source); + + int setBytes = setBytes(destination, 0, source, 0, POSITIONS_PER_PAGE); + + assertEquals(setBytes, POSITIONS_PER_PAGE); + assertCopied(Bytes.asList(source), destination, ARRAY_BYTE_INDEX_SCALE, MoreByteArrays::getByte); + } + + @Test + public static void testSetInts() + { + byte[] destination = new byte[POSITIONS_PER_PAGE * ARRAY_INT_INDEX_SCALE]; + + int copiedBytes = setInts(destination, 0, IntStream.range(0, POSITIONS_PER_PAGE).toArray(), 0, POSITIONS_PER_PAGE); + + assertEquals(copiedBytes, POSITIONS_PER_PAGE * ARRAY_INT_INDEX_SCALE); + assertCopied( + IntStream.range(0, POSITIONS_PER_PAGE).boxed().collect(toImmutableList()), + destination, + ARRAY_INT_INDEX_SCALE, + ByteArrays::getInt); + } + + private static void assertCopied(List expected, byte[] actual, int elementSize, BiFunction readFunction) + { + for (int index = 0; index < expected.size(); index++) { + assertEquals(readFunction.apply(actual, index * elementSize), expected.get(index)); + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestNestedLoopBuildOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestNestedLoopBuildOperator.java index 3ff7b66167c87..93f3ca56bc35c 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestNestedLoopBuildOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestNestedLoopBuildOperator.java @@ -28,10 +28,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.block.BlockAssertions.createLongSequenceBlock; import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestNestedLoopJoinOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestNestedLoopJoinOperator.java index e5b4d5b103578..582f267200017 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestNestedLoopJoinOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestNestedLoopJoinOperator.java @@ -31,6 +31,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.OperatorAssertion.assertOperatorEquals; @@ -39,7 +40,6 @@ import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.google.common.collect.Iterables.concat; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestOperatorStats.java b/presto-main/src/test/java/com/facebook/presto/operator/TestOperatorStats.java index cdb88c31a037f..e22d9d87a4020 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestOperatorStats.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestOperatorStats.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.operator; -import com.facebook.presto.operator.PartitionedOutputOperator.PartitionedOutputInfo; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.operator.repartition.PartitionedOutputInfo; import com.facebook.presto.spi.plan.PlanNodeId; -import io.airlift.json.JsonCodec; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.testng.annotations.Test; @@ -34,6 +34,7 @@ public class TestOperatorStats public static final OperatorStats EXPECTED = new OperatorStats( 0, + 10, 1, 41, new PlanNodeId("test"), @@ -45,6 +46,7 @@ public class TestOperatorStats new Duration(3, NANOSECONDS), new Duration(4, NANOSECONDS), new DataSize(5, BYTE), + 10, new DataSize(6, BYTE), 7, 8d, @@ -75,6 +77,7 @@ public class TestOperatorStats public static final OperatorStats MERGEABLE = new OperatorStats( 0, + 10, 1, 41, new PlanNodeId("test"), @@ -86,6 +89,7 @@ public class TestOperatorStats new Duration(3, NANOSECONDS), new Duration(4, NANOSECONDS), new DataSize(5, BYTE), + 10, new DataSize(6, BYTE), 7, 8d, @@ -128,6 +132,7 @@ public void testJson() public static void assertExpectedOperatorStats(OperatorStats actual) { assertEquals(actual.getStageId(), 0); + assertEquals(actual.getStageExecutionId(), 10); assertEquals(actual.getOperatorId(), 41); assertEquals(actual.getOperatorType(), "test"); @@ -171,6 +176,7 @@ public void testAdd() OperatorStats actual = EXPECTED.add(EXPECTED, EXPECTED); assertEquals(actual.getStageId(), 0); + assertEquals(actual.getStageExecutionId(), 10); assertEquals(actual.getOperatorId(), 41); assertEquals(actual.getOperatorType(), "test"); @@ -212,6 +218,7 @@ public void testAddMergeable() OperatorStats actual = MERGEABLE.add(MERGEABLE, MERGEABLE); assertEquals(actual.getStageId(), 0); + assertEquals(actual.getStageExecutionId(), 10); assertEquals(actual.getOperatorId(), 41); assertEquals(actual.getOperatorType(), "test"); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestOrderByOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestOrderByOperator.java index 984d067df4dab..06a0f90cbb024 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestOrderByOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestOrderByOperator.java @@ -29,6 +29,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.OperatorAssertion.assertOperatorEquals; @@ -40,7 +41,6 @@ import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestPartitionedOutputOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestPartitionedOutputOperator.java new file mode 100644 index 0000000000000..b935dfde6755b --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestPartitionedOutputOperator.java @@ -0,0 +1,218 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.execution.StateMachine; +import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.execution.buffer.PagesSerdeFactory; +import com.facebook.presto.execution.buffer.PartitionedOutputBuffer; +import com.facebook.presto.memory.context.SimpleLocalMemoryContext; +import com.facebook.presto.operator.exchange.LocalPartitionGenerator; +import com.facebook.presto.operator.repartition.PartitionedOutputOperator; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.testing.TestingTaskContext; +import com.facebook.presto.type.TypeRegistry; +import com.google.common.collect.ImmutableList; +import io.airlift.units.DataSize; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Function; + +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.block.BlockAssertions.createLongDictionaryBlock; +import static com.facebook.presto.block.BlockAssertions.createLongSequenceBlock; +import static com.facebook.presto.block.BlockAssertions.createRLEBlock; +import static com.facebook.presto.execution.buffer.BufferState.OPEN; +import static com.facebook.presto.execution.buffer.BufferState.TERMINAL_BUFFER_STATES; +import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.PARTITIONED; +import static com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers; +import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static io.airlift.units.DataSize.Unit.BYTE; +import static io.airlift.units.DataSize.Unit.GIGABYTE; +import static io.airlift.units.DataSize.Unit.MEGABYTE; +import static java.util.concurrent.Executors.newCachedThreadPool; +import static java.util.concurrent.Executors.newScheduledThreadPool; +import static org.testng.Assert.assertEquals; + +public class TestPartitionedOutputOperator +{ + private static final int PAGE_COUNT = 100; + private static final int POSITIONS_PER_PAGE = 1000; + + private static final int PARTITION_COUNT = 512; + private static final DataSize MAX_MEMORY = new DataSize(1, GIGABYTE); + private static final DataSize PARTITION_MAX_MEMORY = new DataSize(5, MEGABYTE); + private static final List TYPES = ImmutableList.of(BIGINT); + private static final List REPLICATION_TYPES = ImmutableList.of(BIGINT, BIGINT); + private static final ExecutorService EXECUTOR = newCachedThreadPool(daemonThreadsNamed("test-EXECUTOR-%s")); + private static final ScheduledExecutorService SCHEDULER = newScheduledThreadPool(1, daemonThreadsNamed("test-%s")); + + private static final Block NULL_BLOCK = new RunLengthEncodedBlock(BIGINT.createBlockBuilder(null, 1).appendNull().build(), POSITIONS_PER_PAGE); + private static final Block TESTING_BLOCK = createLongSequenceBlock(0, POSITIONS_PER_PAGE); + private static final Block TESTING_DICTIONARY_BLOCK = createLongDictionaryBlock(0, POSITIONS_PER_PAGE); + private static final Block TESTING_RLE_BLOCK = createRLEBlock(new Random(0).nextLong(), POSITIONS_PER_PAGE); + private static final Page TESTING_PAGE = new Page(TESTING_BLOCK); + private static final Page TESTING_PAGE_WITH_NULL_BLOCK = new Page(POSITIONS_PER_PAGE, NULL_BLOCK, TESTING_BLOCK); + + @Test + public void testOutputForSimplePage() + { + PartitionedOutputOperator partitionedOutputOperator = createPartitionedOutputOperator(false); + for (int i = 0; i < PAGE_COUNT; i++) { + partitionedOutputOperator.addInput(TESTING_PAGE); + } + partitionedOutputOperator.finish(); + + OperatorContext operatorContext = partitionedOutputOperator.getOperatorContext(); + assertEquals(operatorContext.getOutputDataSize().getTotalCount(), PAGE_COUNT * TESTING_PAGE.getSizeInBytes()); + assertEquals(operatorContext.getOutputPositions().getTotalCount(), PAGE_COUNT * TESTING_PAGE.getPositionCount()); + } + + @Test + public void testOutputForPageWithDictionary() + { + PartitionedOutputOperator partitionedOutputOperator = createPartitionedOutputOperator(false); + for (int i = 0; i < PAGE_COUNT; i++) { + partitionedOutputOperator.addInput(new Page(TESTING_DICTIONARY_BLOCK)); + } + partitionedOutputOperator.finish(); + + OperatorContext operatorContext = partitionedOutputOperator.getOperatorContext(); + assertEquals(operatorContext.getOutputDataSize().getTotalCount(), PAGE_COUNT * TESTING_PAGE.getSizeInBytes()); + assertEquals(operatorContext.getOutputPositions().getTotalCount(), PAGE_COUNT * TESTING_PAGE.getPositionCount()); + } + + @Test + public void testOutputForPageWithRunLength() + { + PartitionedOutputOperator partitionedOutputOperator = createPartitionedOutputOperator(false); + for (int i = 0; i < PAGE_COUNT; i++) { + partitionedOutputOperator.addInput(new Page(TESTING_RLE_BLOCK)); + } + partitionedOutputOperator.finish(); + + OperatorContext operatorContext = partitionedOutputOperator.getOperatorContext(); + assertEquals(operatorContext.getOutputDataSize().getTotalCount(), PAGE_COUNT * TESTING_PAGE.getSizeInBytes()); + assertEquals(operatorContext.getOutputPositions().getTotalCount(), PAGE_COUNT * TESTING_PAGE.getPositionCount()); + } + + @Test + public void testOutputForSimplePageAndReplication() + { + PartitionedOutputOperator partitionedOutputOperator = createPartitionedOutputOperator(true); + for (int i = 0; i < PAGE_COUNT; i++) { + partitionedOutputOperator.addInput(new Page(POSITIONS_PER_PAGE, NULL_BLOCK, TESTING_BLOCK)); + } + partitionedOutputOperator.finish(); + + OperatorContext operatorContext = partitionedOutputOperator.getOperatorContext(); + assertEquals(operatorContext.getOutputDataSize().getTotalCount(), PAGE_COUNT * PARTITION_COUNT * TESTING_PAGE_WITH_NULL_BLOCK.getSizeInBytes()); + assertEquals(operatorContext.getOutputPositions().getTotalCount(), PAGE_COUNT * PARTITION_COUNT * TESTING_PAGE_WITH_NULL_BLOCK.getPositionCount()); + } + + @Test + public void testOutputForPageWithDictionaryAndReplication() + { + PartitionedOutputOperator partitionedOutputOperator = createPartitionedOutputOperator(true); + for (int i = 0; i < PAGE_COUNT; i++) { + partitionedOutputOperator.addInput(new Page(POSITIONS_PER_PAGE, NULL_BLOCK, TESTING_DICTIONARY_BLOCK)); + } + partitionedOutputOperator.finish(); + + OperatorContext operatorContext = partitionedOutputOperator.getOperatorContext(); + assertEquals(operatorContext.getOutputDataSize().getTotalCount(), PAGE_COUNT * PARTITION_COUNT * TESTING_PAGE_WITH_NULL_BLOCK.getSizeInBytes()); + assertEquals(operatorContext.getOutputPositions().getTotalCount(), PAGE_COUNT * PARTITION_COUNT * TESTING_PAGE_WITH_NULL_BLOCK.getPositionCount()); + } + + @Test + public void testOutputForPageWithRunLengthAndReplication() + { + PartitionedOutputOperator partitionedOutputOperator = createPartitionedOutputOperator(true); + for (int i = 0; i < PAGE_COUNT; i++) { + partitionedOutputOperator.addInput(new Page(POSITIONS_PER_PAGE, NULL_BLOCK, TESTING_RLE_BLOCK)); + } + partitionedOutputOperator.finish(); + + OperatorContext operatorContext = partitionedOutputOperator.getOperatorContext(); + assertEquals(operatorContext.getOutputDataSize().getTotalCount(), PAGE_COUNT * PARTITION_COUNT * TESTING_PAGE_WITH_NULL_BLOCK.getSizeInBytes()); + assertEquals(operatorContext.getOutputPositions().getTotalCount(), PAGE_COUNT * PARTITION_COUNT * TESTING_PAGE_WITH_NULL_BLOCK.getPositionCount()); + } + + private static PartitionedOutputOperator createPartitionedOutputOperator(boolean shouldReplicate) + { + PartitionFunction partitionFunction = new LocalPartitionGenerator(new InterpretedHashGenerator(ImmutableList.of(BIGINT), new int[] {0}), PARTITION_COUNT); + PagesSerdeFactory serdeFactory = new PagesSerdeFactory(new BlockEncodingManager(new TypeRegistry()), false); + + DriverContext driverContext = TestingTaskContext.builder(EXECUTOR, SCHEDULER, TEST_SESSION) + .setMemoryPoolSize(MAX_MEMORY) + .setQueryMaxTotalMemory(MAX_MEMORY) + .build() + .addPipelineContext(0, true, true, false) + .addDriverContext(); + + OutputBuffers buffers = createInitialEmptyOutputBuffers(PARTITIONED); + for (int partition = 0; partition < PARTITION_COUNT; partition++) { + buffers = buffers.withBuffer(new OutputBuffers.OutputBufferId(partition), partition); + } + PartitionedOutputBuffer buffer = new PartitionedOutputBuffer( + "task-instance-id", + new StateMachine<>("bufferState", SCHEDULER, OPEN, TERMINAL_BUFFER_STATES), + buffers.withNoMoreBufferIds(), + new DataSize(Long.MAX_VALUE, BYTE), + () -> new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), + SCHEDULER); + buffer.registerLifespanCompletionCallback(ignore -> {}); + + PartitionedOutputOperator.PartitionedOutputFactory operatorFactory; + if (shouldReplicate) { + operatorFactory = new PartitionedOutputOperator.PartitionedOutputFactory( + partitionFunction, + ImmutableList.of(0), + ImmutableList.of(Optional.empty()), + true, + OptionalInt.of(0), + buffer, + PARTITION_MAX_MEMORY); + return (PartitionedOutputOperator) operatorFactory + .createOutputOperator(0, new PlanNodeId("plan-node-0"), REPLICATION_TYPES, Function.identity(), serdeFactory) + .createOperator(driverContext); + } + else { + operatorFactory = new PartitionedOutputOperator.PartitionedOutputFactory( + partitionFunction, + ImmutableList.of(0), + ImmutableList.of(Optional.empty(), Optional.empty()), + false, + OptionalInt.empty(), + buffer, + PARTITION_MAX_MEMORY); + return (PartitionedOutputOperator) operatorFactory + .createOutputOperator(0, new PlanNodeId("plan-node-0"), TYPES, Function.identity(), serdeFactory) + .createOperator(driverContext); + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestPipelineStats.java b/presto-main/src/test/java/com/facebook/presto/operator/TestPipelineStats.java index b7467a1743457..5b623994e6fa0 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestPipelineStats.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestPipelineStats.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.stats.Distribution; +import com.facebook.airlift.stats.Distribution.DistributionSnapshot; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; -import io.airlift.stats.Distribution; -import io.airlift.stats.Distribution.DistributionSnapshot; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestRowNumberOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestRowNumberOperator.java index ac1079445d9f1..c3c94099fdb54 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestRowNumberOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestRowNumberOperator.java @@ -37,6 +37,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.GroupByHashYieldAssertion.createPagesWithDistinctHashKeys; @@ -49,9 +52,6 @@ import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; -import static io.airlift.testing.Assertions.assertGreaterThan; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java index bb3a5c8d1464f..adcde12bebd34 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestScanFilterAndProjectOperator.java @@ -28,11 +28,14 @@ import com.facebook.presto.operator.scalar.AbstractTestFunctions; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorPageSource; +import com.facebook.presto.spi.ConnectorTableHandle; import com.facebook.presto.spi.FixedPageSource; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.RecordPageSource; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.LazyBlock; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.sql.gen.ExpressionCompiler; @@ -50,6 +53,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.block.BlockAssertions.toValues; @@ -67,7 +71,6 @@ import static com.facebook.presto.sql.relational.Expressions.field; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; import static com.facebook.presto.testing.assertions.Assert.assertEquals; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.KILOBYTE; import static java.util.concurrent.Executors.newCachedThreadPool; @@ -80,6 +83,12 @@ public class TestScanFilterAndProjectOperator extends AbstractTestFunctions { + private static final TableHandle TESTING_TABLE_HANDLE = new TableHandle( + new ConnectorId("test"), + new ConnectorTableHandle() {}, + new ConnectorTransactionHandle() {}, + Optional.empty()); + private final MetadataManager metadata = createTestMetadataManager(); private final ExpressionCompiler expressionCompiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); private ExecutorService executor; @@ -98,16 +107,17 @@ public void testPageSource() DriverContext driverContext = newDriverContext(); List projections = ImmutableList.of(field(0, VARCHAR)); - Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections, "key"); - Supplier pageProcessor = expressionCompiler.compilePageProcessor(Optional.empty(), projections); + Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections, "key"); + Supplier pageProcessor = expressionCompiler.compilePageProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections); ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory( 0, new PlanNodeId("test"), new PlanNodeId("0"), - (session, split, columns) -> new FixedPageSource(ImmutableList.of(input)), + (session, split, table, columns) -> new FixedPageSource(ImmutableList.of(input)), cursorProcessor, pageProcessor, + TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(VARCHAR), new DataSize(0, BYTE), @@ -141,16 +151,17 @@ public void testPageSourceMergeOutput() field(0, BIGINT), constant(10L, BIGINT)); List projections = ImmutableList.of(field(0, BIGINT)); - Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.of(filter), projections, "key"); - Supplier pageProcessor = expressionCompiler.compilePageProcessor(Optional.of(filter), projections); + Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(filter), projections, "key"); + Supplier pageProcessor = expressionCompiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(filter), projections); ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory( 0, new PlanNodeId("test"), new PlanNodeId("0"), - (session, split, columns) -> new FixedPageSource(input), + (session, split, table, columns) -> new FixedPageSource(input), cursorProcessor, pageProcessor, + TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), new DataSize(64, KILOBYTE), @@ -184,16 +195,17 @@ public void testPageSourceLazyLoad() DriverContext driverContext = newDriverContext(); List projections = ImmutableList.of(field(0, VARCHAR)); - Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections, "key"); + Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections, "key"); PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new LazyPagePageProjection())); ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory( 0, new PlanNodeId("test"), new PlanNodeId("0"), - (session, split, columns) -> new SinglePagePageSource(input), + (session, split, table, columns) -> new SinglePagePageSource(input), cursorProcessor, () -> pageProcessor, + TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), new DataSize(0, BYTE), @@ -217,16 +229,17 @@ public void testRecordCursorSource() DriverContext driverContext = newDriverContext(); List projections = ImmutableList.of(field(0, VARCHAR)); - Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections, "key"); - Supplier pageProcessor = expressionCompiler.compilePageProcessor(Optional.empty(), projections); + Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections, "key"); + Supplier pageProcessor = expressionCompiler.compilePageProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections); ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory( 0, new PlanNodeId("test"), new PlanNodeId("0"), - (session, split, columns) -> new RecordPageSource(new PageRecordSet(ImmutableList.of(VARCHAR), input)), + (session, split, table, columns) -> new RecordPageSource(new PageRecordSet(ImmutableList.of(VARCHAR), input)), cursorProcessor, pageProcessor, + TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(VARCHAR), new DataSize(0, BYTE), @@ -261,7 +274,7 @@ public void testPageYield() } Metadata metadata = functionAssertions.getMetadata(); FunctionManager functionManager = metadata.getFunctionManager(); - functionManager.addFunctions(functions.build()); + functionManager.registerBuiltInFunctions(functions.build()); // match each column with a projection ExpressionCompiler expressionCompiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); @@ -269,16 +282,17 @@ public void testPageYield() for (int i = 0; i < totalColumns; i++) { projections.add(call("generic_long_page_col", functionManager.lookupFunction("generic_long_page_col" + i, fromTypes(BIGINT)), BIGINT, field(0, BIGINT))); } - Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections.build(), "key"); - Supplier pageProcessor = expressionCompiler.compilePageProcessor(Optional.empty(), projections.build(), MAX_BATCH_SIZE); + Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections.build(), "key"); + Supplier pageProcessor = expressionCompiler.compilePageProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections.build(), MAX_BATCH_SIZE); ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory( 0, new PlanNodeId("test"), new PlanNodeId("0"), - (session, split, columns) -> new FixedPageSource(ImmutableList.of(input)), + (session, split, table, columns) -> new FixedPageSource(ImmutableList.of(input)), cursorProcessor, pageProcessor, + TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), new DataSize(0, BYTE), @@ -324,7 +338,7 @@ public void testRecordCursorYield() // set up generic long function with a callback to force yield Metadata metadata = functionAssertions.getMetadata(); FunctionManager functionManager = metadata.getFunctionManager(); - functionManager.addFunctions(ImmutableList.of(new GenericLongFunction("record_cursor", value -> { + functionManager.registerBuiltInFunctions(ImmutableList.of(new GenericLongFunction("record_cursor", value -> { driverContext.getYieldSignal().forceYieldForTesting(); return value; }))); @@ -335,16 +349,17 @@ public void testRecordCursorYield() functionManager.lookupFunction("generic_long_record_cursor", fromTypes(BIGINT)), BIGINT, field(0, BIGINT))); - Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections, "key"); - Supplier pageProcessor = expressionCompiler.compilePageProcessor(Optional.empty(), projections); + Supplier cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections, "key"); + Supplier pageProcessor = expressionCompiler.compilePageProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections); ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory( 0, new PlanNodeId("test"), new PlanNodeId("0"), - (session, split, columns) -> new RecordPageSource(new PageRecordSet(ImmutableList.of(BIGINT), input)), + (session, split, table, columns) -> new RecordPageSource(new PageRecordSet(ImmutableList.of(BIGINT), input)), cursorProcessor, pageProcessor, + TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), new DataSize(0, BYTE), @@ -420,6 +435,12 @@ public long getCompletedBytes() return 0; } + @Override + public long getCompletedPositions() + { + return 0; + } + @Override public long getReadTimeNanos() { diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestStreamingAggregationOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestStreamingAggregationOperator.java index 10135d716abc3..99761dd4266e4 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestStreamingAggregationOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestStreamingAggregationOperator.java @@ -19,10 +19,10 @@ import com.facebook.presto.operator.StreamingAggregationOperator.StreamingAggregationOperatorFactory; import com.facebook.presto.operator.aggregation.InternalAggregationFunction; import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.JoinCompiler; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.testing.MaterializedResult; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterMethod; @@ -34,6 +34,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.OperatorAssertion.assertOperatorEquals; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -42,7 +43,6 @@ import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.format; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java index b3bcba3ce2318..d3960e17d01a1 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java @@ -13,8 +13,10 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.Session; import com.facebook.presto.execution.Lifespan; +import com.facebook.presto.execution.TaskId; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.operator.TableFinishOperator.LifespanCommitter; import com.facebook.presto.operator.TableFinishOperator.TableFinishOperatorFactory; @@ -23,15 +25,15 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.LongArrayBlockBuilder; import com.facebook.presto.spi.connector.ConnectorOutputMetadata; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.statistics.ColumnStatisticMetadata; import com.facebook.presto.spi.statistics.ComputedStatistics; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; +import com.google.common.util.concurrent.ListenableFuture; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import org.testng.annotations.AfterClass; @@ -44,11 +46,13 @@ import java.util.Optional; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.block.BlockAssertions.assertBlockEquals; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.operator.PageAssertions.assertPageEquals; -import static com.facebook.presto.operator.TableWriterOperator.STATS_START_CHANNEL; +import static com.facebook.presto.operator.TableWriterUtils.STATS_START_CHANNEL; import static com.facebook.presto.spi.statistics.ColumnStatisticType.MAX_VALUE; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; @@ -57,8 +61,7 @@ import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.google.common.util.concurrent.Futures.immediateFuture; import static java.util.concurrent.Executors.newScheduledThreadPool; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.assertj.core.api.Assertions.assertThat; @@ -266,7 +269,7 @@ public void testLifespanCommit() private static byte[] getTableCommitContextBytes(Lifespan lifespan, int stageId, int taskId, boolean lifespanCommitRequired, boolean lastPage) { - return TABLE_COMMIT_CONTEXT_CODEC.toJsonBytes(new TableCommitContext(lifespan, stageId, taskId, lifespanCommitRequired, lastPage)); + return TABLE_COMMIT_CONTEXT_CODEC.toJsonBytes(new TableCommitContext(lifespan, new TaskId("query", stageId, 0, taskId), lifespanCommitRequired, lastPage)); } private static class TestingTableFinisher @@ -303,9 +306,10 @@ private static class TestingLifespanCommitter private List> fragmentsList = new ArrayList<>(); @Override - public void commitLifespan(Collection fragments) + public ListenableFuture commitLifespan(Collection fragments) { fragmentsList.add(fragments); + return immediateFuture(null); } public List> getCommittedFragments() diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java index a35048e1ab73d..1cfd2482ef5a3 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestTableWriterOperator.java @@ -13,9 +13,12 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.RowPagesBuilder; import com.facebook.presto.Session; import com.facebook.presto.execution.Lifespan; +import com.facebook.presto.execution.TaskId; +import com.facebook.presto.execution.scheduler.ExecutionWriterTarget.CreateHandle; import com.facebook.presto.memory.context.MemoryTrackingContext; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.OutputTableHandle; @@ -35,13 +38,11 @@ import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.connector.ConnectorPageSinkProvider; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.Type; import com.facebook.presto.split.PageSinkManager; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -55,6 +56,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; @@ -64,8 +67,6 @@ import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.slice.Slices.wrappedBuffer; import static java.util.concurrent.CompletableFuture.completedFuture; import static java.util.concurrent.Executors.newCachedThreadPool; @@ -255,7 +256,7 @@ public void testStatisticsAggregation() private static Slice getTableCommitContext(boolean lastPage) { - return wrappedBuffer(TABLE_COMMIT_CONTEXT_CODEC.toJsonBytes(new TableCommitContext(Lifespan.taskWide(), 0, 0, false, lastPage))); + return wrappedBuffer(TABLE_COMMIT_CONTEXT_CODEC.toJsonBytes(new TableCommitContext(Lifespan.taskWide(), new TaskId("query", 0, 0, 0), false, lastPage))); } private void assertMemoryIsReleased(TableWriterOperator tableWriterOperator) @@ -307,7 +308,7 @@ private Operator createTableWriterOperator( 0, new PlanNodeId("test"), pageSinkManager, - new TableWriterNode.CreateHandle(new OutputTableHandle( + new CreateHandle(new OutputTableHandle( CONNECTOR_ID, new ConnectorTransactionHandle() {}, new ConnectorOutputTableHandle() {}), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestTaskStats.java b/presto-main/src/test/java/com/facebook/presto/operator/TestTaskStats.java index 37b5a10392949..cbcb44c2f676e 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestTaskStats.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestTaskStats.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.json.JsonCodec; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.joda.time.DateTime; @@ -50,6 +50,7 @@ public class TestTaskStats new DataSize(12, BYTE), new DataSize(13, BYTE), new DataSize(14, BYTE), + 26, new Duration(15, NANOSECONDS), new Duration(16, NANOSECONDS), new Duration(18, NANOSECONDS), @@ -105,6 +106,7 @@ public static void assertExpectedTaskStats(TaskStats actual) assertEquals(actual.getUserMemoryReservation(), new DataSize(12, BYTE)); assertEquals(actual.getRevocableMemoryReservation(), new DataSize(13, BYTE)); assertEquals(actual.getSystemMemoryReservation(), new DataSize(14, BYTE)); + assertEquals(actual.getPeakTotalMemoryInBytes(), 26); assertEquals(actual.getTotalScheduledTime(), new Duration(15, NANOSECONDS)); assertEquals(actual.getTotalCpuTime(), new Duration(16, NANOSECONDS)); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestTopNOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestTopNOperator.java index 0231362a07d09..8d9bc3aa20396 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestTopNOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestTopNOperator.java @@ -28,6 +28,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.OperatorAssertion.assertOperatorEquals; @@ -39,7 +40,6 @@ import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; import static com.facebook.presto.testing.assertions.Assert.assertEquals; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.BYTE; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestTopNRowNumberOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestTopNRowNumberOperator.java index 4782d16eee437..6422a0c21a7b8 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestTopNRowNumberOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestTopNRowNumberOperator.java @@ -34,6 +34,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.GroupByHashYieldAssertion.createPagesWithDistinctHashKeys; @@ -44,8 +46,6 @@ import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertGreaterThan; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestUncheckedByteArrays.java b/presto-main/src/test/java/com/facebook/presto/operator/TestUncheckedByteArrays.java new file mode 100644 index 0000000000000..1558405314a5f --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestUncheckedByteArrays.java @@ -0,0 +1,122 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator; + +import com.google.common.primitives.Bytes; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.BiFunction; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.LongStream; + +import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static sun.misc.Unsafe.ARRAY_BYTE_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_INT_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_LONG_INDEX_SCALE; +import static sun.misc.Unsafe.ARRAY_SHORT_INDEX_SCALE; + +public class TestUncheckedByteArrays +{ + private static final int POSITIONS_PER_PAGE = 10_000; + + private TestUncheckedByteArrays() + {} + + @Test + public static void testSetByte() + { + byte[] source = new byte[POSITIONS_PER_PAGE]; + ThreadLocalRandom.current().nextBytes(source); + + testSequential(Bytes.asList(source), ARRAY_BYTE_INDEX_SCALE, UncheckedByteArrays::setByteUnchecked, UncheckedByteArrays::getByteUnchecked); + testRandom(Bytes.asList(source), ARRAY_BYTE_INDEX_SCALE, UncheckedByteArrays::setByteUnchecked, UncheckedByteArrays::getByteUnchecked); + } + + @Test + public static void testSetShort() + { + List source = IntStream.range(0, POSITIONS_PER_PAGE).mapToObj(i -> (short) ThreadLocalRandom.current().nextInt(256)).collect(Collectors.toList()); + + testSequential(source, ARRAY_SHORT_INDEX_SCALE, UncheckedByteArrays::setShortUnchecked, UncheckedByteArrays::getShortUnchecked); + testRandom(source, ARRAY_SHORT_INDEX_SCALE, UncheckedByteArrays::setShortUnchecked, UncheckedByteArrays::getShortUnchecked); + } + + @Test + public static void testSetInt() + { + List source = IntStream.range(0, POSITIONS_PER_PAGE).boxed().collect(Collectors.toList()); + + testSequential(source, ARRAY_INT_INDEX_SCALE, UncheckedByteArrays::setIntUnchecked, UncheckedByteArrays::getIntUnchecked); + testRandom(source, ARRAY_INT_INDEX_SCALE, UncheckedByteArrays::setIntUnchecked, UncheckedByteArrays::getIntUnchecked); + } + + @Test + public static void testSetLong() + { + List source = LongStream.range(0, POSITIONS_PER_PAGE).boxed().collect(Collectors.toList()); + + testSequential(source, ARRAY_LONG_INDEX_SCALE, UncheckedByteArrays::setLongUnchecked, UncheckedByteArrays::getLongUnchecked); + testRandom(source, ARRAY_LONG_INDEX_SCALE, UncheckedByteArrays::setLongUnchecked, UncheckedByteArrays::getLongUnchecked); + } + + private static void testSequential(List source, int elementSize, TriFunction writeFunction, BiFunction readFunction) + { + byte[] destination = new byte[source.size() * elementSize]; + + int index = 0; + for (T value : source) { + index = writeFunction.apply(destination, index, value); + } + + assertEquals(index, POSITIONS_PER_PAGE * elementSize); + assertCopied(source, destination, elementSize, readFunction); + } + + private static void testRandom(List source, int elementSize, TriFunction writeFunction, BiFunction readFunction) + { + byte[] destination = new byte[source.size() * elementSize]; + List expected = new ArrayList<>(); + + List positions = IntStream.range(0, POSITIONS_PER_PAGE).boxed().collect(Collectors.toList()); + Collections.shuffle(positions); + + int index = 0; + for (int i = 0; i < POSITIONS_PER_PAGE; i++) { + int position = positions.get(i); + index = writeFunction.apply(destination, index, source.get(position)); + expected.add(source.get(position)); + } + + assertEquals(index, POSITIONS_PER_PAGE * elementSize); + assertCopied(expected, destination, elementSize, readFunction); + } + + private static void assertCopied(List expected, byte[] actual, int elementSize, BiFunction readFunction) + { + for (int index = 0; index < expected.size(); index++) { + assertEquals(readFunction.apply(actual, index * elementSize), expected.get(index)); + } + } + + @FunctionalInterface + private interface TriFunction + { + int apply(T t, U u, S s); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestWindowOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestWindowOperator.java index ac1d9eac02c91..0e83d4f79536f 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestWindowOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestWindowOperator.java @@ -41,6 +41,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.OperatorAssertion.assertOperatorEquals; @@ -56,7 +57,6 @@ import static com.facebook.presto.sql.planner.plan.WindowNode.Frame.WindowType.RANGE; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestingExchangeHttpClientHandler.java b/presto-main/src/test/java/com/facebook/presto/operator/TestingExchangeHttpClientHandler.java index e379977ab58d9..560f862e539df 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestingExchangeHttpClientHandler.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestingExchangeHttpClientHandler.java @@ -13,6 +13,11 @@ */ package com.facebook.presto.operator; +import com.facebook.airlift.http.client.HttpStatus; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.testing.TestingHttpClient; +import com.facebook.airlift.http.client.testing.TestingResponse; import com.facebook.presto.execution.buffer.PagesSerde; import com.facebook.presto.execution.buffer.PagesSerdeUtil; import com.facebook.presto.spi.Page; @@ -20,11 +25,6 @@ import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; -import io.airlift.http.client.HttpStatus; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.testing.TestingHttpClient; -import io.airlift.http.client.testing.TestingResponse; import io.airlift.slice.DynamicSliceOutput; import static com.facebook.presto.PrestoMediaTypes.PRESTO_PAGES; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AbstractTestAggregationFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AbstractTestAggregationFunction.java index 1776ca116984c..6bc7a5e556c97 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AbstractTestAggregationFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AbstractTestAggregationFunction.java @@ -64,7 +64,7 @@ public final void destroyTestAggregationFunction() protected void registerFunctions(Plugin plugin) { - functionManager.addFunctions(extractFunctions(plugin.getFunctions())); + functionManager.registerBuiltInFunctions(extractFunctions(plugin.getFunctions())); } protected void registerTypes(Plugin plugin) @@ -77,7 +77,7 @@ protected void registerTypes(Plugin plugin) protected final InternalAggregationFunction getFunction() { List parameterTypes = fromTypeSignatures(Lists.transform(getFunctionParameterTypes(), TypeSignature::parseTypeSignature)); - FunctionHandle functionHandle = functionManager.resolveFunction(session, QualifiedName.of(getFunctionName()), parameterTypes); + FunctionHandle functionHandle = functionManager.resolveFunction(session.getTransactionId(), QualifiedName.of(getFunctionName()), parameterTypes); return functionManager.getAggregateFunctionImplementation(functionHandle); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AbstractTestApproximateCountDistinct.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AbstractTestApproximateCountDistinct.java index 4738b828d25b8..be0985154f43c 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AbstractTestApproximateCountDistinct.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AbstractTestApproximateCountDistinct.java @@ -35,8 +35,8 @@ import java.util.Set; import java.util.concurrent.ThreadLocalRandom; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; -import static io.airlift.testing.Assertions.assertLessThan; import static org.testng.Assert.assertEquals; public abstract class AbstractTestApproximateCountDistinct diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AggregationTestUtils.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AggregationTestUtils.java index 9ed8023ccc09c..b4a5ae952e15d 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AggregationTestUtils.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AggregationTestUtils.java @@ -166,6 +166,11 @@ private static Page[] maskPages(boolean maskValue, Page... pages) return maskedPages; } + public static Object aggregation(InternalAggregationFunction function, Block... blocks) + { + return aggregation(function, new Page(blocks)); + } + public static Object aggregation(InternalAggregationFunction function, Page... pages) { // execute with args in positions: arg0, arg1, arg2 diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestCountNullAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestCountNullAggregation.java index b6974ed793087..5da964cae6afc 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestCountNullAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestCountNullAggregation.java @@ -39,7 +39,7 @@ public class TestCountNullAggregation @BeforeClass public void setup() { - functionManager.addFunctions(new FunctionListBuilder().aggregates(CountNull.class).getFunctions()); + functionManager.registerBuiltInFunctions(new FunctionListBuilder().aggregates(CountNull.class).getFunctions()); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeHyperLogLogAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeHyperLogLogAggregation.java index dd8ba0d5e8570..78852e783d169 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeHyperLogLogAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeHyperLogLogAggregation.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.type.SqlVarbinary; import com.facebook.presto.spi.type.StandardTypes; import com.google.common.collect.ImmutableList; -import io.airlift.stats.cardinality.HyperLogLog; import java.util.List; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeQuantileDigestFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeQuantileDigestFunction.java index d3cb074d2608c..5aed5095dbbe9 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeQuantileDigestFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeQuantileDigestFunction.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; @@ -21,7 +22,6 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeParameter; import com.google.common.collect.ImmutableList; -import io.airlift.stats.QuantileDigest; import org.testng.annotations.Test; import java.util.List; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java index 81c787ec488d2..9c8f95fc52c97 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.operator.scalar.AbstractTestFunctions; @@ -23,7 +24,6 @@ import com.facebook.presto.spi.type.Type; import com.google.common.base.Joiner; import com.google.common.primitives.Floats; -import io.airlift.stats.QuantileDigest; import org.testng.annotations.Test; import java.util.Arrays; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/AbstractTestFixedHistogramAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/AbstractTestFixedHistogramAggregation.java new file mode 100644 index 0000000000000..6c0ee63c78b0d --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/AbstractTestFixedHistogramAggregation.java @@ -0,0 +1,112 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.operator.aggregation.AbstractTestAggregationFunction; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.type.StandardTypes; +import com.google.common.collect.ImmutableList; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.facebook.presto.block.BlockAssertions.createRLEBlock; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; + +abstract class AbstractTestFixedHistogramAggregation + extends AbstractTestAggregationFunction +{ + private static final int NUM_BINS = 5; + protected final String method; + + protected AbstractTestFixedHistogramAggregation(String method) + { + this.method = method; + } + + @Override + public Block[] getSequenceBlocks(int start, int length) + { + int positionCount = 2 * length; + BlockBuilder samples = DOUBLE.createBlockBuilder(null, positionCount); + BlockBuilder weights = DOUBLE.createBlockBuilder(null, positionCount); + for (int weight = 1; weight < 3; weight++) { + for (int i = start; i < start + length; i++) { + int bin = Math.max(Math.min(i, NUM_BINS - 1), 0); + DOUBLE.writeDouble(samples, bin); + DOUBLE.writeDouble(weights, weight); + } + } + + return new Block[] { + createRLEBlock(NUM_BINS, positionCount), + samples.build(), + weights.build(), + createRLEBlock(this.method, positionCount), + createRLEBlock(0.0, positionCount), + createRLEBlock((double) NUM_BINS, positionCount) + }; + } + + @Override + protected String getFunctionName() + { + return "differential_entropy"; + } + + @Override + protected List getFunctionParameterTypes() + { + return ImmutableList.of( + StandardTypes.INTEGER, + StandardTypes.DOUBLE, + StandardTypes.DOUBLE, + StandardTypes.VARCHAR, + StandardTypes.DOUBLE, + StandardTypes.DOUBLE); + } + + protected static void generateSamplesAndWeights(int start, int length, List samples, List weights) + { + for (int weight = 1; weight < 3; weight++) { + for (int i = start; i < start + length; i++) { + int bin = Math.max(Math.min(i, NUM_BINS - 1), 0); + samples.add(Double.valueOf(bin)); + weights.add(Double.valueOf(weight)); + } + } + } + + protected static double calculateEntropy(List samples, List weights) + { + double totalWeight = weights.stream().mapToDouble(weight -> weight).sum(); + if (totalWeight == 0.0) { + return Double.NaN; + } + + Map bucketWeights = new HashMap<>(); + for (int i = 0; i < samples.size(); i++) { + double sample = samples.get(i); + double weight = weights.get(i); + bucketWeights.put(sample, bucketWeights.getOrDefault(sample, 0.0) + weight); + } + + double entropy = bucketWeights.values().stream() + .mapToDouble(weight -> weight == 0.0 ? 0.0 : weight / totalWeight * Math.log(totalWeight / weight)) + .sum(); + return entropy / Math.log(2); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/AbstractTestReservoirAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/AbstractTestReservoirAggregation.java new file mode 100644 index 0000000000000..e90aa2a586620 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/AbstractTestReservoirAggregation.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.operator.aggregation.AbstractTestAggregationFunction; + +import static com.facebook.presto.operator.aggregation.differentialentropy.EntropyCalculations.calculateFromSamplesUsingVasicek; +import static org.testng.Assert.assertTrue; + +abstract class AbstractTestReservoirAggregation + extends AbstractTestAggregationFunction +{ + protected static final int MAX_SAMPLES = 500; + + @Override + protected String getFunctionName() + { + return "differential_entropy"; + } + + @Override + public Double getExpectedValue(int start, int length) + { + assertTrue(2 * length < MAX_SAMPLES); + double[] samples = new double[2 * length]; + for (int i = 0; i < length; i++) { + samples[i] = (double) (start + i); + samples[i + length] = (double) (start + i); + } + return calculateFromSamplesUsingVasicek(samples); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/AbstractTestStateStrategy.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/AbstractTestStateStrategy.java new file mode 100644 index 0000000000000..c690f0d15ac37 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/AbstractTestStateStrategy.java @@ -0,0 +1,75 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import org.testng.annotations.Test; + +import java.util.Random; +import java.util.function.Function; + +import static org.testng.Assert.assertEquals; + +abstract class AbstractTestStateStrategy +{ + protected static final double MIN = 0.0; + protected static final double MAX = 10.0; + + private final Function strategySupplier; + private final boolean weighted; + + protected AbstractTestStateStrategy( + Function strategySupplier, + boolean weighted) + { + this.strategySupplier = strategySupplier; + this.weighted = weighted; + } + + @Test + public void testUniformDistribution() + { + DifferentialEntropyStateStrategy strategy = strategySupplier.apply(2000); + Random random = new Random(13); + for (int i = 0; i < 9_999_999; i++) { + double value = 10 * random.nextFloat(); + if (weighted) { + strategy.add(value, 1.0); + } + else { + strategy.add(value); + } + } + double expected = Math.log(10) / Math.log(2); + assertEquals(strategy.calculateEntropy(), expected, 0.1); + } + + @Test + public void testNormalDistribution() + { + DifferentialEntropyStateStrategy strategy = strategySupplier.apply(200_000); + Random random = new Random(13); + double sigma = 0.5; + for (int i = 0; i < 9_999_999; i++) { + double value = 5 + sigma * random.nextGaussian(); + if (weighted) { + strategy.add(value, 1.0); + } + else { + strategy.add(value); + } + } + double expected = 0.5 * Math.log(2 * Math.PI * Math.E * sigma * sigma) / Math.log(2); + assertEquals(strategy.calculateEntropy(), expected, 0.02); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestEntropyCalculations.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestEntropyCalculations.java new file mode 100644 index 0000000000000..13b8b5b090cd1 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestEntropyCalculations.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import org.testng.annotations.Test; + +import java.util.Random; + +import static com.facebook.presto.operator.aggregation.differentialentropy.EntropyCalculations.calculateFromSamplesUsingVasicek; +import static org.testng.Assert.assertEquals; + +public class TestEntropyCalculations +{ + @Test + public void testUniformDistribution() + { + Random random = new Random(13); + double[] samples = new double[10000000]; + for (int i = 0; i < samples.length; i++) { + samples[i] = random.nextDouble(); + } + assertEquals(calculateFromSamplesUsingVasicek(samples), 0, 0.02); + } + + @Test + public void testNormalDistribution() + { + Random random = new Random(13); + double[] samples = new double[10000000]; + double sigma = 0.5; + for (int i = 0; i < samples.length; i++) { + samples[i] = 5 + sigma * random.nextGaussian(); + } + double expected = 0.5 * Math.log(2 * Math.PI * Math.E * sigma * sigma) / Math.log(2); + assertEquals(calculateFromSamplesUsingVasicek(samples), expected, 0.02); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramJacknifeAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramJacknifeAggregation.java new file mode 100644 index 0000000000000..5f5ddd67e6b9f --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramJacknifeAggregation.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.spi.PrestoException; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.List; + +import static com.facebook.presto.block.BlockAssertions.createDoublesBlock; +import static com.facebook.presto.block.BlockAssertions.createLongsBlock; +import static com.facebook.presto.block.BlockAssertions.createStringsBlock; +import static com.facebook.presto.operator.aggregation.AggregationTestUtils.aggregation; +import static com.facebook.presto.operator.aggregation.differentialentropy.DifferentialEntropyStateStrategy.FIXED_HISTOGRAM_JACKNIFE_METHOD_NAME; + +public class TestFixedHistogramJacknifeAggregation + extends AbstractTestFixedHistogramAggregation +{ + public TestFixedHistogramJacknifeAggregation() + { + super(FIXED_HISTOGRAM_JACKNIFE_METHOD_NAME); + } + + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, bucket count must be non-negative: -200") + public void testIllegalBucketCount() + { + aggregation( + getFunction(), + createLongsBlock(-200), + createDoublesBlock(0.1), + createDoublesBlock(0.2), + createStringsBlock(method), + createDoublesBlock(0.0), + createDoublesBlock(0.2)); + } + + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, weight must be non-negative: -0.2") + public void testNegativeWeight() + { + aggregation( + getFunction(), + createLongsBlock(200), + createDoublesBlock(0.1), + createDoublesBlock(-0.2), + createStringsBlock(method), + createDoublesBlock(0.0), + createDoublesBlock(0.2)); + } + + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, sample must be at least min: sample=-100.0, min=0.0") + public void testTooSmallSample() + { + aggregation( + getFunction(), + createLongsBlock(200), + createDoublesBlock(-100.0), + createDoublesBlock(0.2), + createStringsBlock(method), + createDoublesBlock(0.0), + createDoublesBlock(0.2)); + } + + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, sample must be at most max: sample=300.0, max=0.2") + public void testTooLargeSample() + { + aggregation( + getFunction(), + createLongsBlock(200), + createDoublesBlock(300.0), + createDoublesBlock(0.2), + createStringsBlock(method), + createDoublesBlock(0.0), + createDoublesBlock(0.2)); + } + + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, min must be larger than max: min=0.2, max=0.1") + public void testIllegalMinMax() + { + aggregation( + getFunction(), + createLongsBlock(200), + createDoublesBlock(0.1), + createDoublesBlock(0.2), + createStringsBlock(method), + createDoublesBlock(0.2), + createDoublesBlock(0.1)); + } + + @Override + public Double getExpectedValue(int start, int length) + { + List samples = new ArrayList<>(); + List weights = new ArrayList<>(); + generateSamplesAndWeights(start, length, samples, weights); + + double entropy = samples.size() * calculateEntropy(samples, weights); + for (int i = 0; i < samples.size(); ++i) { + List subSamples = new ArrayList<>(samples); + subSamples.remove(i); + List subWeights = new ArrayList<>(weights); + subWeights.remove(i); + + double holdOutEntropy = (samples.size() - 1) * calculateEntropy(subSamples, subWeights) / samples.size(); + entropy -= holdOutEntropy; + } + return entropy; + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramJacknifeStateStrategy.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramJacknifeStateStrategy.java new file mode 100644 index 0000000000000..97347d5d92197 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramJacknifeStateStrategy.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +public class TestFixedHistogramJacknifeStateStrategy + extends AbstractTestStateStrategy +{ + public TestFixedHistogramJacknifeStateStrategy() + { + super(size -> new FixedHistogramJacknifeStateStrategy(size, AbstractTestStateStrategy.MIN, AbstractTestStateStrategy.MAX), true); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramMleAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramMleAggregation.java new file mode 100644 index 0000000000000..e822efd031abe --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramMleAggregation.java @@ -0,0 +1,119 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.spi.PrestoException; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.List; + +import static com.facebook.presto.block.BlockAssertions.createDoublesBlock; +import static com.facebook.presto.block.BlockAssertions.createLongsBlock; +import static com.facebook.presto.block.BlockAssertions.createStringsBlock; +import static com.facebook.presto.operator.aggregation.AggregationTestUtils.aggregation; +import static com.facebook.presto.operator.aggregation.differentialentropy.DifferentialEntropyStateStrategy.FIXED_HISTOGRAM_MLE_METHOD_NAME; + +public class TestFixedHistogramMleAggregation + extends AbstractTestFixedHistogramAggregation +{ + public TestFixedHistogramMleAggregation() + { + super(FIXED_HISTOGRAM_MLE_METHOD_NAME); + } + + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, bucket count must be non-negative: -200") + public void testIllegalBucketCount() + { + aggregation( + getFunction(), + createLongsBlock(-200), + createDoublesBlock(0.1), + createDoublesBlock(0.2), + createStringsBlock(method), + createDoublesBlock(0.0), + createDoublesBlock(0.2)); + } + + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, weight must be non-negative: -0.2") + public void testNegativeWeight() + { + aggregation( + getFunction(), + createLongsBlock(200), + createDoublesBlock(0.1), + createDoublesBlock(-0.2), + createStringsBlock(method), + createDoublesBlock(0.0), + createDoublesBlock(0.2)); + } + + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, sample must be at least min: sample=-100.0, min=0.0") + public void testTooSmallSample() + { + aggregation( + getFunction(), + createLongsBlock(200), + createDoublesBlock(-100.0), + createDoublesBlock(0.2), + createStringsBlock(method), + createDoublesBlock(0.0), + createDoublesBlock(0.2)); + } + + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, sample must be at most max: sample=300.0, max=0.2") + public void testTooLargeSample() + { + aggregation( + getFunction(), + createLongsBlock(200), + createDoublesBlock(300.0), + createDoublesBlock(0.2), + createStringsBlock(method), + createDoublesBlock(0.0), + createDoublesBlock(0.2)); + } + + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, min must be larger than max: min=0.2, max=0.1") + public void testIllegalMinMax() + { + aggregation( + getFunction(), + createLongsBlock(200), + createDoublesBlock(0.1), + createDoublesBlock(0.2), + createStringsBlock(method), + createDoublesBlock(0.2), + createDoublesBlock(0.1)); + } + + @Override + public Double getExpectedValue(int start, int length) + { + List samples = new ArrayList<>(); + List weights = new ArrayList<>(); + generateSamplesAndWeights(start, length, samples, weights); + return calculateEntropy(samples, weights); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramMleStateStrategy.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramMleStateStrategy.java new file mode 100644 index 0000000000000..3075a24c1eff9 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestFixedHistogramMleStateStrategy.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +public class TestFixedHistogramMleStateStrategy + extends AbstractTestStateStrategy +{ + public TestFixedHistogramMleStateStrategy() + { + super(bucketCount -> new FixedHistogramMleStateStrategy(bucketCount, MIN, MAX), true); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestIllegalMethodAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestIllegalMethodAggregation.java new file mode 100644 index 0000000000000..ba071b0686fe5 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestIllegalMethodAggregation.java @@ -0,0 +1,71 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.operator.aggregation.InternalAggregationFunction; +import com.facebook.presto.spi.PrestoException; +import org.testng.annotations.Test; + +import static com.facebook.presto.block.BlockAssertions.createDoublesBlock; +import static com.facebook.presto.block.BlockAssertions.createLongsBlock; +import static com.facebook.presto.block.BlockAssertions.createStringsBlock; +import static com.facebook.presto.operator.aggregation.AggregationTestUtils.aggregation; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; + +public class TestIllegalMethodAggregation +{ + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, invalid method: no_such_method") + public void testIllegalMethod() + { + FunctionManager functionManager = MetadataManager.createTestMetadataManager().getFunctionManager(); + InternalAggregationFunction function = functionManager.getAggregateFunctionImplementation( + functionManager.lookupFunction( + "differential_entropy", + fromTypes(BIGINT, DOUBLE, DOUBLE, VARCHAR, DOUBLE, DOUBLE))); + aggregation( + function, + createLongsBlock(200), + createDoublesBlock(0.1), + createDoublesBlock(0.2), + createStringsBlock("no_such_method"), + createDoublesBlock(0.0), + createDoublesBlock(1.0)); + } + + @Test + public void testNullMethod() + { + FunctionManager functionManager = MetadataManager.createTestMetadataManager().getFunctionManager(); + InternalAggregationFunction function = functionManager.getAggregateFunctionImplementation( + functionManager.lookupFunction( + "differential_entropy", + fromTypes(BIGINT, DOUBLE, DOUBLE, VARCHAR, DOUBLE, DOUBLE))); + createStringsBlock((String) null); + aggregation( + function, + createLongsBlock(200), + createDoublesBlock(0.1), + createDoublesBlock(-0.2), + createStringsBlock((String) null), + createDoublesBlock(0.0), + createDoublesBlock(1.0)); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestUnweightedReservoirAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestUnweightedReservoirAggregation.java new file mode 100644 index 0000000000000..e5ce85b69136e --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestUnweightedReservoirAggregation.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.type.StandardTypes; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import java.util.List; + +import static com.facebook.presto.block.BlockAssertions.createDoublesBlock; +import static com.facebook.presto.block.BlockAssertions.createLongsBlock; +import static com.facebook.presto.block.BlockAssertions.createRLEBlock; +import static com.facebook.presto.operator.aggregation.AggregationTestUtils.aggregation; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; + +public class TestUnweightedReservoirAggregation + extends AbstractTestReservoirAggregation +{ + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, max samples must be positive: -200") + public void testInvalidMaxSamples() + { + aggregation( + getFunction(), + createLongsBlock(-200), + createDoublesBlock(0.1)); + } + + @Override + public Block[] getSequenceBlocks(int start, int length) + { + int positionCount = 2 * length; + BlockBuilder samples = DOUBLE.createBlockBuilder(null, positionCount); + for (int weight = 1; weight < 3; weight++) { + for (int i = start; i < start + length; i++) { + DOUBLE.writeDouble(samples, i); + } + } + + return new Block[] { + createRLEBlock(AbstractTestReservoirAggregation.MAX_SAMPLES, positionCount), + samples.build() + }; + } + + @Override + protected List getFunctionParameterTypes() + { + return ImmutableList.of(StandardTypes.INTEGER, StandardTypes.DOUBLE); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestUnweightedReservoirSampleStateStrategy.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestUnweightedReservoirSampleStateStrategy.java new file mode 100644 index 0000000000000..bc0b0c7a4e634 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestUnweightedReservoirSampleStateStrategy.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +public class TestUnweightedReservoirSampleStateStrategy + extends AbstractTestStateStrategy +{ + public TestUnweightedReservoirSampleStateStrategy() + { + super(size -> new UnweightedReservoirSampleStateStrategy(size), false); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestWeightedReservoirAggregation.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestWeightedReservoirAggregation.java new file mode 100644 index 0000000000000..1a0fe69ee8bfc --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestWeightedReservoirAggregation.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.type.StandardTypes; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import java.util.List; + +import static com.facebook.presto.block.BlockAssertions.createDoublesBlock; +import static com.facebook.presto.block.BlockAssertions.createLongsBlock; +import static com.facebook.presto.block.BlockAssertions.createRLEBlock; +import static com.facebook.presto.operator.aggregation.AggregationTestUtils.aggregation; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; + +public class TestWeightedReservoirAggregation + extends AbstractTestReservoirAggregation +{ + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, max samples must be positive: -200") + public void testInvalidMaxSamples() + { + aggregation( + getFunction(), + createLongsBlock(-200), + createDoublesBlock(0.1), + createDoublesBlock(0.2)); + } + + @Test( + expectedExceptions = PrestoException.class, + expectedExceptionsMessageRegExp = "In differential_entropy UDF, weight must be non-negative: -0.2") + public void testNegativeWeight() + { + aggregation( + getFunction(), + createLongsBlock(200), + createDoublesBlock(0.1), + createDoublesBlock(-0.2)); + } + + @Override + public Block[] getSequenceBlocks(int start, int length) + { + int positionCount = 2 * length; + BlockBuilder samples = DOUBLE.createBlockBuilder(null, positionCount); + BlockBuilder weights = DOUBLE.createBlockBuilder(null, positionCount); + for (int weight = 1; weight < 3; weight++) { + for (int i = start; i < start + length; i++) { + DOUBLE.writeDouble(samples, i); + DOUBLE.writeDouble(weights, weight); + } + } + + return new Block[] { + createRLEBlock(MAX_SAMPLES, positionCount), + samples.build(), + weights.build() + }; + } + + @Override + protected List getFunctionParameterTypes() + { + return ImmutableList.of(StandardTypes.INTEGER, StandardTypes.DOUBLE, StandardTypes.DOUBLE); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestWeightedReservoirSampleStateStrategy.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestWeightedReservoirSampleStateStrategy.java new file mode 100644 index 0000000000000..e8f76b443bc00 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/differentialentropy/TestWeightedReservoirSampleStateStrategy.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.differentialentropy; + +public class TestWeightedReservoirSampleStateStrategy + extends AbstractTestStateStrategy +{ + public TestWeightedReservoirSampleStateStrategy() + { + super(size -> new WeightedReservoirSampleStateStrategy(size), true); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/fixedhistogram/TestFixedDoubleBreakdownHistogram.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/fixedhistogram/TestFixedDoubleBreakdownHistogram.java index 8756cb3792467..84c1565241b2e 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/fixedhistogram/TestFixedDoubleBreakdownHistogram.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/fixedhistogram/TestFixedDoubleBreakdownHistogram.java @@ -21,7 +21,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; public class TestFixedDoubleBreakdownHistogram { @@ -36,30 +35,20 @@ public void testGetters() assertEquals(histogram.getMax(), 4.0); } - @Test - public void testIllegalBucketWeightCount() + @Test( + expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "bucketCount must be at least 2: -200") + public void testIllegalBucketCount() { - try { - new FixedDoubleBreakdownHistogram(-200, 3.0, 4.0); - fail("exception expected"); - } - catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("bucketCount")); - assertTrue(e.getMessage().contains("must be at least")); - } + new FixedDoubleBreakdownHistogram(-200, 3.0, 4.0); } - @Test + @Test( + expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "min must be smaller than max: 3.0 3.0") public void testIllegalMinMax() { - try { - new FixedDoubleBreakdownHistogram(200, 3.0, 3.0); - fail("exception expected"); - } - catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("min")); - assertTrue(e.getMessage().contains("must be smaller than")); - } + new FixedDoubleBreakdownHistogram(200, 3.0, 3.0); } @Test @@ -72,7 +61,7 @@ public void testBasicOps() histogram.add(3.8, 200.0); histogram.add(3.1, 100.0); assertEquals( - Streams.stream(histogram.iterator()).mapToDouble(FixedDoubleBreakdownHistogram.BucketWeight::getWeight).sum(), + Streams.stream(histogram.iterator()).mapToDouble(FixedDoubleBreakdownHistogram.Bucket::getWeight).sum(), 300.0); } @@ -94,10 +83,10 @@ public void testEqualValuesDifferentWeights() Streams.stream(histogram.iterator()).count(), 3); assertEquals( - Streams.stream(histogram.iterator()).mapToDouble(FixedDoubleBreakdownHistogram.BucketWeight::getWeight).sum(), + Streams.stream(histogram.iterator()).mapToDouble(FixedDoubleBreakdownHistogram.Bucket::getWeight).sum(), 0.9); assertEquals( - Streams.stream(histogram.iterator()).mapToLong(FixedDoubleBreakdownHistogram.BucketWeight::getCount).sum(), + Streams.stream(histogram.iterator()).mapToLong(FixedDoubleBreakdownHistogram.Bucket::getCount).sum(), 4); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/fixedhistogram/TestFixedDoubleHistogram.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/fixedhistogram/TestFixedDoubleHistogram.java index f231e27165198..73dcf42f4d3b6 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/fixedhistogram/TestFixedDoubleHistogram.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/fixedhistogram/TestFixedDoubleHistogram.java @@ -20,8 +20,6 @@ import java.util.stream.IntStream; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; public class TestFixedDoubleHistogram { @@ -35,30 +33,20 @@ public void testGetters() assertEquals(histogram.getMax(), 4.0); } - @Test + @Test( + expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "bucketCount must be at least 2: -200") public void testIllegalBucketCount() { - try { - new FixedDoubleHistogram(-200, 3.0, 4.0); - fail("exception expected"); - } - catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("bucketCount")); - assertTrue(e.getMessage().contains("must be at least")); - } + new FixedDoubleHistogram(-200, 3.0, 4.0); } - @Test + @Test( + expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "min must be smaller than max: 3.0 3.0") public void testIllegalMinMax() { - try { - new FixedDoubleHistogram(200, 3.0, 3.0); - fail("exception expected"); - } - catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("min")); - assertTrue(e.getMessage().contains("must be smaller than")); - } + new FixedDoubleHistogram(200, 3.0, 3.0); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/reservoirsample/TestUnweightedDoubleReservoirSample.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/reservoirsample/TestUnweightedDoubleReservoirSample.java new file mode 100644 index 0000000000000..311c85053fad2 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/reservoirsample/TestUnweightedDoubleReservoirSample.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.reservoirsample; + +import org.testng.annotations.Test; + +import java.util.Arrays; + +import static java.lang.String.format; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestUnweightedDoubleReservoirSample +{ + @Test( + expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "Maximum number of samples must be positive: 0") + public void testIllegalMaxSamples() + { + new UnweightedDoubleReservoirSample(0); + } + + @Test + public void testGetMaxSamples() + { + UnweightedDoubleReservoirSample reservoir = new UnweightedDoubleReservoirSample(200); + + assertEquals(reservoir.getMaxSamples(), 200); + assertEquals(reservoir.getTotalPopulationCount(), 0); + } + + @Test + public void testFew() + { + UnweightedDoubleReservoirSample reservoir = new UnweightedDoubleReservoirSample(200); + + reservoir.add(1.0); + reservoir.add(2.0); + reservoir.add(3.0); + + assertEquals(Arrays.stream(reservoir.getSamples()).sorted().toArray(), new double[] {1.0, 2.0, 3.0}); + assertEquals(reservoir.getTotalPopulationCount(), 3); + } + + @Test + public void testMany() + { + UnweightedDoubleReservoirSample reservoir = new UnweightedDoubleReservoirSample(200); + + long streamLength = 1_000_000; + for (int i = 0; i < streamLength; ++i) { + assertEquals(reservoir.getTotalPopulationCount(), i); + reservoir.add(i); + } + + double[] quantized = new double[4]; + for (double sample : reservoir.getSamples()) { + int index = (int) (4.0 * sample / (streamLength + 1)); + ++quantized[index]; + } + int expectedMin = 25; + for (int i = 0; i < 4; ++i) { + assertTrue(quantized[i] > expectedMin, format("Expected quantized[i] > got: i=%s, quantized=%s, got=%s", i, quantized[i], expectedMin)); + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/reservoirsample/TestWeightedDoubleReservoirSample.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/reservoirsample/TestWeightedDoubleReservoirSample.java new file mode 100644 index 0000000000000..975063e1c1c31 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/reservoirsample/TestWeightedDoubleReservoirSample.java @@ -0,0 +1,109 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation.reservoirsample; + +import org.testng.annotations.Test; + +import java.util.Arrays; + +import static java.lang.String.format; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestWeightedDoubleReservoirSample +{ + @Test( + expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "Maximum number of samples must be positive: 0") + public void testIllegalMaxSamples() + { + new WeightedDoubleReservoirSample(0); + } + + @Test + public void testGetters() + { + WeightedDoubleReservoirSample reservoir = new WeightedDoubleReservoirSample(200); + + assertEquals(reservoir.getMaxSamples(), 200); + assertEquals(reservoir.getTotalPopulationWeight(), 0.0); + } + + @Test + public void testFew() + { + WeightedDoubleReservoirSample reservoir = new WeightedDoubleReservoirSample(200); + + reservoir.add(1.0, 1.0); + reservoir.add(2.0, 1.0); + reservoir.add(3.0, 0.5); + + assertEquals(Arrays.stream(reservoir.getSamples()).sorted().toArray(), new double[] {1.0, 2.0, 3.0}); + assertEquals(reservoir.getTotalPopulationWeight(), 2.5); + } + + @Test + public void testMany() + { + WeightedDoubleReservoirSample reservoir = new WeightedDoubleReservoirSample(200); + + long streamLength = 1_000_000; + for (int i = 0; i < streamLength; ++i) { + assertEquals(reservoir.getTotalPopulationWeight(), i, 0.0001); + reservoir.add(i, 1.0); + } + + double[] quantized = new double[4]; + int count = 0; + for (double sample : reservoir.getSamples()) { + ++count; + int index = (int) (4.0 * sample / (streamLength + 1)); + ++quantized[index]; + } + assertEquals(count, 200, format("Number of samples should be full: got=%s, expected=%s", count, 200)); + int expectedMin = 25; + for (int i = 0; i < 4; ++i) { + assertTrue(quantized[i] > expectedMin, format("Expected quantized[i] > got: i=%s, quantized=%s, got=%s", i, quantized[i], 35)); + } + } + + @Test + public void testManyWeighted() + { + WeightedDoubleReservoirSample reservoir = new WeightedDoubleReservoirSample(200); + + long streamLength = 1_000_000; + double epsilon = 0.00000001; + for (int i = 0; i < streamLength; ++i) { + assertEquals(reservoir.getTotalPopulationWeight(), epsilon * i, epsilon / 100); + reservoir.add(3, epsilon); + } + for (int i = 0; i < streamLength; ++i) { + reservoir.add(i, 9999999999.0); + } + + double[] quantized = new double[4]; + int count = 0; + for (double sample : reservoir.getSamples()) { + ++count; + int index = (int) (4.0 * sample / (streamLength + 1)); + ++quantized[index]; + } + assertEquals(count, 200, format("Number of samples should be full: got=%s, expected=%s", count, 200)); + int expectedMin = 25; + for (int i = 0; i < 4; ++i) { + assertTrue(quantized[i] > expectedMin, format("Expected quantized[i] > got: i=%s, quantized=%s, got=%s", i, quantized[i], expectedMin)); + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/exchange/TestLocalExchange.java b/presto-main/src/test/java/com/facebook/presto/operator/exchange/TestLocalExchange.java index 0f11de6113ac2..80916ba880e92 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/exchange/TestLocalExchange.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/exchange/TestLocalExchange.java @@ -14,6 +14,7 @@ package com.facebook.presto.operator.exchange; import com.facebook.presto.SequencePageBuilder; +import com.facebook.presto.Session; import com.facebook.presto.execution.Lifespan; import com.facebook.presto.operator.InterpretedHashGenerator; import com.facebook.presto.operator.PageAssertions; @@ -23,9 +24,12 @@ import com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeSinkFactoryId; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.DataSize; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -33,6 +37,7 @@ import java.util.Optional; import java.util.function.Consumer; +import static com.facebook.airlift.testing.Assertions.assertContains; import static com.facebook.presto.operator.PipelineExecutionStrategy.GROUPED_EXECUTION; import static com.facebook.presto.operator.PipelineExecutionStrategy.UNGROUPED_EXECUTION; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -41,7 +46,7 @@ import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_PASSTHROUGH_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; -import static io.airlift.testing.Assertions.assertContains; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static io.airlift.units.DataSize.Unit.BYTE; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -56,6 +61,23 @@ public class TestLocalExchange private static final DataSize RETAINED_PAGE_SIZE = new DataSize(createPage(42).getRetainedSizeInBytes(), BYTE); private static final DataSize LOCAL_EXCHANGE_MAX_BUFFERED_BYTES = new DataSize(32, DataSize.Unit.MEGABYTE); + private PartitioningProviderManager partitioningProviderManager; + private Session session; + + @BeforeClass + public void setUp() + { + partitioningProviderManager = new PartitioningProviderManager(); + session = testSessionBuilder().build(); + } + + @AfterClass + public void tearDown() + { + partitioningProviderManager = null; + session = null; + } + @DataProvider public static Object[][] executionStrategy() { @@ -66,6 +88,8 @@ public static Object[][] executionStrategy() public void testGatherSingleWriter(PipelineExecutionStrategy executionStrategy) { LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory( + partitioningProviderManager, + session, SINGLE_DISTRIBUTION, 8, TYPES, @@ -138,6 +162,8 @@ public void testGatherSingleWriter(PipelineExecutionStrategy executionStrategy) public void testBroadcast(PipelineExecutionStrategy executionStrategy) { LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory( + partitioningProviderManager, + session, FIXED_BROADCAST_DISTRIBUTION, 2, TYPES, @@ -225,6 +251,8 @@ public void testBroadcast(PipelineExecutionStrategy executionStrategy) public void testRandom(PipelineExecutionStrategy executionStrategy) { LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory( + partitioningProviderManager, + session, FIXED_ARBITRARY_DISTRIBUTION, 2, TYPES, @@ -273,6 +301,8 @@ public void testRandom(PipelineExecutionStrategy executionStrategy) public void testPassthrough(PipelineExecutionStrategy executionStrategy) { LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory( + partitioningProviderManager, + session, FIXED_PASSTHROUGH_DISTRIBUTION, 2, TYPES, @@ -340,6 +370,8 @@ public void testPassthrough(PipelineExecutionStrategy executionStrategy) public void testPartition(PipelineExecutionStrategy executionStrategy) { LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory( + partitioningProviderManager, + session, FIXED_HASH_DISTRIBUTION, 2, TYPES, @@ -408,6 +440,8 @@ public void writeUnblockWhenAllReadersFinish(PipelineExecutionStrategy execution ImmutableList types = ImmutableList.of(BIGINT); LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory( + partitioningProviderManager, + session, FIXED_BROADCAST_DISTRIBUTION, 2, types, @@ -454,6 +488,8 @@ public void writeUnblockWhenAllReadersFinish(PipelineExecutionStrategy execution public void writeUnblockWhenAllReadersFinishAndPagesConsumed(PipelineExecutionStrategy executionStrategy) { LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory( + partitioningProviderManager, + session, FIXED_BROADCAST_DISTRIBUTION, 2, TYPES, @@ -522,6 +558,8 @@ public void testMismatchedExecutionStrategy() // The most common reason of mismatch is when one of sink/source created the wrong kind of local exchange. // In such case, we want to fail loudly. LocalExchangeFactory ungroupedLocalExchangeFactory = new LocalExchangeFactory( + partitioningProviderManager, + session, FIXED_HASH_DISTRIBUTION, 2, TYPES, @@ -538,6 +576,8 @@ public void testMismatchedExecutionStrategy() } LocalExchangeFactory groupedLocalExchangeFactory = new LocalExchangeFactory( + partitioningProviderManager, + session, FIXED_HASH_DISTRIBUTION, 2, TYPES, diff --git a/presto-main/src/test/java/com/facebook/presto/operator/index/TestTupleFilterProcessor.java b/presto-main/src/test/java/com/facebook/presto/operator/index/TestTupleFilterProcessor.java index 1707b74e4eab6..ec669c22ae91c 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/index/TestTupleFilterProcessor.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/index/TestTupleFilterProcessor.java @@ -62,6 +62,7 @@ public void testFilter() new int[] {0, 1, 2}, new int[] {1, 0, 3}, outputTypes, + SESSION.getSqlFunctionProperties(), new PageFunctionCompiler(createTestMetadataManager(), 0)); PageProcessor tupleFilterProcessor = filterFactory.createPageProcessor(tuplePage, OptionalInt.of(MAX_BATCH_SIZE)).get(); Page actualPage = getOnlyElement( diff --git a/presto-main/src/test/java/com/facebook/presto/operator/project/BenchmarkDictionaryBlockGetSizeInBytes.java b/presto-main/src/test/java/com/facebook/presto/operator/project/BenchmarkDictionaryBlockGetSizeInBytes.java index eb2ff49808443..dcd087ab820a2 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/project/BenchmarkDictionaryBlockGetSizeInBytes.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/project/BenchmarkDictionaryBlockGetSizeInBytes.java @@ -103,7 +103,7 @@ private static Block createMapBlock(int positionCount) for (int i = 0; i < offsets.length; i++) { offsets[i] = mapSize * i; } - return mapType.createBlockFromKeyValue(Optional.empty(), offsets, keyBlock, valueBlock); + return mapType.createBlockFromKeyValue(positionCount, Optional.empty(), offsets, keyBlock, valueBlock); } private static Block createDictionaryBlock(List values) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/project/TestDictionaryAwarePageProjection.java b/presto-main/src/test/java/com/facebook/presto/operator/project/TestDictionaryAwarePageProjection.java index 00931f4b05570..1cf328e308599 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/project/TestDictionaryAwarePageProjection.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/project/TestDictionaryAwarePageProjection.java @@ -31,13 +31,13 @@ import java.util.Arrays; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.block.BlockAssertions.assertBlockEquals; import static com.facebook.presto.block.BlockAssertions.createLongSequenceBlock; import static com.facebook.presto.spi.block.DictionaryId.randomDictionaryId; import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertInstanceOf; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/project/TestPageProcessor.java b/presto-main/src/test/java/com/facebook/presto/operator/project/TestPageProcessor.java index b08aacaf70d00..fe83349b14e2c 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/project/TestPageProcessor.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/project/TestPageProcessor.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.project; +import com.facebook.airlift.testing.TestingTicker; import com.facebook.presto.memory.context.AggregatedMemoryContext; import com.facebook.presto.memory.context.LocalMemoryContext; import com.facebook.presto.metadata.MetadataManager; @@ -31,7 +32,6 @@ import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; import io.airlift.slice.Slices; -import io.airlift.testing.TestingTicker; import io.airlift.units.Duration; import org.openjdk.jol.info.ClassLayout; import org.testng.annotations.Test; @@ -45,6 +45,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.block.BlockAssertions.createLongSequenceBlock; import static com.facebook.presto.block.BlockAssertions.createSlicesBlock; import static com.facebook.presto.block.BlockAssertions.createStringsBlock; @@ -64,7 +65,6 @@ import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.field; import static com.facebook.presto.testing.TestingConnectorSession.SESSION; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.join; import static java.util.Collections.nCopies; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; @@ -394,7 +394,7 @@ public void testExpressionProfiler() TestingTicker testingTicker = new TestingTicker(); PageFunctionCompiler functionCompiler = new PageFunctionCompiler(metadata, 0); - Supplier projectionSupplier = functionCompiler.compileProjection(add10Expression, Optional.empty()); + Supplier projectionSupplier = functionCompiler.compileProjection(SESSION.getSqlFunctionProperties(), add10Expression, Optional.empty()); PageProjection projection = projectionSupplier.get(); Page page = new Page(createLongSequenceBlock(1, 11)); ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, SPLIT_RUN_QUANTA); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/repartition/BenchmarkPartitionedOutputOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/repartition/BenchmarkPartitionedOutputOperator.java new file mode 100644 index 0000000000000..ff5531e09b913 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/repartition/BenchmarkPartitionedOutputOperator.java @@ -0,0 +1,383 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.facebook.presto.Session; +import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.execution.Lifespan; +import com.facebook.presto.execution.StateMachine; +import com.facebook.presto.execution.buffer.BufferState; +import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.execution.buffer.PagesSerdeFactory; +import com.facebook.presto.execution.buffer.PartitionedOutputBuffer; +import com.facebook.presto.execution.buffer.SerializedPage; +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.memory.context.SimpleLocalMemoryContext; +import com.facebook.presto.operator.BucketPartitionFunction; +import com.facebook.presto.operator.DriverContext; +import com.facebook.presto.operator.PageAssertions; +import com.facebook.presto.operator.PartitionFunction; +import com.facebook.presto.operator.PrecomputedHashGenerator; +import com.facebook.presto.operator.exchange.LocalPartitionGenerator; +import com.facebook.presto.operator.repartition.OptimizedPartitionedOutputOperator.OptimizedPartitionedOutputFactory; +import com.facebook.presto.operator.repartition.PartitionedOutputOperator.PartitionedOutputFactory; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.testing.TestingTaskContext; +import com.facebook.presto.type.TypeRegistry; +import com.google.common.collect.ImmutableList; +import io.airlift.units.DataSize; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.VerboseMode; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.IntStream; + +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.presto.block.BlockAssertions.createMapType; +import static com.facebook.presto.execution.buffer.BufferState.OPEN; +import static com.facebook.presto.execution.buffer.BufferState.TERMINAL_BUFFER_STATES; +import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.PARTITIONED; +import static com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers; +import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static com.facebook.presto.operator.PageAssertions.createDictionaryPageWithRandomData; +import static com.facebook.presto.operator.PageAssertions.createRlePageWithRandomData; +import static com.facebook.presto.operator.PageAssertions.updateBlockTypesWithHashBlockAndNullBlock; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DecimalType.createDecimalType; +import static com.facebook.presto.spi.type.Decimals.MAX_SHORT_PRECISION; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RowType.withDefaultFieldNames; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SystemPartitionFunction.HASH; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; +import static io.airlift.units.DataSize.Unit.BYTE; +import static io.airlift.units.DataSize.Unit.GIGABYTE; +import static io.airlift.units.DataSize.Unit.MEGABYTE; +import static java.util.Collections.nCopies; +import static java.util.concurrent.Executors.newCachedThreadPool; +import static java.util.concurrent.Executors.newScheduledThreadPool; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +@State(Scope.Thread) +@OutputTimeUnit(MILLISECONDS) +@Fork(3) +@Warmup(iterations = 10, time = 500, timeUnit = MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = MILLISECONDS) +@BenchmarkMode(Mode.AverageTime) +public class BenchmarkPartitionedOutputOperator +{ + @Benchmark + public void addPage(BenchmarkData data) + { + PartitionedOutputOperator operator = data.createPartitionedOutputOperator(); + for (int i = 0; i < data.pageCount; i++) { + operator.addInput(data.dataPage); + } + operator.finish(); + } + + @Benchmark + public void optimizedAddPage(BenchmarkData data) + { + OptimizedPartitionedOutputOperator operator = data.createOptimizedPartitionedOutputOperator(); + for (int i = 0; i < data.pageCount; i++) { + operator.addInput(data.dataPage); + } + operator.finish(); + } + + @State(Scope.Thread) + public static class BenchmarkData + { + private static final int PARTITION_COUNT = 256; + private static final int POSITION_COUNT = 8192; + private static final DataSize MAX_MEMORY = new DataSize(4, GIGABYTE); + private static final DataSize MAX_PARTITION_BUFFER_SIZE = new DataSize(256, MEGABYTE); + private static final ExecutorService EXECUTOR = newCachedThreadPool(daemonThreadsNamed("test-EXECUTOR-%s")); + private static final ScheduledExecutorService SCHEDULER = newScheduledThreadPool(1, daemonThreadsNamed("test-%s")); + + @SuppressWarnings("unused") + @Param({"true", "false"}) + private boolean enableCompression; + + @SuppressWarnings("unused") + @Param({"1", "2"}) + private int channelCount; + + @SuppressWarnings("unused") + @Param({ + "BIGINT", + "DICTIONARY(BIGINT)", + "RLE(BIGINT)", + "LONG_DECIMAL", + "INTEGER", + "SMALLINT", + "BOOLEAN", + "VARCHAR", + "ARRAY(BIGINT)", + "ARRAY(VARCHAR)", + "ARRAY(ARRAY(BIGINT))", + "MAP(BIGINT,BIGINT)", + "MAP(BIGINT,MAP(BIGINT,BIGINT))", + "ROW(BIGINT,BIGINT)", + "ROW(ARRAY(BIGINT),ARRAY(BIGINT))"}) + private String type; + + @SuppressWarnings("unused") + @Param({"true", "false"}) + private boolean hasNull; + + private List types; + private int pageCount; + private Page dataPage; + + @Setup + public void setup() + { + createPages(type); + } + + private void createPages(String inputType) + { + switch (inputType) { + case "BIGINT": + types = nCopies(channelCount, BIGINT); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 5000; + break; + case "DICTIONARY(BIGINT)": + types = nCopies(channelCount, BIGINT); + dataPage = createDictionaryPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 3000; + break; + case "RLE(BIGINT)": + types = nCopies(channelCount, BIGINT); + dataPage = createRlePageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 3000; + break; + case "LONG_DECIMAL": + types = nCopies(channelCount, createDecimalType(MAX_SHORT_PRECISION + 1)); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 5000; + break; + case "INTEGER": + types = nCopies(channelCount, INTEGER); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 5000; + break; + case "SMALLINT": + types = nCopies(channelCount, SMALLINT); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 5000; + break; + case "BOOLEAN": + types = nCopies(channelCount, BOOLEAN); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 5000; + break; + case "VARCHAR": + types = nCopies(channelCount, VARCHAR); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 5000; + break; + case "ARRAY(BIGINT)": + types = nCopies(channelCount, new ArrayType(BIGINT)); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 1000; + break; + case "ARRAY(VARCHAR)": + types = nCopies(channelCount, new ArrayType(VARCHAR)); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 1000; + break; + case "ARRAY(ARRAY(BIGINT))": + types = nCopies(channelCount, new ArrayType(new ArrayType(BIGINT))); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 1000; + break; + case "MAP(BIGINT,BIGINT)": + types = nCopies(channelCount, createMapType(BIGINT, BIGINT)); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 1000; + break; + case "MAP(BIGINT,MAP(BIGINT,BIGINT))": + types = nCopies(channelCount, createMapType(BIGINT, createMapType(BIGINT, BIGINT))); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 1000; + break; + case "ROW(BIGINT,BIGINT)": + types = nCopies(channelCount, withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT))); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 1000; + break; + case "ROW(ARRAY(BIGINT),ARRAY(BIGINT))": + types = nCopies(channelCount, withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), new ArrayType(BIGINT)))); + dataPage = PageAssertions.createPageWithRandomData(types, POSITION_COUNT, hasNull); + pageCount = 1000; + break; + + default: + throw new UnsupportedOperationException("Unsupported dataType"); + } + // We built the dataPage with added pre-computed hash block at channel 0, so types needs to be udpated + types = updateBlockTypesWithHashBlockAndNullBlock(types, true, false); + } + + private PartitionedOutputBuffer createPartitionedOutputBuffer() + { + OutputBuffers buffers = createInitialEmptyOutputBuffers(PARTITIONED); + for (int partition = 0; partition < PARTITION_COUNT; partition++) { + buffers = buffers.withBuffer(new OutputBuffers.OutputBufferId(partition), partition); + } + PartitionedOutputBuffer buffer = createPartitionedBuffer( + buffers.withNoMoreBufferIds(), + new DataSize(Long.MAX_VALUE, BYTE)); // don't let output buffer block + buffer.registerLifespanCompletionCallback(ignore -> {}); + + return buffer; + } + + private OptimizedPartitionedOutputOperator createOptimizedPartitionedOutputOperator() + { + PartitionFunction partitionFunction = new BucketPartitionFunction( + HASH.createBucketFunction(ImmutableList.of(BIGINT), true, PARTITION_COUNT), + IntStream.range(0, PARTITION_COUNT).toArray()); + + PagesSerdeFactory serdeFactory = new PagesSerdeFactory(new BlockEncodingManager(new TypeRegistry()), enableCompression); + PartitionedOutputBuffer buffer = createPartitionedOutputBuffer(); + + OptimizedPartitionedOutputFactory operatorFactory = new OptimizedPartitionedOutputFactory( + partitionFunction, + ImmutableList.of(0), + ImmutableList.of(Optional.empty()), + false, + OptionalInt.empty(), + buffer, + MAX_PARTITION_BUFFER_SIZE); + + return (OptimizedPartitionedOutputOperator) operatorFactory + .createOutputOperator(0, new PlanNodeId("plan-node-0"), types, Function.identity(), serdeFactory) + .createOperator(createDriverContext()); + } + + private PartitionedOutputOperator createPartitionedOutputOperator() + { + PartitionFunction partitionFunction = new LocalPartitionGenerator(new PrecomputedHashGenerator(0), PARTITION_COUNT); + PagesSerdeFactory serdeFactory = new PagesSerdeFactory(new BlockEncodingManager(new TypeRegistry()), enableCompression); + PartitionedOutputBuffer buffer = createPartitionedOutputBuffer(); + + PartitionedOutputFactory operatorFactory = new PartitionedOutputFactory( + partitionFunction, + ImmutableList.of(0), + ImmutableList.of(Optional.empty()), + false, + OptionalInt.empty(), + buffer, + MAX_PARTITION_BUFFER_SIZE); + + return (PartitionedOutputOperator) operatorFactory + .createOutputOperator(0, new PlanNodeId("plan-node-0"), types, Function.identity(), serdeFactory) + .createOperator(createDriverContext()); + } + + private DriverContext createDriverContext() + { + Session testSession = testSessionBuilder() + .setCatalog("tpch") + .setSchema(TINY_SCHEMA_NAME) + .build(); + + return TestingTaskContext.builder(EXECUTOR, SCHEDULER, testSession) + .setMemoryPoolSize(MAX_MEMORY) + .setQueryMaxTotalMemory(MAX_MEMORY) + .build() + .addPipelineContext(0, true, true, false) + .addDriverContext(); + } + + private TestingPartitionedOutputBuffer createPartitionedBuffer(OutputBuffers buffers, DataSize dataSize) + { + return new TestingPartitionedOutputBuffer( + "task-instance-id", + new StateMachine<>("bufferState", SCHEDULER, OPEN, TERMINAL_BUFFER_STATES), + buffers, + dataSize, + () -> new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), + SCHEDULER); + } + + private static class TestingPartitionedOutputBuffer + extends PartitionedOutputBuffer + { + public TestingPartitionedOutputBuffer( + String taskInstanceId, + StateMachine state, + OutputBuffers outputBuffers, + DataSize maxBufferSize, + Supplier systemMemoryContextSupplier, + Executor notificationExecutor) + { + super(taskInstanceId, state, outputBuffers, maxBufferSize, systemMemoryContextSupplier, notificationExecutor); + } + + // Use a dummy enqueue method to avoid OutOfMemory error + @Override + public void enqueue(Lifespan lifespan, int partitionNumber, List pages) + { + } + } + } + + public static void main(String[] args) + throws RunnerException + { + BenchmarkData data = new BenchmarkData(); + data.setup(); + new BenchmarkPartitionedOutputOperator().optimizedAddPage(data); + + Options options = new OptionsBuilder() + .verbosity(VerboseMode.NORMAL) + .jvmArgs("-Xmx10g") + .include(".*" + BenchmarkPartitionedOutputOperator.class.getSimpleName() + ".*") + .build(); + new Runner(options).run(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/repartition/TestBlockEncodingBuffers.java b/presto-main/src/test/java/com/facebook/presto/operator/repartition/TestBlockEncodingBuffers.java new file mode 100644 index 0000000000000..fab5535d1496a --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/repartition/TestBlockEncodingBuffers.java @@ -0,0 +1,696 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.facebook.presto.block.BlockAssertions.Encoding; +import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.operator.SimpleArrayAllocator; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockFlattener; +import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.DecimalType; +import com.facebook.presto.spi.type.MapType; +import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Closer; +import io.airlift.slice.DynamicSliceOutput; +import io.airlift.slice.SliceOutput; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.invoke.MethodHandle; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.IntStream; + +import static com.facebook.presto.block.BlockAssertions.Encoding.DICTIONARY; +import static com.facebook.presto.block.BlockAssertions.Encoding.RUN_LENGTH; +import static com.facebook.presto.block.BlockAssertions.assertBlockEquals; +import static com.facebook.presto.block.BlockAssertions.createAllNullsBlock; +import static com.facebook.presto.block.BlockAssertions.createMapType; +import static com.facebook.presto.block.BlockAssertions.createRandomBooleansBlock; +import static com.facebook.presto.block.BlockAssertions.createRandomDictionaryBlock; +import static com.facebook.presto.block.BlockAssertions.createRandomIntsBlock; +import static com.facebook.presto.block.BlockAssertions.createRandomLongDecimalsBlock; +import static com.facebook.presto.block.BlockAssertions.createRandomLongsBlock; +import static com.facebook.presto.block.BlockAssertions.createRandomShortDecimalsBlock; +import static com.facebook.presto.block.BlockAssertions.createRandomSmallintsBlock; +import static com.facebook.presto.block.BlockAssertions.createRandomStringBlock; +import static com.facebook.presto.block.BlockAssertions.createRleBlockWithRandomValue; +import static com.facebook.presto.block.BlockAssertions.wrapBlock; +import static com.facebook.presto.block.BlockSerdeUtil.readBlock; +import static com.facebook.presto.operator.repartition.AbstractBlockEncodingBuffer.createBlockEncodingBuffers; +import static com.facebook.presto.operator.repartition.OptimizedPartitionedOutputOperator.decodeBlock; +import static com.facebook.presto.spi.block.ArrayBlock.fromElementBlock; +import static com.facebook.presto.spi.block.MapBlock.fromKeyValueBlock; +import static com.facebook.presto.spi.block.MethodHandleUtil.compose; +import static com.facebook.presto.spi.block.MethodHandleUtil.nativeValueGetter; +import static com.facebook.presto.spi.block.RowBlock.fromFieldBlocks; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DecimalType.createDecimalType; +import static com.facebook.presto.spi.type.Decimals.MAX_SHORT_PRECISION; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RowType.withDefaultFieldNames; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.testing.TestingEnvironment.TYPE_MANAGER; +import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.Math.toIntExact; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static org.testng.Assert.assertEquals; + +public class TestBlockEncodingBuffers +{ + private static final int POSITIONS_PER_BLOCK = 1000; + + @Test + public void testBigint() + { + testBlock(BIGINT, createRandomLongsBlock(POSITIONS_PER_BLOCK, true)); + } + + @Test + public void testLongDecimal() + { + testBlock(createDecimalType(MAX_SHORT_PRECISION + 1), createRandomLongDecimalsBlock(POSITIONS_PER_BLOCK, true)); + } + + @Test + public void testInteger() + { + testBlock(INTEGER, createRandomIntsBlock(POSITIONS_PER_BLOCK, true)); + } + + @Test + public void testSmallint() + { + testBlock(SMALLINT, createRandomSmallintsBlock(POSITIONS_PER_BLOCK, true)); + } + + @Test + public void testBoolean() + { + testBlock(BOOLEAN, createRandomBooleansBlock(POSITIONS_PER_BLOCK, true)); + } + + @Test + public void testVarchar() + { + testBlock(VARCHAR, createRandomStringBlock(POSITIONS_PER_BLOCK, true, 10)); + testBlock(VARCHAR, createRandomStringBlock(POSITIONS_PER_BLOCK, true, 0)); + testBlock(VARCHAR, createRleBlockWithRandomValue(createRandomStringBlock(POSITIONS_PER_BLOCK, true, 0), POSITIONS_PER_BLOCK)); + } + + @Test + public void testArray() + { + testNestedBlock(new ArrayType(BIGINT)); + testNestedBlock(new ArrayType(createDecimalType(MAX_SHORT_PRECISION))); + testNestedBlock(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1))); + testNestedBlock(new ArrayType(INTEGER)); + testNestedBlock(new ArrayType(SMALLINT)); + testNestedBlock(new ArrayType(BOOLEAN)); + testNestedBlock(new ArrayType(VARCHAR)); + + testNestedBlock(new ArrayType(new ArrayType(BIGINT))); + testNestedBlock(new ArrayType(new ArrayType(createDecimalType(MAX_SHORT_PRECISION)))); + testNestedBlock(new ArrayType(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)))); + testNestedBlock(new ArrayType(new ArrayType(INTEGER))); + testNestedBlock(new ArrayType(new ArrayType(SMALLINT))); + testNestedBlock(new ArrayType(new ArrayType(BOOLEAN))); + testNestedBlock(new ArrayType(new ArrayType(VARCHAR))); + + testNestedBlock(new ArrayType(new ArrayType(new ArrayType(BIGINT)))); + testNestedBlock(new ArrayType(new ArrayType(new ArrayType(createDecimalType(MAX_SHORT_PRECISION))))); + testNestedBlock(new ArrayType(new ArrayType(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1))))); + testNestedBlock(new ArrayType(new ArrayType(new ArrayType(INTEGER)))); + testNestedBlock(new ArrayType(new ArrayType(new ArrayType(SMALLINT)))); + testNestedBlock(new ArrayType(new ArrayType(new ArrayType(BOOLEAN)))); + testNestedBlock(new ArrayType(new ArrayType(new ArrayType(VARCHAR)))); + } + + @Test + public void testMap() + { + testNestedBlock(createMapType(BIGINT, BIGINT)); + testNestedBlock(createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1))); + testNestedBlock(createMapType(INTEGER, INTEGER)); + testNestedBlock(createMapType(SMALLINT, SMALLINT)); + testNestedBlock(createMapType(BOOLEAN, BOOLEAN)); + testNestedBlock(createMapType(VARCHAR, VARCHAR)); + + testNestedBlock(createMapType(BIGINT, createMapType(BIGINT, BIGINT))); + testNestedBlock(createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1)))); + testNestedBlock(createMapType(INTEGER, createMapType(INTEGER, INTEGER))); + testNestedBlock(createMapType(SMALLINT, createMapType(SMALLINT, SMALLINT))); + testNestedBlock(createMapType(BOOLEAN, createMapType(BOOLEAN, BOOLEAN))); + testNestedBlock(createMapType(VARCHAR, createMapType(VARCHAR, VARCHAR))); + + testNestedBlock(createMapType(createMapType(createMapType(BIGINT, BIGINT), BIGINT), createMapType(BIGINT, BIGINT))); + testNestedBlock(createMapType( + createMapType(createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1)), createDecimalType(MAX_SHORT_PRECISION + 1)), + createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1)))); + testNestedBlock(createMapType(createMapType(createMapType(INTEGER, INTEGER), INTEGER), createMapType(INTEGER, INTEGER))); + testNestedBlock(createMapType(createMapType(createMapType(SMALLINT, SMALLINT), SMALLINT), createMapType(SMALLINT, SMALLINT))); + testNestedBlock(createMapType(createMapType(createMapType(BOOLEAN, BOOLEAN), BOOLEAN), createMapType(BOOLEAN, BOOLEAN))); + testNestedBlock(createMapType(createMapType(createMapType(VARCHAR, VARCHAR), VARCHAR), createMapType(VARCHAR, VARCHAR))); + + testNestedBlock(createMapType(BIGINT, new ArrayType(BIGINT))); + testNestedBlock(createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)))); + testNestedBlock(createMapType(INTEGER, new ArrayType(INTEGER))); + testNestedBlock(createMapType(SMALLINT, new ArrayType(SMALLINT))); + testNestedBlock(createMapType(BOOLEAN, new ArrayType(BOOLEAN))); + testNestedBlock(createMapType(VARCHAR, new ArrayType(VARCHAR))); + } + + @Test + public void testRow() + { + testNestedBlock(withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT, BIGINT))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1)))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER, INTEGER))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT, SMALLINT))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN, BOOLEAN))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR))); + + testNestedBlock(withDefaultFieldNames(ImmutableList.of(BIGINT, withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT))))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT))))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(INTEGER, withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER))))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(SMALLINT, withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT))))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(BOOLEAN, withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN, BOOLEAN))))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(VARCHAR, withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR))))); + + testNestedBlock(withDefaultFieldNames(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BIGINT)), BIGINT)), BIGINT))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1))), createDecimalType(MAX_SHORT_PRECISION))), createDecimalType(MAX_SHORT_PRECISION - 1)))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(INTEGER)), INTEGER)), INTEGER))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(SMALLINT)), SMALLINT)), SMALLINT))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BOOLEAN)), BOOLEAN)), BOOLEAN))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(VARCHAR)), VARCHAR)), VARCHAR))); + + testNestedBlock(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), new ArrayType(BIGINT)))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1))))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), new ArrayType(INTEGER)))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), new ArrayType(SMALLINT)))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), new ArrayType(BOOLEAN)))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), new ArrayType(VARCHAR)))); + + testNestedBlock(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), createMapType(BIGINT, BIGINT)))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1))))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), createMapType(INTEGER, INTEGER)))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), createMapType(SMALLINT, SMALLINT)))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), createMapType(BOOLEAN, BOOLEAN)))); + testNestedBlock(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), createMapType(VARCHAR, VARCHAR)))); + } + + private void testBlock(Type type, Block block) + { + requireNonNull(type); + + assertSerialized(type, createAllNullsBlock(type, POSITIONS_PER_BLOCK)); + assertSerialized(type, block); + + assertSerialized(type, wrapBlock(block, POSITIONS_PER_BLOCK, ImmutableList.of(DICTIONARY))); + assertSerialized(type, wrapBlock(block, POSITIONS_PER_BLOCK, ImmutableList.of(RUN_LENGTH))); + assertSerialized(type, wrapBlock(block, POSITIONS_PER_BLOCK, ImmutableList.of(DICTIONARY, DICTIONARY, RUN_LENGTH))); + assertSerialized(type, wrapBlock(block, POSITIONS_PER_BLOCK, ImmutableList.of(RUN_LENGTH, DICTIONARY, DICTIONARY))); + assertSerialized(type, wrapBlock(block, POSITIONS_PER_BLOCK, ImmutableList.of(DICTIONARY, RUN_LENGTH, DICTIONARY, RUN_LENGTH))); + assertSerialized(type, wrapBlock(block, POSITIONS_PER_BLOCK, ImmutableList.of(RUN_LENGTH, DICTIONARY, RUN_LENGTH, DICTIONARY))); + + assertSerialized(type, block.getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3)); + + assertSerialized(type, wrapBlock(block, POSITIONS_PER_BLOCK, ImmutableList.of(DICTIONARY)).getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3)); + assertSerialized(type, wrapBlock(block, POSITIONS_PER_BLOCK, ImmutableList.of(RUN_LENGTH)).getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3)); + assertSerialized(type, wrapBlock(block, POSITIONS_PER_BLOCK, ImmutableList.of(DICTIONARY, DICTIONARY, RUN_LENGTH)).getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3)); + assertSerialized(type, wrapBlock(block, POSITIONS_PER_BLOCK, ImmutableList.of(RUN_LENGTH, DICTIONARY, DICTIONARY)).getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3)); + assertSerialized(type, wrapBlock(block, POSITIONS_PER_BLOCK, ImmutableList.of(DICTIONARY, RUN_LENGTH, DICTIONARY, RUN_LENGTH)).getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3)); + assertSerialized(type, wrapBlock(block, POSITIONS_PER_BLOCK, ImmutableList.of(RUN_LENGTH, DICTIONARY, RUN_LENGTH, DICTIONARY)).getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3)); + + assertSerialized(type, wrapBlock(block.getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3), POSITIONS_PER_BLOCK, ImmutableList.of(DICTIONARY))); + assertSerialized(type, wrapBlock(block.getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3), POSITIONS_PER_BLOCK, ImmutableList.of(RUN_LENGTH))); + assertSerialized(type, wrapBlock(block.getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3), POSITIONS_PER_BLOCK, ImmutableList.of(DICTIONARY, DICTIONARY, RUN_LENGTH))); + assertSerialized(type, wrapBlock(block.getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3), POSITIONS_PER_BLOCK, ImmutableList.of(RUN_LENGTH, DICTIONARY, DICTIONARY))); + assertSerialized(type, wrapBlock(block.getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3), POSITIONS_PER_BLOCK, ImmutableList.of(DICTIONARY, RUN_LENGTH, DICTIONARY, RUN_LENGTH))); + assertSerialized(type, wrapBlock(block.getRegion(POSITIONS_PER_BLOCK / 2, POSITIONS_PER_BLOCK / 3), POSITIONS_PER_BLOCK, ImmutableList.of(RUN_LENGTH, DICTIONARY, RUN_LENGTH, DICTIONARY))); + } + + private void testNestedBlock(Type type) + { + requireNonNull(type); + + assertSerialized(type, createAllNullsBlock(type, POSITIONS_PER_BLOCK)); + + BlockStatus blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, false, ImmutableList.of()); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, false, ImmutableList.of(DICTIONARY)); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, false, ImmutableList.of(RUN_LENGTH)); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, false, ImmutableList.of(DICTIONARY, DICTIONARY, RUN_LENGTH)); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, false, ImmutableList.of(RUN_LENGTH, DICTIONARY, DICTIONARY)); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, false, ImmutableList.of(DICTIONARY, RUN_LENGTH, DICTIONARY, RUN_LENGTH)); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, false, ImmutableList.of(RUN_LENGTH, DICTIONARY, RUN_LENGTH, DICTIONARY)); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, true, ImmutableList.of()); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, true, ImmutableList.of(DICTIONARY)); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, true, ImmutableList.of(RUN_LENGTH)); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, true, ImmutableList.of(DICTIONARY, DICTIONARY, RUN_LENGTH)); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, true, ImmutableList.of(RUN_LENGTH, DICTIONARY, DICTIONARY)); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, true, ImmutableList.of(DICTIONARY, RUN_LENGTH, DICTIONARY, RUN_LENGTH)); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + + blockStatus = buildBlockStatusWithType(type, POSITIONS_PER_BLOCK, true, ImmutableList.of(RUN_LENGTH, DICTIONARY, RUN_LENGTH, DICTIONARY)); + assertSerialized(type, blockStatus.block, blockStatus.expectedRowSizes); + } + + private static void assertSerialized(Type type, Block block) + { + assertSerialized(type, block, null); + } + + private static void assertSerialized(Type type, Block block, int[] expectedRowSizes) + { + Closer blockLeaseCloser = Closer.create(); + + BlockFlattener flattener = new BlockFlattener(new SimpleArrayAllocator()); + DecodedBlockNode decodedBlock = decodeBlock(flattener, blockLeaseCloser, block); + + BlockEncodingBuffer buffers = createBlockEncodingBuffers(decodedBlock); + + int[] positions = IntStream.range(0, block.getPositionCount() / 2).toArray(); + copyPositions(decodedBlock, buffers, positions, expectedRowSizes); + + positions = IntStream.range(block.getPositionCount() / 2, block.getPositionCount()).toArray(); + copyPositions(decodedBlock, buffers, positions, expectedRowSizes); + + assertBlockEquals(type, serialize(buffers), block); + + buffers.resetBuffers(); + + positions = IntStream.range(0, block.getPositionCount()).filter(n -> n % 2 == 0).toArray(); + Block expectedBlock = block.copyPositions(positions, 0, positions.length); + copyPositions(decodedBlock, buffers, positions, expectedRowSizes); + + try { + blockLeaseCloser.close(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + + assertBlockEquals(type, serialize(buffers), expectedBlock); + } + + private static void copyPositions(DecodedBlockNode decodedBlock, BlockEncodingBuffer buffer, int[] positions, int[] expectedRowSizes) + { + buffer.setupDecodedBlocksAndPositions(decodedBlock, positions, positions.length); + + ((AbstractBlockEncodingBuffer) buffer).checkValidPositions(); + + if (expectedRowSizes != null) { + int[] actualSizes = new int[positions.length]; + buffer.accumulateSerializedRowSizes(actualSizes); + + int[] expectedSizes = Arrays.stream(positions).map(i -> expectedRowSizes[i]).toArray(); + assertEquals(actualSizes, expectedSizes); + } + + buffer.setNextBatch(0, positions.length); + buffer.appendDataInBatch(); + } + + private static Block serialize(BlockEncodingBuffer buffer) + { + SliceOutput output = new DynamicSliceOutput(toIntExact(buffer.getSerializedSizeInBytes())); + buffer.serializeTo(output); + + BlockEncodingManager blockEncodingSerde = new BlockEncodingManager(TYPE_MANAGER); + return readBlock(blockEncodingSerde, output.slice().getInput()); + } + + private BlockStatus buildBlockStatusWithType(Type type, int positionCount, boolean isView, List wrappings) + { + return buildBlockStatusWithType(type, positionCount, isView, true, wrappings); + } + + private BlockStatus buildBlockStatusWithType(Type type, int positionCount, boolean isView, boolean allowNulls, List wrappings) + { + BlockStatus blockStatus = null; + + if (isView) { + positionCount *= 2; + } + + if (type == BIGINT) { + blockStatus = buildBigintBlockStatus(positionCount, allowNulls); + } + else if (type instanceof DecimalType) { + if (!((DecimalType) type).isShort()) { + blockStatus = buildLongDecimalBlockStatus(positionCount, allowNulls); + } + else { + blockStatus = buildShortDecimalBlockStatus(positionCount, allowNulls); + } + } + else if (type == INTEGER) { + blockStatus = buildIntegerBlockStatus(positionCount, allowNulls); + } + else if (type == SMALLINT) { + blockStatus = buildSmallintBlockStatus(positionCount, allowNulls); + } + else if (type == BOOLEAN) { + blockStatus = buildBooleanBlockStatus(positionCount, allowNulls); + } + else if (type == VARCHAR) { + blockStatus = buildVarcharBlockStatus(positionCount, allowNulls, 10); + } + else { + // Nested types + // Build isNull and offsets of size positionCount + boolean[] isNull = new boolean[positionCount]; + int[] offsets = new int[positionCount + 1]; + for (int position = 0; position < positionCount; position++) { + if (allowNulls && position % 7 == 0) { + isNull[position] = true; + offsets[position + 1] = offsets[position]; + } + else { + offsets[position + 1] = offsets[position] + (type instanceof RowType ? 1 : ThreadLocalRandom.current().nextInt(10) + 1); + } + } + + // Build the nested block of size offsets[positionCount]. + if (type instanceof ArrayType) { + blockStatus = buildArrayBlockStatus((ArrayType) type, positionCount, isView, isNull, offsets, allowNulls, wrappings); + } + else if (type instanceof MapType) { + blockStatus = buildMapBlockStatus((MapType) type, positionCount, isView, isNull, offsets, allowNulls, wrappings); + } + else if (type instanceof RowType) { + blockStatus = buildRowBlockStatus((RowType) type, positionCount, isView, isNull, offsets, allowNulls, wrappings); + } + else { + throw new UnsupportedOperationException(format("type %s is not supported.", type)); + } + } + + if (isView) { + positionCount /= 2; + int offset = positionCount / 2; + Block blockView = blockStatus.block.getRegion(offset, positionCount); + int[] expectedRowSizesView = Arrays.stream(blockStatus.expectedRowSizes, offset, offset + positionCount).toArray(); + blockStatus = new BlockStatus(blockView, expectedRowSizesView); + } + + blockStatus = buildDictRleBlockStatus(blockStatus, positionCount, wrappings); + + return blockStatus; + } + + private static BlockStatus buildBigintBlockStatus(int positionCount, boolean allowNulls) + { + Block block = createRandomLongsBlock(positionCount, allowNulls); + int[] expectedRowSizes = IntStream.generate(() -> LongArrayBlockEncodingBuffer.POSITION_SIZE).limit(positionCount).toArray(); + + return new BlockStatus(block, expectedRowSizes); + } + + private static BlockStatus buildShortDecimalBlockStatus(int positionCount, boolean allowNulls) + { + Block block = createRandomShortDecimalsBlock(positionCount, allowNulls); + int[] expectedRowSizes = IntStream.generate(() -> LongArrayBlockEncodingBuffer.POSITION_SIZE).limit(positionCount).toArray(); + + return new BlockStatus(block, expectedRowSizes); + } + + private static BlockStatus buildLongDecimalBlockStatus(int positionCount, boolean allowNulls) + { + Block block = createRandomLongDecimalsBlock(positionCount, allowNulls); + int[] expectedRowSizes = IntStream.generate(() -> Int128ArrayBlockEncodingBuffer.POSITION_SIZE).limit(positionCount).toArray(); + + return new BlockStatus(block, expectedRowSizes); + } + + private BlockStatus buildIntegerBlockStatus(int positionCount, boolean allowNulls) + { + Block block = createRandomIntsBlock(positionCount, allowNulls); + int[] expectedRowSizes = IntStream.generate(() -> IntArrayBlockEncodingBuffer.POSITION_SIZE).limit(positionCount).toArray(); + + return new BlockStatus(block, expectedRowSizes); + } + + private BlockStatus buildSmallintBlockStatus(int positionCount, boolean allowNulls) + { + Block block = createRandomSmallintsBlock(positionCount, allowNulls); + int[] expectedRowSizes = IntStream.generate(() -> ShortArrayBlockEncodingBuffer.POSITION_SIZE).limit(positionCount).toArray(); + + return new BlockStatus(block, expectedRowSizes); + } + + private BlockStatus buildBooleanBlockStatus(int positionCount, boolean allowNulls) + { + Block block = createRandomBooleansBlock(positionCount, allowNulls); + int[] expectedRowSizes = IntStream.generate(() -> ByteArrayBlockEncodingBuffer.POSITION_SIZE).limit(positionCount).toArray(); + + return new BlockStatus(block, expectedRowSizes); + } + + private BlockStatus buildVarcharBlockStatus(int positionCount, boolean allowNulls, int maxStringLength) + { + Block block = createRandomStringBlock(positionCount, allowNulls, maxStringLength); + + int[] expectedRowSizes = IntStream + .range(0, positionCount) + .map(i -> block.getSliceLength(i) + VariableWidthBlockEncodingBuffer.POSITION_SIZE) + .toArray(); + + return new BlockStatus(block, expectedRowSizes); + } + + private BlockStatus buildDictRleBlockStatus(BlockStatus blockStatus, int positionCount, List wrappings) + { + checkArgument(blockStatus.block.getPositionCount() == positionCount); + + if (wrappings.isEmpty()) { + return blockStatus; + } + + BlockStatus wrappedBlockStatus = blockStatus; + for (int i = wrappings.size() - 1; i >= 0; i--) { + switch (wrappings.get(i)) { + case DICTIONARY: + wrappedBlockStatus = buildDictionaryBlockStatus(blockStatus, positionCount); + break; + case RUN_LENGTH: + wrappedBlockStatus = buildRleBlockStatus(blockStatus, positionCount); + break; + default: + throw new IllegalArgumentException(format("wrappings %s is incorrect", wrappings)); + } + } + return wrappedBlockStatus; + } + + private BlockStatus buildDictionaryBlockStatus(BlockStatus dictionary, int positionCount) + { + DictionaryBlock dictionaryBlock = createRandomDictionaryBlock(dictionary.block, positionCount); + int[] mappedExpectedRowSizes = IntStream.range(0, positionCount).map(i -> dictionary.expectedRowSizes[dictionaryBlock.getId(i)]).toArray(); + return new BlockStatus(dictionaryBlock, mappedExpectedRowSizes); + } + + private BlockStatus buildRleBlockStatus(BlockStatus blockStatus, int positionCount) + { + int[] expectedRowSizes = new int[positionCount]; + // When we contructed the Rle block, we chose the row at the middle. + Arrays.setAll(expectedRowSizes, i -> blockStatus.expectedRowSizes[blockStatus.block.getPositionCount() / 2]); + return new BlockStatus( + createRleBlockWithRandomValue(blockStatus.block, positionCount), + expectedRowSizes); + } + + private BlockStatus buildArrayBlockStatus( + ArrayType arrayType, + int positionCount, + boolean isView, + boolean[] isNull, + int[] offsets, + boolean allowNulls, + List wrappings) + { + requireNonNull(isNull); + requireNonNull(offsets); + + BlockStatus blockStatus; + + BlockStatus valuesBlockStatus = buildBlockStatusWithType( + arrayType.getElementType(), + offsets[positionCount], + isView, + allowNulls, + wrappings); + + int[] expectedRowSizes = IntStream.range(0, positionCount) + .map(i -> ArrayBlockEncodingBuffer.POSITION_SIZE + Arrays.stream(valuesBlockStatus.expectedRowSizes, offsets[i], offsets[i + 1]).sum()) + .toArray(); + + blockStatus = new BlockStatus( + fromElementBlock(positionCount, Optional.of(isNull), offsets, valuesBlockStatus.block), + expectedRowSizes); + return blockStatus; + } + + private BlockStatus buildMapBlockStatus( + MapType mapType, + int positionCount, + boolean isView, + boolean[] isNull, + int[] offsets, + boolean allowNulls, + List wrappings) + { + requireNonNull(isNull); + + BlockStatus blockStatus; + + BlockStatus keyBlockStatus = buildBlockStatusWithType( + mapType.getKeyType(), + offsets[positionCount], + isView, + false, + wrappings); + BlockStatus valueBlockStatus = buildBlockStatusWithType( + mapType.getValueType(), + offsets[positionCount], + isView, + allowNulls, + wrappings); + + int[] expectedKeySizes = keyBlockStatus.expectedRowSizes; + int[] expectedValueSizes = valueBlockStatus.expectedRowSizes; + // Use expectedKeySizes for the total size for both key and values + Arrays.setAll(expectedKeySizes, i -> expectedKeySizes[i] + expectedValueSizes[i]); + int[] expectedRowSizes = IntStream.range(0, positionCount) + .map(i -> MapBlockEncodingBuffer.POSITION_SIZE + Arrays.stream(expectedKeySizes, offsets[i], offsets[i + 1]).sum()) + .toArray(); + + Type keyType = mapType.getKeyType(); + MethodHandle keyNativeEquals = TYPE_MANAGER.resolveOperator(OperatorType.EQUAL, ImmutableList.of(keyType, keyType)); + MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); + MethodHandle keyNativeHashCode = TYPE_MANAGER.resolveOperator(OperatorType.HASH_CODE, ImmutableList.of(keyType)); + MethodHandle keyBlockHashCode = compose(keyNativeHashCode, nativeValueGetter(keyType)); + + blockStatus = new BlockStatus( + fromKeyValueBlock( + positionCount, + Optional.of(isNull), + offsets, + keyBlockStatus.block, + valueBlockStatus.block, + mapType, + keyBlockNativeEquals, + keyNativeHashCode, + keyBlockHashCode), + expectedRowSizes); + return blockStatus; + } + + private BlockStatus buildRowBlockStatus( + RowType rowType, + int positionCount, + boolean isView, + boolean[] isNull, + int[] offsets, + boolean allowNulls, + List wrappings) + { + requireNonNull(isNull); + + BlockStatus blockStatus; + int[] expectedTotalFieldSizes = new int[positionCount]; + + List fieldTypes = rowType.getTypeParameters(); + Block[] fieldBlocks = new Block[fieldTypes.size()]; + + for (int i = 0; i < fieldBlocks.length; i++) { + BlockStatus fieldBlockStatus = buildBlockStatusWithType( + fieldTypes.get(i), + positionCount, + isView, + allowNulls, + wrappings); + fieldBlocks[i] = fieldBlockStatus.block; + Arrays.setAll(expectedTotalFieldSizes, j -> expectedTotalFieldSizes[j] + fieldBlockStatus.expectedRowSizes[j]); + } + + int[] expectedRowSizes = IntStream.range(0, positionCount) + .map(i -> RowBlockEncodingBuffer.POSITION_SIZE + Arrays.stream(expectedTotalFieldSizes, offsets[i], offsets[i + 1]).sum()) + .toArray(); + + blockStatus = new BlockStatus(fromFieldBlocks(positionCount, Optional.of(isNull), fieldBlocks), expectedRowSizes); + return blockStatus; + } + + private static class BlockStatus + { + private final Block block; + private final int[] expectedRowSizes; + + BlockStatus(Block block, int[] expectedRowSizes) + { + this.block = requireNonNull(block, "block is null"); + this.expectedRowSizes = requireNonNull(expectedRowSizes, "expectedRowSizes is null"); + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/repartition/TestOptimizedPartitionedOutputOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/repartition/TestOptimizedPartitionedOutputOperator.java new file mode 100644 index 0000000000000..4530dc97d4a10 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/repartition/TestOptimizedPartitionedOutputOperator.java @@ -0,0 +1,862 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.repartition; + +import com.facebook.presto.Session; +import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.execution.Lifespan; +import com.facebook.presto.execution.StateMachine; +import com.facebook.presto.execution.buffer.BufferState; +import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.execution.buffer.PagesSerde; +import com.facebook.presto.execution.buffer.PagesSerdeFactory; +import com.facebook.presto.execution.buffer.PartitionedOutputBuffer; +import com.facebook.presto.execution.buffer.SerializedPage; +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.memory.context.SimpleLocalMemoryContext; +import com.facebook.presto.operator.DriverContext; +import com.facebook.presto.operator.HashGenerator; +import com.facebook.presto.operator.InterpretedHashGenerator; +import com.facebook.presto.operator.PageAssertions; +import com.facebook.presto.operator.PartitionFunction; +import com.facebook.presto.operator.PrecomputedHashGenerator; +import com.facebook.presto.operator.exchange.LocalPartitionGenerator; +import com.facebook.presto.operator.repartition.OptimizedPartitionedOutputOperator.OptimizedPartitionedOutputFactory; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.VariableWidthBlock; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.testing.TestingTaskContext; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import io.airlift.slice.DynamicSliceOutput; +import io.airlift.slice.Slice; +import io.airlift.units.DataSize; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Random; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.IntStream; + +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.presto.block.BlockAssertions.Encoding.DICTIONARY; +import static com.facebook.presto.block.BlockAssertions.Encoding.RUN_LENGTH; +import static com.facebook.presto.block.BlockAssertions.createMapType; +import static com.facebook.presto.block.BlockAssertions.createRandomLongsBlock; +import static com.facebook.presto.block.BlockAssertions.createRandomStringBlock; +import static com.facebook.presto.block.BlockAssertions.wrapBlock; +import static com.facebook.presto.execution.buffer.BufferState.OPEN; +import static com.facebook.presto.execution.buffer.BufferState.TERMINAL_BUFFER_STATES; +import static com.facebook.presto.execution.buffer.OutputBuffers.BufferType.PARTITIONED; +import static com.facebook.presto.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers; +import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static com.facebook.presto.operator.PageAssertions.assertPageEquals; +import static com.facebook.presto.operator.PageAssertions.mergePages; +import static com.facebook.presto.operator.PageAssertions.updateBlockTypesWithHashBlockAndNullBlock; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DecimalType.createDecimalType; +import static com.facebook.presto.spi.type.Decimals.MAX_SHORT_PRECISION; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RealType.REAL; +import static com.facebook.presto.spi.type.RowType.withDefaultFieldNames; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.testing.TestingEnvironment.TYPE_MANAGER; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; +import static io.airlift.units.DataSize.Unit.BYTE; +import static io.airlift.units.DataSize.Unit.GIGABYTE; +import static io.airlift.units.DataSize.Unit.KILOBYTE; +import static io.airlift.units.DataSize.Unit.MEGABYTE; +import static java.util.concurrent.Executors.newCachedThreadPool; +import static java.util.concurrent.Executors.newScheduledThreadPool; +import static org.testng.Assert.assertEquals; + +public class TestOptimizedPartitionedOutputOperator +{ + private static final ExecutorService EXECUTOR = newCachedThreadPool(daemonThreadsNamed("test-EXECUTOR-%s")); + private static final ScheduledExecutorService SCHEDULER = newScheduledThreadPool(1, daemonThreadsNamed("test-%s")); + private static final DataSize MAX_MEMORY = new DataSize(1, GIGABYTE); + private static final PagesSerde PAGES_SERDE = new PagesSerdeFactory(new BlockEncodingManager(TYPE_MANAGER), false).createPagesSerde(); //testingPagesSerde(); + + private static final int PARTITION_COUNT = 16; + private static final int PAGE_COUNT = 50; + private static final int POSITION_COUNT = 100; + + private static final Random RANDOM = new Random(0); + + @Test + public void testPartitionedSinglePagePrimitiveTypes() + { + testPartitionedSinglePage(ImmutableList.of(BIGINT)); + testPartitionedSinglePage(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1))); + testPartitionedSinglePage(ImmutableList.of(SMALLINT)); + testPartitionedSinglePage(ImmutableList.of(INTEGER)); + testPartitionedSinglePage(ImmutableList.of(REAL)); + testPartitionedSinglePage(ImmutableList.of(BOOLEAN)); + testPartitionedSinglePage(ImmutableList.of(VARCHAR)); + + testPartitionedSinglePage(ImmutableList.of(BIGINT, BIGINT)); + testPartitionedSinglePage(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION - 1))); + testPartitionedSinglePage(ImmutableList.of(SMALLINT, SMALLINT)); + testPartitionedSinglePage(ImmutableList.of(INTEGER, INTEGER)); + testPartitionedSinglePage(ImmutableList.of(REAL, REAL)); + testPartitionedSinglePage(ImmutableList.of(BOOLEAN, BOOLEAN)); + testPartitionedSinglePage(ImmutableList.of(VARCHAR, VARCHAR)); + } + + @Test + public void testPartitionedSinglePageForArray() + { + testPartitionedSinglePage(ImmutableList.of(new ArrayType(BIGINT), new ArrayType(BIGINT))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType((createDecimalType(MAX_SHORT_PRECISION + 1))))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(SMALLINT), new ArrayType(SMALLINT))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(INTEGER), new ArrayType(INTEGER))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(BOOLEAN), new ArrayType(BOOLEAN))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(VARCHAR), new ArrayType(VARCHAR))); + + testPartitionedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(BIGINT)))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1))))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(SMALLINT)))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(INTEGER)))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(BOOLEAN)))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(VARCHAR)))); + + testPartitionedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(BIGINT))))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(SMALLINT))))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(INTEGER))))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(BOOLEAN))))); + testPartitionedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(VARCHAR))))); + } + + @Test + public void testPartitionedSinglePageForMap() + { + testPartitionedSinglePage(ImmutableList.of(createMapType(BIGINT, BIGINT))); + testPartitionedSinglePage(ImmutableList.of(createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1)))); + testPartitionedSinglePage(ImmutableList.of(createMapType(SMALLINT, SMALLINT))); + testPartitionedSinglePage(ImmutableList.of(createMapType(INTEGER, INTEGER))); + testPartitionedSinglePage(ImmutableList.of(createMapType(BOOLEAN, BOOLEAN))); + testPartitionedSinglePage(ImmutableList.of(createMapType(VARCHAR, VARCHAR))); + + testPartitionedSinglePage(ImmutableList.of(createMapType(BIGINT, createMapType(BIGINT, BIGINT)))); + testPartitionedSinglePage(ImmutableList.of(createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1))))); + testPartitionedSinglePage(ImmutableList.of(createMapType(SMALLINT, createMapType(SMALLINT, SMALLINT)))); + testPartitionedSinglePage(ImmutableList.of(createMapType(INTEGER, createMapType(INTEGER, INTEGER)))); + testPartitionedSinglePage(ImmutableList.of(createMapType(BOOLEAN, createMapType(BOOLEAN, BOOLEAN)))); + testPartitionedSinglePage(ImmutableList.of(createMapType(VARCHAR, createMapType(VARCHAR, VARCHAR)))); + + testPartitionedSinglePage(ImmutableList.of(createMapType(createMapType(BIGINT, BIGINT), new ArrayType(createMapType(BIGINT, BIGINT))))); + testPartitionedSinglePage(ImmutableList.of(createMapType(createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType(createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testPartitionedSinglePage(ImmutableList.of(createMapType(createMapType(SMALLINT, SMALLINT), new ArrayType(createMapType(SMALLINT, SMALLINT))))); + testPartitionedSinglePage(ImmutableList.of(createMapType(createMapType(INTEGER, INTEGER), new ArrayType(createMapType(INTEGER, INTEGER))))); + testPartitionedSinglePage(ImmutableList.of(createMapType(createMapType(INTEGER, INTEGER), new ArrayType(createMapType(INTEGER, INTEGER))))); + testPartitionedSinglePage(ImmutableList.of(createMapType(createMapType(INTEGER, INTEGER), new ArrayType(createMapType(INTEGER, INTEGER))))); + } + + @Test + public void testPartitionedSinglePageForRow() + { + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT, BIGINT)))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER, INTEGER)))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT, SMALLINT)))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN, BOOLEAN)))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR)))); + + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BIGINT, withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT)))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT)))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(INTEGER, withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER)))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(SMALLINT, withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT)))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BOOLEAN, withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN, BOOLEAN)))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(VARCHAR, withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR)))))); + + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), new ArrayType(BIGINT))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), new ArrayType(INTEGER))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), new ArrayType(SMALLINT))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), new ArrayType(BOOLEAN))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), new ArrayType(VARCHAR))))); + + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), createMapType(BIGINT, BIGINT))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), createMapType(INTEGER, INTEGER))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), createMapType(SMALLINT, SMALLINT))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), createMapType(BOOLEAN, BOOLEAN))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), createMapType(VARCHAR, VARCHAR))))); + + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), createMapType(BIGINT, withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT))))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), createMapType(BIGINT, withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1)))))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), createMapType(INTEGER, withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER))))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), createMapType(SMALLINT, withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT))))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), createMapType(BOOLEAN, withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN))))))); + testPartitionedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), createMapType(VARCHAR, withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR))))))); + } + + @Test + public void testPartitionedMultiplePagesPrimitiveTypes() + { + testPartitionedMultiplePages(ImmutableList.of(BIGINT)); + testPartitionedMultiplePages(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1))); + testPartitionedMultiplePages(ImmutableList.of(SMALLINT)); + testPartitionedMultiplePages(ImmutableList.of(INTEGER)); + testPartitionedMultiplePages(ImmutableList.of(REAL)); + testPartitionedMultiplePages(ImmutableList.of(BOOLEAN)); + testPartitionedMultiplePages(ImmutableList.of(VARCHAR)); + + testPartitionedMultiplePages(ImmutableList.of(BIGINT, BIGINT)); + testPartitionedMultiplePages(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION - 1))); + testPartitionedMultiplePages(ImmutableList.of(SMALLINT, SMALLINT)); + testPartitionedMultiplePages(ImmutableList.of(INTEGER, INTEGER)); + testPartitionedMultiplePages(ImmutableList.of(REAL, REAL)); + testPartitionedMultiplePages(ImmutableList.of(BOOLEAN, BOOLEAN)); + testPartitionedMultiplePages(ImmutableList.of(VARCHAR, VARCHAR)); + } + + @Test + public void testPartitionedMultiplePagesForArray() + { + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(BIGINT), new ArrayType(BIGINT))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType((createDecimalType(MAX_SHORT_PRECISION + 1))))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(SMALLINT), new ArrayType(SMALLINT))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(INTEGER), new ArrayType(INTEGER))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(BOOLEAN), new ArrayType(BOOLEAN))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(VARCHAR), new ArrayType(VARCHAR))); + + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(BIGINT)))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1))))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(SMALLINT)))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(INTEGER)))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(BOOLEAN)))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(VARCHAR)))); + + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(BIGINT))))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(SMALLINT))))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(INTEGER))))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(BOOLEAN))))); + testPartitionedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(VARCHAR))))); + } + + @Test + public void testPartitionedMultiplePagesForMap() + { + testPartitionedMultiplePages(ImmutableList.of(createMapType(BIGINT, BIGINT))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1)))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(SMALLINT, SMALLINT))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(INTEGER, INTEGER))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(BOOLEAN, BOOLEAN))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(VARCHAR, VARCHAR))); + + testPartitionedMultiplePages(ImmutableList.of(createMapType(BIGINT, createMapType(BIGINT, BIGINT)))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1))))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(SMALLINT, createMapType(SMALLINT, SMALLINT)))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(INTEGER, createMapType(INTEGER, INTEGER)))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(BOOLEAN, createMapType(BOOLEAN, BOOLEAN)))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(VARCHAR, createMapType(VARCHAR, VARCHAR)))); + + testPartitionedMultiplePages(ImmutableList.of(createMapType(createMapType(BIGINT, BIGINT), new ArrayType(createMapType(BIGINT, BIGINT))))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType(createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(createMapType(SMALLINT, SMALLINT), new ArrayType(createMapType(SMALLINT, SMALLINT))))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(createMapType(INTEGER, INTEGER), new ArrayType(createMapType(INTEGER, INTEGER))))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(createMapType(INTEGER, INTEGER), new ArrayType(createMapType(INTEGER, INTEGER))))); + testPartitionedMultiplePages(ImmutableList.of(createMapType(createMapType(INTEGER, INTEGER), new ArrayType(createMapType(INTEGER, INTEGER))))); + } + + @Test + public void testPartitionedMultiplePagesForRow() + { + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT, BIGINT)))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER, INTEGER)))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT, SMALLINT)))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN, BOOLEAN)))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR)))); + + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BIGINT, withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT)))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT)))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(INTEGER, withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER)))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(SMALLINT, withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT)))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BOOLEAN, withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN, BOOLEAN)))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(VARCHAR, withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR)))))); + + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), new ArrayType(BIGINT))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), new ArrayType(INTEGER))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), new ArrayType(SMALLINT))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), new ArrayType(BOOLEAN))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), new ArrayType(VARCHAR))))); + + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), createMapType(BIGINT, BIGINT))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), createMapType(INTEGER, INTEGER))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), createMapType(SMALLINT, SMALLINT))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), createMapType(BOOLEAN, BOOLEAN))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), createMapType(VARCHAR, VARCHAR))))); + + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), createMapType(BIGINT, withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT))))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), createMapType(BIGINT, withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1)))))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), createMapType(INTEGER, withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER))))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), createMapType(SMALLINT, withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT))))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), createMapType(BOOLEAN, withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN))))))); + testPartitionedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), createMapType(VARCHAR, withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR))))))); + } + + @Test + public void testReplicatedSinglePagePrimitiveTypes() + { + testReplicatedSinglePage(ImmutableList.of(BIGINT)); + testReplicatedSinglePage(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1))); + testReplicatedSinglePage(ImmutableList.of(SMALLINT)); + testReplicatedSinglePage(ImmutableList.of(INTEGER)); + testReplicatedSinglePage(ImmutableList.of(BOOLEAN)); + testReplicatedSinglePage(ImmutableList.of(VARCHAR)); + + testReplicatedSinglePage(ImmutableList.of(BIGINT, BIGINT)); + testReplicatedSinglePage(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION - 1))); + testReplicatedSinglePage(ImmutableList.of(SMALLINT, SMALLINT)); + testReplicatedSinglePage(ImmutableList.of(INTEGER, INTEGER)); + testReplicatedSinglePage(ImmutableList.of(BOOLEAN, BOOLEAN)); + testReplicatedSinglePage(ImmutableList.of(VARCHAR, VARCHAR)); + } + + @Test + public void testReplicatedSinglePageForArray() + { + testReplicatedSinglePage(ImmutableList.of(new ArrayType(BIGINT), new ArrayType(BIGINT))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType((createDecimalType(MAX_SHORT_PRECISION + 1))))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(SMALLINT), new ArrayType(SMALLINT))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(INTEGER), new ArrayType(INTEGER))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(BOOLEAN), new ArrayType(BOOLEAN))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(VARCHAR), new ArrayType(VARCHAR))); + + testReplicatedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(BIGINT)))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1))))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(SMALLINT)))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(INTEGER)))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(BOOLEAN)))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(VARCHAR)))); + + testReplicatedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(BIGINT))))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(SMALLINT))))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(INTEGER))))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(BOOLEAN))))); + testReplicatedSinglePage(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(VARCHAR))))); + } + + @Test + public void testReplicatedSinglePageForMap() + { + testReplicatedSinglePage(ImmutableList.of(createMapType(BIGINT, BIGINT))); + testReplicatedSinglePage(ImmutableList.of(createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1)))); + testReplicatedSinglePage(ImmutableList.of(createMapType(SMALLINT, SMALLINT))); + testReplicatedSinglePage(ImmutableList.of(createMapType(INTEGER, INTEGER))); + testReplicatedSinglePage(ImmutableList.of(createMapType(BOOLEAN, BOOLEAN))); + testReplicatedSinglePage(ImmutableList.of(createMapType(VARCHAR, VARCHAR))); + + testReplicatedSinglePage(ImmutableList.of(createMapType(BIGINT, createMapType(BIGINT, BIGINT)))); + testReplicatedSinglePage(ImmutableList.of(createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1))))); + testReplicatedSinglePage(ImmutableList.of(createMapType(SMALLINT, createMapType(SMALLINT, SMALLINT)))); + testReplicatedSinglePage(ImmutableList.of(createMapType(INTEGER, createMapType(INTEGER, INTEGER)))); + testReplicatedSinglePage(ImmutableList.of(createMapType(BOOLEAN, createMapType(BOOLEAN, BOOLEAN)))); + testReplicatedSinglePage(ImmutableList.of(createMapType(VARCHAR, createMapType(VARCHAR, VARCHAR)))); + + testReplicatedSinglePage(ImmutableList.of(createMapType(createMapType(BIGINT, BIGINT), new ArrayType(createMapType(BIGINT, BIGINT))))); + testReplicatedSinglePage(ImmutableList.of(createMapType(createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType(createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testReplicatedSinglePage(ImmutableList.of(createMapType(createMapType(SMALLINT, SMALLINT), new ArrayType(createMapType(SMALLINT, SMALLINT))))); + testReplicatedSinglePage(ImmutableList.of(createMapType(createMapType(INTEGER, INTEGER), new ArrayType(createMapType(INTEGER, INTEGER))))); + testReplicatedSinglePage(ImmutableList.of(createMapType(createMapType(INTEGER, INTEGER), new ArrayType(createMapType(INTEGER, INTEGER))))); + testReplicatedSinglePage(ImmutableList.of(createMapType(createMapType(INTEGER, INTEGER), new ArrayType(createMapType(INTEGER, INTEGER))))); + } + + @Test + public void testReplicatedSinglePageForRow() + { + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT, BIGINT)))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER, INTEGER)))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT, SMALLINT)))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN, BOOLEAN)))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR)))); + + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BIGINT, withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT)))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT)))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(INTEGER, withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER)))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(SMALLINT, withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT)))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BOOLEAN, withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN, BOOLEAN)))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(VARCHAR, withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR)))))); + + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), new ArrayType(BIGINT))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), new ArrayType(INTEGER))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), new ArrayType(SMALLINT))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), new ArrayType(BOOLEAN))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), new ArrayType(VARCHAR))))); + + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), createMapType(BIGINT, BIGINT))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), createMapType(INTEGER, INTEGER))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), createMapType(SMALLINT, SMALLINT))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), createMapType(BOOLEAN, BOOLEAN))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), createMapType(VARCHAR, VARCHAR))))); + + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), createMapType(BIGINT, withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT))))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), createMapType(BIGINT, withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1)))))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), createMapType(INTEGER, withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER))))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), createMapType(SMALLINT, withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT))))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), createMapType(BOOLEAN, withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN))))))); + testReplicatedSinglePage(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), createMapType(VARCHAR, withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR))))))); + } + + @Test + public void testReplicatedMultiplePagesPrimitiveTypes() + { + testReplicatedMultiplePages(ImmutableList.of(BIGINT)); + testReplicatedMultiplePages(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1))); + testReplicatedMultiplePages(ImmutableList.of(SMALLINT)); + testReplicatedMultiplePages(ImmutableList.of(INTEGER)); + testReplicatedMultiplePages(ImmutableList.of(BOOLEAN)); + testReplicatedMultiplePages(ImmutableList.of(VARCHAR)); + + testReplicatedMultiplePages(ImmutableList.of(BIGINT, BIGINT)); + testReplicatedMultiplePages(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION - 1))); + testReplicatedMultiplePages(ImmutableList.of(SMALLINT, SMALLINT)); + testReplicatedMultiplePages(ImmutableList.of(INTEGER, INTEGER)); + testReplicatedMultiplePages(ImmutableList.of(BOOLEAN, BOOLEAN)); + testReplicatedMultiplePages(ImmutableList.of(VARCHAR, VARCHAR)); + } + + @Test + public void testReplicatedMultiplePagesForArray() + { + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(BIGINT), new ArrayType(BIGINT))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType((createDecimalType(MAX_SHORT_PRECISION + 1))))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(SMALLINT), new ArrayType(SMALLINT))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(INTEGER), new ArrayType(INTEGER))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(BOOLEAN), new ArrayType(BOOLEAN))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(VARCHAR), new ArrayType(VARCHAR))); + + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(BIGINT)))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1))))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(SMALLINT)))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(INTEGER)))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(BOOLEAN)))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(VARCHAR)))); + + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(BIGINT))))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(SMALLINT))))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(INTEGER))))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(BOOLEAN))))); + testReplicatedMultiplePages(ImmutableList.of(new ArrayType(new ArrayType(new ArrayType(VARCHAR))))); + } + + @Test + public void testReplicatedMultiplePagesForMap() + { + testReplicatedMultiplePages(ImmutableList.of(createMapType(BIGINT, BIGINT))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1)))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(SMALLINT, SMALLINT))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(INTEGER, INTEGER))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(BOOLEAN, BOOLEAN))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(VARCHAR, VARCHAR))); + + testReplicatedMultiplePages(ImmutableList.of(createMapType(BIGINT, createMapType(BIGINT, BIGINT)))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(createDecimalType(MAX_SHORT_PRECISION + 1), createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1))))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(SMALLINT, createMapType(SMALLINT, SMALLINT)))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(INTEGER, createMapType(INTEGER, INTEGER)))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(BOOLEAN, createMapType(BOOLEAN, BOOLEAN)))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(VARCHAR, createMapType(VARCHAR, VARCHAR)))); + + testReplicatedMultiplePages(ImmutableList.of(createMapType(createMapType(BIGINT, BIGINT), new ArrayType(createMapType(BIGINT, BIGINT))))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType(createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(createMapType(SMALLINT, SMALLINT), new ArrayType(createMapType(SMALLINT, SMALLINT))))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(createMapType(INTEGER, INTEGER), new ArrayType(createMapType(INTEGER, INTEGER))))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(createMapType(INTEGER, INTEGER), new ArrayType(createMapType(INTEGER, INTEGER))))); + testReplicatedMultiplePages(ImmutableList.of(createMapType(createMapType(INTEGER, INTEGER), new ArrayType(createMapType(INTEGER, INTEGER))))); + } + + @Test + public void testReplicatedMultiplePagesForRow() + { + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT, BIGINT)))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER, INTEGER)))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT, SMALLINT)))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN, BOOLEAN)))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR)))); + + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BIGINT, withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT)))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT)))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(INTEGER, withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER)))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(SMALLINT, withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT)))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(BOOLEAN, withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN, BOOLEAN)))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(VARCHAR, withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR)))))); + + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), new ArrayType(BIGINT))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), new ArrayType(INTEGER))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), new ArrayType(SMALLINT))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), new ArrayType(BOOLEAN))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), new ArrayType(VARCHAR))))); + + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), createMapType(BIGINT, BIGINT))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), createMapType(BIGINT, createDecimalType(MAX_SHORT_PRECISION + 1)))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), createMapType(INTEGER, INTEGER))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), createMapType(SMALLINT, SMALLINT))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), createMapType(BOOLEAN, BOOLEAN))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), createMapType(VARCHAR, VARCHAR))))); + + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BIGINT), createMapType(BIGINT, withDefaultFieldNames(ImmutableList.of(BIGINT, BIGINT))))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(createDecimalType(MAX_SHORT_PRECISION + 1)), createMapType(BIGINT, withDefaultFieldNames(ImmutableList.of(createDecimalType(MAX_SHORT_PRECISION + 1), createDecimalType(MAX_SHORT_PRECISION + 1)))))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(INTEGER), createMapType(INTEGER, withDefaultFieldNames(ImmutableList.of(INTEGER, INTEGER))))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(SMALLINT), createMapType(SMALLINT, withDefaultFieldNames(ImmutableList.of(SMALLINT, SMALLINT))))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(BOOLEAN), createMapType(BOOLEAN, withDefaultFieldNames(ImmutableList.of(BOOLEAN, BOOLEAN))))))); + testReplicatedMultiplePages(ImmutableList.of(withDefaultFieldNames(ImmutableList.of(new ArrayType(VARCHAR), createMapType(VARCHAR, withDefaultFieldNames(ImmutableList.of(VARCHAR, VARCHAR))))))); + } + + @Test + public void testEmptyPage() + { + List types = updateBlockTypesWithHashBlockAndNullBlock(ImmutableList.of(BIGINT), true, false); + Page page = PageAssertions.createPageWithRandomData(ImmutableList.of(BIGINT), 0, true, false, true, false, ImmutableList.of()); + + testPartitioned(types, ImmutableList.of(page), new DataSize(128, MEGABYTE)); + } + + @Test + public void testPageWithNoBlocks() + { + List types = updateBlockTypesWithHashBlockAndNullBlock(ImmutableList.of(), false, false); + Page page = PageAssertions.createPageWithRandomData(ImmutableList.of(), 1, false, false, true, false, ImmutableList.of()); + + testPartitionedForZeroBlocks(types, ImmutableList.of(page), new DataSize(128, MEGABYTE)); + } + + /** + * Test BlockFlattener's allocator does not return borrowed arrays among blocks + */ + @Test + public void testPageWithBlocksOfDifferentPositionCounts() + { + Block[] blocks = new Block[4]; + + // PreComputed Hash Block + blocks[0] = createRandomLongsBlock(POSITION_COUNT, false); + + // Create blocks whose base blocks are with increasing number of positions + blocks[1] = wrapBlock(createRandomStringBlock(10, true, 10), POSITION_COUNT, ImmutableList.of(DICTIONARY, DICTIONARY)); + blocks[2] = wrapBlock(createRandomStringBlock(100, true, 10), POSITION_COUNT, ImmutableList.of(DICTIONARY, DICTIONARY)); + blocks[3] = wrapBlock(createRandomStringBlock(1000, true, 10), POSITION_COUNT, ImmutableList.of(DICTIONARY, DICTIONARY)); + + Page page = new Page(blocks); + + List types = ImmutableList.of(BIGINT, VARCHAR, VARCHAR, VARCHAR); + testPartitioned(types, ImmutableList.of(page), new DataSize(128, MEGABYTE)); + testPartitioned(types, ImmutableList.of(page), new DataSize(1, KILOBYTE)); + } + + @Test + public void testPageWithVariableWidthBlocksOfSliceViews() + { + Block[] blocks = new Block[2]; + + // PreComputed Hash Block + blocks[0] = createRandomLongsBlock(POSITION_COUNT, false); + + // Create blocks whose base blocks are with increasing number of positions + blocks[1] = createVariableWidthBlockOverSliceView(POSITION_COUNT); + + Page page = new Page(blocks); + + List types = ImmutableList.of(BIGINT, VARCHAR); + + testPartitioned(types, ImmutableList.of(page), new DataSize(128, MEGABYTE)); + testPartitioned(types, ImmutableList.of(page), new DataSize(1, KILOBYTE)); + } + + private void testPartitionedSinglePage(List targetTypes) + { + List types = updateBlockTypesWithHashBlockAndNullBlock(targetTypes, true, false); + + // Test plain blocks: no block views, no Dicrtionary/RLE blocks + Page page = PageAssertions.createPageWithRandomData(targetTypes, POSITION_COUNT, true, false, true, false, ImmutableList.of()); + + // First test for the cases where the buffer can hold the whole page, then force flushing for every a few rows. + testPartitioned(types, ImmutableList.of(page), new DataSize(128, MEGABYTE)); + testPartitioned(types, ImmutableList.of(page), new DataSize(1, KILOBYTE)); + + // Test block views and Dicrtionary/RLE blocks + page = PageAssertions.createPageWithRandomData(targetTypes, POSITION_COUNT, true, false, true, true, ImmutableList.of(DICTIONARY, RUN_LENGTH, DICTIONARY, RUN_LENGTH)); + testPartitioned(types, ImmutableList.of(page), new DataSize(128, MEGABYTE)); + testPartitioned(types, ImmutableList.of(page), new DataSize(1, KILOBYTE)); + + page = PageAssertions.createPageWithRandomData(targetTypes, POSITION_COUNT, true, false, true, true, ImmutableList.of(RUN_LENGTH, DICTIONARY, RUN_LENGTH, DICTIONARY)); + testPartitioned(types, ImmutableList.of(page), new DataSize(128, MEGABYTE)); + testPartitioned(types, ImmutableList.of(page), new DataSize(1, KILOBYTE)); + } + + private void testPartitionedMultiplePages(List targetTypes) + { + List types = updateBlockTypesWithHashBlockAndNullBlock(targetTypes, true, false); + List pages = new ArrayList<>(); + for (int i = 0; i < PAGE_COUNT; i++) { + pages.add(PageAssertions.createPageWithRandomData(targetTypes, POSITION_COUNT + RANDOM.nextInt(POSITION_COUNT), true, false, true, false, ImmutableList.of())); + pages.add(PageAssertions.createPageWithRandomData(targetTypes, POSITION_COUNT + RANDOM.nextInt(POSITION_COUNT), true, false, true, true, ImmutableList.of(DICTIONARY, DICTIONARY, RUN_LENGTH))); + pages.add(PageAssertions.createPageWithRandomData(targetTypes, POSITION_COUNT + RANDOM.nextInt(POSITION_COUNT), true, false, true, true, ImmutableList.of(RUN_LENGTH, DICTIONARY, DICTIONARY))); + } + + testPartitioned(types, pages, new DataSize(128, MEGABYTE)); + testPartitioned(types, pages, new DataSize(1, KILOBYTE)); + + pages.clear(); + for (int i = 0; i < PAGE_COUNT / 3; i++) { + pages.add(PageAssertions.createPageWithRandomData(targetTypes, POSITION_COUNT + RANDOM.nextInt(POSITION_COUNT), true, false, true, false, ImmutableList.of())); + pages.add(PageAssertions.createPageWithRandomData(targetTypes, POSITION_COUNT + RANDOM.nextInt(POSITION_COUNT), true, false, true, true, ImmutableList.of(DICTIONARY, DICTIONARY, RUN_LENGTH))); + pages.add(PageAssertions.createPageWithRandomData(targetTypes, POSITION_COUNT + RANDOM.nextInt(POSITION_COUNT), true, false, true, true, ImmutableList.of(RUN_LENGTH, DICTIONARY, DICTIONARY))); + } + + testPartitioned(types, pages, new DataSize(128, MEGABYTE)); + testPartitioned(types, pages, new DataSize(1, KILOBYTE)); + } + + private void testReplicatedSinglePage(List targetTypes) + { + // Add a block that only contain null as the last block to force replicating all rows. + List types = updateBlockTypesWithHashBlockAndNullBlock(targetTypes, true, true); + Page page = PageAssertions.createPageWithRandomData(targetTypes, POSITION_COUNT, true, true, true, false, ImmutableList.of()); + testReplicated(types, ImmutableList.of(page), new DataSize(128, MEGABYTE)); + testReplicated(types, ImmutableList.of(page), new DataSize(1, KILOBYTE)); + } + + private void testReplicatedMultiplePages(List targetTypes) + { + List types = updateBlockTypesWithHashBlockAndNullBlock(targetTypes, true, true); + List pages = new ArrayList<>(); + for (int i = 0; i < PAGE_COUNT; i++) { + pages.add(PageAssertions.createPageWithRandomData(targetTypes, POSITION_COUNT + RANDOM.nextInt(POSITION_COUNT), true, true, true, false, ImmutableList.of())); + } + + testReplicated(types, pages, new DataSize(128, MEGABYTE)); + testReplicated(types, pages, new DataSize(1, KILOBYTE)); + } + + private void testPartitionedForZeroBlocks(List types, List pages, DataSize maxMemory) + { + testPartitioned(types, pages, maxMemory, ImmutableList.of(), new InterpretedHashGenerator(ImmutableList.of(), new int[0])); + } + + private void testPartitioned(List types, List pages, DataSize maxMemory) + { + testPartitioned(types, pages, maxMemory, ImmutableList.of(0), new PrecomputedHashGenerator(0)); + } + + private void testPartitioned(List types, List pages, DataSize maxMemory, List partitionChannel, HashGenerator hashGenerator) + { + TestingPartitionedOutputBuffer outputBuffer = createPartitionedOutputBuffer(); + PartitionFunction partitionFunction = new LocalPartitionGenerator(hashGenerator, PARTITION_COUNT); + OptimizedPartitionedOutputOperator operator = createOptimizedPartitionedOutputOperator( + types, + partitionChannel, + partitionFunction, + outputBuffer, + OptionalInt.empty(), + maxMemory); + + Map> expectedPageList = new HashMap<>(); + + for (Page page : pages) { + Map> positionsByPartition = new HashMap<>(); + for (int i = 0; i < page.getPositionCount(); i++) { + int partitionNumber = partitionFunction.getPartition(page, i); + positionsByPartition.computeIfAbsent(partitionNumber, k -> new ArrayList<>()).add(i); + } + + for (Map.Entry> entry : positionsByPartition.entrySet()) { + if (!entry.getValue().isEmpty()) { + expectedPageList.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(copyPositions(page, entry.getValue())); + } + } + + operator.addInput(page); + } + operator.finish(); + + Map expectedPages = Maps.transformValues(expectedPageList, outputPages -> mergePages(types, outputPages)); + Map actualPages = Maps.transformValues(outputBuffer.getPages(), outputPages -> mergePages(types, outputPages)); + + assertEquals(actualPages.size(), expectedPages.size()); + + assertEquals(actualPages.keySet(), expectedPages.keySet()); + + for (Map.Entry entry : expectedPages.entrySet()) { + int key = entry.getKey(); + assertPageEquals(types, actualPages.get(key), entry.getValue()); + } + } + + private void testReplicated(List types, List pages, DataSize maxMemory) + { + TestingPartitionedOutputBuffer outputBuffer = createPartitionedOutputBuffer(); + PartitionFunction partitionFunction = new LocalPartitionGenerator(new PrecomputedHashGenerator(0), PARTITION_COUNT); + OptimizedPartitionedOutputOperator operator = createOptimizedPartitionedOutputOperator( + types, + ImmutableList.of(0), + partitionFunction, + outputBuffer, + OptionalInt.of(types.size() - 1), maxMemory); + + for (Page page : pages) { + operator.addInput(page); + } + operator.finish(); + + Map> acutualPageLists = outputBuffer.getPages(); + + assertEquals(acutualPageLists.size(), PARTITION_COUNT); + + Page expectedPage = mergePages(types, pages); + + acutualPageLists.values().forEach(pageList -> assertPageEquals(types, mergePages(types, pageList), expectedPage)); + } + + private Page copyPositions(Page page, List positions) + { + Block[] blocks = new Block[page.getChannelCount()]; + for (int i = 0; i < blocks.length; i++) { + blocks[i] = page.getBlock(i).copyPositions(positions.stream().mapToInt(j -> j).toArray(), 0, positions.size()); + } + return new Page(positions.size(), blocks); + } + + private TestingPartitionedOutputBuffer createPartitionedOutputBuffer() + { + OutputBuffers buffers = createInitialEmptyOutputBuffers(PARTITIONED); + for (int partition = 0; partition < PARTITION_COUNT; partition++) { + buffers = buffers.withBuffer(new OutputBuffers.OutputBufferId(partition), partition); + } + TestingPartitionedOutputBuffer buffer = createPartitionedBuffer( + buffers.withNoMoreBufferIds(), + new DataSize(Long.MAX_VALUE, BYTE)); // don't let output buffer block + buffer.registerLifespanCompletionCallback(ignore -> {}); + + return buffer; + } + + private OptimizedPartitionedOutputOperator createOptimizedPartitionedOutputOperator( + List types, + List partitionChannel, + PartitionFunction partitionFunction, + PartitionedOutputBuffer buffer, + OptionalInt nullChannel, + DataSize maxMemory) + { + PagesSerdeFactory serdeFactory = new PagesSerdeFactory(new BlockEncodingManager(TYPE_MANAGER), false); + + OptimizedPartitionedOutputFactory operatorFactory = new OptimizedPartitionedOutputFactory( + partitionFunction, + partitionChannel, + ImmutableList.of(Optional.empty()), + false, + nullChannel, + buffer, + maxMemory); + + return (OptimizedPartitionedOutputOperator) operatorFactory + .createOutputOperator(0, new PlanNodeId("plan-node-0"), types, Function.identity(), serdeFactory) + .createOperator(createDriverContext()); + } + + private DriverContext createDriverContext() + { + Session testSession = testSessionBuilder() + .setCatalog("tpch") + .setSchema(TINY_SCHEMA_NAME) + .build(); + + return TestingTaskContext.builder(EXECUTOR, SCHEDULER, testSession) + .setMemoryPoolSize(MAX_MEMORY) + .build() + .addPipelineContext(0, true, true, false) + .addDriverContext(); + } + + private TestingPartitionedOutputBuffer createPartitionedBuffer(OutputBuffers buffers, DataSize dataSize) + { + return new TestingPartitionedOutputBuffer( + "task-instance-id", + new StateMachine<>("bufferState", SCHEDULER, OPEN, TERMINAL_BUFFER_STATES), + buffers, + dataSize, + () -> new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), + SCHEDULER); + } + + private static Block createVariableWidthBlockOverSliceView(int entries) + { + // Create a slice view whose address starts in the middle of the original slice, and length is half of original slice + DynamicSliceOutput dynamicSliceOutput = new DynamicSliceOutput(entries * 2); + for (int i = 0; i < entries * 2; i++) { + dynamicSliceOutput.writeByte(i); + } + Slice slice = dynamicSliceOutput.slice().slice(entries, entries); + + int[] offsets = IntStream.range(0, entries + 1).toArray(); + return new VariableWidthBlock(entries, slice, offsets, Optional.empty()); + } + + private static class TestingPartitionedOutputBuffer + extends PartitionedOutputBuffer + { + private final Map> pages = new HashMap<>(); + + public TestingPartitionedOutputBuffer( + String taskInstanceId, + StateMachine state, + OutputBuffers outputBuffers, + DataSize maxBufferSize, + Supplier systemMemoryContextSupplier, + Executor notificationExecutor) + { + super(taskInstanceId, state, outputBuffers, maxBufferSize, systemMemoryContextSupplier, notificationExecutor); + } + + @Override + public void enqueue(Lifespan lifespan, int partitionNumber, List pages) + { + this.pages.computeIfAbsent(partitionNumber, k -> new ArrayList<>()); + pages.stream().map(PAGES_SERDE::deserialize).forEach(this.pages.get(partitionNumber)::add); + } + + public Map> getPages() + { + return pages; + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java index e9950069dd0bc..ccaa6ba6a1934 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java @@ -14,9 +14,9 @@ package com.facebook.presto.operator.scalar; import com.facebook.presto.Session; +import com.facebook.presto.metadata.BuiltInFunction; import com.facebook.presto.metadata.FunctionListBuilder; import com.facebook.presto.metadata.Metadata; -import com.facebook.presto.metadata.SqlFunction; import com.facebook.presto.metadata.SqlScalarFunction; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.Plugin; @@ -39,13 +39,12 @@ import java.util.List; import java.util.Map; +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.FunctionExtractor.extractFunctions; -import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.type.DecimalType.createDecimalType; -import static io.airlift.testing.Closeables.closeAllRuntimeException; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static org.testng.Assert.assertEquals; @@ -98,7 +97,7 @@ protected void assertFunction(String projection, Type expectedType, Object expec protected void assertOperator(OperatorType operator, String value, Type expectedType, Object expected) { - functionAssertions.assertFunction(format("\"%s\"(%s)", mangleOperatorName(operator), value), expectedType, expected); + functionAssertions.assertFunction(format("\"%s\"(%s)", operator.getFunctionName().getFunctionName(), value), expectedType, expected); } protected void assertDecimalFunction(String statement, SqlDecimal expectedResult) @@ -187,30 +186,30 @@ protected void tryEvaluateWithAll(String projection, Type expectedType) protected void registerScalarFunction(SqlScalarFunction sqlScalarFunction) { Metadata metadata = functionAssertions.getMetadata(); - metadata.getFunctionManager().addFunctions(ImmutableList.of(sqlScalarFunction)); + metadata.getFunctionManager().registerBuiltInFunctions(ImmutableList.of(sqlScalarFunction)); } protected void registerScalar(Class clazz) { Metadata metadata = functionAssertions.getMetadata(); - List functions = new FunctionListBuilder() + List functions = new FunctionListBuilder() .scalars(clazz) .getFunctions(); - metadata.getFunctionManager().addFunctions(functions); + metadata.getFunctionManager().registerBuiltInFunctions(functions); } protected void registerParametricScalar(Class clazz) { Metadata metadata = functionAssertions.getMetadata(); - List functions = new FunctionListBuilder() + List functions = new FunctionListBuilder() .scalar(clazz) .getFunctions(); - metadata.getFunctionManager().addFunctions(functions); + metadata.getFunctionManager().registerBuiltInFunctions(functions); } protected void registerFunctions(Plugin plugin) { - functionAssertions.getMetadata().addFunctions(extractFunctions(plugin.getFunctions())); + functionAssertions.getMetadata().registerBuiltInFunctions(extractFunctions(plugin.getFunctions())); } protected void registerTypes(Plugin plugin) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestRegexpFunctions.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/AbstractTestRegexpFunctions.java similarity index 94% rename from presto-main/src/test/java/com/facebook/presto/operator/scalar/TestRegexpFunctions.java rename to presto-main/src/test/java/com/facebook/presto/operator/scalar/AbstractTestRegexpFunctions.java index 59c728aa1cb41..702646241b633 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestRegexpFunctions.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/AbstractTestRegexpFunctions.java @@ -18,12 +18,11 @@ import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.analyzer.RegexLibrary; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Factory; import org.testng.annotations.Test; import java.util.ArrayList; @@ -33,34 +32,19 @@ import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType; import static com.facebook.presto.spi.type.VarcharType.createVarcharType; -import static com.facebook.presto.sql.analyzer.RegexLibrary.JONI; -import static com.facebook.presto.sql.analyzer.RegexLibrary.RE2J; -public class TestRegexpFunctions +public abstract class AbstractTestRegexpFunctions extends AbstractTestFunctions { - private static final FeaturesConfig JONI_FEATURES_CONFIG = new FeaturesConfig() - .setRegexLibrary(JONI); - - private static final FeaturesConfig RE2J_FEATURES_CONFIG = new FeaturesConfig() - .setRegexLibrary(RE2J); - - @Factory(dataProvider = "featuresConfig") - public TestRegexpFunctions(FeaturesConfig featuresConfig) + AbstractTestRegexpFunctions(RegexLibrary regexLibrary) { - super(featuresConfig); + super(new FeaturesConfig().setRegexLibrary(regexLibrary)); } @BeforeClass public void setUp() { - registerScalar(TestRegexpFunctions.class); - } - - @DataProvider(name = "featuresConfig") - public static Object[][] featuresConfigProvider() - { - return new Object[][] {new Object[] {JONI_FEATURES_CONFIG}, new Object[] {RE2J_FEATURES_CONFIG}}; + registerScalar(AbstractTestRegexpFunctions.class); } @ScalarFunction(deterministic = false) // if not non-deterministic, constant folding code accidentally fix invalid characters @@ -90,7 +74,9 @@ public void testRegexpLike() assertFunction("REGEXP_LIKE('Hello', '^[a-z]+$')", BOOLEAN, false); assertFunction("REGEXP_LIKE('Hello', '^(?i)[a-z]+$')", BOOLEAN, true); assertFunction("REGEXP_LIKE('Hello', '^[a-zA-Z]+$')", BOOLEAN, true); - assertFunction("REGEXP_LIKE('Hello', 'Hello\\b')", BOOLEAN, true); + + // verify word boundaries at end of pattern (https://github.com/airlift/joni/pull/11) + assertFunction("REGEXP_LIKE('test', 'test\\b')", BOOLEAN, true); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayDistinct.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayDistinct.java index 2fbe911217918..57a7b4d7f6544 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayDistinct.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayDistinct.java @@ -107,7 +107,7 @@ public void setup() { MetadataManager metadata = MetadataManager.createTestMetadataManager(); FunctionManager functionManager = metadata.getFunctionManager(); - metadata.addFunctions(extractFunctions(BenchmarkArrayDistinct.class)); + metadata.registerBuiltInFunctions(extractFunctions(BenchmarkArrayDistinct.class)); ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); ImmutableList.Builder projectionsBuilder = ImmutableList.builder(); Block[] blocks = new Block[TYPES.size()]; @@ -120,7 +120,7 @@ public void setup() } ImmutableList projections = projectionsBuilder.build(); - pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); + pageProcessor = compiler.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); page = new Page(blocks); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java index f082be7c6c4af..60728c9036f3a 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java @@ -25,6 +25,7 @@ import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; @@ -61,9 +62,10 @@ import java.util.concurrent.TimeUnit; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.operator.scalar.BenchmarkArrayFilter.ExactArrayFilterFunction.EXACT_ARRAY_FILTER_FUNCTION; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -124,7 +126,7 @@ public void setup() { MetadataManager metadata = MetadataManager.createTestMetadataManager(); FunctionManager functionManager = metadata.getFunctionManager(); - metadata.addFunctions(new FunctionListBuilder().function(EXACT_ARRAY_FILTER_FUNCTION).getFunctions()); + metadata.registerBuiltInFunctions(new FunctionListBuilder().function(EXACT_ARRAY_FILTER_FUNCTION).getFunctions()); ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); ImmutableList.Builder projectionsBuilder = ImmutableList.builder(); Block[] blocks = new Block[TYPES.size()]; @@ -143,7 +145,7 @@ public void setup() } ImmutableList projections = projectionsBuilder.build(); - pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); + pageProcessor = compiler.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); page = new Page(blocks); } @@ -201,7 +203,7 @@ public static final class ExactArrayFilterFunction private ExactArrayFilterFunction() { super(new Signature( - "exact_filter", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "exact_filter"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -229,10 +231,10 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type type = boundVariables.getTypeVariable("T"); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayHashCodeOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayHashCodeOperator.java index 85a45cc88498d..51e340f0f2974 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayHashCodeOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayHashCodeOperator.java @@ -61,7 +61,6 @@ import java.util.concurrent.TimeUnit; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; -import static com.facebook.presto.metadata.OperatorSignatureUtils.mangleOperatorName; import static com.facebook.presto.operator.scalar.CombineHashFunction.getHash; import static com.facebook.presto.spi.function.OperatorType.HASH_CODE; import static com.facebook.presto.spi.type.ArrayType.ARRAY_NULL_ELEMENT_MSG; @@ -105,7 +104,7 @@ public List> arrayHashCode(BenchmarkData data) public static class BenchmarkData { @Param({"$operator$hash_code", "old_hash", "another_hash"}) - private String name = mangleOperatorName(HASH_CODE); + private String name = HASH_CODE.getFunctionName().getFunctionName(); @Param({"BIGINT", "VARCHAR", "DOUBLE", "BOOLEAN"}) private String type = "BIGINT"; @@ -118,8 +117,8 @@ public void setup() { MetadataManager metadata = MetadataManager.createTestMetadataManager(); FunctionManager functionManager = metadata.getFunctionManager(); - metadata.addFunctions(new FunctionListBuilder().scalar(BenchmarkOldArrayHash.class).getFunctions()); - metadata.addFunctions(new FunctionListBuilder().scalar(BenchmarkAnotherArrayHash.class).getFunctions()); + metadata.registerBuiltInFunctions(new FunctionListBuilder().scalar(BenchmarkOldArrayHash.class).getFunctions()); + metadata.registerBuiltInFunctions(new FunctionListBuilder().scalar(BenchmarkAnotherArrayHash.class).getFunctions()); ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); ImmutableList.Builder projectionsBuilder = ImmutableList.builder(); Block[] blocks = new Block[1]; @@ -146,7 +145,7 @@ public void setup() blocks[0] = createChannel(POSITIONS, ARRAY_SIZE, arrayType); ImmutableList projections = projectionsBuilder.build(); - pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); + pageProcessor = compiler.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); page = new Page(blocks); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayIntersect.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayIntersect.java index 6bb2a72a00444..6dd8ffe05c505 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayIntersect.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayIntersect.java @@ -128,7 +128,7 @@ public void setup() new CallExpression(name, functionHandle, arrayType, ImmutableList.of(field(0, arrayType), field(1, arrayType)))); ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); - pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); + pageProcessor = compiler.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); page = new Page(createChannel(POSITIONS, arraySize, elementType), createChannel(POSITIONS, arraySize, elementType)); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayJoin.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayJoin.java index 9d8d63fc28f7d..b7c41459655eb 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayJoin.java @@ -100,7 +100,7 @@ public void setup() constant(Slices.wrappedBuffer(",".getBytes(UTF_8)), VARCHAR)))); pageProcessor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)) - .compilePageProcessor(Optional.empty(), projections) + .compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections) .get(); page = new Page(createChannel(POSITIONS, ARRAY_SIZE)); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySort.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySort.java index 43567151df376..2dec2bf70364d 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySort.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySort.java @@ -105,7 +105,7 @@ public static class BenchmarkData public void setup() { MetadataManager metadata = MetadataManager.createTestMetadataManager(); - metadata.addFunctions(extractFunctions(BenchmarkArraySort.class)); + metadata.registerBuiltInFunctions(extractFunctions(BenchmarkArraySort.class)); ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); ImmutableList.Builder projectionsBuilder = ImmutableList.builder(); Block[] blocks = new Block[TYPES.size()]; @@ -118,7 +118,7 @@ public void setup() } ImmutableList projections = projectionsBuilder.build(); - pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); + pageProcessor = compiler.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); page = new Page(blocks); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySubscript.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySubscript.java index e8c09ef58998e..8e05f66f8b292 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySubscript.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArraySubscript.java @@ -145,7 +145,7 @@ public void setup() } ImmutableList projections = projectionsBuilder.build(); - pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); + pageProcessor = compiler.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); page = new Page(block); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayTransform.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayTransform.java index dd4d013aa30eb..c8fe32eeb3625 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayTransform.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayTransform.java @@ -127,7 +127,7 @@ public void setup() } ImmutableList projections = projectionsBuilder.build(); - pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); + pageProcessor = compiler.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); pageBuilder = new PageBuilder(projections.stream().map(RowExpression::getType).collect(Collectors.toList())); page = new Page(blocks); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkEqualsOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkEqualsOperator.java index 7f056ba150d7e..dfd5093cf1921 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkEqualsOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkEqualsOperator.java @@ -89,7 +89,7 @@ public void setup() metadata, new PageFunctionCompiler(metadata, 0)); RowExpression projection = generateComplexComparisonProjection(functionManager, FIELDS_COUNT, COMPARISONS_COUNT); - compiledProcessor = expressionCompiler.compilePageProcessor(Optional.empty(), ImmutableList.of(projection)).get(); + compiledProcessor = expressionCompiler.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), ImmutableList.of(projection)).get(); } private static RowExpression generateComplexComparisonProjection(FunctionManager functionManager, int fieldsCount, int comparisonsCount) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkJsonToArrayCast.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkJsonToArrayCast.java index 0d750cc173d87..6aea907b7796c 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkJsonToArrayCast.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkJsonToArrayCast.java @@ -120,7 +120,7 @@ public void setup() new CallExpression(CAST.name(), functionHandle, new ArrayType(elementType), ImmutableList.of(field(0, JSON)))); pageProcessor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)) - .compilePageProcessor(Optional.empty(), projections) + .compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections) .get(); page = new Page(createChannel(POSITION_COUNT, ARRAY_SIZE, elementType)); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkJsonToMapCast.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkJsonToMapCast.java index c5ef0ce6fa617..eac0e4114c7fd 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkJsonToMapCast.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkJsonToMapCast.java @@ -120,7 +120,7 @@ public void setup() new CallExpression(CAST.name(), functionHandle, mapType(VARCHAR, valueType), ImmutableList.of(field(0, JSON)))); pageProcessor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)) - .compilePageProcessor(Optional.empty(), projections) + .compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections) .get(); page = new Page(createChannel(POSITION_COUNT, MAP_SIZE, valueType)); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapConcat.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapConcat.java index e839ec1e85531..fd139b1d31564 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapConcat.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapConcat.java @@ -146,7 +146,7 @@ public void setup() ImmutableList.of(field(0, mapType), field(1, mapType)))); ImmutableList projections = projectionsBuilder.build(); - pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); + pageProcessor = compiler.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); page = new Page(leftBlock, rightBlock); } @@ -167,7 +167,7 @@ private static Block createMapBlock(MapType mapType, int positionCount, Block ke for (int i = 0; i < offsets.length; i++) { offsets[i] = mapSize * i; } - return mapType.createBlockFromKeyValue(Optional.empty(), offsets, keyBlock, valueBlock); + return mapType.createBlockFromKeyValue(positionCount, Optional.empty(), offsets, keyBlock, valueBlock); } private static Block createKeyBlock(int positionCount, List keys) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapSubscript.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapSubscript.java index a94f45180efea..22e48d51bb342 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapSubscript.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapSubscript.java @@ -157,7 +157,7 @@ public void setup() } ImmutableList projections = projectionsBuilder.build(); - pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); + pageProcessor = compiler.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); page = new Page(block); } @@ -178,7 +178,7 @@ private static Block createMapBlock(MapType mapType, int positionCount, Block ke for (int i = 0; i < offsets.length; i++) { offsets[i] = mapSize * i; } - return mapType.createBlockFromKeyValue(Optional.empty(), offsets, keyBlock, valueBlock); + return mapType.createBlockFromKeyValue(positionCount, Optional.empty(), offsets, keyBlock, valueBlock); } private static Block createKeyBlock(int positionCount, List keys) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapToMapCast.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapToMapCast.java index 5d8afab160926..aebef8774f986 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapToMapCast.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkMapToMapCast.java @@ -99,7 +99,7 @@ public void setup() new CallExpression(CAST.name(), functionHandle, mapType(BIGINT, DOUBLE), ImmutableList.of(field(0, mapType(DOUBLE, BIGINT))))); pageProcessor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)) - .compilePageProcessor(Optional.empty(), projections) + .compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections) .get(); Block keyBlock = createKeyBlock(POSITION_COUNT, MAP_SIZE); @@ -115,7 +115,7 @@ private static Block createMapBlock(MapType mapType, int positionCount, Block ke for (int i = 0; i < offsets.length; i++) { offsets[i] = mapSize * i; } - return mapType.createBlockFromKeyValue(Optional.empty(), offsets, keyBlock, valueBlock); + return mapType.createBlockFromKeyValue(positionCount, Optional.empty(), offsets, keyBlock, valueBlock); } private static Block createKeyBlock(int positionCount, int mapSize) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkRowToRowCast.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkRowToRowCast.java index 595339e5e5106..24a4a2656a817 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkRowToRowCast.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkRowToRowCast.java @@ -99,7 +99,7 @@ public void setup() new CallExpression(CAST.name(), functionHandle, RowType.anonymous(fromFieldTypes), ImmutableList.of(field(0, RowType.anonymous(toFieldTypes))))); pageProcessor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)) - .compilePageProcessor(Optional.empty(), projections) + .compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections) .get(); Block[] fieldBlocks = fromFieldTypes.stream() diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformKey.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformKey.java index 9731a58881d76..f204861f0ecef 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformKey.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformKey.java @@ -142,7 +142,7 @@ public void setup() Block block = createChannel(POSITIONS, mapType, elementType); ImmutableList projections = projectionsBuilder.build(); - pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); + pageProcessor = compiler.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); page = new Page(block); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformValue.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformValue.java index acf75f4e0b84d..231e2289fbcdf 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformValue.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkTransformValue.java @@ -154,7 +154,7 @@ public void setup() Block block = createChannel(POSITIONS, mapType, elementType); ImmutableList projections = projectionsBuilder.build(); - pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); + pageProcessor = compiler.compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); page = new Page(block); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java index 76c7630cfd2fb..c182c3b41ca41 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java @@ -15,10 +15,10 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.metadata.BuiltInFunction; import com.facebook.presto.metadata.FunctionListBuilder; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.Split; -import com.facebook.presto.metadata.SqlFunction; import com.facebook.presto.operator.DriverContext; import com.facebook.presto.operator.DriverYieldSignal; import com.facebook.presto.operator.FilterAndProjectOperator.FilterAndProjectOperatorFactory; @@ -34,6 +34,7 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableHandle; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.FixedPageSource; import com.facebook.presto.spi.HostAddress; @@ -44,11 +45,15 @@ import com.facebook.presto.spi.RecordPageSource; import com.facebook.presto.spi.RecordSet; import com.facebook.presto.spi.StandardErrorCode; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.predicate.Utils; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.TimeZoneKey; import com.facebook.presto.spi.type.Type; import com.facebook.presto.split.PageSourceProvider; @@ -59,6 +64,7 @@ import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.ExpressionInterpreter; +import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Cast; import com.facebook.presto.sql.tree.DefaultTraversalVisitor; @@ -89,6 +95,7 @@ import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -99,11 +106,14 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.block.BlockAssertions.createBooleansBlock; import static com.facebook.presto.block.BlockAssertions.createDoublesBlock; import static com.facebook.presto.block.BlockAssertions.createIntsBlock; import static com.facebook.presto.block.BlockAssertions.createLongsBlock; +import static com.facebook.presto.block.BlockAssertions.createRowBlock; import static com.facebook.presto.block.BlockAssertions.createSlicesBlock; import static com.facebook.presto.block.BlockAssertions.createStringsBlock; import static com.facebook.presto.block.BlockAssertions.createTimestampsWithTimezoneBlock; @@ -128,9 +138,7 @@ import static com.facebook.presto.sql.relational.SqlToRowExpressionTranslator.translate; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; import static com.facebook.presto.type.UnknownType.UNKNOWN; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.slice.SizeOf.sizeOf; -import static io.airlift.testing.Assertions.assertInstanceOf; import static io.airlift.units.DataSize.Unit.BYTE; import static java.lang.String.format; import static java.util.Collections.emptyList; @@ -151,6 +159,11 @@ public final class FunctionAssertions private static final SqlParser SQL_PARSER = new SqlParser(); + // Increase the number of fields to generate a wide column + private static final int TEST_ROW_NUMBER_OF_FIELDS = 2500; + private static final RowType TEST_ROW_TYPE = createTestRowType(TEST_ROW_NUMBER_OF_FIELDS); + private static final Block TEST_ROW_DATA = createTestRowData(TEST_ROW_TYPE); + private static final Page SOURCE_PAGE = new Page( createLongsBlock(1234L), createStringsBlock("hello"), @@ -161,7 +174,8 @@ public final class FunctionAssertions createStringsBlock((String) null), createTimestampsWithTimezoneBlock(packDateTimeWithZone(new DateTime(1970, 1, 1, 0, 1, 0, 999, DateTimeZone.UTC).getMillis(), TimeZoneKey.getTimeZoneKey("Z"))), createSlicesBlock(Slices.wrappedBuffer((byte) 0xab)), - createIntsBlock(1234)); + createIntsBlock(1234), + TEST_ROW_DATA); private static final Page ZERO_CHANNEL_PAGE = new Page(1); @@ -176,6 +190,7 @@ public final class FunctionAssertions .put(new VariableReferenceExpression("bound_timestamp_with_timezone", TIMESTAMP_WITH_TIME_ZONE), 7) .put(new VariableReferenceExpression("bound_binary_literal", VARBINARY), 8) .put(new VariableReferenceExpression("bound_integer", INTEGER), 9) + .put(new VariableReferenceExpression("bound_row", TEST_ROW_TYPE), 10) .build(); private static final TypeProvider SYMBOL_TYPES = TypeProvider.fromVariables(INPUT_MAPPING.keySet()); @@ -216,15 +231,15 @@ public Metadata getMetadata() return metadata; } - public FunctionAssertions addFunctions(List functionInfos) + public FunctionAssertions addFunctions(List functionInfos) { - metadata.addFunctions(functionInfos); + metadata.registerBuiltInFunctions(functionInfos); return this; } public FunctionAssertions addScalarFunctions(Class clazz) { - metadata.addFunctions(new FunctionListBuilder().scalars(clazz).getFunctions()); + metadata.registerBuiltInFunctions(new FunctionListBuilder().scalars(clazz).getFunctions()); return this; } @@ -444,7 +459,7 @@ public void assertCachedInstanceHasBoundedRetainedSize(String projection) Expression projectionExpression = createExpression(session, projection, metadata, SYMBOL_TYPES); RowExpression projectionRowExpression = toRowExpression(session, projectionExpression); - PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(projectionRowExpression)).get(); + PageProcessor processor = compiler.compilePageProcessor(session.getSqlFunctionProperties(), Optional.empty(), ImmutableList.of(projectionRowExpression)).get(); // This is a heuristic to detect whether the retained size of cachedInstance is bounded. // * The test runs at least 1000 iterations. @@ -573,7 +588,7 @@ private List executeProjectionWithAll(String projection, Type expectedTy } // execute as standalone operator - OperatorFactory operatorFactory = compileFilterProject(Optional.empty(), projectionRowExpression, compiler); + OperatorFactory operatorFactory = compileFilterProject(session.getSqlFunctionProperties(), Optional.empty(), projectionRowExpression, compiler); Object directOperatorValue = selectSingleValue(operatorFactory, expectedType, session); results.add(directOperatorValue); @@ -582,7 +597,7 @@ private List executeProjectionWithAll(String projection, Type expectedTy results.add(interpretedValue); // execute over normal operator - SourceOperatorFactory scanProjectOperatorFactory = compileScanFilterProject(Optional.empty(), projectionRowExpression, compiler); + SourceOperatorFactory scanProjectOperatorFactory = compileScanFilterProject(session.getSqlFunctionProperties(), Optional.empty(), projectionRowExpression, compiler); Object scanOperatorValue = selectSingleValue(scanProjectOperatorFactory, expectedType, createNormalSplit(), session); results.add(scanOperatorValue); @@ -673,12 +688,12 @@ private List executeFilterWithAll(String filter, Session session, boole List results = new ArrayList<>(); // execute as standalone operator - OperatorFactory operatorFactory = compileFilterProject(Optional.of(filterRowExpression), constant(true, BOOLEAN), compiler); + OperatorFactory operatorFactory = compileFilterProject(session.getSqlFunctionProperties(), Optional.of(filterRowExpression), constant(true, BOOLEAN), compiler); results.add(executeFilter(operatorFactory, session)); if (executeWithNoInputColumns) { // execute as standalone operator - operatorFactory = compileFilterWithNoInputColumns(filterRowExpression, compiler); + operatorFactory = compileFilterWithNoInputColumns(session.getSqlFunctionProperties(), filterRowExpression, compiler); results.add(executeFilterWithNoInputColumns(operatorFactory, session)); } @@ -690,7 +705,7 @@ private List executeFilterWithAll(String filter, Session session, boole results.add(interpretedValue); // execute over normal operator - SourceOperatorFactory scanProjectOperatorFactory = compileScanFilterProject(Optional.of(filterRowExpression), constant(true, BOOLEAN), compiler); + SourceOperatorFactory scanProjectOperatorFactory = compileScanFilterProject(session.getSqlFunctionProperties(), Optional.of(filterRowExpression), constant(true, BOOLEAN), compiler); boolean scanOperatorValue = executeFilter(scanProjectOperatorFactory, createNormalSplit(), session); results.add(scanOperatorValue); @@ -854,7 +869,8 @@ private Object interpret(Expression expression, Type expectedType, Session sessi Map, Type> expressionTypes = getExpressionTypes(session, metadata, SQL_PARSER, SYMBOL_TYPES, expression, emptyList(), WarningCollector.NOOP); ExpressionInterpreter evaluator = ExpressionInterpreter.expressionInterpreter(expression, metadata, session, expressionTypes); - Object result = evaluator.evaluate(symbol -> { + Object result = evaluator.evaluate(variable -> { + Symbol symbol = new Symbol(variable.getName()); int position = 0; int channel = INPUT_MAPPING.get(new VariableReferenceExpression(symbol.getName(), SYMBOL_TYPES.get(symbol.toSymbolReference()))); Type type = SYMBOL_TYPES.get(symbol.toSymbolReference()); @@ -892,10 +908,10 @@ else if (javaType == Block.class) { return expectedType.getObjectValue(session.toConnectorSession(), block, 0); } - private static OperatorFactory compileFilterWithNoInputColumns(RowExpression filter, ExpressionCompiler compiler) + private static OperatorFactory compileFilterWithNoInputColumns(SqlFunctionProperties sqlFunctionProperties, RowExpression filter, ExpressionCompiler compiler) { try { - Supplier processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of()); + Supplier processor = compiler.compilePageProcessor(sqlFunctionProperties, Optional.of(filter), ImmutableList.of()); return new FilterAndProjectOperatorFactory(0, new PlanNodeId("test"), processor, ImmutableList.of(), new DataSize(0, BYTE), 0); } @@ -907,10 +923,10 @@ private static OperatorFactory compileFilterWithNoInputColumns(RowExpression fil } } - private static OperatorFactory compileFilterProject(Optional filter, RowExpression projection, ExpressionCompiler compiler) + private static OperatorFactory compileFilterProject(SqlFunctionProperties sqlFunctionProperties, Optional filter, RowExpression projection, ExpressionCompiler compiler) { try { - Supplier processor = compiler.compilePageProcessor(filter, ImmutableList.of(projection)); + Supplier processor = compiler.compilePageProcessor(sqlFunctionProperties, filter, ImmutableList.of(projection)); return new FilterAndProjectOperatorFactory(0, new PlanNodeId("test"), processor, ImmutableList.of(projection.getType()), new DataSize(0, BYTE), 0); } catch (Throwable e) { @@ -921,15 +937,17 @@ private static OperatorFactory compileFilterProject(Optional filt } } - private static SourceOperatorFactory compileScanFilterProject(Optional filter, RowExpression projection, ExpressionCompiler compiler) + private static SourceOperatorFactory compileScanFilterProject(SqlFunctionProperties sqlFunctionProperties, Optional filter, RowExpression projection, ExpressionCompiler compiler) { try { Supplier cursorProcessor = compiler.compileCursorProcessor( + sqlFunctionProperties, filter, ImmutableList.of(projection), SOURCE_ID); Supplier pageProcessor = compiler.compilePageProcessor( + sqlFunctionProperties, filter, ImmutableList.of(projection)); @@ -940,6 +958,11 @@ private static SourceOperatorFactory compileScanFilterProject(Optional, Type> expressionTypes, Map layout) { - return translate(projection, expressionTypes, layout, metadata.getFunctionManager(), metadata.getTypeManager(), session, false); + return translate(projection, expressionTypes, layout, metadata.getFunctionManager(), metadata.getTypeManager(), session); } private static Page getAtMostOnePage(Operator operator, Page sourcePage) @@ -1015,12 +1038,12 @@ private static class TestPageSourceProvider implements PageSourceProvider { @Override - public ConnectorPageSource createPageSource(Session session, Split split, List columns) + public ConnectorPageSource createPageSource(Session session, Split split, TableHandle table, List columns) { assertInstanceOf(split.getConnectorSplit(), FunctionAssertions.TestSplit.class); FunctionAssertions.TestSplit testSplit = (FunctionAssertions.TestSplit) split.getConnectorSplit(); if (testSplit.isRecordSet()) { - RecordSet records = InMemoryRecordSet.builder(ImmutableList.of(BIGINT, VARCHAR, DOUBLE, BOOLEAN, BIGINT, VARCHAR, VARCHAR, TIMESTAMP_WITH_TIME_ZONE, VARBINARY, INTEGER)) + RecordSet records = InMemoryRecordSet.builder(ImmutableList.of(BIGINT, VARCHAR, DOUBLE, BOOLEAN, BIGINT, VARCHAR, VARCHAR, TIMESTAMP_WITH_TIME_ZONE, VARBINARY, INTEGER, TEST_ROW_TYPE)) .addRow( 1234L, "hello", @@ -1031,7 +1054,8 @@ public ConnectorPageSource createPageSource(Session session, Split split, List types = Iterables.cycle( + BIGINT, + INTEGER, + VARCHAR, + DOUBLE, + BOOLEAN, + VARBINARY, + RowType.from(ImmutableList.of(RowType.field("nested_nested_column", VARCHAR)))).iterator(); + + List fields = new ArrayList<>(); + for (int fieldIdx = 0; fieldIdx < numberOfFields; fieldIdx++) { + fields.add(new RowType.Field(Optional.of("nested_column_" + fieldIdx), types.next())); + } + + return RowType.from(fields); + } + + private static Block createTestRowData(RowType rowType) + { + Iterator values = Iterables.cycle( + 1234L, + 34, + "hello", + 12.34d, + true, + Slices.wrappedBuffer((byte) 0xab), + createRowBlock(ImmutableList.of(VARCHAR), Collections.singleton("innerFieldValue").toArray()).getBlock(0)).iterator(); + + final int numFields = rowType.getFields().size(); + Object[] rowValues = new Object[numFields]; + for (int fieldIdx = 0; fieldIdx < numFields; fieldIdx++) { + rowValues[fieldIdx] = values.next(); + } + + return createRowBlock(rowType.getTypeParameters(), rowValues); + } + private static class TestSplit implements ConnectorSplit { diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayCombinationsFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayCombinationsFunction.java new file mode 100644 index 0000000000000..53b75bbd2afc1 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayCombinationsFunction.java @@ -0,0 +1,154 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.spi.type.ArrayType; +import com.google.common.collect.ContiguousSet; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import static com.facebook.presto.operator.scalar.ArrayCombinationsFunction.combinationCount; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.VarcharType.createVarcharType; +import static com.facebook.presto.type.UnknownType.UNKNOWN; +import static com.google.common.math.LongMath.factorial; +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static org.testng.Assert.assertEquals; + +public class TestArrayCombinationsFunction + extends AbstractTestFunctions +{ + @Test + public void testCombinationCount() + { + for (int n = 0; n < 5; n++) { + for (int k = 0; k <= n; k++) { + assertEquals(combinationCount(n, k), factorial(n) / factorial(n - k) / factorial(k)); + } + } + + assertEquals(combinationCount(42, 7), 26978328); + assertEquals(combinationCount(100, 4), 3921225); + } + + @Test + public void testBasic() + { + assertFunction("combinations(ARRAY['bar', 'foo', 'baz', 'foo'], 0)", new ArrayType(new ArrayType(createVarcharType(3))), ImmutableList.of(ImmutableList.of())); + assertFunction("combinations(ARRAY['bar', 'foo', 'baz', 'foo'], 1)", new ArrayType(new ArrayType(createVarcharType(3))), ImmutableList.of( + ImmutableList.of("bar"), + ImmutableList.of("foo"), + ImmutableList.of("baz"), + ImmutableList.of("foo"))); + assertFunction("combinations(ARRAY['bar', 'foo', 'baz', 'foo'], 2)", new ArrayType(new ArrayType(createVarcharType(3))), ImmutableList.of( + ImmutableList.of("bar", "foo"), + ImmutableList.of("bar", "baz"), + ImmutableList.of("foo", "baz"), + ImmutableList.of("bar", "foo"), + ImmutableList.of("foo", "foo"), + ImmutableList.of("baz", "foo"))); + assertFunction("combinations(ARRAY['bar', 'foo', 'baz', 'foo'], 3)", new ArrayType(new ArrayType(createVarcharType(3))), ImmutableList.of( + ImmutableList.of("bar", "foo", "baz"), + ImmutableList.of("bar", "foo", "foo"), + ImmutableList.of("bar", "baz", "foo"), + ImmutableList.of("foo", "baz", "foo"))); + assertFunction("combinations(ARRAY['bar', 'foo', 'baz', 'foo'], 4)", new ArrayType(new ArrayType(createVarcharType(3))), ImmutableList.of( + ImmutableList.of("bar", "foo", "baz", "foo"))); + assertFunction("combinations(ARRAY['bar', 'foo', 'baz', 'foo'], 5)", new ArrayType(new ArrayType(createVarcharType(3))), ImmutableList.of()); + assertFunction("combinations(ARRAY['a', 'bb', 'ccc', 'dddd'], 2)", new ArrayType(new ArrayType(createVarcharType(4))), ImmutableList.of( + ImmutableList.of("a", "bb"), + ImmutableList.of("a", "ccc"), + ImmutableList.of("bb", "ccc"), + ImmutableList.of("a", "dddd"), + ImmutableList.of("bb", "dddd"), + ImmutableList.of("ccc", "dddd"))); + } + + @Test + public void testLimits() + { + assertInvalidFunction("combinations(sequence(1, 40), -1)", INVALID_FUNCTION_ARGUMENT, "combination size must not be negative: -1"); + assertInvalidFunction("combinations(sequence(1, 40), 10)", INVALID_FUNCTION_ARGUMENT, "combination size must not exceed 5: 10"); + assertInvalidFunction("combinations(sequence(1, 100), 5)", INVALID_FUNCTION_ARGUMENT, "combinations exceed max size"); + } + + @Test + public void testCardinality() + { + for (int n = 0; n < 5; n++) { + for (int k = 0; k <= n; k++) { + String array = "ARRAY" + ContiguousSet.closedOpen(0, n).asList(); + assertFunction(format("cardinality(combinations(%s, %s))", array, k), BIGINT, factorial(n) / factorial(n - k) / factorial(k)); + } + } + } + + @Test + public void testNull() + { + assertFunction("combinations(CAST(NULL AS array(bigint)), 2)", new ArrayType(new ArrayType(BIGINT)), null); + + assertFunction("combinations(ARRAY['foo', NULL, 'bar'], 2)", new ArrayType(new ArrayType(createVarcharType(3))), ImmutableList.of( + asList("foo", null), + asList("foo", "bar"), + asList(null, "bar"))); + + assertFunction("combinations(ARRAY [NULL, NULL, NULL], 2)", new ArrayType(new ArrayType(UNKNOWN)), ImmutableList.of( + asList(null, null), + asList(null, null), + asList(null, null))); + + assertFunction("combinations(ARRAY [NULL, 3, NULL], 2)", new ArrayType(new ArrayType(INTEGER)), ImmutableList.of( + asList(null, 3), + asList(null, null), + asList(3, null))); + } + + @Test + public void testTypeCombinations() + { + assertFunction("combinations(ARRAY[1, 2, 3], 2)", new ArrayType(new ArrayType(INTEGER)), ImmutableList.of( + ImmutableList.of(1, 2), + ImmutableList.of(1, 3), + ImmutableList.of(2, 3))); + assertFunction("combinations(ARRAY[1.1E0, 2.1E0, 3.1E0], 2)", new ArrayType(new ArrayType(DOUBLE)), ImmutableList.of( + ImmutableList.of(1.1, 2.1), + ImmutableList.of(1.1, 3.1), + ImmutableList.of(2.1, 3.1))); + assertFunction("combinations(ARRAY[true, false, true], 2)", new ArrayType(new ArrayType(BOOLEAN)), ImmutableList.of( + ImmutableList.of(true, false), + ImmutableList.of(true, true), + ImmutableList.of(false, true))); + + assertFunction("combinations(ARRAY[ARRAY['A1', 'A2'], ARRAY['B1'], ARRAY['C1', 'C2']], 2)", new ArrayType(new ArrayType(new ArrayType(createVarcharType(2)))), ImmutableList.of( + ImmutableList.of(ImmutableList.of("A1", "A2"), ImmutableList.of("B1")), + ImmutableList.of(ImmutableList.of("A1", "A2"), ImmutableList.of("C1", "C2")), + ImmutableList.of(ImmutableList.of("B1"), ImmutableList.of("C1", "C2")))); + + assertFunction("combinations(ARRAY['\u4FE1\u5FF5\u7231', '\u5E0C\u671B', '\u671B'], 2)", new ArrayType(new ArrayType(createVarcharType(3))), ImmutableList.of( + ImmutableList.of("\u4FE1\u5FF5\u7231", "\u5E0C\u671B"), + ImmutableList.of("\u4FE1\u5FF5\u7231", "\u671B"), + ImmutableList.of("\u5E0C\u671B", "\u671B"))); + + assertFunction("combinations(ARRAY[], 2)", new ArrayType(new ArrayType(UNKNOWN)), ImmutableList.of()); + assertFunction("combinations(ARRAY[''], 2)", new ArrayType(new ArrayType(createVarcharType(0))), ImmutableList.of()); + assertFunction("combinations(ARRAY['', ''], 2)", new ArrayType(new ArrayType(createVarcharType(0))), ImmutableList.of(ImmutableList.of("", ""))); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayIntersectFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayIntersectFunction.java new file mode 100644 index 0000000000000..46f9db468397d --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayIntersectFunction.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.spi.type.ArrayType; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.type.UnknownType.UNKNOWN; +import static java.util.Arrays.asList; + +public class TestArrayIntersectFunction + extends AbstractTestFunctions +{ + @Test + public void testBasic() + { + assertFunction("array_intersect(ARRAY[1, 5, 3], ARRAY[3])", new ArrayType(INTEGER), ImmutableList.of(3)); + assertFunction("array_intersect(ARRAY[CAST(1 as BIGINT), 5, 3], ARRAY[5])", new ArrayType(BIGINT), ImmutableList.of(5L)); + assertFunction("array_intersect(ARRAY[CAST('x' as VARCHAR), 'y', 'z'], ARRAY['x', 'y'])", new ArrayType(VARCHAR), ImmutableList.of("x", "y")); + assertFunction("array_intersect(ARRAY[true, false, null], ARRAY[true, null])", new ArrayType(BOOLEAN), asList(true, null)); + assertFunction("array_intersect(ARRAY[1.1E0, 5.4E0, 3.9E0], ARRAY[5, 5.4E0])", new ArrayType(DOUBLE), ImmutableList.of(5.4)); + } + + @Test + public void testEmpty() + { + assertFunction("array_intersect(ARRAY[], ARRAY[])", new ArrayType(UNKNOWN), ImmutableList.of()); + assertFunction("array_intersect(ARRAY[], ARRAY[1, 3])", new ArrayType(INTEGER), ImmutableList.of()); + assertFunction("array_intersect(ARRAY[CAST('abc' as VARCHAR)], ARRAY[])", new ArrayType(VARCHAR), ImmutableList.of()); + } + + @Test + public void testNull() + { + assertFunction("array_intersect(ARRAY[NULL], NULL)", new ArrayType(UNKNOWN), null); + assertFunction("array_intersect(NULL, NULL)", new ArrayType(UNKNOWN), null); + assertFunction("array_intersect(NULL, ARRAY[NULL])", new ArrayType(UNKNOWN), null); + assertFunction("array_intersect(ARRAY[NULL], ARRAY[NULL])", new ArrayType(UNKNOWN), asList(false ? 1 : null)); + assertFunction("array_intersect(ARRAY[], ARRAY[NULL])", new ArrayType(UNKNOWN), ImmutableList.of()); + assertFunction("array_intersect(ARRAY[NULL], ARRAY[])", new ArrayType(UNKNOWN), ImmutableList.of()); + } + + @Test + public void testDuplicates() + { + assertFunction("array_intersect(ARRAY[1, 5, 3, 5, 1], ARRAY[3, 3, 5])", new ArrayType(INTEGER), ImmutableList.of(5, 3)); + assertFunction("array_intersect(ARRAY[CAST(1 as BIGINT), 5, 5, 3, 3, 3, 1], ARRAY[3, 5])", new ArrayType(BIGINT), ImmutableList.of(5L, 3L)); + assertFunction("array_intersect(ARRAY[CAST('x' as VARCHAR), 'x', 'y', 'z'], ARRAY['x', 'y', 'x'])", new ArrayType(VARCHAR), ImmutableList.of("x", "y")); + assertFunction("array_intersect(ARRAY[true, false, null, true, false, null], ARRAY[true, true, true])", new ArrayType(BOOLEAN), asList(true)); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayMatchFunctions.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayMatchFunctions.java new file mode 100644 index 0000000000000..a7062dee06d79 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArrayMatchFunctions.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.spi.type.BooleanType; +import org.testng.annotations.Test; + +public class TestArrayMatchFunctions + extends AbstractTestFunctions +{ + @Test + public void testAllMatch() + { + assertFunction("all_match(ARRAY [5, 7, 9], x -> x % 2 = 1)", BooleanType.BOOLEAN, true); + assertFunction("all_match(ARRAY [true, false, true], x -> x)", BooleanType.BOOLEAN, false); + assertFunction("all_match(ARRAY ['abc', 'ade', 'afg'], x -> substr(x, 1, 1) = 'a')", BooleanType.BOOLEAN, true); + assertFunction("all_match(ARRAY [], x -> true)", BooleanType.BOOLEAN, true); + assertFunction("all_match(ARRAY [true, true, NULL], x -> x)", BooleanType.BOOLEAN, null); + assertFunction("all_match(ARRAY [true, false, NULL], x -> x)", BooleanType.BOOLEAN, false); + assertFunction("all_match(ARRAY [NULL, NULL, NULL], x -> x > 1)", BooleanType.BOOLEAN, null); + assertFunction("all_match(ARRAY [NULL, NULL, NULL], x -> x IS NULL)", BooleanType.BOOLEAN, true); + assertFunction("all_match(ARRAY [MAP(ARRAY[1,2], ARRAY[3,4]), MAP(ARRAY[1,2,3], ARRAY[3,4,5])], x -> cardinality(x) > 1)", BooleanType.BOOLEAN, true); + } + + @Test + public void testAnyMatch() + { + assertFunction("any_match(ARRAY [5, 8, 10], x -> x % 2 = 1)", BooleanType.BOOLEAN, true); + assertFunction("any_match(ARRAY [false, false, false], x -> x)", BooleanType.BOOLEAN, false); + assertFunction("any_match(ARRAY ['abc', 'def', 'ghi'], x -> substr(x, 1, 1) = 'a')", BooleanType.BOOLEAN, true); + assertFunction("any_match(ARRAY [], x -> true)", BooleanType.BOOLEAN, false); + assertFunction("any_match(ARRAY [false, false, NULL], x -> x)", BooleanType.BOOLEAN, null); + assertFunction("any_match(ARRAY [true, false, NULL], x -> x)", BooleanType.BOOLEAN, true); + assertFunction("any_match(ARRAY [NULL, NULL, NULL], x -> x > 1)", BooleanType.BOOLEAN, null); + assertFunction("any_match(ARRAY [true, false, NULL], x -> x IS NULL)", BooleanType.BOOLEAN, true); + assertFunction("any_match(ARRAY [MAP(ARRAY[1,2], ARRAY[3,4]), MAP(ARRAY[1,2,3], ARRAY[3,4,5])], x -> cardinality(x) > 4)", BooleanType.BOOLEAN, false); + } + + @Test + public void testNoneMatch() + { + assertFunction("none_match(ARRAY [5, 8, 10], x -> x % 2 = 1)", BooleanType.BOOLEAN, false); + assertFunction("none_match(ARRAY [false, false, false], x -> x)", BooleanType.BOOLEAN, true); + assertFunction("none_match(ARRAY ['abc', 'def', 'ghi'], x -> substr(x, 1, 1) = 'a')", BooleanType.BOOLEAN, false); + assertFunction("none_match(ARRAY [], x -> true)", BooleanType.BOOLEAN, true); + assertFunction("none_match(ARRAY [false, false, NULL], x -> x)", BooleanType.BOOLEAN, null); + assertFunction("none_match(ARRAY [true, false, NULL], x -> x)", BooleanType.BOOLEAN, false); + assertFunction("none_match(ARRAY [NULL, NULL, NULL], x -> x > 1)", BooleanType.BOOLEAN, null); + assertFunction("none_match(ARRAY [true, false, NULL], x -> x IS NULL)", BooleanType.BOOLEAN, false); + assertFunction("none_match(ARRAY [MAP(ARRAY[1,2], ARRAY[3,4]), MAP(ARRAY[1,2,3], ARRAY[3,4,5])], x -> cardinality(x) > 4)", BooleanType.BOOLEAN, true); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArraySortFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArraySortFunction.java new file mode 100644 index 0000000000000..eea82618f92b8 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestArraySortFunction.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.spi.type.ArrayType; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.VarcharType.createVarcharType; +import static java.util.Arrays.asList; + +public class TestArraySortFunction + extends AbstractTestFunctions +{ + @Test + public void testArraySort() + { + assertFunction("array_sort(ARRAY [5, 20, null, 5, 3, 50]) ", new ArrayType(INTEGER), + asList(3, 5, 5, 20, 50, null)); + assertFunction("array_sort(array['x', 'a', 'a', 'a', 'a', 'm', 'j', 'p'])", + new ArrayType(createVarcharType(1)), ImmutableList.of("a", "a", "a", "a", "j", "m", "p", "x")); + assertFunction("array_sort(sequence(-4, 3))", new ArrayType(BIGINT), + asList(-4L, -3L, -2L, -1L, 0L, 1L, 2L, 3L)); + assertFunction("array_sort(reverse(sequence(-4, 3)))", new ArrayType(BIGINT), + asList(-4L, -3L, -2L, -1L, 0L, 1L, 2L, 3L)); + assertFunction("repeat(1,4)", new ArrayType(INTEGER), asList(1, 1, 1, 1)); + assertFunction("cast(array[] as array)", new ArrayType(INTEGER), asList()); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestDateTimeFunctionsBase.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestDateTimeFunctionsBase.java index 50a09d3b3d9f9..06fb1830f7adf 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestDateTimeFunctionsBase.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestDateTimeFunctionsBase.java @@ -160,7 +160,7 @@ public void testCurrentDateTimezone() private void assertCurrentDateAtInstant(TimeZoneKey timeZoneKey, long instant) { long expectedDays = epochDaysInZone(timeZoneKey, instant); - long dateTimeCalculation = currentDate(new TestingConnectorSession("test", Optional.empty(), Optional.empty(), timeZoneKey, US, instant, ImmutableList.of(), ImmutableMap.of(), isLegacyTimestamp(session))); + long dateTimeCalculation = currentDate(new TestingConnectorSession("test", Optional.empty(), Optional.empty(), timeZoneKey, US, instant, ImmutableList.of(), ImmutableMap.of(), isLegacyTimestamp(session), Optional.empty())); assertEquals(dateTimeCalculation, expectedDays); } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestFailureFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestFailureFunction.java index 8e4f6e727e40d..f07a19291eff7 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestFailureFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestFailureFunction.java @@ -13,14 +13,18 @@ */ package com.facebook.presto.operator.scalar; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.client.FailureInfo; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.util.Failures; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.spi.StandardErrorCode.DIVISION_BY_ZERO; +import static com.facebook.presto.testing.assertions.Assert.assertEquals; import static com.facebook.presto.type.UnknownType.UNKNOWN; +import static org.testng.Assert.assertTrue; public class TestFailureFunction extends AbstractTestFunctions @@ -33,13 +37,20 @@ public void testFailure() assertFunction("fail(json_parse('" + FAILURE_INFO + "'))", UNKNOWN, null); } - @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "/ by zero") + @Test public void testQuery() { // The other test does not exercise this function during execution (i.e. inside a page processor). // It only verifies constant folding works. try (LocalQueryRunner runner = new LocalQueryRunner(TEST_SESSION)) { - runner.execute("select if(x, 78, 0/0) from (values rand() >= 0, rand() < 0) t(x)"); + try { + runner.execute("select if(x, 78, 0/0) from (values rand() >= 0, rand() < 0) t(x)"); + throw new RuntimeException("Should throw PrestoException"); + } + catch (PrestoException e) { + assertEquals(e.getErrorCode(), DIVISION_BY_ZERO.toErrorCode()); + assertTrue(e.getMessage().contains("/ by zero")); + } } } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestHyperLogLogFunctions.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestHyperLogLogFunctions.java index 766f4dc8567d2..d9dd24b886707 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestHyperLogLogFunctions.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestHyperLogLogFunctions.java @@ -13,11 +13,11 @@ * limitations under the License. */ +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.io.BaseEncoding; import io.airlift.slice.Slice; -import io.airlift.stats.cardinality.HyperLogLog; import org.testng.annotations.Test; import java.util.Iterator; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestIpPrefixFunctions.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestIpPrefixFunctions.java new file mode 100644 index 0000000000000..2fe1d39a8bace --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestIpPrefixFunctions.java @@ -0,0 +1,60 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import org.testng.annotations.Test; + +import static com.facebook.presto.type.IpPrefixType.IPPREFIX; + +public class TestIpPrefixFunctions + extends AbstractTestFunctions +{ + @Test + public void testIpAddressIpPrefix() + { + assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); + assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 32)", IPPREFIX, "1.2.3.4/32"); + assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 0)", IPPREFIX, "0.0.0.0/0"); + assertFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); + assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 64)", IPPREFIX, "64:ff9b::/64"); + assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 127)", IPPREFIX, "64:ff9b::16/127"); + assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 128)", IPPREFIX, "64:ff9b::17/128"); + assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 0)", IPPREFIX, "::/0"); + assertInvalidFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', -1)", "IPv4 subnet size must be in range [0, 32]"); + assertInvalidFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', 33)", "IPv4 subnet size must be in range [0, 32]"); + assertInvalidFunction("IP_PREFIX(IPADDRESS '64:ff9b::10', -1)", "IPv6 subnet size must be in range [0, 128]"); + assertInvalidFunction("IP_PREFIX(IPADDRESS '64:ff9b::10', 129)", "IPv6 subnet size must be in range [0, 128]"); + } + + @Test + public void testStringIpPrefix() + { + assertFunction("IP_PREFIX('1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); + assertFunction("IP_PREFIX('1.2.3.4', 32)", IPPREFIX, "1.2.3.4/32"); + assertFunction("IP_PREFIX('1.2.3.4', 0)", IPPREFIX, "0.0.0.0/0"); + assertFunction("IP_PREFIX('::ffff:1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); + assertFunction("IP_PREFIX('64:ff9b::17', 64)", IPPREFIX, "64:ff9b::/64"); + assertFunction("IP_PREFIX('64:ff9b::17', 127)", IPPREFIX, "64:ff9b::16/127"); + assertFunction("IP_PREFIX('64:ff9b::17', 128)", IPPREFIX, "64:ff9b::17/128"); + assertFunction("IP_PREFIX('64:ff9b::17', 0)", IPPREFIX, "::/0"); + assertInvalidFunction("IP_PREFIX('::ffff:1.2.3.4', -1)", "IPv4 subnet size must be in range [0, 32]"); + assertInvalidFunction("IP_PREFIX('::ffff:1.2.3.4', 33)", "IPv4 subnet size must be in range [0, 32]"); + assertInvalidFunction("IP_PREFIX('64:ff9b::10', -1)", "IPv6 subnet size must be in range [0, 128]"); + assertInvalidFunction("IP_PREFIX('64:ff9b::10', 129)", "IPv6 subnet size must be in range [0, 128]"); + assertInvalidCast("IP_PREFIX('localhost', 24)", "Cannot cast value to IPADDRESS: localhost"); + assertInvalidCast("IP_PREFIX('64::ff9b::10', 24)", "Cannot cast value to IPADDRESS: 64::ff9b::10"); + assertInvalidCast("IP_PREFIX('64:face:book::10', 24)", "Cannot cast value to IPADDRESS: 64:face:book::10"); + assertInvalidCast("IP_PREFIX('123.456.789.012', 24)", "Cannot cast value to IPADDRESS: 123.456.789.012"); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestJoniRegexpFunctions.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestJoniRegexpFunctions.java new file mode 100644 index 0000000000000..372e6b50438ee --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestJoniRegexpFunctions.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import static com.facebook.presto.sql.analyzer.RegexLibrary.JONI; + +public class TestJoniRegexpFunctions + extends AbstractTestRegexpFunctions +{ + public TestJoniRegexpFunctions() + { + super(JONI); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestLambdaExpression.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestLambdaExpression.java index 72ca397d55a87..a2e0fdd33bd11 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestLambdaExpression.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestLambdaExpression.java @@ -50,7 +50,7 @@ private TestLambdaExpression(Session session) @BeforeClass public void setUp() { - functionAssertions.getMetadata().addFunctions(ImmutableList.of(APPLY_FUNCTION, INVOKE_FUNCTION)); + functionAssertions.getMetadata().registerBuiltInFunctions(ImmutableList.of(APPLY_FUNCTION, INVOKE_FUNCTION)); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestPageProcessorCompiler.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestPageProcessorCompiler.java index ceca36252ee92..fe6e210cb2d90 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestPageProcessorCompiler.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestPageProcessorCompiler.java @@ -38,6 +38,7 @@ import java.util.Optional; +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.block.BlockAssertions.createLongDictionaryBlock; import static com.facebook.presto.block.BlockAssertions.createRLEBlock; import static com.facebook.presto.block.BlockAssertions.createSlicesBlock; @@ -86,15 +87,15 @@ public void testNoCaching() projectionsBuilder.add(new CallExpression("concat", functionHandle, arrayType, ImmutableList.of(field(0, arrayType), field(1, arrayType)))); ImmutableList projections = projectionsBuilder.build(); - PageProcessor pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); - PageProcessor pageProcessor2 = compiler.compilePageProcessor(Optional.empty(), projections).get(); + PageProcessor pageProcessor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); + PageProcessor pageProcessor2 = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.empty(), projections).get(); assertTrue(pageProcessor != pageProcessor2); } @Test public void testSanityRLE() { - PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, BIGINT), field(1, VARCHAR)), MAX_BATCH_SIZE).get(); + PageProcessor processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.empty(), ImmutableList.of(field(0, BIGINT), field(1, VARCHAR)), MAX_BATCH_SIZE).get(); Slice varcharValue = Slices.utf8Slice("hello"); Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100)); @@ -126,7 +127,7 @@ public void testSanityFilterOnDictionary() FunctionHandle lessThan = functionManager.resolveOperator(LESS_THAN, fromTypes(BIGINT, BIGINT)); CallExpression filter = new CallExpression(LESS_THAN.name(), lessThan, BOOLEAN, ImmutableList.of(lengthVarchar, constant(10L, BIGINT))); - PageProcessor processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field(0, VARCHAR)), MAX_BATCH_SIZE).get(); + PageProcessor processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(filter), ImmutableList.of(field(0, VARCHAR)), MAX_BATCH_SIZE).get(); Page page = new Page(createDictionaryBlock(createExpectedValues(10), 100)); Page outputPage = getOnlyElement( @@ -165,7 +166,7 @@ public void testSanityFilterOnRLE() FunctionHandle lessThan = functionManager.resolveOperator(LESS_THAN, fromTypes(BIGINT, BIGINT)); CallExpression filter = new CallExpression(LESS_THAN.name(), lessThan, BOOLEAN, ImmutableList.of(field(0, BIGINT), constant(10L, BIGINT))); - PageProcessor processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field(0, BIGINT)), MAX_BATCH_SIZE).get(); + PageProcessor processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(filter), ImmutableList.of(field(0, BIGINT)), MAX_BATCH_SIZE).get(); Page page = new Page(createRLEBlock(5L, 100)); Page outputPage = getOnlyElement( @@ -186,7 +187,7 @@ public void testSanityFilterOnRLE() @Test public void testSanityColumnarDictionary() { - PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, VARCHAR)), MAX_BATCH_SIZE).get(); + PageProcessor processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.empty(), ImmutableList.of(field(0, VARCHAR)), MAX_BATCH_SIZE).get(); Page page = new Page(createDictionaryBlock(createExpectedValues(10), 100)); Page outputPage = getOnlyElement( @@ -214,7 +215,7 @@ public void testNonDeterministicProject() InputReferenceExpression col0 = field(0, BIGINT); CallExpression lessThanRandomExpression = new CallExpression(LESS_THAN.name(), lessThan, BOOLEAN, ImmutableList.of(col0, random)); - PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(lessThanRandomExpression), MAX_BATCH_SIZE).get(); + PageProcessor processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.empty(), ImmutableList.of(lessThanRandomExpression), MAX_BATCH_SIZE).get(); assertFalse(new RowExpressionDeterminismEvaluator(metadataManager.getFunctionManager()).isDeterministic(lessThanRandomExpression)); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestParametricScalarImplementationValidation.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestParametricScalarImplementationValidation.java index 31f5dcc815207..29be0a32f5b9f 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestParametricScalarImplementationValidation.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestParametricScalarImplementationValidation.java @@ -20,8 +20,8 @@ import java.lang.invoke.MethodHandle; import java.util.Optional; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.util.Reflection.methodHandle; import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; @@ -35,7 +35,7 @@ public void testConnectorSessionPosition() { // Without cached instance factory MethodHandle validFunctionMethodHandle = methodHandle(TestParametricScalarImplementationValidation.class, "validConnectorSessionParameterPosition", ConnectorSession.class, long.class, long.class); - ScalarFunctionImplementation validFunction = new ScalarFunctionImplementation( + BuiltInScalarFunctionImplementation validFunction = new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), @@ -44,7 +44,7 @@ public void testConnectorSessionPosition() assertEquals(validFunction.getMethodHandle(), validFunctionMethodHandle); try { - ScalarFunctionImplementation invalidFunction = new ScalarFunctionImplementation( + BuiltInScalarFunctionImplementation invalidFunction = new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), @@ -58,7 +58,7 @@ public void testConnectorSessionPosition() // With cached instance factory MethodHandle validFunctionWithInstanceFactoryMethodHandle = methodHandle(TestParametricScalarImplementationValidation.class, "validConnectorSessionParameterPosition", Object.class, ConnectorSession.class, long.class, long.class); - ScalarFunctionImplementation validFunctionWithInstanceFactory = new ScalarFunctionImplementation( + BuiltInScalarFunctionImplementation validFunctionWithInstanceFactory = new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), @@ -68,7 +68,7 @@ public void testConnectorSessionPosition() assertEquals(validFunctionWithInstanceFactory.getMethodHandle(), validFunctionWithInstanceFactoryMethodHandle); try { - ScalarFunctionImplementation invalidFunctionWithInstanceFactory = new ScalarFunctionImplementation( + BuiltInScalarFunctionImplementation invalidFunctionWithInstanceFactory = new BuiltInScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestProvidedBlockBuilderReturnPlaceConvention.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestProvidedBlockBuilderReturnPlaceConvention.java index cf07237e1bd2d..69bc83c23b345 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestProvidedBlockBuilderReturnPlaceConvention.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestProvidedBlockBuilderReturnPlaceConvention.java @@ -16,11 +16,12 @@ import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ReturnPlaceConvention; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ScalarImplementationChoice; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ReturnPlaceConvention; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ScalarImplementationChoice; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; @@ -35,10 +36,11 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ReturnPlaceConvention.PROVIDED_BLOCKBUILDER; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ReturnPlaceConvention.PROVIDED_BLOCKBUILDER; import static com.facebook.presto.operator.scalar.TestProvidedBlockBuilderReturnPlaceConvention.FunctionWithProvidedBlockReturnPlaceConvention1.PROVIDED_BLOCKBUILDER_CONVENTION1; import static com.facebook.presto.operator.scalar.TestProvidedBlockBuilderReturnPlaceConvention.FunctionWithProvidedBlockReturnPlaceConvention2.PROVIDED_BLOCKBUILDER_CONVENTION2; import static com.facebook.presto.spi.function.Signature.typeVariable; @@ -153,7 +155,7 @@ public static class FunctionWithProvidedBlockReturnPlaceConvention1 protected FunctionWithProvidedBlockReturnPlaceConvention1() { super(new Signature( - "identity1", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "identity1"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -163,7 +165,7 @@ protected FunctionWithProvidedBlockReturnPlaceConvention1() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type type = boundVariables.getTypeVariable("T"); MethodHandle methodHandleStack = MethodHandles.identity(type.getJavaType()); @@ -187,7 +189,7 @@ else if (type.getJavaType() == Block.class) { throw new UnsupportedOperationException(); } - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( ImmutableList.of( new ScalarImplementationChoice( false, @@ -273,7 +275,7 @@ public static class FunctionWithProvidedBlockReturnPlaceConvention2 protected FunctionWithProvidedBlockReturnPlaceConvention2() { super(new Signature( - "identity2", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "identity2"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -283,7 +285,7 @@ protected FunctionWithProvidedBlockReturnPlaceConvention2() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { Type type = boundVariables.getTypeVariable("T"); MethodHandle methodHandleStack = MethodHandles.identity(wrap(type.getJavaType())); @@ -307,7 +309,7 @@ else if (type.getJavaType() == Block.class) { throw new UnsupportedOperationException(); } - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( ImmutableList.of( new ScalarImplementationChoice( true, diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestRe2jRegexpFunctions.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestRe2jRegexpFunctions.java new file mode 100644 index 0000000000000..cf6d78e9a47f0 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestRe2jRegexpFunctions.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import static com.facebook.presto.sql.analyzer.RegexLibrary.RE2J; + +public class TestRe2jRegexpFunctions + extends AbstractTestRegexpFunctions +{ + public TestRe2jRegexpFunctions() + { + super(RE2J); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestArrayOfRowsUnnester.java b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestArrayOfRowsUnnester.java new file mode 100644 index 0000000000000..04ff0420a2516 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestArrayOfRowsUnnester.java @@ -0,0 +1,232 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.block.PageBuilderStatus; +import com.facebook.presto.spi.type.RowType; +import io.airlift.slice.Slice; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static com.facebook.presto.block.ColumnarTestUtils.assertBlock; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.array; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.column; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.computeExpectedUnnestedOutput; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.createArrayBlockOfRowBlocks; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.getFieldElements; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.nullExists; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.toSlices; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.validateTestInput; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestArrayOfRowsUnnester +{ + @Test + public void testWithoutMisAlignments() + { + int fieldCount = 2; + int[] unnestedLengths = {2, 1}; + + Slice[][][] elements = column( + array(toSlices("0.0.0", "0.0.1"), toSlices("0.1.0", "0.1.1")), + array(toSlices("1.0.0", "1.0.1"))); + + // Test output building incrementally and get final output. + Block[] blocks = testArrayOfRowsUnnester(unnestedLengths, unnestedLengths, elements, fieldCount); + // Check final state. No misalignment, so all blocks should be DictionaryBlocks. + assertEquals(blocks.length, 2); + assertTrue(blocks[0] instanceof DictionaryBlock); + assertTrue(blocks[1] instanceof DictionaryBlock); + } + + @Test + public void testWithMisAlignments() + { + int fieldCount = 2; + int[] unnestedLengths = {2, 1}; + + Slice[][][] elements = column( + array(toSlices("0.0.0", "0.0.1"), toSlices(null, "0.1.1")), + array(toSlices("1.0.0", "1.0.1"))); + + // requiredCounts[1] causes a misalignment + int[] requiredCounts = {2, 2}; + + // Test output building incrementally and get final output blocks + Block[] blocks = testArrayOfRowsUnnester(requiredCounts, unnestedLengths, elements, fieldCount); + + // Check final state. A misalignment has occurred. + assertEquals(blocks.length, 2); + // Misaligned, and has a null entry (elements[0][1][0]) + assertTrue(blocks[0] instanceof DictionaryBlock); + // Misaligned, and does not have any null entries + assertFalse(blocks[1] instanceof DictionaryBlock); + } + + @Test + public void testWithNullRowElement() + { + int fieldCount = 3; + int[] requiredOutputCounts = {3, 1, 1, 0, 0}; + int[] unnestedLengths = {3, 1, 1, 0, 0}; + + Slice[][][] elements = column(array(toSlices("0.0.0", "0.0.1", null), null, toSlices("0.2.0", "0.2.1", "0.2.2")), + array(toSlices("1.0.0", "1.0.1", "1.0.2")), + array(toSlices(null, "2.0.1", "2.0.2")), + null, + array()); + + // Test output building incrementally and get final output blocks + Block[] blocks = testArrayOfRowsUnnester(requiredOutputCounts, unnestedLengths, elements, fieldCount); + + // Check final state. All blocks are misaligned because of NULL row object elements[0][1]. + assertEquals(blocks.length, 3); + // Misaligned, but has a null entry. (elements[0][0][1]) + assertTrue(blocks[0] instanceof DictionaryBlock); + // Misaligned, and doesn't have a null entry. + assertFalse(blocks[1] instanceof DictionaryBlock); + // Misaligned, but has a null entry. (elements[0][0][1]) + assertTrue(blocks[2] instanceof DictionaryBlock); + } + + /** + * Test operations of ArrayOfRowUnnester incrementally on the input. + * Output final blocks after the whole input has been processed. + * + * Input 3d array {@code elements} stores values from a column with type >. + * elements[i] corresponds to a position in this column, represents one array of row(....). + * elements[i][j] represents one row(....) object in the array. + * elements[i][j][k] represents value of kth field in row(...) object. + */ + private static Block[] testArrayOfRowsUnnester(int[] requiredOutputCounts, int[] unnestedLengths, Slice[][][] elements, int fieldCount) + { + validateTestInput(requiredOutputCounts, unnestedLengths, elements, fieldCount); + + int positionCount = requiredOutputCounts.length; + + // True if there is a null Row element inside the array at this position + boolean[] containsNullRowElement = new boolean[positionCount]; + + // Populate containsNullRowElement + for (int i = 0; i < positionCount; i++) { + containsNullRowElement[i] = false; + if (elements[i] != null) { + for (int j = 0; j < elements[i].length; j++) { + if (elements[i][j] == null) { + containsNullRowElement[i] = true; + } + } + } + } + + // Check for null elements in individual input fields + boolean[] nullsPresent = new boolean[fieldCount]; + for (int field = 0; field < fieldCount; field++) { + nullsPresent[field] = nullExists(elements[field]); + } + + // Create the unnester and input block + RowType rowType = RowType.anonymous(Collections.nCopies(fieldCount, VARCHAR)); + Unnester arrayofRowsUnnester = new ArrayOfRowsUnnester(rowType); + Block arrayBlockOfRows = createArrayBlockOfRowBlocks(elements, rowType); + + Block[] blocks = null; + + // Verify output being produced after processing every position. (quadratic) + for (int inputTestCount = 1; inputTestCount <= elements.length; inputTestCount++) { + // Reset input and prepare for new output + PageBuilderStatus status = new PageBuilderStatus(); + arrayofRowsUnnester.resetInput(arrayBlockOfRows); + assertEquals(arrayofRowsUnnester.getInputEntryCount(), elements.length); + + arrayofRowsUnnester.startNewOutput(status, 10); + boolean misAligned = false; + + // Process inputTestCount positions + for (int i = 0; i < inputTestCount; i++) { + arrayofRowsUnnester.processCurrentAndAdvance(requiredOutputCounts[i]); + int elementsSize = (elements[i] != null ? elements[i].length : 0); + + // Check for misalignment, caused by either + // (1) required output count being greater than unnested length OR + // (2) null Row element + if ((requiredOutputCounts[i] > elementsSize) || containsNullRowElement[i]) { + misAligned = true; + } + } + + // Build output block and verify + blocks = arrayofRowsUnnester.buildOutputBlocksAndFlush(); + assertEquals(blocks.length, rowType.getFields().size()); + + // Verify output blocks for individual fields + for (int field = 0; field < blocks.length; field++) { + assertTrue((blocks[field] instanceof DictionaryBlock) || (!nullsPresent[field] && misAligned)); + assertFalse((blocks[field] instanceof DictionaryBlock) && (!nullsPresent[field] && misAligned)); + Slice[][] fieldElements = getFieldElements(elements, field); + Slice[] expectedOutput = computeExpectedUnnestedOutput(fieldElements, requiredOutputCounts, 0, inputTestCount); + assertBlock(blocks[field], expectedOutput); + } + } + + return blocks; + } + + @Test + public void testTrimmedBlocks() + { + int fieldCount = 3; + int[] unnestedLengths = {1, 2, 1}; + + Slice[][][] elements = column( + array(toSlices("0.0.0", "0.0.1", "0.0.2")), + array(toSlices("1.0.0", "1.0.1", "1.0.2"), toSlices("1.1.0", "1.1.1", "1.1.2")), + array(toSlices("2.0.0", "2.0.1", "2.0.2"))); + + RowType rowType = RowType.anonymous(Collections.nCopies(fieldCount, VARCHAR)); + Block arrayOfRowBlock = createArrayBlockOfRowBlocks(elements, rowType); + + // Remove the first element from the arrayBlock for testing + int startElement = 1; + + Slice[][][] truncatedSlices = Arrays.copyOfRange(elements, startElement, elements.length - startElement + 1); + int[] truncatedUnnestedLengths = Arrays.copyOfRange(unnestedLengths, startElement, elements.length - startElement + 1); + Block truncatedBlock = arrayOfRowBlock.getRegion(startElement, truncatedSlices.length); + + Unnester arrayOfRowsUnnester = new ArrayOfRowsUnnester(rowType); + arrayOfRowsUnnester.resetInput(truncatedBlock); + + arrayOfRowsUnnester.startNewOutput(new PageBuilderStatus(), 20); + + // Process all input entries in the truncated block + for (int i = 0; i < truncatedBlock.getPositionCount(); i++) { + arrayOfRowsUnnester.processCurrentAndAdvance(truncatedUnnestedLengths[i]); + } + + Block[] output = arrayOfRowsUnnester.buildOutputBlocksAndFlush(); + assertEquals(Arrays.asList(truncatedSlices).stream().mapToInt(slice -> slice.length).sum(), output[0].getPositionCount()); + + for (int i = 0; i < fieldCount; i++) { + Slice[] expectedOutput = computeExpectedUnnestedOutput(getFieldElements(truncatedSlices, i), truncatedUnnestedLengths, 0, truncatedSlices.length); + assertBlock(output[i], expectedOutput); + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestArrayUnnester.java b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestArrayUnnester.java new file mode 100644 index 0000000000000..2e8363c18a471 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestArrayUnnester.java @@ -0,0 +1,163 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.block.PageBuilderStatus; +import io.airlift.slice.Slice; +import org.testng.annotations.Test; + +import java.util.Arrays; + +import static com.facebook.presto.block.ColumnarTestUtils.assertBlock; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.array; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.column; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.computeExpectedUnnestedOutput; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.createArrayBlock; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.getFieldElements; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.nullExists; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.toSlices; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.validateTestInput; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestArrayUnnester +{ + @Test + public void testWithNullElement() + { + int[] unnestedLengths = {2, 1, 0, 1, 0}; + int[] requiredOutputCounts = {2, 2, 1, 1, 1}; + + Slice[][][] elements = column( + array(toSlices("0.0.0"), toSlices("0.1.0")), + array(toSlices("1.0.0")), + null, + array(toSlices(null)), + array()); + + Block[] blocks = testArrayUnnester(requiredOutputCounts, unnestedLengths, elements); + // Misalignment occurs, but a null element exists. + assertTrue(blocks[0] instanceof DictionaryBlock); + } + + @Test + public void testWithoutNullElement() + { + int[] unnestedLengths = {1, 1, 1, 0, 0}; + int[] requiredOutputCounts = {2, 2, 1, 1, 1}; + + Slice[][][] elements = column( + array(toSlices("0.0.0")), + array(toSlices("1.0.0")), + array(toSlices("2.0.0")), + null, + array()); + + Block[] blocks = testArrayUnnester(requiredOutputCounts, unnestedLengths, elements); + // Misalignment occurs, and there is no null element + assertFalse(blocks[0] instanceof DictionaryBlock); + } + + private static Block[] testArrayUnnester(int[] requiredOutputCounts, int[] unnestedLengths, Slice[][][] elements) + { + Slice[][] slices = getFieldElements(elements, 0); + validateTestInput(requiredOutputCounts, unnestedLengths, elements, 1); + + // Check if there is a null element in the input + boolean nullPresent = nullExists(slices); + + // Initialize unnester + Unnester arrayUnnester = new ArrayUnnester(VARCHAR); + Block arrayBlock = createArrayBlock(slices); + + Block[] blocks = null; + + // Verify output being produced after processing every position. (quadratic) + for (int inputTestCount = 1; inputTestCount <= elements.length; inputTestCount++) { + // Reset input + arrayUnnester.resetInput(arrayBlock); + assertEquals(arrayUnnester.getInputEntryCount(), elements.length); + + // Prepare for new output + PageBuilderStatus status = new PageBuilderStatus(); + arrayUnnester.startNewOutput(status, 10); + boolean misAligned = false; + + // Process inputTestCount positions + for (int i = 0; i < inputTestCount; i++) { + int elementsSize = (elements[i] != null ? elements[i].length : 0); + assertEquals(arrayUnnester.getCurrentUnnestedLength(), elementsSize); + arrayUnnester.processCurrentAndAdvance(requiredOutputCounts[i]); + + if (requiredOutputCounts[i] > elementsSize) { + misAligned = true; + } + } + + // Verify output + blocks = arrayUnnester.buildOutputBlocksAndFlush(); + assertEquals(blocks.length, 1); + assertTrue((blocks[0] instanceof DictionaryBlock) || (!nullPresent && misAligned)); + assertFalse((blocks[0] instanceof DictionaryBlock) && (!nullPresent && misAligned)); + Slice[] expectedOutput = computeExpectedUnnestedOutput(slices, requiredOutputCounts, 0, inputTestCount); + assertBlock(blocks[0], expectedOutput); + } + + return blocks; + } + + @Test + public void testTrimmedBlocks() + { + int[] unnestedLengths = {2, 1, 2, 3, 1}; + + Slice[][][] elements = column( + array(toSlices("0.0.0"), toSlices("0.1.0")), + array(toSlices("1.0.0")), + array(toSlices("2.0.0"), toSlices("2.1.0")), + array(toSlices("3.0.0"), toSlices("3.1.0"), toSlices("3.2.0")), + array(toSlices("4.0.0"))); + + Slice[][] slices = getFieldElements(elements, 0); + Block arrayBlock = createArrayBlock(slices); + + // Remove the first element from the arrayBlock for testing + int startElement = 1; + + Slice[][] truncatedSlices = Arrays.copyOfRange(slices, startElement, slices.length - startElement + 1); + int[] truncatedUnnestedLengths = Arrays.copyOfRange(unnestedLengths, startElement, slices.length - startElement + 1); + Block truncatedBlock = arrayBlock.getRegion(startElement, truncatedSlices.length); + assertBlock(truncatedBlock, truncatedSlices); + + Unnester arrayUnnester = new ArrayUnnester(VARCHAR); + arrayUnnester.resetInput(truncatedBlock); + + arrayUnnester.startNewOutput(new PageBuilderStatus(), 20); + + // Process all input entries in the truncated block + for (int i = 0; i < truncatedBlock.getPositionCount(); i++) { + arrayUnnester.processCurrentAndAdvance(truncatedUnnestedLengths[i]); + } + + Block[] output = arrayUnnester.buildOutputBlocksAndFlush(); + assertEquals(Arrays.asList(truncatedSlices).stream().mapToInt(slice -> slice.length).sum(), output[0].getPositionCount()); + + Slice[] expectedOutput = computeExpectedUnnestedOutput(truncatedSlices, truncatedUnnestedLengths, 0, truncatedSlices.length); + assertBlock(output[0], expectedOutput); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestMapUnnester.java b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestMapUnnester.java new file mode 100644 index 0000000000000..07a7e5aa9aaa4 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestMapUnnester.java @@ -0,0 +1,184 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.block.PageBuilderStatus; +import io.airlift.slice.Slice; +import org.testng.annotations.Test; + +import java.util.Arrays; + +import static com.facebook.presto.block.ColumnarTestUtils.assertBlock; +import static com.facebook.presto.block.TestColumnarMap.createBlockBuilderWithValues; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.array; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.column; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.computeExpectedUnnestedOutput; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.getFieldElements; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.nullExists; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.toSlices; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.validateTestInput; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestMapUnnester +{ + @Test + public void testSimple() + { + int[] unnestedLengths = {1, 2, 0}; + int[] requiredOutputCounts = unnestedLengths; + + Slice[][][] elements = column( + array(toSlices("0.0.0", "0.0.1")), + array(toSlices("1.0.0", "1.0.1"), toSlices("1.1.0", "1.1.1")), + null); + + Block[] blocks = testMapUnnester(unnestedLengths, requiredOutputCounts, elements); + + // Check final state. Both the blocks should be dictionary blocks, since there is no misalignment. + assertEquals(blocks.length, 2); + assertTrue(blocks[0] instanceof DictionaryBlock); + assertTrue(blocks[1] instanceof DictionaryBlock); + } + + @Test + public void testMisaligned() + { + int[] unnestedLengths = {1, 2, 0, 0}; + int[] requiredOutputCounts = {1, 3, 0, 1}; + + Slice[][][] elements = column( + array(toSlices("0.0.0", "0.0.1")), + array(toSlices("1.0.0", "1.0.1"), toSlices("1.1.0", null)), + null, + null); + + Block[] blocks = testMapUnnester(requiredOutputCounts, unnestedLengths, elements); + + // Check final state. Misalignment has occurred. + assertEquals(blocks.length, 2); + // Misaligned, does not have a null entry. + assertFalse(blocks[0] instanceof DictionaryBlock); + // Misaligned, but has a null entry elements[1][1][1] + assertTrue(blocks[1] instanceof DictionaryBlock); + } + + private static Block[] testMapUnnester(int[] requiredOutputCounts, int[] unnestedLengths, Slice[][][] elements) + { + validateTestInput(requiredOutputCounts, unnestedLengths, elements, 2); + int positionCount = unnestedLengths.length; + + for (int index = 0; index < positionCount; index++) { + if (elements[index] != null) { + for (int i = 0; i < elements[index].length; i++) { + assertTrue(elements[index][i] != null, "entry cannot be null"); + assertEquals(elements[index][i].length, 2); + } + } + } + + // Check for null elements in individual input fields + boolean[] nullsPresent = new boolean[2]; + nullsPresent[0] = nullExists(elements[0]); + nullsPresent[1] = nullExists(elements[1]); + assertFalse(nullsPresent[0]); + + // Create the unnester and input block + Unnester mapUnnester = new MapUnnester(VARCHAR, VARCHAR); + Block mapBlock = createBlockBuilderWithValues(elements).build(); + + Block[] blocks = null; + + // Verify output being produced after processing every position. (quadratic) + for (int inputTestCount = 1; inputTestCount <= elements.length; inputTestCount++) { + // Reset input and prepare for new output + PageBuilderStatus status = new PageBuilderStatus(); + mapUnnester.resetInput(mapBlock); + assertEquals(mapUnnester.getInputEntryCount(), elements.length); + + mapUnnester.startNewOutput(status, 10); + boolean misAligned = false; + + // Process inputTestCount positions + for (int i = 0; i < inputTestCount; i++) { + mapUnnester.processCurrentAndAdvance(requiredOutputCounts[i]); + int elementsSize = (elements[i] != null ? elements[i].length : 0); + + // Check for misalignment, caused by required output count being greater than unnested length + if ((requiredOutputCounts[i] > elementsSize)) { + misAligned = true; + } + } + + // Build output + blocks = mapUnnester.buildOutputBlocksAndFlush(); + assertEquals(blocks.length, 2); + + // Verify output blocks for individual fields + for (int field = 0; field < blocks.length; field++) { + assertTrue((blocks[field] instanceof DictionaryBlock) || (!nullsPresent[field] && misAligned)); + assertFalse((blocks[field] instanceof DictionaryBlock) && (!nullsPresent[field] && misAligned)); + Slice[][] fieldElements = getFieldElements(elements, field); + Slice[] expectedOutput = computeExpectedUnnestedOutput(fieldElements, requiredOutputCounts, 0, inputTestCount); + assertBlock(blocks[field], expectedOutput); + } + } + + return blocks; + } + + @Test + public void testTrimmedBlocks() + { + int[] unnestedLengths = {1, 2, 1}; + + Slice[][][] elements = column( + array(toSlices("0.0.0", "0.0.1")), + array(toSlices("1.0.0", "1.0.1"), toSlices("1.1.0", "1.1.1")), + array(toSlices("2.0.0", "2.0.1"))); + + Block mapBlock = createBlockBuilderWithValues(elements).build(); + + // Remove the first element from the arrayBlock for testing + int startElement = 1; + + Slice[][][] truncatedSlices = Arrays.copyOfRange(elements, startElement, elements.length - startElement + 1); + int[] truncatedUnnestedLengths = Arrays.copyOfRange(unnestedLengths, startElement, elements.length - startElement + 1); + Block truncatedBlock = mapBlock.getRegion(startElement, elements.length - startElement); + assertBlock(truncatedBlock, truncatedSlices); + + Unnester mapUnnester = new MapUnnester(VARCHAR, VARCHAR); + mapUnnester.resetInput(truncatedBlock); + + mapUnnester.startNewOutput(new PageBuilderStatus(), 20); + + // Process all input entries in the truncated block + for (int i = 0; i < truncatedBlock.getPositionCount(); i++) { + mapUnnester.processCurrentAndAdvance(truncatedUnnestedLengths[i]); + } + + Block[] output = mapUnnester.buildOutputBlocksAndFlush(); + assertEquals(Arrays.asList(truncatedSlices).stream().mapToInt(slice -> slice.length).sum(), output[0].getPositionCount()); + + Slice[] expectedOutput0 = computeExpectedUnnestedOutput(getFieldElements(truncatedSlices, 0), truncatedUnnestedLengths, 0, truncatedSlices.length); + assertBlock(output[0], expectedOutput0); + + Slice[] expectedOutput1 = computeExpectedUnnestedOutput(getFieldElements(truncatedSlices, 1), truncatedUnnestedLengths, 0, truncatedSlices.length); + assertBlock(output[1], expectedOutput1); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestReplicatedBlockBuilder.java b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestReplicatedBlockBuilder.java new file mode 100644 index 0000000000000..f7fcb89b6b678 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestReplicatedBlockBuilder.java @@ -0,0 +1,113 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.DictionaryBlock; +import io.airlift.slice.Slice; +import org.testng.annotations.Test; + +import java.util.Collections; + +import static com.facebook.presto.block.ColumnarTestUtils.assertBlock; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.createReplicatedOutputSlice; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.createSimpleBlock; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.toSlices; +import static com.facebook.presto.operator.unnest.UnnestOperatorBlockUtil.calculateNewArraySize; +import static io.airlift.slice.Slices.utf8Slice; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestReplicatedBlockBuilder +{ + @Test + public void testReplicateOutput() + { + String[] values = {"a", "b", "c", null, null, "e"}; + int[] count = {1, 0, 2, 2, 0, 3}; + testReplication(toSlices(values), count); + } + + @Test + public void testReplicateEmptyOutput() + { + String[] values = {"a", null, "b"}; + int[] count = {0, 0, 0}; + testReplication(toSlices(values), count); + } + + private static void testReplication(Slice[] values, int[] counts) + { + assertEquals(values.length, counts.length); + + ReplicatedBlockBuilder replicateBlockBuilder = new ReplicatedBlockBuilder(); + Block valuesBlock = createSimpleBlock(values); + replicateBlockBuilder.resetInputBlock(valuesBlock); + replicateBlockBuilder.startNewOutput(100); + + for (int i = 0; i < counts.length; i++) { + replicateBlockBuilder.appendRepeated(i, counts[i]); + } + + Block outputBlock = replicateBlockBuilder.buildOutputAndFlush(); + assertBlock(outputBlock, createReplicatedOutputSlice(values, counts)); + assertTrue(outputBlock instanceof DictionaryBlock); + } + + @Test + public void testCapacityIncrease() + { + assertSmallCapacityIncrease(4, 1, 4); + assertBigCapacityIncrease(50, 49, 100); + } + + /** + * verify capacity increase when required new capacity is <= the value returned from {@link UnnestOperatorBlockUtil#calculateNewArraySize} + */ + private static void assertSmallCapacityIncrease(int initialSize, int firstAppendCount, int secondAppendCount) + { + assertTrue(firstAppendCount <= initialSize); + assertTrue(firstAppendCount + secondAppendCount > initialSize); + assertTrue(firstAppendCount + secondAppendCount <= calculateNewArraySize(initialSize)); + assertCapacityIncrease(initialSize, firstAppendCount, secondAppendCount, new ReplicatedBlockBuilder()); + } + + /** + * verify capacity increase when required new capacity is > the value returned from {@link UnnestOperatorBlockUtil#calculateNewArraySize} + */ + private static void assertBigCapacityIncrease(int initialSize, int firstAppendCount, int secondAppendCount) + { + assertTrue(firstAppendCount <= initialSize); + assertTrue(firstAppendCount + secondAppendCount > initialSize); + assertTrue(firstAppendCount + secondAppendCount > calculateNewArraySize(initialSize)); + assertCapacityIncrease(initialSize, firstAppendCount, secondAppendCount, new ReplicatedBlockBuilder()); + } + + private static void assertCapacityIncrease(int initialSize, int firstAppendCount, int secondAppendCount, ReplicatedBlockBuilder replicatedBlockBuilder) + { + Slice[] values = {null, utf8Slice("a")}; + Block inputBlock = createSimpleBlock(values); + int repeatingIndex = 1; + + replicatedBlockBuilder.resetInputBlock(inputBlock); + replicatedBlockBuilder.startNewOutput(initialSize); + + replicatedBlockBuilder.appendRepeated(repeatingIndex, firstAppendCount); + replicatedBlockBuilder.appendRepeated(repeatingIndex, secondAppendCount); + + Block output = replicatedBlockBuilder.buildOutputAndFlush(); + int totalCount = firstAppendCount + secondAppendCount; + assertBlock(output, Collections.nCopies(totalCount, values[repeatingIndex]).toArray(new Slice[totalCount])); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestUnnestBlockBuilder.java b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestUnnestBlockBuilder.java new file mode 100644 index 0000000000000..1ad7e43ba419e --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestUnnestBlockBuilder.java @@ -0,0 +1,158 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.block.PageBuilderStatus; +import io.airlift.slice.Slice; +import org.testng.annotations.Test; + +import java.util.Arrays; + +import static com.facebook.presto.block.ColumnarTestUtils.assertBlock; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.createSimpleBlock; +import static com.facebook.presto.operator.unnest.TestUnnesterUtil.toSlices; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static io.airlift.slice.Slices.utf8Slice; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestUnnestBlockBuilder +{ + @Test + public void testWithNullElements() + { + testUnnestBlockBuilderMethods(new String[]{"a", "b", "c", null, "d", "e"}); + } + + @Test + public void testWithoutNullElements() + { + testUnnestBlockBuilderMethods(new String[]{"a", "b", "c", "d", "e"}); + } + + @Test + public void testCapacityIncrease() + { + Slice[] values = new Slice[100]; + for (int i = 0; i < values.length; i++) { + values[i] = utf8Slice("a"); + } + + UnnestBlockBuilder unnestBlockBuilder = new UnnestBlockBuilder(VARCHAR); + Block valuesBlock = createSimpleBlock(values); + unnestBlockBuilder.resetInputBlock(valuesBlock); + + unnestBlockBuilder.startNewOutput(new PageBuilderStatus(), 20); + unnestBlockBuilder.appendRange(0, values.length); + assertBlock(unnestBlockBuilder.buildOutputAndFlush(), values); + + unnestBlockBuilder.clearCurrentOutput(); + } + + @Test + public void testEmptyOutput() + { + Slice[] values = toSlices(new String[]{"a", "b"}); + UnnestBlockBuilder unnestBlockBuilder = new UnnestBlockBuilder(VARCHAR); + Block valuesBlock = createSimpleBlock(values); + unnestBlockBuilder.resetInputBlock(valuesBlock); + unnestBlockBuilder.startNewOutput(new PageBuilderStatus(), 10); + + // Nothing added to the output + Block block = unnestBlockBuilder.buildOutputAndFlush(); + assertTrue(block instanceof DictionaryBlock); + assertBlock(block, new Slice[]{}); + } + + private static void testUnnestBlockBuilderMethods(String[] stringValues) + { + Slice[] values = toSlices(stringValues); + UnnestBlockBuilder unnestBlockBuilder = new UnnestBlockBuilder(VARCHAR); + Block valuesBlock = createSimpleBlock(values); + unnestBlockBuilder.resetInputBlock(valuesBlock); + + testAppendSingleElement(unnestBlockBuilder, values); + testAppendRange(unnestBlockBuilder, values); + testAppendNull(unnestBlockBuilder, values); + } + + private static void testAppendSingleElement(UnnestBlockBuilder unnestBlockBuilder, Slice[] values) + { + // Test unnestBlockBuilder output over the course of building output + for (int testCount = 1; testCount <= values.length; testCount++) { + unnestBlockBuilder.startNewOutput(new PageBuilderStatus(), 10); + + for (int i = 0; i < testCount; i++) { + unnestBlockBuilder.appendElement(i); + } + + Block block = unnestBlockBuilder.buildOutputAndFlush(); + assertTrue(block instanceof DictionaryBlock); + assertBlock(block, Arrays.copyOf(values, testCount)); + } + } + + private static void testAppendRange(UnnestBlockBuilder unnestBlockBuilder, Slice[] values) + { + unnestBlockBuilder.startNewOutput(new PageBuilderStatus(), 10); + assertTrue(values.length >= 3, "test requires at least 3 elements in values"); + int startIndex = 1; + int length = values.length - 2; + unnestBlockBuilder.appendRange(startIndex, length); + Block block = unnestBlockBuilder.buildOutputAndFlush(); + assertTrue(block instanceof DictionaryBlock); + assertBlock(block, Arrays.copyOfRange(values, startIndex, startIndex + length)); + } + + private static void testAppendNull(UnnestBlockBuilder unnestBlockBuilder, Slice[] values) + { + assertTrue(values.length >= 1, "values should have at least one element"); + + int nullIndex = -1; + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + nullIndex = i; + break; + } + } + + // Test-1: appending a non-null element + unnestBlockBuilder.startNewOutput(new PageBuilderStatus(), 10); + unnestBlockBuilder.appendElement(0); + Block block = unnestBlockBuilder.buildOutputAndFlush(); + assertTrue(block instanceof DictionaryBlock); + assertBlock(block, new Slice[]{values[0]}); + + // Test-2: appending a non-null element and a null. + unnestBlockBuilder.startNewOutput(new PageBuilderStatus(), 10); + unnestBlockBuilder.appendElement(0); + unnestBlockBuilder.appendNull(); + block = unnestBlockBuilder.buildOutputAndFlush(); + assertTrue((block instanceof DictionaryBlock) || (nullIndex == -1)); + assertFalse((block instanceof DictionaryBlock) && (nullIndex == -1)); + assertBlock(block, new Slice[]{values[0], null}); + + // Test-3: appending a non-null element, a null, and another non-null element. + unnestBlockBuilder.startNewOutput(new PageBuilderStatus(), 10); + unnestBlockBuilder.appendElement(0); + unnestBlockBuilder.appendNull(); + unnestBlockBuilder.appendElement(0); + block = unnestBlockBuilder.buildOutputAndFlush(); + assertTrue((block instanceof DictionaryBlock) || (nullIndex == -1)); + assertFalse((block instanceof DictionaryBlock) && (nullIndex == -1)); + assertBlock(block, new Slice[]{values[0], null, values[0]}); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestUnnestOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestUnnestOperator.java similarity index 97% rename from presto-main/src/test/java/com/facebook/presto/operator/TestUnnestOperator.java rename to presto-main/src/test/java/com/facebook/presto/operator/unnest/TestUnnestOperator.java index d501c5ae011e5..27b33309bd97f 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestUnnestOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestUnnestOperator.java @@ -11,9 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.operator; +package com.facebook.presto.operator.unnest; import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.operator.DriverContext; +import com.facebook.presto.operator.OperatorFactory; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.type.ArrayType; @@ -30,6 +32,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; @@ -42,7 +45,6 @@ import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; import static com.facebook.presto.util.StructuralTestUtil.arrayBlockOf; import static com.facebook.presto.util.StructuralTestUtil.mapBlockOf; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.Double.NEGATIVE_INFINITY; import static java.lang.Double.NaN; import static java.lang.Double.POSITIVE_INFINITY; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestUnnesterUtil.java b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestUnnesterUtil.java new file mode 100644 index 0000000000000..d1b4f01047269 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/unnest/TestUnnesterUtil.java @@ -0,0 +1,248 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.unnest; + +import com.facebook.presto.spi.block.ArrayBlockBuilder; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.type.RowType; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; + +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestUnnesterUtil +{ + private TestUnnesterUtil() {} + + public static Block createSimpleBlock(Slice[] values) + { + BlockBuilder elementBlockBuilder = VARCHAR.createBlockBuilder(null, values.length); + for (Slice v : values) { + if (v == null) { + elementBlockBuilder.appendNull(); + } + else { + VARCHAR.writeSlice(elementBlockBuilder, v); + } + } + return elementBlockBuilder.build(); + } + + public static Block createArrayBlock(Slice[][] values) + { + BlockBuilder blockBuilder = new ArrayBlockBuilder(VARCHAR, null, 100, 100); + for (Slice[] expectedValue : values) { + if (expectedValue == null) { + blockBuilder.appendNull(); + } + else { + Block elementBlock = createSimpleBlock(expectedValue); + blockBuilder.appendStructure(elementBlock); + } + } + return blockBuilder.build(); + } + + public static Block createArrayBlockOfRowBlocks(Slice[][][] elements, RowType rowType) + { + BlockBuilder arrayBlockBuilder = new ArrayBlockBuilder(rowType, null, 100, 100); + for (int i = 0; i < elements.length; i++) { + if (elements[i] == null) { + arrayBlockBuilder.appendNull(); + } + else { + Slice[][] expectedValues = elements[i]; + BlockBuilder elementBlockBuilder = rowType.createBlockBuilder(null, elements[i].length); + for (Slice[] expectedValue : expectedValues) { + if (expectedValue == null) { + elementBlockBuilder.appendNull(); + } + else { + BlockBuilder entryBuilder = elementBlockBuilder.beginBlockEntry(); + for (Slice v : expectedValue) { + if (v == null) { + entryBuilder.appendNull(); + } + else { + VARCHAR.writeSlice(entryBuilder, v); + } + } + elementBlockBuilder.closeEntry(); + } + } + arrayBlockBuilder.appendStructure(elementBlockBuilder.build()); + } + } + return arrayBlockBuilder.build(); + } + + public static boolean nullExists(Slice[][] elements) + { + for (int i = 0; i < elements.length; i++) { + if (elements[i] != null) { + for (int j = 0; j < elements[i].length; j++) { + if (elements[i][j] == null) { + return true; + } + } + } + } + return false; + } + + public static Slice[] computeExpectedUnnestedOutput(Slice[][] elements, int[] requiredOutputCounts, int startPosition, int length) + { + checkArgument(startPosition >= 0 && length >= 0); + checkArgument(startPosition + length - 1 < requiredOutputCounts.length); + checkArgument(elements.length == requiredOutputCounts.length); + + int outputCount = 0; + for (int i = 0; i < length; i++) { + int position = startPosition + i; + int arrayLength = elements[position] == null ? 0 : elements[position].length; + checkArgument(requiredOutputCounts[position] >= arrayLength); + outputCount += requiredOutputCounts[position]; + } + + Slice[] expectedOutput = new Slice[outputCount]; + int offset = 0; + + for (int i = 0; i < length; i++) { + int position = startPosition + i; + int arrayLength = elements[position] == null ? 0 : elements[position].length; + + int requiredCount = requiredOutputCounts[position]; + + for (int j = 0; j < arrayLength; j++) { + expectedOutput[offset++] = elements[position][j]; + } + for (int j = 0; j < (requiredCount - arrayLength); j++) { + expectedOutput[offset++] = null; + } + } + + return expectedOutput; + } + + /** + * Extract elements corresponding to a specific field from 3D slices + */ + public static Slice[][] getFieldElements(Slice[][][] slices, int fieldNo) + { + Slice[][] output = new Slice[slices.length][]; + + for (int i = 0; i < slices.length; i++) { + if (slices[i] != null) { + output[i] = new Slice[slices[i].length]; + + for (int j = 0; j < slices[i].length; j++) { + if (slices[i][j] != null) { + output[i][j] = slices[i][j][fieldNo]; + } + else { + output[i][j] = null; + } + } + } + else { + output[i] = null; + } + } + + return output; + } + + public static void validateTestInput(int[] requiredOutputCounts, int[] unnestedLengths, Slice[][][] slices, int fieldCount) + { + requireNonNull(requiredOutputCounts, "output counts array is null"); + requireNonNull(unnestedLengths, "unnested lengths is null"); + requireNonNull(slices, "slices array is null"); + + // verify lengths + int positionCount = slices.length; + assertEquals(requiredOutputCounts.length, positionCount); + assertEquals(unnestedLengths.length, positionCount); + + // Unnested array lengths must be <= required output count + for (int i = 0; i < requiredOutputCounts.length; i++) { + assertTrue(unnestedLengths[i] <= requiredOutputCounts[i]); + } + + // Elements should have the right shape for every field + for (int index = 0; index < positionCount; index++) { + Slice[][] entry = slices[index]; + + int entryLength = entry != null ? entry.length : 0; + assertEquals(entryLength, unnestedLengths[index]); + + // Verify number of fields + for (int i = 0; i < entryLength; i++) { + if (entry[i] != null) { + assertEquals(entry[i].length, fieldCount); + } + } + } + } + + public static Slice[] createReplicatedOutputSlice(Slice[] input, int[] counts) + { + assertEquals(input.length, counts.length); + + int outputLength = 0; + for (int i = 0; i < input.length; i++) { + outputLength += counts[i]; + } + + Slice[] output = new Slice[outputLength]; + int offset = 0; + for (int i = 0; i < input.length; i++) { + for (int j = 0; j < counts[i]; j++) { + output[offset++] = input[i]; + } + } + + return output; + } + + static Slice[][][] column(Slice[][]... arraysOfRow) + { + return arraysOfRow; + } + + static Slice[][] array(Slice[]... rows) + { + return rows; + } + + static Slice[] toSlices(String... values) + { + if (values == null) { + return null; + } + + Slice[] slices = new Slice[values.length]; + for (int i = 0; i < values.length; i++) { + if (values[i] != null) { + slices[i] = Slices.utf8Slice(values[i]); + } + } + + return slices; + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/window/AbstractTestWindowFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/window/AbstractTestWindowFunction.java index 7bea7add9ffe7..817ed7354d5b8 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/window/AbstractTestWindowFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/window/AbstractTestWindowFunction.java @@ -20,9 +20,9 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.testing.Closeables.closeAllRuntimeException; @Test public abstract class AbstractTestWindowFunction diff --git a/presto-main/src/test/java/com/facebook/presto/operator/window/TestFirstValueFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/window/TestFirstValueFunction.java index 12d665885a080..745b533208ff0 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/window/TestFirstValueFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/window/TestFirstValueFunction.java @@ -129,4 +129,272 @@ public void testFirstValueBounded() .row(null, null, 7L) .build()); } + + @Test + public void testFirstValueUnboundedIgnoreNulls() + { + assertWindowQuery("first_value(orderdate) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, INTEGER, VARCHAR, VARCHAR) + .row(3, "F", "1993-10-14") + .row(5, "F", "1993-10-14") + .row(6, "F", "1993-10-14") + .row(33, "F", "1993-10-14") + .row(1, "O", "1996-01-02") + .row(2, "O", "1996-01-02") + .row(4, "O", "1996-01-02") + .row(7, "O", "1996-01-02") + .row(32, "O", "1996-01-02") + .row(34, "O", "1996-01-02") + .build()); + assertWindowQueryWithNulls("first_value(orderdate) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, VARCHAR) + .row(3L, "F", "1993-10-14") + .row(5L, "F", "1993-10-14") + .row(6L, "F", "1993-10-14") + .row(null, "F", "1993-10-14") + .row(34L, "O", "1998-07-21") + .row(null, "O", "1998-07-21") + .row(1L, null, null) + .row(7L, null, "1996-01-10") + .row(null, null, "1996-01-10") + .row(null, null, "1996-01-10") + .build()); + assertWindowQueryWithNulls("first_value(orderdate) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey " + + "ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, VARCHAR) + .row(3L, "F", "1993-10-14") + .row(5L, "F", "1993-10-14") + .row(6L, "F", "1993-10-14") + .row(null, "F", "1993-10-14") + .row(34L, "O", "1998-07-21") + .row(null, "O", "1998-07-21") + .row(1L, null, "1996-01-10") + .row(7L, null, "1996-01-10") + .row(null, null, "1996-01-10") + .row(null, null, "1996-01-10") + .build()); + + assertWindowQuery("first_value(orderkey) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, INTEGER, VARCHAR, INTEGER) + .row(3, "F", 3) + .row(5, "F", 3) + .row(6, "F", 3) + .row(33, "F", 3) + .row(1, "O", 1) + .row(2, "O", 1) + .row(4, "O", 1) + .row(7, "O", 1) + .row(32, "O", 1) + .row(34, "O", 1) + .build()); + assertWindowQueryWithNulls("first_value(orderkey) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 3L) + .row(5L, "F", 3L) + .row(6L, "F", 3L) + .row(null, "F", 3L) + .row(34L, "O", 34L) + .row(null, "O", 34L) + .row(1L, null, 1L) + .row(7L, null, 1L) + .row(null, null, 1L) + .row(null, null, 1L) + .build()); + assertWindowQueryWithNulls("first_value(orderkey) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey NULLS FIRST " + + "ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(null, "F", 3L) + .row(3L, "F", 3L) + .row(5L, "F", 3L) + .row(6L, "F", 3L) + .row(null, "O", 34L) + .row(34L, "O", 34L) + .row(null, null, 1L) + .row(null, null, 1L) + .row(1L, null, 1L) + .row(7L, null, 1L) + .build()); + + // Timestamp + assertWindowQuery("date_format(first_value(cast(orderdate as TIMESTAMP)) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey), '%Y-%m-%d')", + resultBuilder(TEST_SESSION, INTEGER, VARCHAR, VARCHAR) + .row(3, "F", "1993-10-14") + .row(5, "F", "1993-10-14") + .row(6, "F", "1993-10-14") + .row(33, "F", "1993-10-14") + .row(1, "O", "1996-01-02") + .row(2, "O", "1996-01-02") + .row(4, "O", "1996-01-02") + .row(7, "O", "1996-01-02") + .row(32, "O", "1996-01-02") + .row(34, "O", "1996-01-02") + .build()); + } + + @Test + public void testFirstValueBoundedIgnoreNulls() + { + assertWindowQuery("first_value(orderkey) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey " + + "ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING)", + resultBuilder(TEST_SESSION, INTEGER, VARCHAR, INTEGER) + .row(3, "F", 3) + .row(5, "F", 3) + .row(6, "F", 3) + .row(33, "F", 5) + .row(1, "O", 1) + .row(2, "O", 1) + .row(4, "O", 1) + .row(7, "O", 2) + .row(32, "O", 4) + .row(34, "O", 7) + .build()); + assertWindowQueryWithNulls("first_value(orderkey) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey NULLS FIRST " + + "ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(null, "F", 3L) + .row(3L, "F", 3L) + .row(5L, "F", 3L) + .row(6L, "F", 5L) + .row(null, "O", 34L) + .row(34L, "O", 34L) + .row(null, null, null) + .row(null, null, 1L) + .row(1L, null, 1L) + .row(7L, null, 1L) + .build()); + } + + @Test + public void testFirstValueUnboundedRespectNulls() + { + assertWindowQuery("first_value(orderdate) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, INTEGER, VARCHAR, VARCHAR) + .row(3, "F", "1993-10-14") + .row(5, "F", "1993-10-14") + .row(6, "F", "1993-10-14") + .row(33, "F", "1993-10-14") + .row(1, "O", "1996-01-02") + .row(2, "O", "1996-01-02") + .row(4, "O", "1996-01-02") + .row(7, "O", "1996-01-02") + .row(32, "O", "1996-01-02") + .row(34, "O", "1996-01-02") + .build()); + assertWindowQueryWithNulls("first_value(orderdate) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, VARCHAR) + .row(3L, "F", "1993-10-14") + .row(5L, "F", "1993-10-14") + .row(6L, "F", "1993-10-14") + .row(null, "F", "1993-10-14") + .row(34L, "O", "1998-07-21") + .row(null, "O", "1998-07-21") + .row(1L, null, null) + .row(7L, null, null) + .row(null, null, null) + .row(null, null, null) + .build()); + assertWindowQueryWithNulls("first_value(orderdate) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey " + + "ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, VARCHAR) + .row(3L, "F", "1993-10-14") + .row(5L, "F", "1993-10-14") + .row(6L, "F", "1993-10-14") + .row(null, "F", "1993-10-14") + .row(34L, "O", "1998-07-21") + .row(null, "O", "1998-07-21") + .row(1L, null, null) + .row(7L, null, null) + .row(null, null, null) + .row(null, null, null) + .build()); + + assertWindowQuery("first_value(orderkey) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, INTEGER, VARCHAR, INTEGER) + .row(3, "F", 3) + .row(5, "F", 3) + .row(6, "F", 3) + .row(33, "F", 3) + .row(1, "O", 1) + .row(2, "O", 1) + .row(4, "O", 1) + .row(7, "O", 1) + .row(32, "O", 1) + .row(34, "O", 1) + .build()); + assertWindowQueryWithNulls("first_value(orderkey) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 3L) + .row(5L, "F", 3L) + .row(6L, "F", 3L) + .row(null, "F", 3L) + .row(34L, "O", 34L) + .row(null, "O", 34L) + .row(1L, null, 1L) + .row(7L, null, 1L) + .row(null, null, 1L) + .row(null, null, 1L) + .build()); + assertWindowQueryWithNulls("first_value(orderkey) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey NULLS FIRST " + + "ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(null, "F", null) + .row(3L, "F", null) + .row(5L, "F", null) + .row(6L, "F", null) + .row(null, "O", null) + .row(34L, "O", null) + .row(null, null, null) + .row(null, null, null) + .row(1L, null, null) + .row(7L, null, null) + .build()); + + // Timestamp + assertWindowQuery("date_format(first_value(cast(orderdate as TIMESTAMP)) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey), '%Y-%m-%d')", + resultBuilder(TEST_SESSION, INTEGER, VARCHAR, VARCHAR) + .row(3, "F", "1993-10-14") + .row(5, "F", "1993-10-14") + .row(6, "F", "1993-10-14") + .row(33, "F", "1993-10-14") + .row(1, "O", "1996-01-02") + .row(2, "O", "1996-01-02") + .row(4, "O", "1996-01-02") + .row(7, "O", "1996-01-02") + .row(32, "O", "1996-01-02") + .row(34, "O", "1996-01-02") + .build()); + } + + @Test + public void testFirstValueBoundedRespectNulls() + { + assertWindowQuery("first_value(orderkey) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey " + + "ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING)", + resultBuilder(TEST_SESSION, INTEGER, VARCHAR, INTEGER) + .row(3, "F", 3) + .row(5, "F", 3) + .row(6, "F", 3) + .row(33, "F", 5) + .row(1, "O", 1) + .row(2, "O", 1) + .row(4, "O", 1) + .row(7, "O", 2) + .row(32, "O", 4) + .row(34, "O", 7) + .build()); + assertWindowQueryWithNulls("first_value(orderkey) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey NULLS FIRST " + + "ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(null, "F", null) + .row(3L, "F", null) + .row(5L, "F", 3L) + .row(6L, "F", 5L) + .row(null, "O", null) + .row(34L, "O", null) + .row(null, null, null) + .row(null, null, null) + .row(1L, null, null) + .row(7L, null, 1L) + .build()); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/window/TestLagFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/window/TestLagFunction.java index 3ce8213d94dd1..ba5af5dbb4110 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/window/TestLagFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/window/TestLagFunction.java @@ -121,6 +121,7 @@ public void testLagFunction() .row(32, "O", 4) .row(34, "O", 7) .build()); + assertWindowQueryWithNulls("lag(orderkey, 2, -1) OVER (PARTITION BY orderstatus ORDER BY orderkey)", resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) .row(3L, "F", -1L) @@ -191,4 +192,107 @@ public void testLagFunction() .row(34, "O", "1998-07-21") .build()); } + + @Test + public void testLagFunctionWithNullTreatment() + { + assertWindowQueryWithNulls("lag(orderkey, 1, -1) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", -1L) + .row(5L, "F", 3L) + .row(6L, "F", 5L) + .row(null, "F", 6L) + .row(34L, "O", -1L) + .row(null, "O", 34L) + .row(1L, null, -1L) + .row(7L, null, 1L) + .row(null, null, 7L) + .row(null, null, null) + .build()); + + assertWindowQueryWithNulls("lag(orderstatus, 1, null) RESPECT NULLS OVER (ORDER BY orderkey, orderstatus)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, VARCHAR) + .row(1L, null, null) + .row(3L, "F", null) + .row(5L, "F", "F") + .row(6L, "F", "F") + .row(7L, null, "F") + .row(34L, "O", null) + .row(null, "F", "O") + .row(null, "O", "F") + .row(null, null, "O") + .row(null, null, null) + .build()); + + assertWindowQueryWithNulls("lag(orderstatus, 0) RESPECT NULLS OVER (ORDER BY orderkey, orderstatus)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, VARCHAR) + .row(1L, null, null) + .row(3L, "F", "F") + .row(5L, "F", "F") + .row(6L, "F", "F") + .row(7L, null, null) + .row(34L, "O", "O") + .row(null, "F", "F") + .row(null, "O", "O") + .row(null, null, null) + .row(null, null, null) + .build()); + + assertWindowQueryWithNulls("lag(orderkey, 1, -1) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", -1L) + .row(5L, "F", 3L) + .row(6L, "F", 5L) + .row(null, "F", 6L) + .row(34L, "O", -1L) + .row(null, "O", 34L) + .row(1L, null, -1L) + .row(7L, null, 1L) + .row(null, null, 7L) + .row(null, null, 7L) + .build()); + + assertWindowQueryWithNulls("lag(orderstatus, 1, null) IGNORE NULLS OVER (ORDER BY orderkey, orderstatus)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, VARCHAR) + .row(1L, null, null) + .row(3L, "F", null) + .row(5L, "F", "F") + .row(6L, "F", "F") + .row(7L, null, "F") + .row(34L, "O", "F") + .row(null, "F", "O") + .row(null, "O", "F") + .row(null, null, "O") + .row(null, null, "O") + .build()); + + assertWindowQueryWithNulls("lag(orderstatus, 0) IGNORE NULLS OVER (ORDER BY orderkey, orderstatus)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, VARCHAR) + .row(1L, null, null) + .row(3L, "F", "F") + .row(5L, "F", "F") + .row(6L, "F", "F") + .row(7L, null, null) + .row(34L, "O", "O") + .row(null, "F", "F") + .row(null, "O", "O") + .row(null, null, null) + .row(null, null, null) + .build()); + + assertWindowQueryWithNulls("lag(orderkey, 1, -1) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey), " + + "lag(orderkey, 1, -1) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", -1L, -1L) + .row(5L, "F", 3L, 3L) + .row(6L, "F", 5L, 5L) + .row(null, "F", 6L, 6L) + .row(34L, "O", -1L, -1L) + .row(null, "O", 34L, 34L) + .row(1L, null, -1L, -1L) + .row(7L, null, 1L, 1L) + .row(null, null, 7L, 7L) + .row(null, null, null, 7L) + .build()); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/window/TestLastValueFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/window/TestLastValueFunction.java index 8a6818cfae05c..58ab31865e39c 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/window/TestLastValueFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/window/TestLastValueFunction.java @@ -96,6 +96,42 @@ public void testLastValueUnbounded() .build()); } + @Test + public void testLastValueUnboundedIgnoreNulls() + { + assertUnboundedWindowQueryWithNulls("last_value(orderkey) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 6L) + .row(5L, "F", 6L) + .row(6L, "F", 6L) + .row(null, "F", 6L) + .row(34L, "O", 34L) + .row(null, "O", 34L) + .row(1L, null, 7L) + .row(7L, null, 7L) + .row(null, null, 7L) + .row(null, null, 7L) + .build()); + } + + @Test + public void testLastValueUnboundedRespectNulls() + { + assertUnboundedWindowQueryWithNulls("last_value(orderkey) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", null) + .row(5L, "F", null) + .row(6L, "F", null) + .row(null, "F", null) + .row(34L, "O", null) + .row(null, "O", null) + .row(1L, null, null) + .row(7L, null, null) + .row(null, null, null) + .row(null, null, null) + .build()); + } + @Test public void testLastValueBounded() { @@ -128,4 +164,70 @@ public void testLastValueBounded() .row(null, null, null) .build()); } + + @Test + public void testLastValueBoundedIgnoreNulls() + { + assertWindowQueryWithNulls("last_value(orderkey) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey " + + "ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 6L) + .row(5L, "F", 6L) + .row(6L, "F", 6L) + .row(null, "F", 6L) + .row(34L, "O", 34L) + .row(null, "O", 34L) + .row(1L, null, 7L) + .row(7L, null, 7L) + .row(null, null, 7L) + .row(null, null, 7L) + .build()); + assertWindowQueryWithNulls("last_value(orderkey) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey " + + "ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 5L) + .row(5L, "F", 6L) + .row(6L, "F", 6L) + .row(null, "F", 6L) + .row(34L, "O", 34L) + .row(null, "O", 34L) + .row(1L, null, 7L) + .row(7L, null, 7L) + .row(null, null, 7L) + .row(null, null, null) + .build()); + } + + @Test + public void testLastValueBoundedRespectNulls() + { + assertWindowQueryWithNulls("last_value(orderkey) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey " + + "ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 6L) + .row(5L, "F", null) + .row(6L, "F", null) + .row(null, "F", null) + .row(34L, "O", null) + .row(null, "O", null) + .row(1L, null, null) + .row(7L, null, null) + .row(null, null, null) + .row(null, null, null) + .build()); + assertWindowQueryWithNulls("last_value(orderkey) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey " + + "ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 5L) + .row(5L, "F", 6L) + .row(6L, "F", null) + .row(null, "F", null) + .row(34L, "O", null) + .row(null, "O", null) + .row(1L, null, 7L) + .row(7L, null, null) + .row(null, null, null) + .row(null, null, null) + .build()); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/window/TestLeadFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/window/TestLeadFunction.java index d6d6d03cc7015..1906b29b5b7ac 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/window/TestLeadFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/window/TestLeadFunction.java @@ -121,6 +121,7 @@ public void testLeadFunction() .row(32, "O", -1) .row(34, "O", -1) .build()); + assertWindowQueryWithNulls("lead(orderkey, 2, -1) OVER (PARTITION BY orderstatus ORDER BY orderkey)", resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) .row(3L, "F", 6L) @@ -191,4 +192,107 @@ public void testLeadFunction() .row(34, "O", "1998-07-21") .build()); } + + @Test + public void testLeadFunctionWithNullTreatment() + { + assertWindowQueryWithNulls("lead(orderkey, 1, -1) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 5L) + .row(5L, "F", 6L) + .row(6L, "F", null) + .row(null, "F", -1L) + .row(34L, "O", null) + .row(null, "O", -1L) + .row(1L, null, 7L) + .row(7L, null, null) + .row(null, null, null) + .row(null, null, -1L) + .build()); + + assertWindowQueryWithNulls("lead(orderstatus, 1, null) RESPECT NULLS OVER (ORDER BY orderkey, orderstatus)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, VARCHAR) + .row(1L, null, "F") + .row(3L, "F", "F") + .row(5L, "F", "F") + .row(6L, "F", null) + .row(7L, null, "O") + .row(34L, "O", "F") + .row(null, "F", "O") + .row(null, "O", null) + .row(null, null, null) + .row(null, null, null) + .build()); + + assertWindowQueryWithNulls("lead(orderstatus, 0) RESPECT NULLS OVER (ORDER BY orderkey, orderstatus)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, VARCHAR) + .row(1L, null, null) + .row(3L, "F", "F") + .row(5L, "F", "F") + .row(6L, "F", "F") + .row(7L, null, null) + .row(34L, "O", "O") + .row(null, "F", "F") + .row(null, "O", "O") + .row(null, null, null) + .row(null, null, null) + .build()); + + assertWindowQueryWithNulls("lead(orderkey, 1, -1) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 5L) + .row(5L, "F", 6L) + .row(6L, "F", -1L) + .row(null, "F", -1L) + .row(34L, "O", -1L) + .row(null, "O", -1L) + .row(1L, null, 7L) + .row(7L, null, -1L) + .row(null, null, -1L) + .row(null, null, -1L) + .build()); + + assertWindowQueryWithNulls("lead(orderkey, 1, null) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 5L) + .row(5L, "F", 6L) + .row(6L, "F", null) + .row(null, "F", null) + .row(34L, "O", null) + .row(null, "O", null) + .row(1L, null, 7L) + .row(7L, null, null) + .row(null, null, null) + .row(null, null, null) + .build()); + + assertWindowQueryWithNulls("lead(orderkey, 0) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 3L) + .row(5L, "F", 5L) + .row(6L, "F", 6L) + .row(null, "F", null) + .row(34L, "O", 34L) + .row(null, "O", null) + .row(1L, null, 1L) + .row(7L, null, 7L) + .row(null, null, null) + .row(null, null, null) + .build()); + + assertWindowQueryWithNulls("lead(orderkey, 1, -1) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey), " + + "lead(orderkey, 1, -1) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 5L, 5L) + .row(5L, "F", 6L, 6L) + .row(6L, "F", null, -1L) + .row(null, "F", -1L, -1L) + .row(34L, "O", null, -1L) + .row(null, "O", -1L, -1L) + .row(1L, null, 7L, 7L) + .row(7L, null, null, -1L) + .row(null, null, null, -1L) + .row(null, null, -1L, -1L) + .build()); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/window/TestNthValueFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/window/TestNthValueFunction.java index 748998afc0f1e..2ba50684a2ef9 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/window/TestNthValueFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/window/TestNthValueFunction.java @@ -115,6 +115,42 @@ public void testNthValueUnbounded() .build()); } + @Test + public void testNthValueUnboundedIgnoreNulls() + { + assertUnboundedWindowQueryWithNulls("nth_value(orderkey, 3) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, VARCHAR) + .row(3L, "F", 6L) + .row(5L, "F", 6L) + .row(6L, "F", 6L) + .row(null, "F", 6L) + .row(34L, "O", null) + .row(null, "O", null) + .row(1L, null, null) + .row(7L, null, null) + .row(null, null, null) + .row(null, null, null) + .build()); + } + + @Test + public void testNthValueUnboundedRespectNulls() + { + assertUnboundedWindowQueryWithNulls("nth_value(orderkey, 3) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, VARCHAR) + .row(3L, "F", 6L) + .row(5L, "F", 6L) + .row(6L, "F", 6L) + .row(null, "F", 6L) + .row(34L, "O", null) + .row(null, "O", null) + .row(1L, null, null) + .row(7L, null, null) + .row(null, null, null) + .row(null, null, null) + .build()); + } + @Test public void testNthValueBounded() { @@ -162,4 +198,42 @@ public void testNthValueBounded() .row(34, "O", "1996-12-01") .build()); } + + @Test + public void testNthValueBoundedIgnoreNulls() + { + assertWindowQueryWithNulls("nth_value(orderkey, 3) IGNORE NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey " + + "ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", 6L) + .row(5L, "F", 6L) + .row(6L, "F", 6L) + .row(null, "F", null) + .row(34L, "O", null) + .row(null, "O", null) + .row(1L, null, null) + .row(7L, null, null) + .row(null, null, null) + .row(null, null, null) + .build()); + } + + @Test + public void testNthValueBoundedRespectNulls() + { + assertWindowQueryWithNulls("nth_value(orderkey, 4) RESPECT NULLS OVER (PARTITION BY orderstatus ORDER BY orderkey " + + "ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING)", + resultBuilder(TEST_SESSION, BIGINT, VARCHAR, BIGINT) + .row(3L, "F", null) + .row(5L, "F", null) + .row(6L, "F", null) + .row(null, "F", null) + .row(34L, "O", null) + .row(null, "O", null) + .row(1L, null, null) + .row(7L, null, null) + .row(null, null, null) + .row(null, null, null) + .build()); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/window/WindowAssertions.java b/presto-main/src/test/java/com/facebook/presto/operator/window/WindowAssertions.java index a6317dcdbbbe0..c837ed08a8b58 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/window/WindowAssertions.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/window/WindowAssertions.java @@ -17,7 +17,7 @@ import com.facebook.presto.testing.MaterializedResult; import org.intellij.lang.annotations.Language; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static java.lang.String.format; public final class WindowAssertions diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestBasicQueryInfo.java b/presto-main/src/test/java/com/facebook/presto/server/TestBasicQueryInfo.java index 5c4305b69f176..b7d1c3d621fea 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestBasicQueryInfo.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestBasicQueryInfo.java @@ -66,6 +66,7 @@ public void testConstructor() Duration.valueOf("11m"), 13, 14, + 21, 15, 16, 17, @@ -97,6 +98,7 @@ public void testConstructor() DataSize.valueOf("36GB"), ImmutableList.of(new StageGcStatistics( 101, + 1002, 102, 103, 104, @@ -106,7 +108,6 @@ public void testConstructor() ImmutableList.of()), Optional.empty(), Optional.empty(), - Optional.empty(), ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of(), diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestEmbeddedDiscoveryConfig.java b/presto-main/src/test/java/com/facebook/presto/server/TestEmbeddedDiscoveryConfig.java index f1f61dd8d22d3..d2dff5e844280 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestEmbeddedDiscoveryConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestEmbeddedDiscoveryConfig.java @@ -18,9 +18,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestEmbeddedDiscoveryConfig { diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestFailureDetectorConfig.java b/presto-main/src/test/java/com/facebook/presto/server/TestFailureDetectorConfig.java index 8ce57d8a19f3e..7fae640b0fae6 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestFailureDetectorConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestFailureDetectorConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.facebook.presto.failureDetector.FailureDetectorConfig; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.Duration; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestGenerateTokenFilter.java b/presto-main/src/test/java/com/facebook/presto/server/TestGenerateTokenFilter.java index 6e58c4d8d61dc..30d9ef8a9a3aa 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestGenerateTokenFilter.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestGenerateTokenFilter.java @@ -13,16 +13,16 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpRequestFilter; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.StringResponseHandler.StringResponse; +import com.facebook.airlift.http.client.jetty.JettyHttpClient; import com.facebook.presto.server.testing.TestingPrestoServer; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; import com.google.inject.Key; import com.google.inject.Module; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpRequestFilter; -import io.airlift.http.client.Request; -import io.airlift.http.client.StringResponseHandler.StringResponse; -import io.airlift.http.client.jetty.JettyHttpClient; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -37,13 +37,13 @@ import java.lang.annotation.Target; import java.util.List; -import static io.airlift.http.client.HttpClientBinder.httpClientBinder; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.http.client.StringResponseHandler.createStringResponseHandler; -import static io.airlift.http.client.TraceTokenRequestFilter.TRACETOKEN_HEADER; -import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; -import static io.airlift.testing.Assertions.assertInstanceOf; -import static io.airlift.testing.Closeables.closeQuietly; +import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.airlift.http.client.StringResponseHandler.createStringResponseHandler; +import static com.facebook.airlift.http.client.TraceTokenRequestFilter.TRACETOKEN_HEADER; +import static com.facebook.airlift.jaxrs.JaxrsBinder.jaxrsBinder; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; +import static com.facebook.airlift.testing.Closeables.closeQuietly; import static java.lang.annotation.RetentionPolicy.RUNTIME; import static javax.servlet.http.HttpServletResponse.SC_OK; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestHttpRequestSessionContext.java b/presto-main/src/test/java/com/facebook/presto/server/TestHttpRequestSessionContext.java index 7d77630c5385f..58a38cd35da27 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestHttpRequestSessionContext.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestHttpRequestSessionContext.java @@ -22,6 +22,8 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.WebApplicationException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.util.Optional; import static com.facebook.presto.SystemSessionProperties.HASH_PARTITION_COUNT; @@ -31,7 +33,6 @@ import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_INFO; import static com.facebook.presto.client.PrestoHeaders.PRESTO_EXTRA_CREDENTIAL; import static com.facebook.presto.client.PrestoHeaders.PRESTO_LANGUAGE; -import static com.facebook.presto.client.PrestoHeaders.PRESTO_PATH; import static com.facebook.presto.client.PrestoHeaders.PRESTO_PREPARED_STATEMENT; import static com.facebook.presto.client.PrestoHeaders.PRESTO_ROLE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SCHEMA; @@ -52,7 +53,6 @@ public void testSessionContext() .put(PRESTO_SOURCE, "testSource") .put(PRESTO_CATALOG, "testCatalog") .put(PRESTO_SCHEMA, "testSchema") - .put(PRESTO_PATH, "testPath") .put(PRESTO_LANGUAGE, "zh-TW") .put(PRESTO_TIME_ZONE, "Asia/Taipei") .put(PRESTO_CLIENT_INFO, "client-info") @@ -71,7 +71,6 @@ public void testSessionContext() assertEquals(context.getSource(), "testSource"); assertEquals(context.getCatalog(), "testCatalog"); assertEquals(context.getSchema(), "testSchema"); - assertEquals(context.getPath(), "testPath"); assertEquals(context.getIdentity(), new Identity("testUser", Optional.empty())); assertEquals(context.getClientInfo(), "client-info"); assertEquals(context.getLanguage(), "zh-TW"); @@ -94,7 +93,6 @@ public void testPreparedStatementsHeaderDoesNotParse() .put(PRESTO_SOURCE, "testSource") .put(PRESTO_CATALOG, "testCatalog") .put(PRESTO_SCHEMA, "testSchema") - .put(PRESTO_PATH, "testPath") .put(PRESTO_LANGUAGE, "zh-TW") .put(PRESTO_TIME_ZONE, "Asia/Taipei") .put(PRESTO_CLIENT_INFO, "null") @@ -103,4 +101,51 @@ public void testPreparedStatementsHeaderDoesNotParse() "testRemote"); new HttpRequestSessionContext(request); } + + @Test + public void testExtraCredentials() + { + HttpServletRequest request = new MockHttpServletRequest( + ImmutableListMultimap.builder() + .put(PRESTO_USER, "testUser") + .put(PRESTO_SOURCE, "testSource") + .put(PRESTO_CATALOG, "testCatalog") + .put(PRESTO_SCHEMA, "testSchema") + .put(PRESTO_LANGUAGE, "zh-TW") + .put(PRESTO_TIME_ZONE, "Asia/Taipei") + .put(PRESTO_CLIENT_INFO, "client-info") + .put(PRESTO_SESSION, QUERY_MAX_MEMORY + "=1GB") + .put(PRESTO_SESSION, JOIN_DISTRIBUTION_TYPE + "=partitioned," + HASH_PARTITION_COUNT + " = 43") + .put(PRESTO_PREPARED_STATEMENT, "query1=select * from foo,query2=select * from bar") + .put(PRESTO_ROLE, "foo_connector=ALL") + .put(PRESTO_ROLE, "bar_connector=NONE") + .put(PRESTO_ROLE, "foobar_connector=ROLE{role}") + .put(PRESTO_EXTRA_CREDENTIAL, "test.token.key1=" + urlEncode("bar=ab===,d")) + .put(PRESTO_EXTRA_CREDENTIAL, "test.token.key2=bar=ab===") + .put(PRESTO_EXTRA_CREDENTIAL, "test.json=" + urlEncode("{\"a\" : \"b\", \"c\" : \"d=\"}") + ", test.token.key3 = abc=cd") + .put(PRESTO_EXTRA_CREDENTIAL, "test.token.abc=xyz") + .build(), + "testRemote"); + + HttpRequestSessionContext context = new HttpRequestSessionContext(request); + assertEquals( + context.getIdentity().getExtraCredentials(), + ImmutableMap.builder() + .put("test.token.key1", "bar=ab===,d") + .put("test.token.key2", "bar=ab===") + .put("test.token.key3", "abc=cd") + .put("test.json", "{\"a\" : \"b\", \"c\" : \"d=\"}") + .put("test.token.abc", "xyz") + .build()); + } + + private static String urlEncode(String value) + { + try { + return URLEncoder.encode(value, "UTF-8"); + } + catch (UnsupportedEncodingException e) { + throw new AssertionError(e); + } + } } diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestInternalCommunicationConfig.java b/presto-main/src/test/java/com/facebook/presto/server/TestInternalCommunicationConfig.java index 90a9ebb9f2a24..a152a536aafb4 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestInternalCommunicationConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestInternalCommunicationConfig.java @@ -14,13 +14,15 @@ package com.facebook.presto.server; import com.google.common.collect.ImmutableMap; +import io.airlift.units.DataSize; import org.testng.annotations.Test; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.units.DataSize.Unit.MEGABYTE; public class TestInternalCommunicationConfig { @@ -31,9 +33,13 @@ public void testDefaults() .setHttpsRequired(false) .setKeyStorePath(null) .setKeyStorePassword(null) + .setTrustStorePath(null) .setKerberosEnabled(false) + .setIncludedCipherSuites(null) + .setExcludeCipherSuites(null) .setKerberosUseCanonicalHostname(true) - .setBinaryTransportEnabled(false)); + .setBinaryTransportEnabled(false) + .setMaxTaskUpdateSize(new DataSize(16, MEGABYTE))); } @Test @@ -42,19 +48,27 @@ public void testExplicitPropertyMappings() Map properties = new ImmutableMap.Builder() .put("internal-communication.https.required", "true") .put("internal-communication.https.keystore.path", "/a") + .put("internal-communication.https.trust-store-path", "/a") .put("internal-communication.https.keystore.key", "key") + .put("internal-communication.https.included-cipher", "cipher") + .put("internal-communication.https.excluded-cipher", "") .put("internal-communication.kerberos.enabled", "true") .put("internal-communication.kerberos.use-canonical-hostname", "false") .put("experimental.internal-communication.binary-transport-enabled", "true") + .put("experimental.internal-communication.max-task-update-size", "512MB") .build(); InternalCommunicationConfig expected = new InternalCommunicationConfig() .setHttpsRequired(true) .setKeyStorePath("/a") .setKeyStorePassword("key") + .setTrustStorePath("/a") + .setIncludedCipherSuites("cipher") + .setExcludeCipherSuites("") .setKerberosEnabled(true) .setKerberosUseCanonicalHostname(false) - .setBinaryTransportEnabled(true); + .setBinaryTransportEnabled(true) + .setMaxTaskUpdateSize(new DataSize(512, MEGABYTE)); assertFullMapping(properties, expected); } diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestNodeResource.java b/presto-main/src/test/java/com/facebook/presto/server/TestNodeResource.java index a5ab19243567f..464470bea2721 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestNodeResource.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestNodeResource.java @@ -13,20 +13,20 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.jetty.JettyHttpClient; import com.facebook.presto.server.testing.TestingPrestoServer; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.jetty.JettyHttpClient; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.util.List; +import static com.facebook.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; +import static com.facebook.airlift.testing.Closeables.closeQuietly; import static com.facebook.presto.failureDetector.HeartbeatFailureDetector.Stats; -import static io.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.json.JsonCodec.listJsonCodec; -import static io.airlift.testing.Closeables.closeQuietly; import static org.testng.Assert.assertTrue; @Test(singleThreaded = true) diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestPluginManagerConfig.java b/presto-main/src/test/java/com/facebook/presto/server/TestPluginManagerConfig.java index 789608b2d2b1f..1fad7c4bfa9e3 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestPluginManagerConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestPluginManagerConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.resolver.ArtifactResolver; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestQueryProgressStats.java b/presto-main/src/test/java/com/facebook/presto/server/TestQueryProgressStats.java index 3849ab82068df..44e6b4fbb442a 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestQueryProgressStats.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestQueryProgressStats.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.server; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import org.testng.annotations.Test; import java.util.OptionalDouble; diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestQueryResource.java b/presto-main/src/test/java/com/facebook/presto/server/TestQueryResource.java index 014254b9aa393..1951b2ddc6c2e 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestQueryResource.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestQueryResource.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.jetty.JettyHttpClient; import com.facebook.presto.client.QueryResults; import com.facebook.presto.server.testing.TestingPrestoServer; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.Request; -import io.airlift.http.client.jetty.JettyHttpClient; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -25,16 +25,16 @@ import java.net.URI; import java.util.List; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.airlift.http.client.Request.Builder.preparePost; +import static com.facebook.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; +import static com.facebook.airlift.testing.Closeables.closeQuietly; import static com.facebook.presto.client.PrestoHeaders.PRESTO_USER; import static com.facebook.presto.testing.assertions.Assert.assertEquals; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.http.client.Request.Builder.preparePost; -import static io.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.json.JsonCodec.listJsonCodec; -import static io.airlift.testing.Closeables.closeQuietly; import static java.nio.charset.StandardCharsets.UTF_8; import static org.testng.Assert.fail; diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestQuerySessionSupplier.java b/presto-main/src/test/java/com/facebook/presto/server/TestQuerySessionSupplier.java index 80e690a7f6e95..3eae59faf8a33 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestQuerySessionSupplier.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestQuerySessionSupplier.java @@ -19,11 +19,6 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; import com.facebook.presto.sql.SqlEnvironmentConfig; -import com.facebook.presto.sql.SqlPath; -import com.facebook.presto.sql.SqlPathElement; -import com.facebook.presto.sql.tree.Identifier; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -31,19 +26,15 @@ import javax.servlet.http.HttpServletRequest; -import java.util.List; import java.util.Locale; -import java.util.Optional; import static com.facebook.presto.SystemSessionProperties.HASH_PARTITION_COUNT; import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; import static com.facebook.presto.SystemSessionProperties.QUERY_MAX_MEMORY; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CATALOG; -import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_CAPABILITIES; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_INFO; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_TAGS; import static com.facebook.presto.client.PrestoHeaders.PRESTO_LANGUAGE; -import static com.facebook.presto.client.PrestoHeaders.PRESTO_PATH; import static com.facebook.presto.client.PrestoHeaders.PRESTO_PREPARED_STATEMENT; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SCHEMA; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SESSION; @@ -62,7 +53,6 @@ public class TestQuerySessionSupplier .put(PRESTO_SOURCE, "testSource") .put(PRESTO_CATALOG, "testCatalog") .put(PRESTO_SCHEMA, "testSchema") - .put(PRESTO_PATH, "testPath") .put(PRESTO_LANGUAGE, "zh-TW") .put(PRESTO_TIME_ZONE, "Asia/Taipei") .put(PRESTO_CLIENT_INFO, "client-info") @@ -89,7 +79,6 @@ public void testCreateSession() assertEquals(session.getSource().get(), "testSource"); assertEquals(session.getCatalog().get(), "testCatalog"); assertEquals(session.getSchema().get(), "testSchema"); - assertEquals(session.getPath().getRawPath().get(), "testPath"); assertEquals(session.getLocale(), Locale.TAIWAN); assertEquals(session.getTimeZoneKey(), getTimeZoneKey("Asia/Taipei")); assertEquals(session.getRemoteUserAddress().get(), "testRemote"); @@ -127,27 +116,6 @@ public void testEmptyClientTags() assertEquals(context2.getClientTags(), ImmutableSet.of()); } - @Test - public void testClientCapabilities() - { - HttpServletRequest request1 = new MockHttpServletRequest( - ImmutableListMultimap.builder() - .put(PRESTO_USER, "testUser") - .put(PRESTO_CLIENT_CAPABILITIES, "foo, bar") - .build(), - "remoteAddress"); - HttpRequestSessionContext context1 = new HttpRequestSessionContext(request1); - assertEquals(context1.getClientCapabilities(), ImmutableSet.of("foo", "bar")); - - HttpServletRequest request2 = new MockHttpServletRequest( - ImmutableListMultimap.builder() - .put(PRESTO_USER, "testUser") - .build(), - "remoteAddress"); - HttpRequestSessionContext context2 = new HttpRequestSessionContext(request2); - assertEquals(context2.getClientCapabilities(), ImmutableSet.of()); - } - @Test(expectedExceptions = PrestoException.class) public void testInvalidTimeZone() { @@ -165,31 +133,4 @@ public void testInvalidTimeZone() new SqlEnvironmentConfig()); sessionSupplier.createSession(new QueryId("test_query_id"), context); } - - @Test - public void testSqlPathCreation() - { - ImmutableList.Builder correctValues = ImmutableList.builder(); - correctValues.add(new SqlPathElement( - Optional.of(new Identifier("normal")), - new Identifier("schema"))); - correctValues.add(new SqlPathElement( - Optional.of(new Identifier("who.uses.periods")), - new Identifier("in.schema.names"))); - correctValues.add(new SqlPathElement( - Optional.of(new Identifier("same,deal")), - new Identifier("with,commas"))); - correctValues.add(new SqlPathElement( - Optional.of(new Identifier("aterrible")), - new Identifier("thing!@#$%^&*()"))); - List expected = correctValues.build(); - - SqlPath path = new SqlPath(Optional.of("normal.schema," - + "\"who.uses.periods\".\"in.schema.names\"," - + "\"same,deal\".\"with,commas\"," - + "aterrible.\"thing!@#$%^&*()\"")); - - assertEquals(path.getParsedPath(), expected); - assertEquals(path.toString(), Joiner.on(", ").join(expected)); - } } diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java b/presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java index a2bd04c3eec75..8eaa8c032ae50 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java @@ -117,6 +117,7 @@ private QueryInfo createQueryInfo(String queryId, QueryState state, String query 13, 14, 15, + 16, 100, 17, 18, @@ -149,7 +150,6 @@ private QueryInfo createQueryInfo(String queryId, QueryState state, String query ImmutableList.of()), Optional.empty(), Optional.empty(), - Optional.empty(), ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of(), diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfoResource.java b/presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfoResource.java index 8f602941d6fd4..a2c433d654eb5 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfoResource.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestQueryStateInfoResource.java @@ -13,31 +13,31 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.UnexpectedResponseException; +import com.facebook.airlift.http.client.jetty.JettyHttpClient; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.client.QueryResults; import com.facebook.presto.server.testing.TestingPrestoServer; import com.facebook.presto.tpch.TpchPlugin; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.Request; -import io.airlift.http.client.UnexpectedResponseException; -import io.airlift.http.client.jetty.JettyHttpClient; -import io.airlift.json.JsonCodec; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.util.List; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.airlift.http.client.Request.Builder.preparePost; +import static com.facebook.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; +import static com.facebook.airlift.testing.Closeables.closeQuietly; import static com.facebook.presto.client.PrestoHeaders.PRESTO_USER; import static com.facebook.presto.execution.QueryState.RUNNING; import static com.facebook.presto.testing.assertions.Assert.assertEquals; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.http.client.Request.Builder.preparePost; -import static io.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.json.JsonCodec.listJsonCodec; -import static io.airlift.testing.Closeables.closeQuietly; import static java.nio.charset.StandardCharsets.UTF_8; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestServer.java b/presto-main/src/test/java/com/facebook/presto/server/TestServer.java index 55216d2ec0dba..d4a65699046cd 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestServer.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestServer.java @@ -13,6 +13,14 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.http.client.FullJsonResponseHandler.JsonResponse; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpUriBuilder; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.StatusResponseHandler; +import com.facebook.airlift.http.client.jetty.JettyHttpClient; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.testing.Closeables; import com.facebook.presto.client.QueryError; import com.facebook.presto.client.QueryResults; import com.facebook.presto.server.testing.TestingPrestoServer; @@ -20,14 +28,6 @@ import com.facebook.presto.spi.type.TimeZoneNotSupportedException; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.http.client.FullJsonResponseHandler.JsonResponse; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpUriBuilder; -import io.airlift.http.client.Request; -import io.airlift.http.client.StatusResponseHandler; -import io.airlift.http.client.jetty.JettyHttpClient; -import io.airlift.json.JsonCodec; -import io.airlift.testing.Closeables; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -35,12 +35,18 @@ import java.net.URI; import java.util.List; +import static com.facebook.airlift.http.client.FullJsonResponseHandler.createFullJsonResponseHandler; +import static com.facebook.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.airlift.http.client.Request.Builder.preparePost; +import static com.facebook.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; +import static com.facebook.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.SystemSessionProperties.HASH_PARTITION_COUNT; import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; import static com.facebook.presto.SystemSessionProperties.QUERY_MAX_MEMORY; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CATALOG; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_INFO; -import static com.facebook.presto.client.PrestoHeaders.PRESTO_PATH; import static com.facebook.presto.client.PrestoHeaders.PRESTO_PREPARED_STATEMENT; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SCHEMA; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SESSION; @@ -50,13 +56,6 @@ import static com.facebook.presto.client.PrestoHeaders.PRESTO_TRANSACTION_ID; import static com.facebook.presto.client.PrestoHeaders.PRESTO_USER; import static com.facebook.presto.spi.StandardErrorCode.INCOMPATIBLE_CLIENT; -import static io.airlift.http.client.FullJsonResponseHandler.createFullJsonResponseHandler; -import static io.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.http.client.Request.Builder.preparePost; -import static io.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; -import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; -import static io.airlift.json.JsonCodec.jsonCodec; import static java.nio.charset.StandardCharsets.UTF_8; import static javax.ws.rs.core.Response.Status.OK; import static org.testng.Assert.assertEquals; @@ -96,7 +95,6 @@ public void testInvalidSessionError() .setHeader(PRESTO_SOURCE, "source") .setHeader(PRESTO_CATALOG, "catalog") .setHeader(PRESTO_SCHEMA, "schema") - .setHeader(PRESTO_PATH, "path") .setHeader(PRESTO_TIME_ZONE, invalidTimeZone) .build(); @@ -135,7 +133,6 @@ public void testQuery() .setHeader(PRESTO_SOURCE, "source") .setHeader(PRESTO_CATALOG, "catalog") .setHeader(PRESTO_SCHEMA, "schema") - .setHeader(PRESTO_PATH, "path") .setHeader(PRESTO_CLIENT_INFO, "{\"clientVersion\":\"testVersion\"}") .addHeader(PRESTO_SESSION, QUERY_MAX_MEMORY + "=1GB") .addHeader(PRESTO_SESSION, JOIN_DISTRIBUTION_TYPE + "=partitioned," + HASH_PARTITION_COUNT + " = 43") diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestServerConfig.java b/presto-main/src/test/java/com/facebook/presto/server/TestServerConfig.java index de9c5c7fbd786..d2cd18da833da 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestServerConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestServerConfig.java @@ -13,15 +13,15 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.Duration; import org.testng.annotations.Test; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static java.util.concurrent.TimeUnit.MINUTES; public class TestServerConfig diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestSessionPropertyDefaults.java b/presto-main/src/test/java/com/facebook/presto/server/TestSessionPropertyDefaults.java index b03464bca4849..c4e00fea26217 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestSessionPropertyDefaults.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestSessionPropertyDefaults.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.server; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.Session; import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.spi.QueryId; @@ -21,7 +22,6 @@ import com.facebook.presto.spi.session.SessionPropertyConfigurationManagerFactory; import com.facebook.presto.spi.session.TestingSessionPropertyConfigurationManagerFactory; import com.google.common.collect.ImmutableMap; -import io.airlift.node.NodeInfo; import org.testng.annotations.Test; import java.util.Optional; diff --git a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestBackoff.java b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestBackoff.java index dfa2b152ce5c2..886bd7a6433c4 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestBackoff.java +++ b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestBackoff.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.server.remotetask; +import com.facebook.airlift.testing.TestingTicker; import com.google.common.collect.ImmutableList; -import io.airlift.testing.TestingTicker; import io.airlift.units.Duration; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java index f1dc36ae1a8e7..1db63dd9e640e 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java +++ b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java @@ -13,6 +13,12 @@ */ package com.facebook.presto.server.remotetask; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.http.client.testing.TestingHttpClient; +import com.facebook.airlift.jaxrs.JsonMapper; +import com.facebook.airlift.jaxrs.testing.JaxrsTestingHttpProcessor; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.client.NodeVersion; import com.facebook.presto.execution.Lifespan; import com.facebook.presto.execution.NodeTaskMap; @@ -27,6 +33,7 @@ import com.facebook.presto.execution.TaskTestUtils; import com.facebook.presto.execution.TestSqlTaskManager; import com.facebook.presto.execution.buffer.OutputBuffers; +import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.metadata.HandleJsonModule; import com.facebook.presto.metadata.HandleResolver; import com.facebook.presto.metadata.InternalNode; @@ -55,12 +62,6 @@ import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.Scopes; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.http.client.testing.TestingHttpClient; -import io.airlift.jaxrs.JsonMapper; -import io.airlift.jaxrs.testing.JaxrsTestingHttpProcessor; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonModule; import io.airlift.units.Duration; import org.testng.annotations.Test; @@ -81,12 +82,16 @@ import java.net.URI; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.OptionalInt; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BooleanSupplier; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CURRENT_STATE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_WAIT; @@ -98,9 +103,6 @@ import static com.facebook.presto.testing.assertions.Assert.assertEquals; import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.inject.multibindings.Multibinder.newSetBinder; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.lang.Math.min; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -212,14 +214,15 @@ private RemoteTask createRemoteTask(HttpRemoteTaskFactory httpRemoteTaskFactory) { return httpRemoteTaskFactory.createRemoteTask( TEST_SESSION, - new TaskId("test", 1, 2), + new TaskId("test", 1, 0, 2), new InternalNode("node-id", URI.create("http://fake.invalid/"), new NodeVersion("version"), false), TaskTestUtils.PLAN_FRAGMENT, ImmutableMultimap.of(), OptionalInt.empty(), createInitialEmptyOutputBuffers(OutputBuffers.BufferType.BROADCAST), new NodeTaskMap.PartitionedSplitCountTracker(i -> {}), - true); + true, + new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())); } private static HttpRemoteTaskFactory createHttpRemoteTaskFactory(TestingTaskResource testingTaskResource) diff --git a/presto-main/src/test/java/com/facebook/presto/server/security/TestJsonWebTokenConfig.java b/presto-main/src/test/java/com/facebook/presto/server/security/TestJsonWebTokenConfig.java index 86535d56831d6..0492a888d842e 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/security/TestJsonWebTokenConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/server/security/TestJsonWebTokenConfig.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.server.security; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import org.testng.annotations.Test; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; public class TestJsonWebTokenConfig { diff --git a/presto-main/src/test/java/com/facebook/presto/server/security/TestKerberosConfig.java b/presto-main/src/test/java/com/facebook/presto/server/security/TestKerberosConfig.java index 73bb78d99c110..b9d1635d98e2a 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/security/TestKerberosConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/server/security/TestKerberosConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.server.security; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import org.testng.annotations.Test; import java.io.File; diff --git a/presto-main/src/test/java/com/facebook/presto/server/security/TestSecurityConfig.java b/presto-main/src/test/java/com/facebook/presto/server/security/TestSecurityConfig.java index 96beefb828f7c..40fbd9e71b2d3 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/security/TestSecurityConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/server/security/TestSecurityConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.server.security; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import org.testng.annotations.Test; import java.util.Map; diff --git a/presto-main/src/test/java/com/facebook/presto/server/smile/ImmutablePerson.java b/presto-main/src/test/java/com/facebook/presto/server/smile/ImmutablePerson.java index 1e8cb9ef06178..8c413e1151cd6 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/smile/ImmutablePerson.java +++ b/presto-main/src/test/java/com/facebook/presto/server/smile/ImmutablePerson.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.server.smile; +import com.facebook.airlift.json.JsonCodec; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; import java.util.List; import java.util.Map; diff --git a/presto-main/src/test/java/com/facebook/presto/server/smile/Person.java b/presto-main/src/test/java/com/facebook/presto/server/smile/Person.java index 73a6f91a4e156..5b3b275cfc2a6 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/smile/Person.java +++ b/presto-main/src/test/java/com/facebook/presto/server/smile/Person.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.server.smile; +import com.facebook.airlift.json.JsonCodec; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; import java.util.List; import java.util.Map; diff --git a/presto-main/src/test/java/com/facebook/presto/server/smile/TestFullSmileResponseHandler.java b/presto-main/src/test/java/com/facebook/presto/server/smile/TestFullSmileResponseHandler.java index 5cbb67983ce9b..b20a5e03d8f62 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/smile/TestFullSmileResponseHandler.java +++ b/presto-main/src/test/java/com/facebook/presto/server/smile/TestFullSmileResponseHandler.java @@ -13,21 +13,21 @@ */ package com.facebook.presto.server.smile; +import com.facebook.airlift.http.client.HttpStatus; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.testing.TestingResponse; import com.facebook.presto.server.smile.FullSmileResponseHandler.SmileResponse; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableListMultimap; import com.google.common.net.MediaType; -import io.airlift.http.client.HttpStatus; -import io.airlift.http.client.Response; -import io.airlift.http.client.testing.TestingResponse; import org.testng.annotations.Test; +import static com.facebook.airlift.http.client.HttpStatus.INTERNAL_SERVER_ERROR; +import static com.facebook.airlift.http.client.HttpStatus.OK; +import static com.facebook.airlift.http.client.testing.TestingResponse.contentType; import static com.facebook.presto.server.smile.FullSmileResponseHandler.createFullSmileResponseHandler; import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; -import static io.airlift.http.client.HttpStatus.INTERNAL_SERVER_ERROR; -import static io.airlift.http.client.HttpStatus.OK; -import static io.airlift.http.client.testing.TestingResponse.contentType; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/server/smile/TestSmileSupport.java b/presto-main/src/test/java/com/facebook/presto/server/smile/TestSmileSupport.java index 609f1ca7834ef..d3e9e7d45e5e3 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/smile/TestSmileSupport.java +++ b/presto-main/src/test/java/com/facebook/presto/server/smile/TestSmileSupport.java @@ -42,8 +42,8 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; import static com.google.common.base.MoreObjects.toStringHelper; -import static io.airlift.json.JsonBinder.jsonBinder; import static java.util.Objects.requireNonNull; import static org.joda.time.DateTimeZone.UTC; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/spiller/TestGenericPartitioningSpiller.java b/presto-main/src/test/java/com/facebook/presto/spiller/TestGenericPartitioningSpiller.java index a581677f69ff6..4660e7d7a39df 100644 --- a/presto-main/src/test/java/com/facebook/presto/spiller/TestGenericPartitioningSpiller.java +++ b/presto-main/src/test/java/com/facebook/presto/spiller/TestGenericPartitioningSpiller.java @@ -40,6 +40,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.function.IntPredicate; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.operator.PageAssertions.assertPageEquals; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; @@ -47,7 +48,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static java.lang.Math.toIntExact; import static java.nio.file.Files.createTempDirectory; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; diff --git a/presto-main/src/test/java/com/facebook/presto/spiller/TestNodeSpillConfig.java b/presto-main/src/test/java/com/facebook/presto/spiller/TestNodeSpillConfig.java index f30e20707f0d9..3af86d6fbdfc6 100644 --- a/presto-main/src/test/java/com/facebook/presto/spiller/TestNodeSpillConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/spiller/TestNodeSpillConfig.java @@ -13,15 +13,15 @@ */ package com.facebook.presto.spiller; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.DataSize; import org.testng.annotations.Test; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; diff --git a/presto-main/src/test/java/com/facebook/presto/split/TestBufferingSplitSource.java b/presto-main/src/test/java/com/facebook/presto/split/TestBufferingSplitSource.java index 2bbfd7e73e1e5..5987e184c2f7e 100644 --- a/presto-main/src/test/java/com/facebook/presto/split/TestBufferingSplitSource.java +++ b/presto-main/src/test/java/com/facebook/presto/split/TestBufferingSplitSource.java @@ -21,12 +21,12 @@ import java.util.concurrent.Future; +import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; +import static com.facebook.airlift.testing.Assertions.assertContains; import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; import static com.facebook.presto.split.MockSplitSource.Action.FAIL; import static com.facebook.presto.split.MockSplitSource.Action.FINISH; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; -import static io.airlift.testing.Assertions.assertContains; import static java.util.Objects.requireNonNull; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java index 4b6297625f4f0..962f9af9fa418 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java @@ -29,6 +29,7 @@ import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Decimals; import com.facebook.presto.spi.type.SqlTimestampWithTimeZone; import com.facebook.presto.spi.type.Type; @@ -39,7 +40,7 @@ import com.facebook.presto.sql.planner.RowExpressionInterpreter; import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.relational.optimizer.ExpressionOptimizer; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.ExpressionRewriter; import com.facebook.presto.sql.tree.ExpressionTreeRewriter; @@ -71,6 +72,9 @@ import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.scalar.ApplyFunction.APPLY_FUNCTION; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.SERIALIZABLE; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DateType.DATE; @@ -89,7 +93,6 @@ import static com.facebook.presto.sql.planner.ExpressionInterpreter.expressionInterpreter; import static com.facebook.presto.sql.planner.ExpressionInterpreter.expressionOptimizer; import static com.facebook.presto.sql.planner.RowExpressionInterpreter.rowExpressionInterpreter; -import static com.facebook.presto.testing.TestingConnectorSession.SESSION; import static com.facebook.presto.type.IntervalDayTimeType.INTERVAL_DAY_TIME; import static com.facebook.presto.util.DateTimeZoneIndex.getDateTimeZone; import static io.airlift.slice.Slices.utf8Slice; @@ -127,6 +130,7 @@ public class TestExpressionInterpreter .put("unbound_boolean", BOOLEAN) .put("unbound_date", DATE) .put("unbound_time", TIME) + .put("unbound_array", new ArrayType(BIGINT)) .put("unbound_timestamp", TIMESTAMP) .put("unbound_interval", INTERVAL_DAY_TIME) .put("unbound_pattern", VARCHAR) @@ -141,7 +145,7 @@ public class TestExpressionInterpreter @BeforeClass public void setup() { - METADATA.getFunctionManager().addFunctions(ImmutableList.of(APPLY_FUNCTION)); + METADATA.getFunctionManager().registerBuiltInFunctions(ImmutableList.of(APPLY_FUNCTION)); } @Test @@ -546,13 +550,6 @@ public void testCurrentUser() assertOptimizedEquals("current_user", "'" + TEST_SESSION.getUser() + "'"); } - @Test - public void testCurrentPath() - throws Exception - { - assertOptimizedEquals("current_path", "'" + TEST_SESSION.getPath() + "'"); - } - @Test public void testCastToString() { @@ -893,9 +890,9 @@ public void testSearchCase() "end"); assertOptimizedMatches("case when 0 / 0 = 0 then 1 end", - "case when cast(fail() as boolean) then 1 end"); + "case when cast(fail(8, 'ignored failure message') as boolean) then 1 end"); - assertOptimizedMatches("if(false, 1, 0 / 0)", "cast(fail() as integer)"); + assertOptimizedMatches("if(false, 1, 0 / 0)", "cast(fail(8, 'ignored failure message') as integer)"); assertOptimizedEquals("case " + "when false then 2.2 " + @@ -1116,7 +1113,7 @@ public void testSimpleCase() "" + "case BIGINT '1' " + "when unbound_long then 1 " + - "when cast(fail() AS integer) then 2 " + + "when cast(fail(8, 'ignored failure message') AS integer) then 2 " + "else 1 " + "end"); @@ -1127,8 +1124,8 @@ public void testSimpleCase() "end", "" + "case 1 " + - "when cast(fail() as integer) then 1 " + - "when cast(fail() as integer) then 2 " + + "when cast(fail(8, 'ignored failure message') as integer) then 1 " + + "when cast(fail(8, 'ignored failure message') as integer) then 2 " + "else 1 " + "end"); @@ -1167,7 +1164,7 @@ public void testCoalesce() assertOptimizedEquals("coalesce(2 * 3 * unbound_integer, 1.0E0/2.0E0, null)", "coalesce(6 * unbound_integer, 0.5E0)"); assertOptimizedEquals("coalesce(unbound_integer, 2, 1.0E0/2.0E0, 12.34E0, null)", "coalesce(unbound_integer, 2.0E0, 0.5E0, 12.34E0)"); assertOptimizedMatches("coalesce(0 / 0 > 1, unbound_boolean, 0 / 0 = 0)", - "coalesce(cast(fail() as boolean), unbound_boolean)"); + "coalesce(cast(fail(8, 'ignored failure message') as boolean), unbound_boolean)"); assertOptimizedMatches("coalesce(unbound_long, unbound_long)", "unbound_long"); assertOptimizedMatches("coalesce(2 * unbound_long, 2 * unbound_long)", "BIGINT '2' * unbound_long"); assertOptimizedMatches("coalesce(unbound_long, unbound_long2, unbound_long)", "coalesce(unbound_long, unbound_long2)"); @@ -1311,6 +1308,7 @@ public void testLikeOptimization() assertOptimizedEquals("'abc' LIKE bound_pattern", "false"); assertOptimizedEquals("unbound_string LIKE bound_pattern", "unbound_string LIKE bound_pattern"); + assertDoNotOptimize("unbound_string LIKE 'abc%'", SERIALIZABLE); assertOptimizedEquals("unbound_string LIKE unbound_pattern ESCAPE unbound_string", "unbound_string LIKE unbound_pattern ESCAPE unbound_string"); } @@ -1328,9 +1326,18 @@ public void testInvalidLike() @Test public void testLambda() { + assertDoNotOptimize("transform(unbound_array, x -> x + x)", OPTIMIZED); assertOptimizedEquals("transform(ARRAY[1, 5], x -> x + x)", "transform(ARRAY[1, 5], x -> x + x)"); assertOptimizedEquals("transform(sequence(1, 5), x -> x + x)", "transform(sequence(1, 5), x -> x + x)"); - evaluate("transform(ARRAY[1, 5], x -> x + x)", true); + assertRowExpressionOptimizedEquals( + OPTIMIZED, + "transform(sequence(1, unbound_long), x -> cast(json_parse('[1, 2]') AS ARRAY)[1] + x)", + "transform(sequence(1, unbound_long), x -> 1 + x)"); + assertRowExpressionOptimizedEquals( + OPTIMIZED, + "transform(sequence(1, unbound_long), x -> cast(json_parse('[1, 2]') AS ARRAY)[1] + 1)", + "transform(sequence(1, unbound_long), x -> 2)"); + assertEquals(evaluate("reduce(ARRAY[1, 5], 0, (x, y) -> x + y, x -> x)", true), 6L); } @Test @@ -1348,22 +1355,22 @@ public void testFailedExpressionOptimization() assertOptimizedEquals("if(unbound_boolean, 0 / 0, 1)", "CASE WHEN unbound_boolean THEN 0 / 0 ELSE 1 END"); assertOptimizedMatches("CASE unbound_long WHEN 1 THEN 1 WHEN 0 / 0 THEN 2 END", - "CASE unbound_long WHEN BIGINT '1' THEN 1 WHEN cast(fail() as bigint) THEN 2 END"); + "CASE unbound_long WHEN BIGINT '1' THEN 1 WHEN cast(fail(8, 'ignored failure message') as bigint) THEN 2 END"); assertOptimizedMatches("CASE unbound_boolean WHEN true THEN 1 ELSE 0 / 0 END", - "CASE unbound_boolean WHEN true THEN 1 ELSE cast(fail() as integer) END"); + "CASE unbound_boolean WHEN true THEN 1 ELSE cast(fail(8, 'ignored failure message') as integer) END"); assertOptimizedMatches("CASE bound_long WHEN unbound_long THEN 1 WHEN 0 / 0 THEN 2 ELSE 1 END", - "CASE BIGINT '1234' WHEN unbound_long THEN 1 WHEN cast(fail() as bigint) THEN 2 ELSE 1 END"); + "CASE BIGINT '1234' WHEN unbound_long THEN 1 WHEN cast(fail(8, 'ignored failure message') as bigint) THEN 2 ELSE 1 END"); assertOptimizedMatches("case when unbound_boolean then 1 when 0 / 0 = 0 then 2 end", - "case when unbound_boolean then 1 when cast(fail() as boolean) then 2 end"); + "case when unbound_boolean then 1 when cast(fail(8, 'ignored failure message') as boolean) then 2 end"); assertOptimizedMatches("case when unbound_boolean then 1 else 0 / 0 end", - "case when unbound_boolean then 1 else cast(fail() as integer) end"); + "case when unbound_boolean then 1 else cast(fail(8, 'ignored failure message') as integer) end"); assertOptimizedMatches("case when unbound_boolean then 0 / 0 else 1 end", - "case when unbound_boolean then cast(fail() as integer) else 1 end"); + "case when unbound_boolean then cast(fail(8, 'ignored failure message') as integer) else 1 end"); } @Test(expectedExceptions = PrestoException.class) @@ -1373,8 +1380,13 @@ public void testOptimizeDivideByZero() } @Test - public void testMassiveArrayConstructor() + public void testMassiveArray() { + assertRowExpressionOptimizedEquals( + OPTIMIZED, + "SEQUENCE(1, 999)", + format("ARRAY [%s]", Joiner.on(", ").join(IntStream.range(1, 1000).mapToObj(i -> "(BIGINT '" + i + "')").iterator()))); + assertDoNotOptimize("SEQUENCE(1, 1000)", SERIALIZABLE); optimize(format("ARRAY [%s]", Joiner.on(", ").join(IntStream.range(0, 10_000).mapToObj(i -> "(bound_long + " + i + ")").iterator()))); optimize(format("ARRAY [%s]", Joiner.on(", ").join(IntStream.range(0, 10_000).mapToObj(i -> "(bound_integer + " + i + ")").iterator()))); optimize(format("ARRAY [%s]", Joiner.on(", ").join(IntStream.range(0, 10_000).mapToObj(i -> "'" + i + "'").iterator()))); @@ -1513,6 +1525,17 @@ private static void assertOptimizedEquals(@Language("SQL") String actual, @Langu assertEquals(optimize(actual), optimize(expected)); } + private static void assertRowExpressionOptimizedEquals(Level level, @Language("SQL") String actual, @Language("SQL") String expected) + { + Object actualResult = optimize(toRowExpression(expression(actual)), level); + Object expectedResult = optimize(toRowExpression(expression(expected)), level); + if (actualResult instanceof Block && expectedResult instanceof Block) { + assertEquals(blockToSlice((Block) actualResult), blockToSlice((Block) expectedResult)); + return; + } + assertEquals(actualResult, expectedResult); + } + private static void assertOptimizedMatches(@Language("SQL") String actual, @Language("SQL") String expected) { // replaces FunctionCalls to FailureFunction by fail() @@ -1529,29 +1552,64 @@ private static Object optimize(@Language("SQL") String expression) { assertRoundTrip(expression); - Expression parsedExpression = FunctionAssertions.createExpression(expression, METADATA, SYMBOL_TYPES); + Expression parsedExpression = expression(expression); + Object expressionResult = optimize(parsedExpression); + + RowExpression rowExpression = toRowExpression(parsedExpression); + Object rowExpressionResult = optimize(rowExpression, OPTIMIZED); + assertExpressionAndRowExpressionEquals(expressionResult, rowExpressionResult); + return expressionResult; + } - Map, Type> expressionTypes = getExpressionTypes(TEST_SESSION, METADATA, SQL_PARSER, SYMBOL_TYPES, parsedExpression, emptyList(), WarningCollector.NOOP); - ExpressionInterpreter interpreter = expressionOptimizer(parsedExpression, METADATA, TEST_SESSION, expressionTypes); + private static Expression expression(String expression) + { + return FunctionAssertions.createExpression(expression, METADATA, SYMBOL_TYPES); + } + + private static RowExpression toRowExpression(Expression expression) + { + return TRANSLATOR.translate(expression, SYMBOL_TYPES); + } - Object expressionResult = interpreter.optimize(symbol -> { + private static Object optimize(Expression expression) + { + Map, Type> expressionTypes = getExpressionTypes(TEST_SESSION, METADATA, SQL_PARSER, SYMBOL_TYPES, expression, emptyList(), WarningCollector.NOOP); + ExpressionInterpreter interpreter = expressionOptimizer(expression, METADATA, TEST_SESSION, expressionTypes); + return interpreter.optimize(variable -> { + Symbol symbol = new Symbol(variable.getName()); Object value = symbolConstant(symbol); if (value == null) { return symbol.toSymbolReference(); } return value; }); - RowExpression rowExpression = TRANSLATOR.translate(parsedExpression, SYMBOL_TYPES); - Object rowExpressionResult = new RowExpressionInterpreter(rowExpression, METADATA, TEST_SESSION.toConnectorSession(), true).optimize(symbol -> { + } + + private static Object optimize(RowExpression expression, Level level) + { + return new RowExpressionInterpreter(expression, METADATA, TEST_SESSION.toConnectorSession(), level).optimize(variable -> { + Symbol symbol = new Symbol(variable.getName()); Object value = symbolConstant(symbol); if (value == null) { return new VariableReferenceExpression(symbol.getName(), SYMBOL_TYPES.get(symbol.toSymbolReference())); } return value; }); + } - assertExpressionAndRowExpressionEquals(expressionResult, rowExpressionResult); - return expressionResult; + private static void assertDoNotOptimize(@Language("SQL") String expression, Level optimizationLevel) + { + assertRoundTrip(expression); + Expression translatedExpression = expression(expression); + RowExpression rowExpression = toRowExpression(translatedExpression); + + Object expressionResult = optimize(translatedExpression); + if (expressionResult instanceof Expression) { + expressionResult = toRowExpression((Expression) expressionResult); + } + Object rowExpressionResult = optimize(rowExpression, optimizationLevel); + assertRowExpressionEvaluationEquals(expressionResult, rowExpressionResult); + assertRowExpressionEvaluationEquals(rowExpressionResult, rowExpression); } private static Object symbolConstant(Symbol symbol) @@ -1594,7 +1652,7 @@ private static void assertExpressionAndRowExpressionEquals(Object expressionResu // It is tricky to check the equivalence of an expression and a row expression. // We rely on the optimized translator to fill the gap. RowExpression translated = TRANSLATOR.translateAndOptimize((Expression) expressionResult, SYMBOL_TYPES); - assertRowExpressionEvaluationEquals(translated, new ExpressionOptimizer(METADATA.getFunctionManager(), SESSION).optimize((RowExpression) rowExpressionResult)); + assertRowExpressionEvaluationEquals(translated, rowExpressionResult); } else { // We have constants; directly compare @@ -1612,6 +1670,10 @@ private static void assertRowExpressionEvaluationEquals(Object left, Object righ assertTrue(left instanceof RowExpression); // assertEquals(((RowExpression) left).getType(), ((RowExpression) right).getType()); if (left instanceof ConstantExpression) { + if (isRemovableCast(right)) { + assertRowExpressionEvaluationEquals(left, ((CallExpression) right).getArguments().get(0)); + return; + } assertTrue(right instanceof ConstantExpression); assertRowExpressionEvaluationEquals(((ConstantExpression) left).getValue(), ((ConstantExpression) left).getValue()); } @@ -1654,6 +1716,17 @@ else if (left instanceof SpecialFormExpression) { } } + private static boolean isRemovableCast(Object value) + { + if (value instanceof CallExpression && + new FunctionResolution(METADATA.getFunctionManager()).isCastFunction(((CallExpression) value).getFunctionHandle())) { + Type targetType = ((CallExpression) value).getType(); + Type sourceType = ((CallExpression) value).getArguments().get(0).getType(); + return METADATA.getTypeManager().canCoerce(sourceType, targetType); + } + return false; + } + private static Slice blockToSlice(Block block) { // This function is strictly for testing use only @@ -1702,7 +1775,7 @@ private static class FailedFunctionRewriter public Expression rewriteFunctionCall(FunctionCall node, Object context, ExpressionTreeRewriter treeRewriter) { if (node.getName().equals(QualifiedName.of("fail"))) { - return new FunctionCall(QualifiedName.of("fail"), ImmutableList.of()); + return new FunctionCall(QualifiedName.of("fail"), ImmutableList.of(node.getArguments().get(0), new StringLiteral("ignored failure message"))); } return node; } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java b/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java index 90b0b6f6711a2..a74d960c62f35 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestRowExpressionSerde.java @@ -13,6 +13,10 @@ */ package com.facebook.presto.sql; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonModule; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.block.BlockJsonSerde; import com.facebook.presto.execution.warnings.WarningCollector; @@ -37,6 +41,7 @@ import com.facebook.presto.sql.analyzer.Scope; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.NodeRef; @@ -48,19 +53,19 @@ import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.Scopes; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonModule; import io.airlift.slice.Slice; -import io.airlift.stats.cardinality.HyperLogLog; import org.intellij.lang.annotations.Language; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.util.Map; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.spi.function.OperatorType.SUBSCRIPT; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.ROW_CONSTRUCTOR; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -79,9 +84,6 @@ import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.specialForm; import static com.google.inject.multibindings.Multibinder.newSetBinder; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.lang.Float.floatToIntBits; import static java.util.Collections.emptyList; import static org.testng.Assert.assertEquals; @@ -93,7 +95,7 @@ public class TestRowExpressionSerde private final Metadata metadata = MetadataManager.createTestMetadataManager(); private JsonCodec codec; - @BeforeMethod + @BeforeClass public void setUp() throws Exception { @@ -163,7 +165,7 @@ public void testArrayGet() @Test public void testRowLiteral() { - assertEquals(getRoundTrip("ROW(1, 1.1)", true), + assertEquals(getRoundTrip("ROW(1, 1.1)", false), specialForm( ROW_CONSTRUCTOR, RowType.anonymous( @@ -265,7 +267,12 @@ private JsonCodec getJsonCodec() private RowExpression translate(Expression expression, boolean optimize) { - return SqlToRowExpressionTranslator.translate(expression, getExpressionTypes(expression), ImmutableMap.of(), metadata.getFunctionManager(), metadata.getTypeManager(), TEST_SESSION, optimize); + RowExpression rowExpression = SqlToRowExpressionTranslator.translate(expression, getExpressionTypes(expression), ImmutableMap.of(), metadata.getFunctionManager(), metadata.getTypeManager(), TEST_SESSION); + if (optimize) { + RowExpressionOptimizer optimizer = new RowExpressionOptimizer(metadata); + return optimizer.optimize(rowExpression, OPTIMIZED, TEST_SESSION.toConnectorSession()); + } + return rowExpression; } private Map, Type> getExpressionTypes(Expression expression) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestSqlEnvironmentConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/TestSqlEnvironmentConfig.java index 9eb2565c5c6f3..308ebbe09495f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestSqlEnvironmentConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestSqlEnvironmentConfig.java @@ -13,15 +13,14 @@ */ package com.facebook.presto.sql; -import com.facebook.presto.sql.parser.ParsingException; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestSqlEnvironmentConfig { @@ -29,7 +28,6 @@ public class TestSqlEnvironmentConfig public void testDefaults() { assertRecordedDefaults(recordDefaults(SqlEnvironmentConfig.class) - .setPath(null) .setForcedSessionTimeZone(null)); } @@ -37,21 +35,12 @@ public void testDefaults() public void testExplicitPropertyMappings() { Map properties = new ImmutableMap.Builder() - .put("sql.path", "a.b, c.d") .put("sql.forced-session-time-zone", "UTC") .build(); SqlEnvironmentConfig expected = new SqlEnvironmentConfig() - .setPath("a.b, c.d") .setForcedSessionTimeZone("UTC"); assertFullMapping(properties, expected); } - - @Test(expectedExceptions = ParsingException.class, expectedExceptionsMessageRegExp = "\\Qline 1:9: mismatched input '.'. Expecting: ',', \\E") - public void testInvalidPath() - { - SqlEnvironmentConfig config = new SqlEnvironmentConfig().setPath("too.many.qualifiers"); - new SqlPath(config.getPath()).getParsedPath(); - } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestingRowExpressionTranslator.java b/presto-main/src/test/java/com/facebook/presto/sql/TestingRowExpressionTranslator.java index 4d93c839a3ca3..e6cb9d073b8fc 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestingRowExpressionTranslator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestingRowExpressionTranslator.java @@ -20,10 +20,12 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.analyzer.ExpressionAnalyzer; import com.facebook.presto.sql.analyzer.Scope; +import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.ExpressionInterpreter; import com.facebook.presto.sql.planner.LiteralEncoder; -import com.facebook.presto.sql.planner.NoOpSymbolResolver; +import com.facebook.presto.sql.planner.NoOpVariableResolver; import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.NodeRef; @@ -32,6 +34,7 @@ import java.util.Map; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static java.util.Collections.emptyList; public class TestingRowExpressionTranslator @@ -60,6 +63,11 @@ public RowExpression translateAndOptimize(Expression expression, TypeProvider ty return translateAndOptimize(expression, getExpressionTypes(expression, typeProvider)); } + public RowExpression translate(String sql, Map types) + { + return translate(ExpressionUtils.rewriteIdentifiersToSymbolReferences(new SqlParser().createExpression(sql)), TypeProvider.viewOf(types)); + } + public RowExpression translate(Expression expression, TypeProvider typeProvider) { return SqlToRowExpressionTranslator.translate( @@ -68,13 +76,14 @@ public RowExpression translate(Expression expression, TypeProvider typeProvider) ImmutableMap.of(), metadata.getFunctionManager(), metadata.getTypeManager(), - TEST_SESSION, - false); + TEST_SESSION); } public RowExpression translateAndOptimize(Expression expression, Map, Type> types) { - return SqlToRowExpressionTranslator.translate(expression, types, ImmutableMap.of(), metadata.getFunctionManager(), metadata.getTypeManager(), TEST_SESSION, true); + RowExpression rowExpression = SqlToRowExpressionTranslator.translate(expression, types, ImmutableMap.of(), metadata.getFunctionManager(), metadata.getTypeManager(), TEST_SESSION); + RowExpressionOptimizer optimizer = new RowExpressionOptimizer(metadata); + return optimizer.optimize(rowExpression, OPTIMIZED, TEST_SESSION.toConnectorSession()); } Expression simplifyExpression(Expression expression) @@ -83,7 +92,7 @@ Expression simplifyExpression(Expression expression) Map, Type> expressionTypes = getExpressionTypes(expression, TypeProvider.empty()); ExpressionInterpreter interpreter = ExpressionInterpreter.expressionOptimizer(expression, metadata, TEST_SESSION, expressionTypes); - Object value = interpreter.optimize(NoOpSymbolResolver.INSTANCE); + Object value = interpreter.optimize(NoOpVariableResolver.INSTANCE); return literalEncoder.toExpression(value, expressionTypes.get(NodeRef.of(expression))); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java index 5eb3079a85822..a23d4a14c1644 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.analyzer; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.Session; import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.block.BlockEncodingManager; @@ -57,7 +58,6 @@ import com.facebook.presto.transaction.TransactionManager; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import org.intellij.lang.annotations.Language; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -1562,7 +1562,7 @@ public void setup() new AnalyzePropertyManager(), transactionManager); - metadata.getFunctionManager().addFunctions(ImmutableList.of(APPLY_FUNCTION)); + metadata.getFunctionManager().registerBuiltInFunctions(ImmutableList.of(APPLY_FUNCTION)); Catalog tpchTestCatalog = createTestingCatalog(TPCH_CATALOG, TPCH_CONNECTOR_ID); catalogManager.registerCatalog(tpchTestCatalog); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 5d1e4d1228411..db1105f3c44bf 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -13,18 +13,20 @@ */ package com.facebook.presto.sql.analyzer; +import com.facebook.airlift.configuration.ConfigurationFactory; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.facebook.presto.operator.aggregation.arrayagg.ArrayAggGroupImplementation; import com.facebook.presto.operator.aggregation.histogram.HistogramGroupImplementation; import com.facebook.presto.operator.aggregation.multimapagg.MultimapAggGroupImplementation; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.ConfigurationFactory; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.testng.annotations.Test; import java.util.Map; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.BROADCAST; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.PARTITIONED; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.ELIMINATE_CROSS_JOINS; @@ -34,8 +36,6 @@ import static com.facebook.presto.sql.analyzer.FeaturesConfig.SPILL_ENABLED; import static com.facebook.presto.sql.analyzer.RegexLibrary.JONI; import static com.facebook.presto.sql.analyzer.RegexLibrary.RE2J; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.KILOBYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; @@ -116,8 +116,13 @@ public void testDefaults() .setLegacyUnnestArrayRows(false) .setJsonSerdeCodeGenerationEnabled(false) .setPushLimitThroughOuterJoin(true) - .setMaxConcurrentMaterializations(10) - .setPushdownSubfieldsEnabled(false)); + .setMaxConcurrentMaterializations(3) + .setPushdownSubfieldsEnabled(false) + .setTableWriterMergeOperatorEnabled(true) + .setOptimizeFullOuterJoinWithCoalesce(true) + .setIndexLoaderTimeout(new Duration(20, SECONDS)) + .setOptimizedRepartitioningEnabled(false) + .setListNonBuiltInFunctions(false)); } @Test @@ -194,6 +199,11 @@ public void testExplicitPropertyMappings() .put("optimizer.push-limit-through-outer-join", "false") .put("max-concurrent-materializations", "5") .put("experimental.pushdown-subfields-enabled", "true") + .put("experimental.table-writer-merge-operator-enabled", "false") + .put("optimizer.optimize-full-outer-join-with-coalesce", "false") + .put("index-loader-timeout", "10s") + .put("experimental.optimized-repartitioning", "true") + .put("list-non-built-in-functions", "true") .build(); FeaturesConfig expected = new FeaturesConfig() @@ -266,7 +276,12 @@ public void testExplicitPropertyMappings() .setJsonSerdeCodeGenerationEnabled(true) .setPushLimitThroughOuterJoin(false) .setMaxConcurrentMaterializations(5) - .setPushdownSubfieldsEnabled(true); + .setPushdownSubfieldsEnabled(true) + .setTableWriterMergeOperatorEnabled(false) + .setOptimizeFullOuterJoinWithCoalesce(false) + .setIndexLoaderTimeout(new Duration(10, SECONDS)) + .setOptimizedRepartitioningEnabled(true) + .setListNonBuiltInFunctions(true); assertFullMapping(properties, expected); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/gen/BenchmarkPageProcessor.java b/presto-main/src/test/java/com/facebook/presto/sql/gen/BenchmarkPageProcessor.java index bbd369ecb6d4d..e18853b9eb558 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/gen/BenchmarkPageProcessor.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/gen/BenchmarkPageProcessor.java @@ -44,6 +44,7 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN_OR_EQUAL; @@ -89,7 +90,7 @@ public void setup() MetadataManager metadata = createTestMetadataManager(); FunctionManager functionManager = metadata.getFunctionManager(); compiledProcessor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)) - .compilePageProcessor(Optional.of(createFilterExpression(functionManager)), ImmutableList.of(createProjectExpression(functionManager))) + .compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(createFilterExpression(functionManager)), ImmutableList.of(createProjectExpression(functionManager))) .get(); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/gen/InCodeGeneratorBenchmark.java b/presto-main/src/test/java/com/facebook/presto/sql/gen/InCodeGeneratorBenchmark.java index 9a164ca5de4bd..6d11bbcc2ad82 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/gen/InCodeGeneratorBenchmark.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/gen/InCodeGeneratorBenchmark.java @@ -127,7 +127,7 @@ public void setup() RowExpression filter = specialForm(IN, BOOLEAN, arguments); MetadataManager metadata = MetadataManager.createTestMetadataManager(); - processor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)).compilePageProcessor(Optional.of(filter), ImmutableList.of(project)).get(); + processor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)).compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.of(filter), ImmutableList.of(project)).get(); } @Benchmark diff --git a/presto-main/src/test/java/com/facebook/presto/sql/gen/PageProcessorBenchmark.java b/presto-main/src/test/java/com/facebook/presto/sql/gen/PageProcessorBenchmark.java index a135f9e4760af..9d9173669f7d6 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/gen/PageProcessorBenchmark.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/gen/PageProcessorBenchmark.java @@ -30,6 +30,7 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.NodeRef; @@ -62,6 +63,7 @@ import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.operator.scalar.FunctionAssertions.createExpression; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypes; @@ -121,10 +123,10 @@ public void setup() PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0); inputPage = createPage(types, dictionaryBlocks); - pageProcessor = new ExpressionCompiler(metadata, pageFunctionCompiler).compilePageProcessor(Optional.of(getFilter(type)), projections).get(); + pageProcessor = new ExpressionCompiler(metadata, pageFunctionCompiler).compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(getFilter(type)), projections).get(); recordSet = new PageRecordSet(types, inputPage); - cursorProcessor = new ExpressionCompiler(metadata, pageFunctionCompiler).compileCursorProcessor(Optional.of(getFilter(type)), projections, "key").get(); + cursorProcessor = new ExpressionCompiler(metadata, pageFunctionCompiler).compileCursorProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(getFilter(type)), projections, "key").get(); } @Benchmark @@ -180,7 +182,9 @@ private RowExpression rowExpression(String value) Expression expression = createExpression(value, METADATA, TypeProvider.copyOf(symbolTypes)); Map, Type> expressionTypes = getExpressionTypes(TEST_SESSION, METADATA, SQL_PARSER, TypeProvider.copyOf(symbolTypes), expression, emptyList(), WarningCollector.NOOP); - return SqlToRowExpressionTranslator.translate(expression, expressionTypes, sourceLayout, METADATA.getFunctionManager(), METADATA.getTypeManager(), TEST_SESSION, true); + RowExpression rowExpression = SqlToRowExpressionTranslator.translate(expression, expressionTypes, sourceLayout, METADATA.getFunctionManager(), METADATA.getTypeManager(), TEST_SESSION); + RowExpressionOptimizer optimizer = new RowExpressionOptimizer(METADATA); + return optimizer.optimize(rowExpression, OPTIMIZED, TEST_SESSION.toConnectorSession()); } private static Page createPage(List types, boolean dictionary) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestExpressionCompiler.java b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestExpressionCompiler.java index 989a1da69fb92..918bdce92ce4a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestExpressionCompiler.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestExpressionCompiler.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.sql.gen; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.log.Logging; import com.facebook.presto.operator.scalar.BitwiseFunctions; import com.facebook.presto.operator.scalar.DateTimeFunctions; import com.facebook.presto.operator.scalar.FunctionAssertions; @@ -40,8 +42,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import io.airlift.joni.Regex; -import io.airlift.log.Logger; -import io.airlift.log.Logging; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import io.airlift.units.Duration; @@ -66,6 +66,8 @@ import java.util.Set; import java.util.stream.LongStream; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.operator.scalar.JoniRegexpCasts.joniRegexp; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -88,9 +90,7 @@ import static com.facebook.presto.util.StructuralTestUtil.mapType; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.slice.Slices.utf8Slice; -import static io.airlift.testing.Closeables.closeAllRuntimeException; import static java.lang.Math.cos; import static java.lang.Runtime.getRuntime; import static java.lang.String.format; @@ -621,6 +621,35 @@ public void testBinaryOperatorsString() Futures.allAsList(futures).get(); } + @Test + public void testNestedColumnFilter() + { + assertFilter("bound_row.nested_column_0 = 1234", true); + assertFilter("bound_row.nested_column_0 = 1223", false); + assertFilter("bound_row.nested_column_1 = 34", true); + assertFilter("bound_row.nested_column_1 = 33", false); + assertFilter("bound_row.nested_column_2 = 'hello'", true); + assertFilter("bound_row.nested_column_2 = 'value1'", false); + assertFilter("bound_row.nested_column_3 = 12.34", true); + assertFilter("bound_row.nested_column_3 = 34.34", false); + assertFilter("bound_row.nested_column_4 = true", true); + assertFilter("bound_row.nested_column_4 = false", false); + assertFilter("bound_row.nested_column_6.nested_nested_column = 'innerFieldValue'", true); + assertFilter("bound_row.nested_column_6.nested_nested_column != 'innerFieldValue'", false); + + // combination of types in one filter + assertFilter( + ImmutableList.of( + "bound_row.nested_column_0 = 1234", "bound_row.nested_column_7 >= 1234", + "bound_row.nested_column_1 = 34", "bound_row.nested_column_8 >= 33", + "bound_row.nested_column_2 = 'hello'", "bound_row.nested_column_9 >= 'hello'", + "bound_row.nested_column_3 = 12.34", "bound_row.nested_column_10 >= 12.34", + "bound_row.nested_column_4 = true", "NOT (bound_row.nested_column_11 = false)", + "bound_row.nested_column_6.nested_nested_column = 'innerFieldValue'", "bound_row.nested_column_13.nested_nested_column LIKE 'innerFieldValue'") + .stream().collect(joining(" AND ")), + true); + } + private static VarcharType varcharType(String... values) { return varcharType(Arrays.asList(values)); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestInCodeGenerator.java b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestInCodeGenerator.java index 0f9ad633b80c3..cb36ffd3ac955 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestInCodeGenerator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestInCodeGenerator.java @@ -44,9 +44,9 @@ public void testInteger() { FunctionManager functionManager = createTestMetadataManager().getFunctionManager(); List values = new ArrayList<>(); - values.add(constant(Integer.MIN_VALUE, INTEGER)); - values.add(constant(Integer.MAX_VALUE, INTEGER)); - values.add(constant(3, INTEGER)); + values.add(constant((long) Integer.MIN_VALUE, INTEGER)); + values.add(constant((long) Integer.MAX_VALUE, INTEGER)); + values.add(constant(3L, INTEGER)); assertEquals(checkSwitchGenerationCase(INTEGER, values), DIRECT_SWITCH); values.add(constant(null, INTEGER)); @@ -59,11 +59,11 @@ public void testInteger() assertEquals(checkSwitchGenerationCase(INTEGER, values), DIRECT_SWITCH); for (int i = 6; i <= 32; ++i) { - values.add(constant(i, INTEGER)); + values.add(constant((long) i, INTEGER)); } assertEquals(checkSwitchGenerationCase(INTEGER, values), DIRECT_SWITCH); - values.add(constant(33, INTEGER)); + values.add(constant(33L, INTEGER)); assertEquals(checkSwitchGenerationCase(INTEGER, values), SET_CONTAINS); } @@ -138,9 +138,9 @@ public void testDouble() public void testVarchar() { List values = new ArrayList<>(); - values.add(constant(Slices.utf8Slice("1"), DOUBLE)); - values.add(constant(Slices.utf8Slice("2"), DOUBLE)); - values.add(constant(Slices.utf8Slice("3"), DOUBLE)); + values.add(constant(Slices.utf8Slice("1"), VARCHAR)); + values.add(constant(Slices.utf8Slice("2"), VARCHAR)); + values.add(constant(Slices.utf8Slice("3"), VARCHAR)); assertEquals(checkSwitchGenerationCase(VARCHAR, values), HASH_SWITCH); values.add(constant(null, VARCHAR)); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestPageFunctionCompiler.java b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestPageFunctionCompiler.java index d93e37474911d..9e6c07d8faf12 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestPageFunctionCompiler.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestPageFunctionCompiler.java @@ -56,7 +56,7 @@ public void testFailureDoesNotCorruptFutureResults() { PageFunctionCompiler functionCompiler = new PageFunctionCompiler(createTestMetadataManager(), 0); - Supplier projectionSupplier = functionCompiler.compileProjection(ADD_10_EXPRESSION, Optional.empty()); + Supplier projectionSupplier = functionCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.empty()); PageProjection projection = projectionSupplier.get(); // process good page and verify we got the expected number of result rows @@ -88,7 +88,7 @@ public void testGeneratedClassName() String planNodeId = "7"; String stageId = "20170707_223500_67496_zguwn.2"; String classSuffix = stageId + "_" + planNodeId; - Supplier projectionSupplier = functionCompiler.compileProjection(ADD_10_EXPRESSION, Optional.of(classSuffix)); + Supplier projectionSupplier = functionCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.of(classSuffix)); PageProjection projection = projectionSupplier.get(); Work work = projection.project(SESSION, new DriverYieldSignal(), createLongBlockPage(0), SelectedPositions.positionsRange(0, 1)); // class name should look like PageProjectionOutput_20170707_223500_67496_zguwn_2_7_XX @@ -100,31 +100,31 @@ public void testCache() { PageFunctionCompiler cacheCompiler = new PageFunctionCompiler(createTestMetadataManager(), 100); assertSame( - cacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.empty()), - cacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.empty())); + cacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.empty()), + cacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.empty())); assertSame( - cacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.of("hint")), - cacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.of("hint"))); + cacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.of("hint")), + cacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.of("hint"))); assertSame( - cacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.of("hint")), - cacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.of("hint2"))); + cacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.of("hint")), + cacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.of("hint2"))); assertSame( - cacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.empty()), - cacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.of("hint2"))); + cacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.empty()), + cacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.of("hint2"))); PageFunctionCompiler noCacheCompiler = new PageFunctionCompiler(createTestMetadataManager(), 0); assertNotSame( - noCacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.empty()), - noCacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.empty())); + noCacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.empty()), + noCacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.empty())); assertNotSame( - noCacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.of("hint")), - noCacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.of("hint"))); + noCacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.of("hint")), + noCacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.of("hint"))); assertNotSame( - noCacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.of("hint")), - noCacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.of("hint2"))); + noCacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.of("hint")), + noCacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.of("hint2"))); assertNotSame( - noCacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.empty()), - noCacheCompiler.compileProjection(ADD_10_EXPRESSION, Optional.of("hint2"))); + noCacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.empty()), + noCacheCompiler.compileProjection(SESSION.getSqlFunctionProperties(), ADD_10_EXPRESSION, Optional.of("hint2"))); } private Block project(PageProjection projection, Page page, SelectedPositions selectedPositions) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestRowExpressionPredicateCompiler.java b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestRowExpressionPredicateCompiler.java index 41e922ae519bd..9b3bb83be6420 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestRowExpressionPredicateCompiler.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestRowExpressionPredicateCompiler.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.sql.gen; -import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; @@ -45,8 +45,8 @@ public class TestRowExpressionPredicateCompiler { - private FunctionManager functionManager = createTestMetadataManager().getFunctionManager(); - private FunctionResolution functionResolution = new FunctionResolution(functionManager); + private Metadata metadata = createTestMetadataManager(); + private FunctionResolution functionResolution = new FunctionResolution(metadata.getFunctionManager()); @Test public void test() @@ -65,8 +65,8 @@ public void test() call("b - a", functionResolution.arithmeticFunction(SUBTRACT, BIGINT, BIGINT), BIGINT, b, a), constant(0L, BIGINT)); - PredicateCompiler compiler = new RowExpressionPredicateCompiler(functionManager, 10_000); - Predicate compiledSum = compiler.compilePredicate(sum).get(); + PredicateCompiler compiler = new RowExpressionPredicateCompiler(metadata, 10_000); + Predicate compiledSum = compiler.compilePredicate(SESSION.getSqlFunctionProperties(), sum).get(); assertEquals(Arrays.asList(1, 0), Ints.asList(compiledSum.getInputChannels())); @@ -84,7 +84,7 @@ public void test() BOOLEAN, call("b * 2", functionResolution.arithmeticFunction(MULTIPLY, BIGINT, BIGINT), BIGINT, b, constant(2L, BIGINT)), constant(10L, BIGINT)); - Predicate compiledTimesTwo = compiler.compilePredicate(timesTwo).get(); + Predicate compiledTimesTwo = compiler.compilePredicate(SESSION.getSqlFunctionProperties(), timesTwo).get(); assertEquals(Arrays.asList(1), Ints.asList(compiledTimesTwo.getInputChannels())); @@ -107,11 +107,11 @@ public void testCache() call("a * 2", functionResolution.arithmeticFunction(MULTIPLY, BIGINT, BIGINT), BIGINT, new InputReferenceExpression(1, BIGINT), constant(2L, BIGINT)), constant(10L, BIGINT)); - PredicateCompiler compiler = new RowExpressionPredicateCompiler(functionManager, 10_000); - assertSame(compiler.compilePredicate(predicate), compiler.compilePredicate(predicate)); + PredicateCompiler compiler = new RowExpressionPredicateCompiler(metadata, 10_000); + assertSame(compiler.compilePredicate(SESSION.getSqlFunctionProperties(), predicate), compiler.compilePredicate(SESSION.getSqlFunctionProperties(), predicate)); - PredicateCompiler noCacheCompiler = new RowExpressionPredicateCompiler(functionManager, 0); - assertNotSame(noCacheCompiler.compilePredicate(predicate), noCacheCompiler.compilePredicate(predicate)); + PredicateCompiler noCacheCompiler = new RowExpressionPredicateCompiler(metadata, 0); + assertNotSame(noCacheCompiler.compilePredicate(SESSION.getSqlFunctionProperties(), predicate), noCacheCompiler.compilePredicate(SESSION.getSqlFunctionProperties(), predicate)); } private static Block createLongBlock(long... values) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestVarArgsToArrayAdapterGenerator.java b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestVarArgsToArrayAdapterGenerator.java index 35b37184a97eb..706a0a227f1e1 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestVarArgsToArrayAdapterGenerator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestVarArgsToArrayAdapterGenerator.java @@ -18,8 +18,9 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; import com.facebook.presto.operator.scalar.AbstractTestFunctions; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.TypeManager; import com.google.common.base.Joiner; @@ -31,8 +32,9 @@ import java.util.Optional; import java.util.stream.IntStream; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.sql.gen.TestVarArgsToArrayAdapterGenerator.TestVarArgsSum.VAR_ARGS_SUM; import static com.facebook.presto.sql.gen.VarArgsToArrayAdapterGenerator.generateVarArgsToArrayAdapter; @@ -77,7 +79,7 @@ public static class TestVarArgsSum private TestVarArgsSum() { super(new Signature( - "var_args_sum", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "var_args_sum"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), @@ -105,7 +107,7 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { VarArgsToArrayAdapterGenerator.MethodHandleAndConstructor methodHandleAndConstructor = generateVarArgsToArrayAdapter( long.class, @@ -113,7 +115,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in arity, METHOD_HANDLE, USER_STATE_FACTORY); - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, nCopies(arity, valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandleAndConstructor.getMethodHandle(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalize.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalize.java index 9642a7309d48e..fba123db13353 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalize.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalize.java @@ -75,7 +75,7 @@ public void testDuplicatesInWindowOrderBy() .addFunction(functionCall("row_number", Optional.empty(), ImmutableList.of())), values("A"))), ImmutableList.of( - new UnaliasSymbolReferences(), + new UnaliasSymbolReferences(getMetadata().getFunctionManager()), new IterativeOptimizer( new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCompilerConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCompilerConfig.java index 52d343bf4ffb3..1768bec96fc52 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCompilerConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCompilerConfig.java @@ -18,9 +18,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestCompilerConfig { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java index 9aafa965476de..a86de1f44eaec 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestEffectivePredicateExtractor.java @@ -19,22 +19,23 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.block.SortOrder; -import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SortNode; -import com.facebook.presto.sql.planner.plan.TopNNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.tree.BooleanLiteral; import com.facebook.presto.sql.tree.ComparisonExpression; @@ -53,7 +54,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; @@ -71,14 +71,15 @@ import java.util.Set; import java.util.UUID; +import static com.facebook.presto.spi.plan.AggregationNode.globalAggregation; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; +import static com.facebook.presto.spi.plan.LimitNode.Step.FINAL; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.ExpressionUtils.and; import static com.facebook.presto.sql.ExpressionUtils.combineConjuncts; import static com.facebook.presto.sql.ExpressionUtils.or; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; import static com.facebook.presto.sql.planner.optimizations.AggregationNodeUtils.count; -import static com.facebook.presto.sql.planner.plan.AggregationNode.globalAggregation; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; import static com.facebook.presto.sql.tree.BooleanLiteral.FALSE_LITERAL; import static com.facebook.presto.sql.tree.BooleanLiteral.TRUE_LITERAL; @@ -149,8 +150,6 @@ public void setUp() @Test public void testAggregation() { - FunctionCall functionCall = new FunctionCall(QualifiedName.of("count"), ImmutableList.of()); - FunctionHandle functionHandle = metadata.getFunctionManager().lookupFunction("count", ImmutableList.of()); PlanNode node = new AggregationNode(newId(), filter(baseTableScan, and( @@ -243,7 +242,7 @@ public void testTopN() equals(AE, BE), equals(BE, CE), lessThan(CE, bigintLiteral(10)))), - 1, new OrderingScheme(ImmutableList.of(AV), ImmutableMap.of(AV, SortOrder.ASC_NULLS_LAST)), TopNNode.Step.PARTIAL); + 1, new OrderingScheme(ImmutableList.of(new Ordering(AV, SortOrder.ASC_NULLS_LAST))), TopNNode.Step.PARTIAL); Expression effectivePredicate = effectivePredicateExtractor.extract(node, types); @@ -265,7 +264,7 @@ public void testLimit() equals(BE, CE), lessThan(CE, bigintLiteral(10)))), 1, - false); + FINAL); Expression effectivePredicate = effectivePredicateExtractor.extract(node, types); @@ -286,7 +285,8 @@ public void testSort() equals(AE, BE), equals(BE, CE), lessThan(CE, bigintLiteral(10)))), - new OrderingScheme(ImmutableList.of(AV), ImmutableMap.of(AV, SortOrder.ASC_NULLS_LAST))); + new OrderingScheme(ImmutableList.of(new Ordering(AV, SortOrder.ASC_NULLS_LAST))), + false); Expression effectivePredicate = effectivePredicateExtractor.extract(node, types); @@ -309,9 +309,7 @@ public void testWindow() lessThan(CE, bigintLiteral(10)))), new WindowNode.Specification( ImmutableList.of(AV), - Optional.of(new OrderingScheme( - ImmutableList.of(AV), - ImmutableMap.of(AV, SortOrder.ASC_NULLS_LAST)))), + Optional.of(new OrderingScheme(ImmutableList.of(new Ordering(AV, SortOrder.ASC_NULLS_LAST))))), ImmutableMap.of(), Optional.empty(), ImmutableSet.of(), @@ -388,13 +386,13 @@ public void testTableScan() @Test public void testUnion() { - ImmutableListMultimap variableMapping = ImmutableListMultimap.of(AV, BV, AV, CV, AV, EV); PlanNode node = new UnionNode(newId(), ImmutableList.of( filter(baseTableScan, greaterThan(AE, bigintLiteral(10))), filter(baseTableScan, and(greaterThan(AE, bigintLiteral(10)), lessThan(AE, bigintLiteral(100)))), filter(baseTableScan, and(greaterThan(AE, bigintLiteral(10)), lessThan(AE, bigintLiteral(100))))), - variableMapping); + ImmutableList.of(AV), + ImmutableMap.of(AV, ImmutableList.of(BV, CV, EV))); Expression effectivePredicate = effectivePredicateExtractor.extract(node, types); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestExpressionDomainTranslator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestExpressionDomainTranslator.java index 61742c1ed71e1..8c8eb1cb47c3e 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestExpressionDomainTranslator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestExpressionDomainTranslator.java @@ -1442,7 +1442,7 @@ private static Expression cast(Expression expression, Type type) private static FunctionCall colorLiteral(long value) { - return new FunctionCall(QualifiedName.of(getMagicLiteralFunctionSignature(COLOR).getName()), ImmutableList.of(bigintLiteral(value))); + return new FunctionCall(QualifiedName.of(getMagicLiteralFunctionSignature(COLOR).getNameSuffix()), ImmutableList.of(bigintLiteral(value))); } private Expression varbinaryLiteral(Slice value) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLocalExecutionPlanner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLocalExecutionPlanner.java index a8b5a5e5f14bc..14ad4a7a46a82 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLocalExecutionPlanner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLocalExecutionPlanner.java @@ -22,9 +22,9 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.spi.StandardErrorCode.COMPILER_ERROR; -import static io.airlift.testing.Closeables.closeAllRuntimeException; import static java.util.Collections.nCopies; import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java index 6be4376d1af42..0e5d32e1700cf 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.Session; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.TableScanNode; @@ -24,7 +25,6 @@ import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.optimizations.AddLocalExchanges; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.DistinctLimitNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; @@ -52,6 +52,9 @@ import static com.facebook.presto.SystemSessionProperties.OPTIMIZE_HASH_GENERATION; import static com.facebook.presto.spi.StandardErrorCode.SUBQUERY_MULTIPLE_ROWS; import static com.facebook.presto.spi.block.SortOrder.ASC_NULLS_LAST; +import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.spi.predicate.Domain.singleValue; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarcharType.createVarcharType; @@ -80,13 +83,11 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.specification; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.strictTableScan; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.topN; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.topNRowNumber; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.window; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.FINAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.PARTIAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.LOCAL; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; @@ -993,4 +994,57 @@ public void testEqualityInference() "p_comment = '42'", tableScan("partsupp", ImmutableMap.of("p_suppkey", "suppkey", "p_partkey", "partkey", "p_comment", "comment"))))))); } + + @Test + public void testLimitZero() + { + assertPlan( + "SELECT orderkey FROM orders LIMIT 0", + output( + values("orderkey_0"))); + + assertPlan( + "SELECT orderkey FROM orders ORDER BY orderkey ASC LIMIT 0", + output( + values("orderkey_0"))); + + assertPlan( + "SELECT orderkey FROM orders GROUP BY 1 ORDER BY 1 DESC LIMIT 0", + output( + values("orderkey_0"))); + + assertPlan( + "SELECT DISTINCT orderkey FROM orders LIMIT 0", + output( + values("orderkey_0"))); + + assertPlan( + "SELECT * FROM (SELECT regionkey FROM region GROUP BY regionkey) r1, region r2 WHERE r2.regionkey > r1.regionkey LIMIT 0", + output( + values("expr_8", "expr_9", "expr_10", "expr_11"))); + } + + @Test + public void testTopN() + { + ImmutableList orderBy = ImmutableList.of(sort("ORDERKEY", DESCENDING, LAST)); + assertDistributedPlan( + "SELECT orderkey FROM orders ORDER BY orderkey DESC LIMIT 1", + output( + topN(1, orderBy, + anyTree( + topN(1, orderBy, + tableScan("orders", ImmutableMap.of( + "ORDERKEY", "orderkey"))))))); + + assertDistributedPlan( + "SELECT orderkey FROM orders GROUP BY 1 ORDER BY 1 DESC LIMIT 1", + output( + topN(1, orderBy, + anyTree( + topN(1, orderBy, + anyTree( + tableScan("orders", ImmutableMap.of( + "ORDERKEY", "orderkey")))))))); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestNullabilityAnalyzer.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestNullabilityAnalyzer.java new file mode 100644 index 0000000000000..fad11fa4daf2f --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestNullabilityAnalyzer.java @@ -0,0 +1,104 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner; + +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.TestingRowExpressionTranslator; +import com.facebook.presto.sql.parser.ParsingOptions; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.planner.iterative.rule.LambdaCaptureDesugaringRewriter; +import com.facebook.presto.sql.tree.Expression; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Collection; + +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.RowType.field; +import static com.facebook.presto.sql.ExpressionUtils.rewriteIdentifiersToSymbolReferences; +import static org.testng.Assert.assertEquals; + +public class TestNullabilityAnalyzer +{ + private static final Metadata METADATA = MetadataManager.createTestMetadataManager(); + private static final TestingRowExpressionTranslator TRANSLATOR = new TestingRowExpressionTranslator(METADATA); + private static final TypeProvider TYPES = TypeProvider.viewOf(new ImmutableMap.Builder() + .put("a", BIGINT) + .put("b", new ArrayType(BIGINT)) + .put("c", RowType.from(ImmutableList.of(field("field_1", BIGINT)))) + .build()); + + private static final NullabilityAnalyzer analyzer = new NullabilityAnalyzer(METADATA.getFunctionManager(), METADATA.getTypeManager()); + + @Test + void test() + { + assertNullability("TRY_CAST(JSON '123' AS VARCHAR)", true); + assertNullability("TRY_CAST(a AS VARCHAR)", true); + assertNullability("CAST(a AS VARCHAR)", true); + + //TODO following two tests should return false but we are not yet smart enough to infer it. + assertNullability("TRY_CAST('123' AS VARCHAR)", true); + assertNullability("CAST('123' AS VARCHAR)", true); + + assertNullability("a = 1", false); + assertNullability("(a/9+1)*5-10 > 10", false); + assertNullability("1", false); + assertNullability("a", false); + assertNullability("TRY(a + 1)", true); + + assertNullability("IF(a > 10, 1)", true); + assertNullability("a IN (1, NULL)", true); + assertNullability("CASE WHEN a> 10 THEN 1 END", true); + assertNullability("c.field_1", true); + assertNullability("b[0]", true); + assertNullability("a BETWEEN 1 AND 2", false); + + // nested + assertNullability("1 = TRY(a + 1)", true); + } + + private void assertNullability(String expression, boolean mayReturnNullForNotNullInput) + { + Expression rawExpression = rewriteIdentifiersToSymbolReferences(new SqlParser().createExpression(expression, new ParsingOptions())); + Expression desugaredExpression = new TestingDesugarExpressions(TYPES.allVariables()).rewrite(rawExpression); + RowExpression rowExpression = TRANSLATOR.translate(desugaredExpression, TYPES); + assertEquals(NullabilityAnalyzer.mayReturnNullOnNonNullInput(rawExpression), mayReturnNullForNotNullInput); + assertEquals(analyzer.mayReturnNullOnNonNullInput(rowExpression), mayReturnNullForNotNullInput); + } + + private static class TestingDesugarExpressions + { + private final PlanVariableAllocator variableAllocator; + + public TestingDesugarExpressions(Collection variables) + { + this.variableAllocator = new PlanVariableAllocator(variables); + } + + public Expression rewrite(Expression expression) + { + expression = DesugarTryExpressionRewriter.rewrite(expression); + expression = LambdaCaptureDesugaringRewriter.rewrite(expression, variableAllocator); + return expression; + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java index 45283b9ad6267..f45cb9724b801 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java @@ -13,17 +13,17 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; import com.facebook.presto.sql.planner.optimizations.PredicatePushDown; +import com.facebook.presto.sql.planner.optimizations.RowExpressionPredicatePushDown; +import com.facebook.presto.sql.planner.optimizations.StatsRecordingPlanOptimizer; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; import com.facebook.presto.sql.planner.plan.WindowNode; -import com.facebook.presto.sql.tree.ComparisonExpression; -import com.facebook.presto.sql.tree.LongLiteral; -import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -31,6 +31,7 @@ import java.util.List; import java.util.Optional; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.any; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.assignUniqueId; @@ -49,7 +50,7 @@ import static com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED; import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; import static com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT; -import static com.facebook.presto.sql.tree.ComparisonExpression.Operator.EQUAL; +import static com.facebook.presto.sql.relational.Expressions.constant; public class TestPredicatePushdown extends BasePlanTest @@ -423,14 +424,26 @@ public void testNonDeterministicPredicateNotPushedDown() ImmutableMap.of("CUST_KEY", "custkey")))))))); } + @Override + protected void assertPlan(String sql, PlanMatchPattern pattern) + { + // TODO remove tests with filtered optimizer once we only have RowExpressionPredicatePushDown + // Currently we have mixture of Expression/RowExpression based push down, so we disable one of them to make sure test covers both code path. + assertPlan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, pattern, planOptimizer -> !(planOptimizer instanceof StatsRecordingPlanOptimizer) || + !(((StatsRecordingPlanOptimizer) planOptimizer).getDelegate() instanceof PredicatePushDown)); + assertPlan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, pattern, planOptimizer -> !(planOptimizer instanceof StatsRecordingPlanOptimizer) || + !(((StatsRecordingPlanOptimizer) planOptimizer).getDelegate() instanceof RowExpressionPredicatePushDown)); + assertPlan(sql, LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED, pattern); + } + @Test public void testPredicatePushDownCreatesValidJoin() { RuleTester tester = new RuleTester(); - tester.assertThat(new PredicatePushDown(tester.getMetadata(), tester.getSqlParser())) + tester.assertThat(new RowExpressionPredicatePushDown(tester.getMetadata(), tester.getSqlParser())) .on(p -> p.join(INNER, - p.filter(new ComparisonExpression(EQUAL, new SymbolReference("a1"), new LongLiteral("1")), + p.filter(p.comparison(OperatorType.EQUAL, p.variable("a1"), constant(1L, INTEGER)), p.values(p.variable("a1"))), p.values(p.variable("b1")), ImmutableList.of(new EquiJoinClause(p.variable("a1"), p.variable("b1"))), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestQuantifiedComparison.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestQuantifiedComparison.java index b290746912d5a..1a2619355f1ff 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestQuantifiedComparison.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestQuantifiedComparison.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.assertions.BasePlanTest; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java index 08028a5e57daa..7872db2fb4735 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionDomainTranslator.java @@ -46,16 +46,16 @@ import java.util.concurrent.TimeUnit; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.expressions.LogicalRowExpressions.FALSE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.and; +import static com.facebook.presto.expressions.LogicalRowExpressions.or; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN_OR_EQUAL; import static com.facebook.presto.spi.function.OperatorType.LESS_THAN_OR_EQUAL; import static com.facebook.presto.spi.function.OperatorType.NOT_EQUAL; import static com.facebook.presto.spi.predicate.TupleDomain.withColumnDomains; import static com.facebook.presto.spi.relation.DomainTranslator.BASIC_COLUMN_EXTRACTOR; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.FALSE_CONSTANT; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.and; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.or; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IN; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IS_NULL; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -928,8 +928,8 @@ public void testConjunctExpression() assertEquals( toPredicate(fromPredicate(expression).getTupleDomain()), and( - greaterThan(C_BIGINT, bigintLiteral(0)), - greaterThan(C_DOUBLE, doubleLiteral(0)))); + greaterThan(C_DOUBLE, doubleLiteral(0)), + greaterThan(C_BIGINT, bigintLiteral(0)))); } @Test @@ -1130,6 +1130,15 @@ public void testCharComparedToVarcharExpression() assertUnsupportedPredicate(equal(cast(C_CHAR, charType), cast(stringLiteral("abc12345678"), charType))); } + @Test + public void testBooleanAll() + { + RowExpression rowExpression = or(or(equal(C_BOOLEAN, constant(true, BOOLEAN)), equal(C_BOOLEAN, constant(false, BOOLEAN))), isNull(C_BOOLEAN)); + ExtractionResult result = fromPredicate(rowExpression); + TupleDomain tupleDomain = result.getTupleDomain(); + assertTrue(tupleDomain.isAll()); + } + private void assertPredicateTranslates(RowExpression expression, TupleDomain tupleDomain) { ExtractionResult result = fromPredicate(expression); @@ -1217,7 +1226,7 @@ private static RowExpression isNull(RowExpression expression) private RowExpression cast(RowExpression expression, Type toType) { FunctionHandle cast = metadata.getFunctionManager().lookupCast(CastType.CAST, expression.getType().getTypeSignature(), toType.getTypeSignature()); - return call(CastType.CAST.getCastName(), cast, toType, expression); + return call(CastType.CAST.name(), cast, toType, expression); } private RowExpression not(RowExpression expression) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionEqualityInference.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionEqualityInference.java new file mode 100644 index 0000000000000..22ab85dc3bf8f --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionEqualityInference.java @@ -0,0 +1,524 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner; + +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.sql.TestingRowExpressionTranslator; +import com.facebook.presto.sql.relational.Expressions; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.expressions.LogicalRowExpressions.and; +import static com.facebook.presto.spi.function.OperatorType.ADD; +import static com.facebook.presto.spi.function.OperatorType.MULTIPLY; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.RowType.field; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; +import static com.facebook.presto.sql.planner.RowExpressionEqualityInference.Builder.isInferenceCandidate; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.sql.relational.Expressions.constant; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Predicates.not; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +public class TestRowExpressionEqualityInference +{ + private static final Metadata METADATA = MetadataManager.createTestMetadataManager(); + private static final TestingRowExpressionTranslator ROW_EXPRESSION_TRANSLATOR = new TestingRowExpressionTranslator(METADATA); + + @Test + public void testTransitivity() + { + RowExpressionEqualityInference.Builder builder = new RowExpressionEqualityInference.Builder(METADATA); + addEquality("a1", "b1", builder); + addEquality("b1", "c1", builder); + addEquality("d1", "c1", builder); + + addEquality("a2", "b2", builder); + addEquality("b2", "a2", builder); + addEquality("b2", "c2", builder); + addEquality("d2", "b2", builder); + addEquality("c2", "d2", builder); + + RowExpressionEqualityInference inference = builder.build(); + + assertEquals( + inference.rewriteExpression(someExpression("a1", "a2"), matchesVariables("d1", "d2")), + someExpression("d1", "d2")); + + assertEquals( + inference.rewriteExpression(someExpression("a1", "c1"), matchesVariables("b1")), + someExpression("b1", "b1")); + + assertEquals( + inference.rewriteExpression(someExpression("a1", "a2"), matchesVariables("b1", "d2", "c3")), + someExpression("b1", "d2")); + + // Both starting expressions should canonicalize to the same expression + assertEquals( + inference.getScopedCanonical(variable("a2"), matchesVariables("c2", "d2")), + inference.getScopedCanonical(variable("b2"), matchesVariables("c2", "d2"))); + RowExpression canonical = inference.getScopedCanonical(variable("a2"), matchesVariables("c2", "d2")); + + // Given multiple translatable candidates, should choose the canonical + assertEquals( + inference.rewriteExpression(someExpression("a2", "b2"), matchesVariables("c2", "d2")), + someExpression(canonical, canonical)); + } + + @Test + public void testTriviallyRewritable() + { + RowExpressionEqualityInference.Builder builder = new RowExpressionEqualityInference.Builder(METADATA); + RowExpression expression = builder.build() + .rewriteExpression(someExpression("a1", "a2"), matchesVariables("a1", "a2")); + + assertEquals(expression, someExpression("a1", "a2")); + } + + @Test + public void testUnrewritable() + { + RowExpressionEqualityInference.Builder builder = new RowExpressionEqualityInference.Builder(METADATA); + addEquality("a1", "b1", builder); + addEquality("a2", "b2", builder); + RowExpressionEqualityInference inference = builder.build(); + + assertNull(inference.rewriteExpression(someExpression("a1", "a2"), matchesVariables("b1", "c1"))); + assertNull(inference.rewriteExpression(someExpression("c1", "c2"), matchesVariables("a1", "a2"))); + } + + @Test + public void testParseEqualityExpression() + { + RowExpressionEqualityInference inference = new RowExpressionEqualityInference.Builder(METADATA) + .addEquality(equals("a1", "b1")) + .addEquality(equals("a1", "c1")) + .addEquality(equals("c1", "a1")) + .build(); + + RowExpression expression = inference.rewriteExpression(someExpression("a1", "b1"), matchesVariables("c1")); + assertEquals(expression, someExpression("c1", "c1")); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testInvalidEqualityExpression1() + { + new RowExpressionEqualityInference.Builder(METADATA) + .addEquality(equals("a1", "a1")); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testInvalidEqualityExpression2() + { + new RowExpressionEqualityInference.Builder(METADATA) + .addEquality(someExpression("a1", "b1")); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testInvalidEqualityExpression3() + { + RowExpressionEqualityInference.Builder builder = new RowExpressionEqualityInference.Builder(METADATA); + addEquality("a1", "a1", builder); + } + + @Test + public void testExtractInferrableEqualities() + { + RowExpressionEqualityInference inference = new RowExpressionEqualityInference.Builder(METADATA) + .extractInferenceCandidates(and(equals("a1", "b1"), equals("b1", "c1"), someExpression("c1", "d1"))) + .build(); + + // Able to rewrite to c1 due to equalities + assertEquals(variable("c1"), inference.rewriteExpression(variable("a1"), matchesVariables("c1"))); + + // But not be able to rewrite to d1 which is not connected via equality + assertNull(inference.rewriteExpression(variable("a1"), matchesVariables("d1"))); + } + + @Test + public void testEqualityPartitionGeneration() + { + RowExpressionEqualityInference.Builder builder = new RowExpressionEqualityInference.Builder(METADATA); + builder.addEquality(variable("a1"), variable("b1")); + builder.addEquality(add("a1", "a1"), multiply(variable("a1"), number(2))); + builder.addEquality(variable("b1"), variable("c1")); + builder.addEquality(add("a1", "a1"), variable("c1")); + builder.addEquality(add("a1", "b1"), variable("c1")); + + RowExpressionEqualityInference inference = builder.build(); + + RowExpressionEqualityInference.EqualityPartition emptyScopePartition = inference.generateEqualitiesPartitionedBy(Predicates.alwaysFalse()); + // Cannot generate any scope equalities with no matching symbols + assertTrue(emptyScopePartition.getScopeEqualities().isEmpty()); + // All equalities should be represented in the inverse scope + assertFalse(emptyScopePartition.getScopeComplementEqualities().isEmpty()); + // There should be no equalities straddling the scope + assertTrue(emptyScopePartition.getScopeStraddlingEqualities().isEmpty()); + + RowExpressionEqualityInference.EqualityPartition equalityPartition = inference.generateEqualitiesPartitionedBy(matchesVariables("c1")); + + // There should be equalities in the scope, that only use c1 and are all inferrable equalities + assertFalse(equalityPartition.getScopeEqualities().isEmpty()); + assertTrue(Iterables.all(equalityPartition.getScopeEqualities(), matchesVariableScope(matchesVariables("c1")))); + assertTrue(Iterables.all(equalityPartition.getScopeEqualities(), isInferenceCandidate(METADATA))); + + // There should be equalities in the inverse scope, that never use c1 and are all inferrable equalities + assertFalse(equalityPartition.getScopeComplementEqualities().isEmpty()); + assertTrue(Iterables.all(equalityPartition.getScopeComplementEqualities(), matchesVariableScope(not(matchesVariables("c1"))))); + assertTrue(Iterables.all(equalityPartition.getScopeComplementEqualities(), isInferenceCandidate(METADATA))); + + // There should be equalities in the straddling scope, that should use both c1 and not c1 symbols + assertFalse(equalityPartition.getScopeStraddlingEqualities().isEmpty()); + assertTrue(Iterables.any(equalityPartition.getScopeStraddlingEqualities(), matchesStraddlingScope(matchesVariables("c1")))); + assertTrue(Iterables.all(equalityPartition.getScopeStraddlingEqualities(), isInferenceCandidate(METADATA))); + + // There should be a "full cover" of all of the equalities used + // THUS, we should be able to plug the generated equalities back in and get an equivalent set of equalities back the next time around + RowExpressionEqualityInference newInference = new RowExpressionEqualityInference.Builder(METADATA) + .addAllEqualities(equalityPartition.getScopeEqualities()) + .addAllEqualities(equalityPartition.getScopeComplementEqualities()) + .addAllEqualities(equalityPartition.getScopeStraddlingEqualities()) + .build(); + + RowExpressionEqualityInference.EqualityPartition newEqualityPartition = newInference.generateEqualitiesPartitionedBy(matchesVariables("c1")); + + assertEquals(setCopy(equalityPartition.getScopeEqualities()), setCopy(newEqualityPartition.getScopeEqualities())); + assertEquals(setCopy(equalityPartition.getScopeComplementEqualities()), setCopy(newEqualityPartition.getScopeComplementEqualities())); + assertEquals(setCopy(equalityPartition.getScopeStraddlingEqualities()), setCopy(newEqualityPartition.getScopeStraddlingEqualities())); + } + + @Test + public void testMultipleEqualitySetsPredicateGeneration() + { + RowExpressionEqualityInference.Builder builder = new RowExpressionEqualityInference.Builder(METADATA); + addEquality("a1", "b1", builder); + addEquality("b1", "c1", builder); + addEquality("c1", "d1", builder); + + addEquality("a2", "b2", builder); + addEquality("b2", "c2", builder); + addEquality("c2", "d2", builder); + + RowExpressionEqualityInference inference = builder.build(); + + // Generating equalities for disjoint groups + RowExpressionEqualityInference.EqualityPartition equalityPartition = inference.generateEqualitiesPartitionedBy(variableBeginsWith("a", "b")); + + // There should be equalities in the scope, that only use a* and b* symbols and are all inferrable equalities + assertFalse(equalityPartition.getScopeEqualities().isEmpty()); + assertTrue(Iterables.all(equalityPartition.getScopeEqualities(), matchesVariableScope(variableBeginsWith("a", "b")))); + assertTrue(Iterables.all(equalityPartition.getScopeEqualities(), isInferenceCandidate(METADATA))); + + // There should be equalities in the inverse scope, that never use a* and b* symbols and are all inferrable equalities + assertFalse(equalityPartition.getScopeComplementEqualities().isEmpty()); + assertTrue(Iterables.all(equalityPartition.getScopeComplementEqualities(), matchesVariableScope(not(variableBeginsWith("a", "b"))))); + assertTrue(Iterables.all(equalityPartition.getScopeComplementEqualities(), isInferenceCandidate(METADATA))); + + // There should be equalities in the straddling scope, that should use both c1 and not c1 symbols + assertFalse(equalityPartition.getScopeStraddlingEqualities().isEmpty()); + assertTrue(Iterables.any(equalityPartition.getScopeStraddlingEqualities(), matchesStraddlingScope(variableBeginsWith("a", "b")))); + assertTrue(Iterables.all(equalityPartition.getScopeStraddlingEqualities(), isInferenceCandidate(METADATA))); + + // Again, there should be a "full cover" of all of the equalities used + // THUS, we should be able to plug the generated equalities back in and get an equivalent set of equalities back the next time around + RowExpressionEqualityInference newInference = new RowExpressionEqualityInference.Builder(METADATA) + .addAllEqualities(equalityPartition.getScopeEqualities()) + .addAllEqualities(equalityPartition.getScopeComplementEqualities()) + .addAllEqualities(equalityPartition.getScopeStraddlingEqualities()) + .build(); + + RowExpressionEqualityInference.EqualityPartition newEqualityPartition = newInference.generateEqualitiesPartitionedBy(variableBeginsWith("a", "b")); + + assertEquals(setCopy(equalityPartition.getScopeEqualities()), setCopy(newEqualityPartition.getScopeEqualities())); + assertEquals(setCopy(equalityPartition.getScopeComplementEqualities()), setCopy(newEqualityPartition.getScopeComplementEqualities())); + assertEquals(setCopy(equalityPartition.getScopeStraddlingEqualities()), setCopy(newEqualityPartition.getScopeStraddlingEqualities())); + } + + @Test + public void testSubExpressionRewrites() + { + RowExpressionEqualityInference.Builder builder = new RowExpressionEqualityInference.Builder(METADATA); + builder.addEquality(variable("a1"), add("b", "c")); // a1 = b + c + builder.addEquality(variable("a2"), multiply(variable("b"), add("b", "c"))); // a2 = b * (b + c) + builder.addEquality(variable("a3"), multiply(variable("a1"), add("b", "c"))); // a3 = a1 * (b + c) + RowExpressionEqualityInference inference = builder.build(); + + // Expression (b + c) should get entirely rewritten as a1 + assertEquals(inference.rewriteExpression(add("b", "c"), variableBeginsWith("a")), variable("a1")); + + // Only the sub-expression (b + c) should get rewritten in terms of a* + assertEquals(inference.rewriteExpression(multiply(variable("ax"), add("b", "c")), variableBeginsWith("a")), multiply(variable("ax"), variable("a1"))); + + // To be compliant, could rewrite either the whole expression, or just the sub-expression. Rewriting larger expressions are preferred + assertEquals(inference.rewriteExpression(multiply(variable("a1"), add("b", "c")), variableBeginsWith("a")), variable("a3")); + } + + @Test + public void testConstantEqualities() + { + RowExpressionEqualityInference.Builder builder = new RowExpressionEqualityInference.Builder(METADATA); + addEquality("a1", "b1", builder); + addEquality("b1", "c1", builder); + builder.addEquality(variable("c1"), number(1)); + RowExpressionEqualityInference inference = builder.build(); + + // Should always prefer a constant if available (constant is part of all scopes) + assertEquals(inference.rewriteExpression(variable("a1"), matchesVariables("a1", "b1")), number(1)); + + // All scope equalities should utilize the constant if possible + RowExpressionEqualityInference.EqualityPartition equalityPartition = inference.generateEqualitiesPartitionedBy(matchesVariables("a1", "b1")); + assertEquals(equalitiesAsSets(equalityPartition.getScopeEqualities()), + set(set(variable("a1"), number(1)), set(variable("b1"), number(1)))); + assertEquals(equalitiesAsSets(equalityPartition.getScopeComplementEqualities()), + set(set(variable("c1"), number(1)))); + + // There should be no scope straddling equalities as the full set of equalities should be already represented by the scope and inverse scope + assertTrue(equalityPartition.getScopeStraddlingEqualities().isEmpty()); + } + + @Test + public void testEqualityGeneration() + { + RowExpressionEqualityInference.Builder builder = new RowExpressionEqualityInference.Builder(METADATA); + builder.addEquality(variable("a1"), add("b", "c")); // a1 = b + c + builder.addEquality(variable("e1"), add("b", "d")); // e1 = b + d + addEquality("c", "d", builder); + RowExpressionEqualityInference inference = builder.build(); + + RowExpression scopedCanonical = inference.getScopedCanonical(variable("e1"), variableBeginsWith("a")); + assertEquals(scopedCanonical, variable("a1")); + } + + @Test(dataProvider = "testRowExpressions") + public void testExpressionsThatMayReturnNullOnNonNullInput(RowExpression candidate) + { + RowExpressionEqualityInference.Builder builder = new RowExpressionEqualityInference.Builder(METADATA); + builder.extractInferenceCandidates(equals(variable("b"), variable("x"))); + builder.extractInferenceCandidates(equals(variable("a"), candidate)); + + RowExpressionEqualityInference inference = builder.build(); + List equalities = inference.generateEqualitiesPartitionedBy(matchesVariables("b")).getScopeStraddlingEqualities(); + assertEquals(equalities.size(), 1); + assertTrue(equalities.get(0).equals(equals(variable("x"), variable("b"))) || equalities.get(0).equals(equals(variable("b"), variable("x")))); + } + + @DataProvider(name = "testRowExpressions") + public Object[][] toRowExpressionProvider() + { + return new Object[][] { + {ROW_EXPRESSION_TRANSLATOR.translate("try_cast(b AS BIGINT)", ImmutableMap.of("b", VARCHAR))}, + {ROW_EXPRESSION_TRANSLATOR.translate("\"$internal$try\"(() -> b)", ImmutableMap.of("b", BIGINT))}, + {ROW_EXPRESSION_TRANSLATOR.translate("nullif(b, 1)", ImmutableMap.of("b", BIGINT))}, + {ROW_EXPRESSION_TRANSLATOR.translate("if(b = 1, 1)", ImmutableMap.of("b", BIGINT))}, + {ROW_EXPRESSION_TRANSLATOR.translate("b.x", ImmutableMap.of("b", RowType.from(ImmutableList.of(field("x", BIGINT)))))}, + {ROW_EXPRESSION_TRANSLATOR.translate("IF(b in (NULL), b)", ImmutableMap.of("b", BIGINT))}, + {ROW_EXPRESSION_TRANSLATOR.translate("case b when 1 then 1 END", ImmutableMap.of("b", BIGINT))}, //simple case + {ROW_EXPRESSION_TRANSLATOR.translate("case when b is not NULL then 1 END", ImmutableMap.of("b", BIGINT))}, //search case + {ROW_EXPRESSION_TRANSLATOR.translate("ARRAY [NULL][b]", ImmutableMap.of("b", BIGINT))}}; + } + + private static Predicate matchesVariableScope(final Predicate variableScope) + { + return expression -> Iterables.all(VariablesExtractor.extractUnique(expression), variableScope); + } + + private static Predicate matchesStraddlingScope(final Predicate variableScope) + { + return expression -> { + Set variables = VariablesExtractor.extractUnique(expression); + return Iterables.any(variables, variableScope) && Iterables.any(variables, not(variableScope)); + }; + } + + private static void addEquality(String symbol1, String symbol2, RowExpressionEqualityInference.Builder builder) + { + builder.addEquality(variable(symbol1), variable(symbol2)); + } + + private static RowExpression someExpression(String symbol1, String symbol2) + { + return someExpression(variable(symbol1), variable(symbol2)); + } + + private static RowExpression someExpression(RowExpression expression1, RowExpression expression2) + { + return compare(OperatorType.GREATER_THAN, expression1, expression2); + } + + private static RowExpression add(String symbol1, String symbol2) + { + return arithmeticOperation(ADD, variable(symbol1), variable(symbol2)); + } + + private static RowExpression add(RowExpression expression1, RowExpression expression2) + { + return arithmeticOperation(ADD, expression1, expression2); + } + + private static RowExpression multiply(String symbol1, String symbol2) + { + return arithmeticOperation(MULTIPLY, variable(symbol1), variable(symbol2)); + } + + private static RowExpression multiply(RowExpression expression1, RowExpression expression2) + { + return arithmeticOperation(MULTIPLY, expression1, expression2); + } + + private static RowExpression equals(String symbol1, String symbol2) + { + return compare(OperatorType.EQUAL, variable(symbol1), variable(symbol2)); + } + + private static RowExpression equals(RowExpression expression1, RowExpression expression2) + { + return compare(OperatorType.EQUAL, expression1, expression2); + } + + private static VariableReferenceExpression variable(String name) + { + return Expressions.variable(name, BIGINT); + } + + private static RowExpression number(long number) + { + return constant(number, BIGINT); + } + + private static Predicate matchesVariables(String... variables) + { + return matchesVariables(Arrays.asList(variables)); + } + + private static Predicate matchesVariables(Collection variables) + { + final Set symbolSet = variables.stream() + .map(name -> new VariableReferenceExpression(name, BIGINT)) + .collect(toImmutableSet()); + + return Predicates.in(symbolSet); + } + + private static Predicate variableBeginsWith(String... prefixes) + { + return variableBeginsWith(Arrays.asList(prefixes)); + } + + private static Predicate variableBeginsWith(final Iterable prefixes) + { + return variable -> { + for (String prefix : prefixes) { + if (variable.getName().startsWith(prefix)) { + return true; + } + } + return false; + }; + } + + private static Set> equalitiesAsSets(Iterable expressions) + { + ImmutableSet.Builder> builder = ImmutableSet.builder(); + for (RowExpression expression : expressions) { + builder.add(equalityAsSet(expression)); + } + return builder.build(); + } + + private static Set equalityAsSet(RowExpression expression) + { + Preconditions.checkArgument(isOperation(expression, OperatorType.EQUAL)); + return ImmutableSet.of(getLeft(expression), getRight(expression)); + } + + private static Set set(E... elements) + { + return setCopy(Arrays.asList(elements)); + } + + private static Set setCopy(Iterable elements) + { + return ImmutableSet.copyOf(elements); + } + + private static CallExpression compare(OperatorType type, RowExpression left, RowExpression right) + { + return call( + type.getFunctionName().getFunctionName(), + METADATA.getFunctionManager().resolveOperator(type, fromTypes(left.getType(), right.getType())), + BOOLEAN, + left, + right); + } + + private static RowExpression getLeft(RowExpression expression) + { + checkArgument(expression instanceof CallExpression && ((CallExpression) expression).getArguments().size() == 2, "must be binary call expression"); + return ((CallExpression) expression).getArguments().get(0); + } + + private static RowExpression getRight(RowExpression expression) + { + checkArgument(expression instanceof CallExpression && ((CallExpression) expression).getArguments().size() == 2, "must be binary call expression"); + return ((CallExpression) expression).getArguments().get(1); + } + + private static CallExpression arithmeticOperation(OperatorType type, RowExpression left, RowExpression right) + { + return call( + type.getFunctionName().getFunctionName(), + METADATA.getFunctionManager().resolveOperator(type, fromTypes(left.getType(), right.getType())), + left.getType(), + left, + right); + } + + private static boolean isOperation(RowExpression expression, OperatorType type) + { + if (expression instanceof CallExpression) { + CallExpression call = (CallExpression) expression; + Optional expressionOperatorType = METADATA.getFunctionManager().getFunctionMetadata(call.getFunctionHandle()).getOperatorType(); + if (expressionOperatorType.isPresent()) { + return expressionOperatorType.get() == type; + } + } + return false; + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionFormatter.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionFormatter.java index ea7d2e80d20ae..2c03afbc823e6 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionFormatter.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionFormatter.java @@ -84,7 +84,7 @@ public class TestRowExpressionFormatter { private static final TypeManager typeManager = new TypeRegistry(); private static final FunctionManager functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - private static final RowExpressionFormatter FORMATTER = new RowExpressionFormatter(TEST_SESSION.toConnectorSession(), functionManager); + private static final RowExpressionFormatter FORMATTER = new RowExpressionFormatter(functionManager); private static final VariableReferenceExpression C_BIGINT = new VariableReferenceExpression("c_bigint", BIGINT); private static final VariableReferenceExpression C_BIGINT_ARRAY = new VariableReferenceExpression("c_bigint_array", new ArrayType(BIGINT)); @@ -137,7 +137,7 @@ public void testConstants() assertEquals(format(constantExpression), "VARBINARY 12 34 56"); // color - constantExpression = constant(256, COLOR); + constantExpression = constant(256L, COLOR); assertEquals(format(constantExpression), "COLOR 256"); // long and short decimals @@ -211,7 +211,7 @@ public void testCalls() RowExpression subscriptExpression = call(SUBSCRIPT.name(), functionManager.resolveOperator(SUBSCRIPT, fromTypes(arrayType, elementType)), elementType, - ImmutableList.of(C_BIGINT_ARRAY, constant(0, INTEGER))); + ImmutableList.of(C_BIGINT_ARRAY, constant(0L, INTEGER))); callExpression = subscriptExpression; assertEquals(format(callExpression), "c_bigint_array[INTEGER 0]"); @@ -220,7 +220,7 @@ public void testCalls() CAST.name(), functionManager.lookupCast(CastType.CAST, TINYINT.getTypeSignature(), BIGINT.getTypeSignature()), BIGINT, - constant(1, TINYINT)); + constant(1L, TINYINT)); assertEquals(format(callExpression), "CAST(TINYINT 1 AS bigint)"); // between @@ -229,8 +229,8 @@ public void testCalls() functionManager.resolveOperator(BETWEEN, fromTypes(BIGINT, BIGINT, BIGINT)), BOOLEAN, subscriptExpression, - constant(1, BIGINT), - constant(5, BIGINT)); + constant(1L, BIGINT), + constant(5L, BIGINT)); assertEquals(format(callExpression), "c_bigint_array[INTEGER 0] BETWEEN (BIGINT 1) AND (BIGINT 5)"); // other @@ -238,7 +238,7 @@ public void testCalls() HASH_CODE.name(), functionManager.resolveOperator(HASH_CODE, fromTypes(BIGINT)), BIGINT, - constant(1, BIGINT)); + constant(1L, BIGINT)); assertEquals(format(callExpression), "HASH_CODE(BIGINT 1)"); } @@ -288,7 +288,7 @@ public void testComplex() expression1 = call(SUBSCRIPT.name(), functionManager.resolveOperator(SUBSCRIPT, fromTypes(arrayType, elementType)), elementType, - ImmutableList.of(C_BIGINT_ARRAY, constant(5, INTEGER))); + ImmutableList.of(C_BIGINT_ARRAY, constant(5L, INTEGER))); expression2 = call( NEGATION.name(), functionManager.resolveOperator(NEGATION, fromTypes(expression1.getType())), @@ -320,6 +320,6 @@ private static CallExpression createCallExpression(OperatorType type) private static String format(RowExpression expression) { - return FORMATTER.formatRowExpression(expression); + return FORMATTER.formatRowExpression(TEST_SESSION.toConnectorSession(), expression); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionPredicateExtractor.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionPredicateExtractor.java new file mode 100644 index 0000000000000..5ba572b7848ab --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionPredicateExtractor.java @@ -0,0 +1,798 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner; + +import com.facebook.presto.expressions.LogicalRowExpressions; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; +import com.facebook.presto.spi.predicate.Domain; +import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.plan.JoinNode; +import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.facebook.presto.sql.planner.plan.SortNode; +import com.facebook.presto.sql.planner.plan.WindowNode; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; +import com.facebook.presto.testing.TestingHandle; +import com.facebook.presto.testing.TestingMetadata; +import com.facebook.presto.testing.TestingTransactionHandle; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +import static com.facebook.presto.expressions.LogicalRowExpressions.FALSE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.and; +import static com.facebook.presto.expressions.LogicalRowExpressions.or; +import static com.facebook.presto.spi.function.OperatorType.EQUAL; +import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; +import static com.facebook.presto.spi.function.OperatorType.LESS_THAN; +import static com.facebook.presto.spi.function.OperatorType.LESS_THAN_OR_EQUAL; +import static com.facebook.presto.spi.plan.AggregationNode.globalAggregation; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; +import static com.facebook.presto.spi.plan.LimitNode.Step.FINAL; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IS_NULL; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; +import static com.facebook.presto.sql.planner.RowExpressionEqualityInference.Builder.nonInferrableConjuncts; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; +import static com.facebook.presto.sql.planner.optimizations.AggregationNodeUtils.count; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.sql.relational.Expressions.constant; +import static com.facebook.presto.sql.relational.Expressions.specialForm; +import static org.testng.Assert.assertEquals; + +public class TestRowExpressionPredicateExtractor +{ + private static final TableHandle DUAL_TABLE_HANDLE = new TableHandle( + new ConnectorId("test"), + new TestingMetadata.TestingTableHandle(), + TestingTransactionHandle.create(), + Optional.empty()); + + private static final TableHandle DUAL_TABLE_HANDLE_WITH_LAYOUT = new TableHandle( + new ConnectorId("test"), + new TestingMetadata.TestingTableHandle(), + TestingTransactionHandle.create(), + Optional.of(TestingHandle.INSTANCE)); + + private static final VariableReferenceExpression AV = new VariableReferenceExpression("a", BIGINT); + private static final VariableReferenceExpression BV = new VariableReferenceExpression("b", BIGINT); + private static final VariableReferenceExpression CV = new VariableReferenceExpression("c", BIGINT); + private static final VariableReferenceExpression DV = new VariableReferenceExpression("d", BIGINT); + private static final VariableReferenceExpression EV = new VariableReferenceExpression("e", BIGINT); + private static final VariableReferenceExpression FV = new VariableReferenceExpression("f", BIGINT); + private static final VariableReferenceExpression GV = new VariableReferenceExpression("g", BIGINT); + + private final Metadata metadata = MetadataManager.createTestMetadataManager(); + private final LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions( + new RowExpressionDeterminismEvaluator(metadata.getFunctionManager()), + new FunctionResolution(metadata.getFunctionManager()), + metadata.getFunctionManager()); + private final RowExpressionPredicateExtractor effectivePredicateExtractor = new RowExpressionPredicateExtractor( + new RowExpressionDomainTranslator(metadata), + metadata.getFunctionManager(), + metadata.getTypeManager()); + + private Map scanAssignments; + private TableScanNode baseTableScan; + + @BeforeMethod + public void setUp() + { + scanAssignments = ImmutableMap.builder() + .put(AV, new TestingMetadata.TestingColumnHandle("a")) + .put(BV, new TestingMetadata.TestingColumnHandle("b")) + .put(CV, new TestingMetadata.TestingColumnHandle("c")) + .put(DV, new TestingMetadata.TestingColumnHandle("d")) + .put(EV, new TestingMetadata.TestingColumnHandle("e")) + .put(FV, new TestingMetadata.TestingColumnHandle("f")) + .build(); + + Map assignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV, DV, EV, FV))); + baseTableScan = new TableScanNode( + newId(), + DUAL_TABLE_HANDLE, + ImmutableList.copyOf(assignments.keySet()), + assignments, + TupleDomain.all(), + TupleDomain.all()); + } + + @Test + public void testAggregation() + { + PlanNode node = new AggregationNode(newId(), + filter(baseTableScan, + and( + equals(AV, DV), + equals(BV, EV), + equals(CV, FV), + lessThan(DV, bigintLiteral(10)), + lessThan(CV, DV), + greaterThan(AV, bigintLiteral(2)), + equals(EV, FV))), + ImmutableMap.of( + CV, count(metadata.getFunctionManager()), + DV, count(metadata.getFunctionManager())), + singleGroupingSet(ImmutableList.of(AV, BV, CV)), + ImmutableList.of(), + AggregationNode.Step.FINAL, + Optional.empty(), + Optional.empty()); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // Rewrite in terms of group by symbols + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts( + lessThan(AV, bigintLiteral(10)), + lessThan(BV, AV), + greaterThan(AV, bigintLiteral(2)), + equals(BV, CV))); + } + + @Test + public void testGroupByEmpty() + { + PlanNode node = new AggregationNode( + newId(), + filter(baseTableScan, FALSE_CONSTANT), + ImmutableMap.of(), + globalAggregation(), + ImmutableList.of(), + AggregationNode.Step.FINAL, + Optional.empty(), + Optional.empty()); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + assertEquals(effectivePredicate, TRUE_CONSTANT); + } + + @Test + public void testFilter() + { + PlanNode node = filter(baseTableScan, + and( + greaterThan(AV, call(metadata.getFunctionManager(), "rand", DOUBLE, ImmutableList.of())), + lessThan(BV, bigintLiteral(10)))); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // Non-deterministic functions should be purged + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts(lessThan(BV, bigintLiteral(10)))); + } + + @Test + public void testProject() + { + PlanNode node = new ProjectNode(newId(), + filter(baseTableScan, + and( + equals(AV, BV), + equals(BV, CV), + lessThan(CV, bigintLiteral(10)))), + assignment(DV, AV, EV, CV)); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // Rewrite in terms of project output symbols + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts( + lessThan(DV, bigintLiteral(10)), + equals(DV, EV))); + } + + @Test + public void testTopN() + { + PlanNode node = new TopNNode(newId(), + filter(baseTableScan, + and( + equals(AV, BV), + equals(BV, CV), + lessThan(CV, bigintLiteral(10)))), + 1, new OrderingScheme(ImmutableList.of(new Ordering(AV, SortOrder.ASC_NULLS_FIRST))), TopNNode.Step.PARTIAL); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // Pass through + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts( + equals(AV, BV), + equals(BV, CV), + lessThan(CV, bigintLiteral(10)))); + } + + @Test + public void testLimit() + { + PlanNode node = new LimitNode(newId(), + filter(baseTableScan, + and( + equals(AV, BV), + equals(BV, CV), + lessThan(CV, bigintLiteral(10)))), + 1, + FINAL); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // Pass through + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts( + equals(AV, BV), + equals(BV, CV), + lessThan(CV, bigintLiteral(10)))); + } + + @Test + public void testSort() + { + PlanNode node = new SortNode(newId(), + filter(baseTableScan, + and( + equals(AV, BV), + equals(BV, CV), + lessThan(CV, bigintLiteral(10)))), + new OrderingScheme(ImmutableList.of(new Ordering(AV, SortOrder.ASC_NULLS_LAST))), + false); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // Pass through + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts( + equals(AV, BV), + equals(BV, CV), + lessThan(CV, bigintLiteral(10)))); + } + + @Test + public void testWindow() + { + PlanNode node = new WindowNode(newId(), + filter(baseTableScan, + and( + equals(AV, BV), + equals(BV, CV), + lessThan(CV, bigintLiteral(10)))), + new WindowNode.Specification( + ImmutableList.of(AV), + Optional.of(new OrderingScheme( + ImmutableList.of(new Ordering(AV, SortOrder.ASC_NULLS_LAST))))), + ImmutableMap.of(), + Optional.empty(), + ImmutableSet.of(), + 0); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // Pass through + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts( + equals(AV, BV), + equals(BV, CV), + lessThan(CV, bigintLiteral(10)))); + } + + @Test + public void testTableScan() + { + // Effective predicate is True if there is no effective predicate + Map assignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV, DV))); + PlanNode node = new TableScanNode( + newId(), + DUAL_TABLE_HANDLE, + ImmutableList.copyOf(assignments.keySet()), + assignments, + TupleDomain.all(), + TupleDomain.all()); + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + assertEquals(effectivePredicate, TRUE_CONSTANT); + + node = new TableScanNode( + newId(), + DUAL_TABLE_HANDLE_WITH_LAYOUT, + ImmutableList.copyOf(assignments.keySet()), + assignments, + TupleDomain.none(), + TupleDomain.all()); + effectivePredicate = effectivePredicateExtractor.extract(node); + assertEquals(effectivePredicate, FALSE_CONSTANT); + + node = new TableScanNode( + newId(), + DUAL_TABLE_HANDLE_WITH_LAYOUT, + ImmutableList.copyOf(assignments.keySet()), + assignments, + TupleDomain.withColumnDomains(ImmutableMap.of(scanAssignments.get(AV), Domain.singleValue(BIGINT, 1L))), + TupleDomain.all()); + effectivePredicate = effectivePredicateExtractor.extract(node); + assertEquals(normalizeConjuncts(effectivePredicate), normalizeConjuncts(equals(bigintLiteral(1L), AV))); + + node = new TableScanNode( + newId(), + DUAL_TABLE_HANDLE_WITH_LAYOUT, + ImmutableList.copyOf(assignments.keySet()), + assignments, + TupleDomain.withColumnDomains(ImmutableMap.of( + scanAssignments.get(AV), Domain.singleValue(BIGINT, 1L), + scanAssignments.get(BV), Domain.singleValue(BIGINT, 2L))), + TupleDomain.all()); + effectivePredicate = effectivePredicateExtractor.extract(node); + assertEquals(normalizeConjuncts(effectivePredicate), normalizeConjuncts(equals(bigintLiteral(2L), BV), equals(bigintLiteral(1L), AV))); + + node = new TableScanNode( + newId(), + DUAL_TABLE_HANDLE, + ImmutableList.copyOf(assignments.keySet()), + assignments, + TupleDomain.all(), + TupleDomain.all()); + effectivePredicate = effectivePredicateExtractor.extract(node); + assertEquals(effectivePredicate, TRUE_CONSTANT); + } + + @Test + public void testUnion() + { + PlanNode node = new UnionNode(newId(), + ImmutableList.of( + filter(baseTableScan, greaterThan(AV, bigintLiteral(10))), + filter(baseTableScan, and(greaterThan(AV, bigintLiteral(10)), lessThan(AV, bigintLiteral(100)))), + filter(baseTableScan, and(greaterThan(AV, bigintLiteral(10)), lessThan(AV, bigintLiteral(100))))), + ImmutableList.of(AV), + ImmutableMap.of(AV, ImmutableList.of(BV, CV, EV))); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // Only the common conjuncts can be inferred through a Union + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts(greaterThan(AV, bigintLiteral(10)))); + } + + @Test + public void testInnerJoin() + { + ImmutableList.Builder criteriaBuilder = ImmutableList.builder(); + criteriaBuilder.add(new JoinNode.EquiJoinClause(AV, DV)); + criteriaBuilder.add(new JoinNode.EquiJoinClause(BV, EV)); + List criteria = criteriaBuilder.build(); + + Map leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV))); + TableScanNode leftScan = tableScanNode(leftAssignments); + + Map rightAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(DV, EV, FV))); + TableScanNode rightScan = tableScanNode(rightAssignments); + + FilterNode left = filter(leftScan, + and( + lessThan(BV, AV), + lessThan(CV, bigintLiteral(10)), + equals(GV, bigintLiteral(10)))); + FilterNode right = filter(rightScan, + and( + equals(DV, EV), + lessThan(FV, bigintLiteral(100)))); + + PlanNode node = new JoinNode(newId(), + JoinNode.Type.INNER, + left, + right, + criteria, + ImmutableList.builder() + .addAll(left.getOutputVariables()) + .addAll(right.getOutputVariables()) + .build(), + Optional.of(lessThanOrEqual(BV, EV)), + Optional.empty(), + Optional.empty(), + Optional.empty()); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // All predicates having output symbol should be carried through + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts(lessThan(BV, AV), + lessThan(CV, bigintLiteral(10)), + equals(DV, EV), + lessThan(FV, bigintLiteral(100)), + equals(AV, DV), + equals(BV, EV), + lessThanOrEqual(BV, EV))); + } + + @Test + public void testInnerJoinPropagatesPredicatesViaEquiConditions() + { + Map leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV))); + TableScanNode leftScan = tableScanNode(leftAssignments); + + Map rightAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(DV, EV, FV))); + TableScanNode rightScan = tableScanNode(rightAssignments); + + FilterNode left = filter(leftScan, equals(AV, bigintLiteral(10))); + + // predicates on "a" column should be propagated to output symbols via join equi conditions + PlanNode node = new JoinNode(newId(), + JoinNode.Type.INNER, + left, + rightScan, + ImmutableList.of(new JoinNode.EquiJoinClause(AV, DV)), + ImmutableList.builder() + .addAll(rightScan.getOutputVariables()) + .build(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + assertEquals( + normalizeConjuncts(effectivePredicate), + normalizeConjuncts(equals(DV, bigintLiteral(10)))); + } + + @Test + public void testInnerJoinWithFalseFilter() + { + Map leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV))); + TableScanNode leftScan = tableScanNode(leftAssignments); + + Map rightAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(DV, EV, FV))); + TableScanNode rightScan = tableScanNode(rightAssignments); + + PlanNode node = new JoinNode(newId(), + JoinNode.Type.INNER, + leftScan, + rightScan, + ImmutableList.of(new JoinNode.EquiJoinClause(AV, DV)), + ImmutableList.builder() + .addAll(leftScan.getOutputVariables()) + .addAll(rightScan.getOutputVariables()) + .build(), + Optional.of(FALSE_CONSTANT), + Optional.empty(), + Optional.empty(), + Optional.empty()); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + assertEquals(effectivePredicate, FALSE_CONSTANT); + } + + @Test + public void testLeftJoin() + { + ImmutableList.Builder criteriaBuilder = ImmutableList.builder(); + criteriaBuilder.add(new JoinNode.EquiJoinClause(AV, DV)); + criteriaBuilder.add(new JoinNode.EquiJoinClause(BV, EV)); + List criteria = criteriaBuilder.build(); + + Map leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV))); + TableScanNode leftScan = tableScanNode(leftAssignments); + + Map rightAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(DV, EV, FV))); + TableScanNode rightScan = tableScanNode(rightAssignments); + + FilterNode left = filter(leftScan, + and( + lessThan(BV, AV), + lessThan(CV, bigintLiteral(10)), + equals(GV, bigintLiteral(10)))); + FilterNode right = filter(rightScan, + and( + equals(DV, EV), + lessThan(FV, bigintLiteral(100)))); + PlanNode node = new JoinNode(newId(), + JoinNode.Type.LEFT, + left, + right, + criteria, + ImmutableList.builder() + .addAll(left.getOutputVariables()) + .addAll(right.getOutputVariables()) + .build(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // All right side symbols having output symbols should be checked against NULL + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts(lessThan(BV, AV), + lessThan(CV, bigintLiteral(10)), + or(equals(DV, EV), and(isNull(DV), isNull(EV))), + or(lessThan(FV, bigintLiteral(100)), isNull(FV)), + or(equals(AV, DV), isNull(DV)), + or(equals(BV, EV), isNull(EV)))); + } + + @Test + public void testLeftJoinWithFalseInner() + { + List criteria = ImmutableList.of(new JoinNode.EquiJoinClause(AV, DV)); + + Map leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV))); + TableScanNode leftScan = tableScanNode(leftAssignments); + + Map rightAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(DV, EV, FV))); + TableScanNode rightScan = tableScanNode(rightAssignments); + + FilterNode left = filter(leftScan, + and( + lessThan(BV, AV), + lessThan(CV, bigintLiteral(10)), + equals(GV, bigintLiteral(10)))); + FilterNode right = filter(rightScan, FALSE_CONSTANT); + PlanNode node = new JoinNode(newId(), + JoinNode.Type.LEFT, + left, + right, + criteria, + ImmutableList.builder() + .addAll(left.getOutputVariables()) + .addAll(right.getOutputVariables()) + .build(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // False literal on the right side should be ignored + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts(lessThan(BV, AV), + lessThan(CV, bigintLiteral(10)), + or(equals(AV, DV), isNull(DV)))); + } + + @Test + public void testRightJoin() + { + ImmutableList.Builder criteriaBuilder = ImmutableList.builder(); + criteriaBuilder.add(new JoinNode.EquiJoinClause(AV, DV)); + criteriaBuilder.add(new JoinNode.EquiJoinClause(BV, EV)); + List criteria = criteriaBuilder.build(); + + Map leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV))); + TableScanNode leftScan = tableScanNode(leftAssignments); + + Map rightAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(DV, EV, FV))); + TableScanNode rightScan = tableScanNode(rightAssignments); + + FilterNode left = filter(leftScan, + and( + lessThan(BV, AV), + lessThan(CV, bigintLiteral(10)), + equals(GV, bigintLiteral(10)))); + FilterNode right = filter(rightScan, + and( + equals(DV, EV), + lessThan(FV, bigintLiteral(100)))); + PlanNode node = new JoinNode(newId(), + JoinNode.Type.RIGHT, + left, + right, + criteria, + ImmutableList.builder() + .addAll(left.getOutputVariables()) + .addAll(right.getOutputVariables()) + .build(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // All left side symbols should be checked against NULL + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts(or(lessThan(BV, AV), and(isNull(BV), isNull(AV))), + or(lessThan(CV, bigintLiteral(10)), isNull(CV)), + equals(DV, EV), + lessThan(FV, bigintLiteral(100)), + or(equals(AV, DV), isNull(AV)), + or(equals(BV, EV), isNull(BV)))); + } + + @Test + public void testRightJoinWithFalseInner() + { + List criteria = ImmutableList.of(new JoinNode.EquiJoinClause(AV, DV)); + + Map leftAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(AV, BV, CV))); + TableScanNode leftScan = tableScanNode(leftAssignments); + + Map rightAssignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(DV, EV, FV))); + TableScanNode rightScan = tableScanNode(rightAssignments); + + FilterNode left = filter(leftScan, FALSE_CONSTANT); + FilterNode right = filter(rightScan, + and( + equals(DV, EV), + lessThan(FV, bigintLiteral(100)))); + PlanNode node = new JoinNode(newId(), + JoinNode.Type.RIGHT, + left, + right, + criteria, + ImmutableList.builder() + .addAll(left.getOutputVariables()) + .addAll(right.getOutputVariables()) + .build(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // False literal on the left side should be ignored + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts(equals(DV, EV), + lessThan(FV, bigintLiteral(100)), + or(equals(AV, DV), isNull(AV)))); + } + + @Test + public void testSemiJoin() + { + PlanNode node = new SemiJoinNode(newId(), + filter(baseTableScan, and(greaterThan(AV, bigintLiteral(10)), lessThan(AV, bigintLiteral(100)))), + filter(baseTableScan, greaterThan(AV, bigintLiteral(5))), + AV, BV, CV, + Optional.empty(), + Optional.empty(), + Optional.empty()); + + RowExpression effectivePredicate = effectivePredicateExtractor.extract(node); + + // Currently, only pull predicates through the source plan + assertEquals(normalizeConjuncts(effectivePredicate), + normalizeConjuncts(and(greaterThan(AV, bigintLiteral(10)), lessThan(AV, bigintLiteral(100))))); + } + + private static TableScanNode tableScanNode(Map scanAssignments) + { + return new TableScanNode( + newId(), + DUAL_TABLE_HANDLE, + ImmutableList.copyOf(scanAssignments.keySet()), + scanAssignments, + TupleDomain.all(), + TupleDomain.all()); + } + + private static PlanNodeId newId() + { + return new PlanNodeId(UUID.randomUUID().toString()); + } + + private static FilterNode filter(PlanNode source, RowExpression predicate) + { + return new FilterNode(newId(), source, predicate); + } + + private static RowExpression bigintLiteral(long number) + { + return constant(number, BIGINT); + } + + private RowExpression equals(RowExpression expression1, RowExpression expression2) + { + return compare(EQUAL, expression1, expression2); + } + + private RowExpression lessThan(RowExpression expression1, RowExpression expression2) + { + return compare(LESS_THAN, expression1, expression2); + } + + private RowExpression lessThanOrEqual(RowExpression expression1, RowExpression expression2) + { + return compare(LESS_THAN_OR_EQUAL, expression1, expression2); + } + + private RowExpression greaterThan(RowExpression expression1, RowExpression expression2) + { + return compare(GREATER_THAN, expression1, expression2); + } + + private RowExpression compare(OperatorType type, RowExpression left, RowExpression right) + { + return call( + type.getFunctionName().getFunctionName(), + metadata.getFunctionManager().resolveOperator(type, fromTypes(left.getType(), right.getType())), + BOOLEAN, + left, + right); + } + + private static RowExpression isNull(RowExpression expression) + { + return specialForm(IS_NULL, BOOLEAN, expression); + } + + private Set normalizeConjuncts(RowExpression... conjuncts) + { + return normalizeConjuncts(Arrays.asList(conjuncts)); + } + + private Set normalizeConjuncts(Collection conjuncts) + { + return normalizeConjuncts(logicalRowExpressions.combineConjuncts(conjuncts)); + } + + private Set normalizeConjuncts(RowExpression predicate) + { + // Normalize the predicate by identity so that the EqualityInference will produce stable rewrites in this test + // and thereby produce comparable Sets of conjuncts from this method. + + // Equality inference rewrites and equality generation will always be stable across multiple runs in the same JVM + RowExpressionEqualityInference inference = RowExpressionEqualityInference.createEqualityInference(metadata, predicate); + + Set rewrittenSet = new HashSet<>(); + for (RowExpression expression : nonInferrableConjuncts(metadata, predicate)) { + RowExpression rewritten = inference.rewriteExpression(expression, Predicates.alwaysTrue()); + Preconditions.checkState(rewritten != null, "Rewrite with full symbol scope should always be possible"); + rewrittenSet.add(rewritten); + } + rewrittenSet.addAll(inference.generateEqualitiesPartitionedBy(Predicates.alwaysTrue()).getScopeEqualities()); + + return rewrittenSet; + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionRewriter.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionRewriter.java index cf34198709e80..bf33c7eeb9f12 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionRewriter.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionRewriter.java @@ -14,13 +14,13 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.expressions.RowExpressionRewriter; +import com.facebook.presto.expressions.RowExpressionTreeRewriter; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; -import com.facebook.presto.spi.relation.RowExpressionRewriter; -import com.facebook.presto.spi.relation.RowExpressionTreeRewriter; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.type.TypeRegistry; @@ -28,9 +28,9 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression; import static com.facebook.presto.spi.function.OperatorType.ADD; import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; -import static com.facebook.presto.spi.relation.RowExpressionNodeInliner.replaceExpression; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionVariableInliner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionVariableInliner.java index bd585a6f6d3b7..41e105f3d31e5 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionVariableInliner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionVariableInliner.java @@ -13,7 +13,9 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.spi.CatalogSchemaName; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -26,7 +28,17 @@ public class TestRowExpressionVariableInliner { - private static final FunctionHandle TEST_FUNCTION = () -> null; + private static class TestFunctionHandle + implements FunctionHandle + { + @Override + public CatalogSchemaName getFunctionNamespace() + { + return QualifiedFunctionName.of(new CatalogSchemaName("a", "b"), "c").getFunctionNamespace(); + } + } + + private static final FunctionHandle TEST_FUNCTION = new TestFunctionHandle(); @Test public void testInlineVariable() diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java index fa7131c6c74ba..163d9b29b3143 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java @@ -19,20 +19,20 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.VarcharType; import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; -import com.facebook.presto.sql.planner.plan.Assignments; -import com.facebook.presto.sql.planner.plan.ProjectNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.planner.sanity.TypeValidator; import com.facebook.presto.sql.tree.Cast; @@ -42,11 +42,9 @@ import com.facebook.presto.testing.TestingMetadata.TestingTableHandle; import com.facebook.presto.testing.TestingTransactionHandle; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ListMultimap; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.util.Map; @@ -55,21 +53,20 @@ import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.DateType.DATE; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.sql.planner.plan.WindowNode.Frame.BoundType.UNBOUNDED_FOLLOWING; import static com.facebook.presto.sql.planner.plan.WindowNode.Frame.BoundType.UNBOUNDED_PRECEDING; import static com.facebook.presto.sql.planner.plan.WindowNode.Frame.WindowType.RANGE; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; -@Test(singleThreaded = true) public class TestTypeValidator { private static final TableHandle TEST_TABLE_HANDLE = new TableHandle( @@ -90,7 +87,7 @@ public class TestTypeValidator private VariableReferenceExpression variableD; private VariableReferenceExpression variableE; - @BeforeMethod + @BeforeClass public void setUp() { variableAllocator = new PlanVariableAllocator(); @@ -138,15 +135,12 @@ public void testValidProject() public void testValidUnion() { VariableReferenceExpression output = variableAllocator.newVariable("output", DATE); - ListMultimap mappings = ImmutableListMultimap.builder() - .put(output, variableD) - .put(output, variableD) - .build(); PlanNode node = new UnionNode( newId(), ImmutableList.of(baseTableScan, baseTableScan), - mappings); + ImmutableList.of(output), + ImmutableMap.of(output, ImmutableList.of(variableD, variableD))); assertTypesValid(node); } @@ -166,7 +160,7 @@ public void testValidWindow() Optional.empty(), Optional.empty()); - WindowNode.Function function = new WindowNode.Function(call("sum", functionHandle, DOUBLE, variableC), frame); + WindowNode.Function function = new WindowNode.Function(call("sum", functionHandle, DOUBLE, variableC), frame, false); WindowNode.Specification specification = new WindowNode.Specification(ImmutableList.of(), Optional.empty()); @@ -307,7 +301,7 @@ public void testInvalidWindowFunctionCall() Optional.empty(), Optional.empty()); - WindowNode.Function function = new WindowNode.Function(call("sum", functionHandle, BIGINT, variableA), frame); + WindowNode.Function function = new WindowNode.Function(call("sum", functionHandle, BIGINT, variableA), frame, false); WindowNode.Specification specification = new WindowNode.Specification(ImmutableList.of(), Optional.empty()); @@ -338,7 +332,7 @@ public void testInvalidWindowFunctionSignature() Optional.empty(), Optional.empty()); - WindowNode.Function function = new WindowNode.Function(call("sum", functionHandle, BIGINT, variableC), frame); + WindowNode.Function function = new WindowNode.Function(call("sum", functionHandle, BIGINT, variableC), frame, false); WindowNode.Specification specification = new WindowNode.Specification(ImmutableList.of(), Optional.empty()); @@ -358,15 +352,16 @@ public void testInvalidWindowFunctionSignature() public void testInvalidUnion() { VariableReferenceExpression output = variableAllocator.newVariable("output", DATE); - ListMultimap mappings = ImmutableListMultimap.builder() - .put(output, variableD) - .put(output, variableA) // should be a symbol with DATE type - .build(); PlanNode node = new UnionNode( newId(), ImmutableList.of(baseTableScan, baseTableScan), - mappings); + ImmutableList.of(output), + ImmutableMap.of( + output, + ImmutableList.of( + variableD, + variableA))); // should be a symbol with DATE type assertTypesValid(node); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestingWriterTarget.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestingWriterTarget.java index ca45d957f9430..0ed382c524e36 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestingWriterTarget.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestingWriterTarget.java @@ -14,11 +14,28 @@ package com.facebook.presto.sql.planner; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.sql.planner.plan.TableWriterNode; public class TestingWriterTarget extends TableWriterNode.WriterTarget { + private static final ConnectorId CONNECTOR_ID = new ConnectorId("test"); + private static final SchemaTableName SCHEMA_TABLE_NAME = new SchemaTableName("test-schema", "test-table"); + + @Override + public ConnectorId getConnectorId() + { + return CONNECTOR_ID; + } + + @Override + public SchemaTableName getSchemaTableName() + { + return SCHEMA_TABLE_NAME; + } + @Override public String toString() { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/AggregationFunctionMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/AggregationFunctionMatcher.java index a85ca09fbe34a..5b1bbc0ee676e 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/AggregationFunctionMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/AggregationFunctionMatcher.java @@ -16,12 +16,12 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.sql.tree.OrderBy; @@ -71,7 +71,7 @@ public Optional getAssignedVariable(PlanNode node, private static boolean verifyAggregation(FunctionManager functionManager, Aggregation aggregation, FunctionCall expectedCall) { - return functionManager.getFunctionMetadata(aggregation.getFunctionHandle()).getName().equalsIgnoreCase(expectedCall.getName().getSuffix()) && + return functionManager.getFunctionMetadata(aggregation.getFunctionHandle()).getName().getFunctionName().equalsIgnoreCase(expectedCall.getName().getSuffix()) && aggregation.getArguments().size() == expectedCall.getArguments().size() && Streams.zip( aggregation.getArguments().stream(), @@ -92,11 +92,11 @@ private static boolean verifyAggregationOrderBy(Optional orderin private static boolean verifyAggregationOrderBy(OrderingScheme orderingScheme, OrderBy expectedSortOrder) { - if (orderingScheme.getOrderBy().size() != expectedSortOrder.getSortItems().size()) { + if (orderingScheme.getOrderByVariables().size() != expectedSortOrder.getSortItems().size()) { return false; } for (int i = 0; i < expectedSortOrder.getSortItems().size(); i++) { - VariableReferenceExpression orderingVariable = orderingScheme.getOrderBy().get(i); + VariableReferenceExpression orderingVariable = orderingScheme.getOrderByVariables().get(i); if (expectedSortOrder.getSortItems().get(i).getSortKey().equals(new SymbolReference(orderingVariable.getName())) && toSortOrder(expectedSortOrder.getSortItems().get(i)).equals(orderingScheme.getOrdering(orderingVariable))) { continue; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/AggregationMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/AggregationMatcher.java index 7552597160b4c..ca94f6ef51920 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/AggregationMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/AggregationMatcher.java @@ -16,11 +16,11 @@ import com.facebook.presto.Session; import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.Symbol; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import java.util.Collection; import java.util.List; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/AggregationStepMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/AggregationStepMatcher.java index c4c165a1f969c..702c529f86514 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/AggregationStepMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/AggregationStepMatcher.java @@ -16,9 +16,9 @@ import com.facebook.presto.Session; import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import static com.facebook.presto.sql.planner.assertions.MatchResult.NO_MATCH; import static com.facebook.presto.sql.planner.assertions.MatchResult.match; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/BasePlanTest.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/BasePlanTest.java index 76b0d3d3d0677..fef314641389d 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/BasePlanTest.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/BasePlanTest.java @@ -15,12 +15,14 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.sql.planner.LogicalPlanner; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.iterative.IterativeOptimizer; import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantIdentityProjections; +import com.facebook.presto.sql.planner.iterative.rule.TranslateExpressions; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; import com.facebook.presto.sql.planner.optimizations.PruneUnreferencedOutputs; import com.facebook.presto.sql.planner.optimizations.UnaliasSymbolReferences; @@ -38,8 +40,8 @@ import java.util.function.Consumer; import java.util.function.Predicate; +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.testing.Closeables.closeAllRuntimeException; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; @@ -138,7 +140,7 @@ protected void assertPlan(String sql, LogicalPlanner.Stage stage, PlanMatchPatte sql, ImmutableList.builder() .addAll(optimizers) - .add(queryRunner.translateExpressions()).build(), // To avoid assert plan failure not printing out plan (#12885) + .add(getExpressionTranslator()).build(), // To avoid assert plan failure not printing out plan (#12885) stage, WarningCollector.NOOP); PlanAssert.assertPlan(transactionSession, queryRunner.getMetadata(), queryRunner.getStatsCalculator(), actualPlan, pattern); @@ -159,14 +161,14 @@ protected void assertDistributedPlan(String sql, Session session, PlanMatchPatte protected void assertMinimallyOptimizedPlan(@Language("SQL") String sql, PlanMatchPattern pattern) { List optimizers = ImmutableList.of( - new UnaliasSymbolReferences(), + new UnaliasSymbolReferences(queryRunner.getMetadata().getFunctionManager()), new PruneUnreferencedOutputs(), new IterativeOptimizer( new RuleStatsRecorder(), queryRunner.getStatsCalculator(), queryRunner.getCostCalculator(), ImmutableSet.of(new RemoveRedundantIdentityProjections())), - queryRunner.translateExpressions()); // To avoid assert plan failure not printing out plan (#12885) + getExpressionTranslator()); // To avoid assert plan failure not printing out plan (#12885) assertPlan(sql, LogicalPlanner.Stage.OPTIMIZED, pattern, optimizers); } @@ -210,6 +212,21 @@ protected Plan plan(String sql, LogicalPlanner.Stage stage, boolean forceSingleN } } + protected Metadata getMetadata() + { + return getQueryRunner().getMetadata(); + } + + // Translate all OriginalExpression in planNodes to RowExpression so that we can do plan pattern asserting and printing on RowExpression only. + protected PlanOptimizer getExpressionTranslator() + { + return new IterativeOptimizer( + new RuleStatsRecorder(), + getQueryRunner().getStatsCalculator(), + getQueryRunner().getCostCalculator(), + ImmutableSet.copyOf(new TranslateExpressions(getMetadata(), getQueryRunner().getSqlParser()).rules())); + } + public interface LocalQueryRunnerSupplier { LocalQueryRunner get() diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/ExpressionMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/ExpressionMatcher.java index a6f52a3a25850..79c2d33b4ae64 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/ExpressionMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/ExpressionMatcher.java @@ -16,11 +16,11 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.plan.ApplyNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.tree.Expression; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/LimitMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/LimitMatcher.java index 5179e7af6f825..10ae84e36afb5 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/LimitMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/LimitMatcher.java @@ -16,8 +16,8 @@ import com.facebook.presto.Session; import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import static com.google.common.base.Preconditions.checkState; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanAssert.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanAssert.java index 39cf8eefb6e13..6cebe5051a587 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanAssert.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanAssert.java @@ -43,14 +43,13 @@ public static void assertPlan(Session session, Metadata metadata, StatsCalculato public static void assertPlan(Session session, Metadata metadata, StatsProvider statsProvider, Plan actual, Lookup lookup, PlanMatchPattern pattern, Function planSanitizer) { MatchResult matches = actual.getRoot().accept(new PlanMatchingVisitor(session, metadata, statsProvider, lookup), pattern); + // TODO (Issue #13231) add back printing unresolved plan once we have no need to translate OriginalExpression to RowExpression if (!matches.isMatch()) { - String formattedPlan = textLogicalPlan(planSanitizer.apply(actual.getRoot()), actual.getTypes(), metadata.getFunctionManager(), StatsAndCosts.empty(), session, 0); PlanNode resolvedPlan = resolveGroupReferences(actual.getRoot(), lookup); String resolvedFormattedPlan = textLogicalPlan(planSanitizer.apply(resolvedPlan), actual.getTypes(), metadata.getFunctionManager(), StatsAndCosts.empty(), session, 0); throw new AssertionError(format( - "Plan does not match, expected [\n\n%s\n] but found [\n\n%s\n] which resolves to [\n\n%s\n]", + "Plan does not match, expected [\n\n%s\n] but found [\n\n%s\n]", pattern, - formattedPlan, resolvedFormattedPlan)); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java index a26572025409b..becb7cf6bffa3 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java @@ -17,36 +17,36 @@ import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Step; +import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.iterative.GroupReference; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; -import com.facebook.presto.sql.planner.plan.ExceptNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.GroupIdNode; import com.facebook.presto.sql.planner.plan.IndexSourceNode; -import com.facebook.presto.sql.planner.plan.IntersectNode; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; import com.facebook.presto.sql.planner.plan.WindowNode.Frame.BoundType; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchingVisitor.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchingVisitor.java index ac5610253ffda..c2414acf435bc 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchingVisitor.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchingVisitor.java @@ -16,14 +16,14 @@ import com.facebook.presto.Session; import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.GroupReference; import com.facebook.presto.sql.planner.iterative.Lookup; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; -import com.facebook.presto.sql.planner.plan.ProjectNode; import java.util.List; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java index b406c5bbc82a9..c174fe18a7994 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java @@ -45,7 +45,6 @@ import com.facebook.presto.sql.tree.Node; import com.facebook.presto.sql.tree.NotExpression; import com.facebook.presto.sql.tree.NullLiteral; -import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.SimpleCaseExpression; import com.facebook.presto.sql.tree.StringLiteral; import com.facebook.presto.sql.tree.SymbolReference; @@ -209,6 +208,11 @@ protected Boolean visitComparisonExpression(ComparisonExpression expected, RowEx OperatorType actualOperatorType = functionMetadata.getOperatorType().get(); OperatorType expectedOperatorType = getOperatorType(expected.getOperator()); if (expectedOperatorType.equals(actualOperatorType)) { + if (actualOperatorType == EQUAL) { + return (process(expected.getLeft(), ((CallExpression) actual).getArguments().get(0)) && process(expected.getRight(), ((CallExpression) actual).getArguments().get(1))) + || (process(expected.getLeft(), ((CallExpression) actual).getArguments().get(1)) && process(expected.getRight(), ((CallExpression) actual).getArguments().get(0))); + } + // TODO support other comparison operators return process(expected.getLeft(), ((CallExpression) actual).getArguments().get(0)) && process(expected.getRight(), ((CallExpression) actual).getArguments().get(1)); } } @@ -479,7 +483,7 @@ protected Boolean visitFunctionCall(FunctionCall expected, RowExpression actual) } CallExpression actualFunction = (CallExpression) actual; - if (!expected.getName().equals(QualifiedName.of(metadata.getFunctionManager().getFunctionMetadata(actualFunction.getFunctionHandle()).getName()))) { + if (!expected.getName().getSuffix().equals(metadata.getFunctionManager().getFunctionMetadata(actualFunction.getFunctionHandle()).getName().getFunctionName())) { return false; } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/SpecificationProvider.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/SpecificationProvider.java index 300be38e191ec..3150f41b84bcd 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/SpecificationProvider.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/SpecificationProvider.java @@ -14,8 +14,9 @@ package com.facebook.presto.sql.planner.assertions; import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.plan.WindowNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -54,14 +55,13 @@ public WindowNode.Specification getExpectedValue(SymbolAliases aliases) Optional orderingScheme = Optional.empty(); if (!orderBy.isEmpty()) { orderingScheme = Optional.of(new OrderingScheme( - orderBy - .stream() - .map(alias -> new VariableReferenceExpression(alias.toSymbol(aliases).getName(), UNKNOWN)) - .collect(toImmutableList()), orderings .entrySet() .stream() - .collect(toImmutableMap(entry -> new VariableReferenceExpression(entry.getKey().toSymbol(aliases).getName(), UNKNOWN), Map.Entry::getValue)))); + .map(entry -> new Ordering( + new VariableReferenceExpression(entry.getKey().toSymbol(aliases).getName(), UNKNOWN), + entry.getValue())) + .collect(toImmutableList()))); } return new WindowNode.Specification( @@ -91,15 +91,15 @@ public static boolean matchSpecification(WindowNode.Specification actual, Window { return actual.getPartitionBy().stream().map(VariableReferenceExpression::getName).collect(toImmutableList()) .equals(expected.getPartitionBy().stream().map(VariableReferenceExpression::getName).collect(toImmutableList())) && - actual.getOrderingScheme().map(orderingScheme -> orderingScheme.getOrderBy().stream() + actual.getOrderingScheme().map(orderingScheme -> orderingScheme.getOrderByVariables().stream() .map(VariableReferenceExpression::getName) .collect(toImmutableSet()) - .equals(expected.getOrderingScheme().get().getOrderBy().stream() + .equals(expected.getOrderingScheme().get().getOrderByVariables().stream() .map(VariableReferenceExpression::getName) .collect(toImmutableSet())) && - orderingScheme.getOrderings().entrySet().stream() + orderingScheme.getOrderingsMap().entrySet().stream() .collect(toImmutableMap(entry -> entry.getKey().getName(), Map.Entry::getValue)) - .equals(expected.getOrderingScheme().get().getOrderings().entrySet().stream() + .equals(expected.getOrderingScheme().get().getOrderingsMap().entrySet().stream() .collect(toImmutableMap(entry -> entry.getKey().getName(), Map.Entry::getValue)))) .orElse(true); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/StrictAssignedSymbolsMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/StrictAssignedSymbolsMatcher.java index f2ff151dd739b..79c31cadcbf8c 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/StrictAssignedSymbolsMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/StrictAssignedSymbolsMatcher.java @@ -16,8 +16,8 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.google.common.collect.ImmutableSet; import java.util.Collection; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/SymbolAliases.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/SymbolAliases.java index 5542fb7293679..ef812fb4b9922 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/SymbolAliases.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/SymbolAliases.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.sql.planner.assertions; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.collect.ImmutableMap; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/TopNMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/TopNMatcher.java index 025f2d68483e1..1cc81591a7ded 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/TopNMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/TopNMatcher.java @@ -17,8 +17,8 @@ import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern.Ordering; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.google.common.collect.ImmutableList; import java.util.List; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/Util.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/Util.java index 2fbaab8bfabaf..3b8bde973f301 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/Util.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/Util.java @@ -17,16 +17,20 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.TupleDomain; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern.Ordering; +import com.google.common.collect.Maps; import java.util.List; import java.util.Map; import java.util.Optional; +import static com.google.common.collect.Iterators.getOnlyElement; + final class Util { private Util() {} @@ -70,17 +74,18 @@ static boolean domainsMatch( static boolean orderingSchemeMatches(List expectedOrderBy, OrderingScheme orderingScheme, SymbolAliases symbolAliases) { - if (expectedOrderBy.size() != orderingScheme.getOrderBy().size()) { + if (expectedOrderBy.size() != orderingScheme.getOrderByVariables().size()) { return false; } for (int i = 0; i < expectedOrderBy.size(); ++i) { Ordering ordering = expectedOrderBy.get(i); Symbol symbol = Symbol.from(symbolAliases.get(ordering.getField())); - if (!symbol.equals(new Symbol(orderingScheme.getOrderBy().get(i).getName()))) { + if (!symbol.equals(new Symbol(orderingScheme.getOrderByVariables().get(i).getName()))) { return false; } - if (!ordering.getSortOrder().equals(orderingScheme.getOrdering(symbol))) { + SortOrder sortOrder = getOnlyElement(Maps.filterKeys(orderingScheme.getOrderingsMap(), variable -> variable.getName().equals(symbol.getName())).values().iterator()); + if (!ordering.getSortOrder().equals(sortOrder)) { return false; } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/WindowFunctionMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/WindowFunctionMatcher.java index 4292f091c8464..a60c914057de2 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/WindowFunctionMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/WindowFunctionMatcher.java @@ -74,7 +74,7 @@ public Optional getAssignedVariable(PlanNode node, List matchedOutputs = windowNode.getWindowFunctions().entrySet().stream() .filter(assignment -> { - if (!expectedCall.getName().equals(QualifiedName.of(metadata.getFunctionManager().getFunctionMetadata(assignment.getValue().getFunctionCall().getFunctionHandle()).getName()))) { + if (!expectedCall.getName().equals(QualifiedName.of(metadata.getFunctionManager().getFunctionMetadata(assignment.getValue().getFunctionCall().getFunctionHandle()).getName().getFunctionName()))) { return false; } if (!functionHandle.map(assignment.getValue().getFunctionHandle()::equals).orElse(true)) { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/TestIterativeOptimizer.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/TestIterativeOptimizer.java index 96c2f40c3967a..36157d270b7c7 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/TestIterativeOptimizer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/TestIterativeOptimizer.java @@ -19,9 +19,9 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.tpch.TpchConnectorFactory; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/TestRuleIndex.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/TestRuleIndex.java index 1cd296eb139af..2800992fff808 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/TestRuleIndex.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/TestRuleIndex.java @@ -16,13 +16,13 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.Assignments; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.tree.BooleanLiteral; import com.google.common.collect.ImmutableSet; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestAddIntermediateAggregations.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestAddIntermediateAggregations.java index 5b7641ae6cf58..c0434b180a4c7 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestAddIntermediateAggregations.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestAddIntermediateAggregations.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.sql.planner.assertions.ExpectedValueProvider; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.tree.FunctionCall; import com.google.common.collect.ImmutableList; @@ -27,6 +27,9 @@ import static com.facebook.presto.SystemSessionProperties.ENABLE_INTERMEDIATE_AGGREGATIONS; import static com.facebook.presto.SystemSessionProperties.TASK_CONCURRENCY; +import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.INTERMEDIATE; +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anySymbol; @@ -35,9 +38,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.FINAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.INTERMEDIATE; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignmentsAsSymbolReferences; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.LOCAL; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestDetermineJoinDistributionType.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestDetermineJoinDistributionType.java index c682642dbcd2b..611974ab1233a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestDetermineJoinDistributionType.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestDetermineJoinDistributionType.java @@ -94,10 +94,10 @@ private void testDetermineDistributionType(JoinDistributionType sessionDistribut joinType, p.values( ImmutableList.of(p.variable("A1")), - ImmutableList.of(constantExpressions(BIGINT, 10), constantExpressions(BIGINT, 11))), + ImmutableList.of(constantExpressions(BIGINT, 10L), constantExpressions(BIGINT, 11L))), p.values( ImmutableList.of(p.variable("B1")), - ImmutableList.of(constantExpressions(BIGINT, 50), constantExpressions(BIGINT, 11))), + ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) @@ -130,10 +130,10 @@ private void testRepartitionRightOuter(JoinDistributionType sessionDistributedJo joinType, p.values( ImmutableList.of(p.variable("A1")), - ImmutableList.of(constantExpressions(BIGINT, 10), constantExpressions(BIGINT, 11))), + ImmutableList.of(constantExpressions(BIGINT, 10L), constantExpressions(BIGINT, 11L))), p.values( ImmutableList.of(p.variable("B1")), - ImmutableList.of(constantExpressions(BIGINT, 50), constantExpressions(BIGINT, 11))), + ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) @@ -156,11 +156,11 @@ public void testReplicateScalar() INNER, p.values( ImmutableList.of(p.variable("A1")), - ImmutableList.of(constantExpressions(BIGINT, 10), constantExpressions(BIGINT, 11))), + ImmutableList.of(constantExpressions(BIGINT, 10L), constantExpressions(BIGINT, 11L))), p.enforceSingleRow( p.values( ImmutableList.of(p.variable("B1")), - ImmutableList.of(constantExpressions(BIGINT, 50), constantExpressions(BIGINT, 11)))), + ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L)))), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())) @@ -189,10 +189,10 @@ private void testReplicateNoEquiCriteria(Type joinType) joinType, p.values( ImmutableList.of(p.variable("A1")), - ImmutableList.of(constantExpressions(BIGINT, 10), constantExpressions(BIGINT, 11))), + ImmutableList.of(constantExpressions(BIGINT, 10L), constantExpressions(BIGINT, 11L))), p.values( ImmutableList.of(p.variable("B1")), - ImmutableList.of(constantExpressions(BIGINT, 50), constantExpressions(BIGINT, 11))), + ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), ImmutableList.of(), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.of(castToRowExpression("A1 * B1 > 100")))) @@ -215,10 +215,10 @@ public void testRetainDistributionType() INNER, p.values( ImmutableList.of(p.variable("A1")), - ImmutableList.of(constantExpressions(BIGINT, 10), constantExpressions(BIGINT, 11))), + ImmutableList.of(constantExpressions(BIGINT, 10L), constantExpressions(BIGINT, 11L))), p.values( ImmutableList.of(p.variable("B1")), - ImmutableList.of(constantExpressions(BIGINT, 50), constantExpressions(BIGINT, 11))), + ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestDetermineSemiJoinDistributionType.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestDetermineSemiJoinDistributionType.java index 6259b515d5308..e3ec6c8eb8477 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestDetermineSemiJoinDistributionType.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestDetermineSemiJoinDistributionType.java @@ -82,10 +82,10 @@ public void testRetainDistributionType() p.semiJoin( p.values( ImmutableList.of(p.variable("A1")), - ImmutableList.of(constantExpressions(BIGINT, 10), constantExpressions(BIGINT, 11))), + ImmutableList.of(constantExpressions(BIGINT, 10L), constantExpressions(BIGINT, 11L))), p.values( ImmutableList.of(p.variable("B1")), - ImmutableList.of(constantExpressions(BIGINT, 50), constantExpressions(BIGINT, 11))), + ImmutableList.of(constantExpressions(BIGINT, 50L), constantExpressions(BIGINT, 11L))), p.variable("A1"), p.variable("B1"), p.variable("output"), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEliminateCrossJoins.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEliminateCrossJoins.java index 4dc4403f458da..e1a332b556ff9 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEliminateCrossJoins.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEliminateCrossJoins.java @@ -15,6 +15,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.GroupReference; @@ -23,7 +24,6 @@ import com.facebook.presto.sql.planner.optimizations.joins.JoinGraph; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.JoinNode.EquiJoinClause; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.tree.ArithmeticUnaryExpression; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.SymbolReference; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroLimit.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroLimit.java index f9d2a8094e9ab..682362b4459e9 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroLimit.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroLimit.java @@ -49,8 +49,8 @@ public void test() p.values( ImmutableList.of(p.variable("a"), p.variable("b")), ImmutableList.of( - constantExpressions(BIGINT, 1, 10), - constantExpressions(BIGINT, 2, 11)))))) + constantExpressions(BIGINT, 1L, 10L), + constantExpressions(BIGINT, 2L, 11L)))))) // TODO: verify contents .matches(values(ImmutableMap.of())); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroSample.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroSample.java index 054784c90e0ef..2ed4c8cdd6051 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroSample.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroSample.java @@ -52,8 +52,8 @@ public void test() p.values( ImmutableList.of(p.variable("a"), p.variable("b")), ImmutableList.of( - constantExpressions(BIGINT, 1, 10), - constantExpressions(BIGINT, 2, 11)))))) + constantExpressions(BIGINT, 1L, 10L), + constantExpressions(BIGINT, 2L, 11L)))))) // TODO: verify contents .matches(values(ImmutableMap.of())); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestInlineProjections.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestInlineProjections.java index 3db7ef0441ffc..a5494efbc9317 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestInlineProjections.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestInlineProjections.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.sql.planner.assertions.ExpressionMatcher; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.Assignments; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -33,7 +33,7 @@ public class TestInlineProjections @Test public void test() { - tester().assertThat(new InlineProjections()) + tester().assertThat(new InlineProjections(getFunctionManager())) .on(p -> p.project( Assignments.builder() @@ -70,10 +70,55 @@ public void test() values(ImmutableMap.of("x", 0))))); } + @Test + public void testRowExpression() + { + // TODO add testing to expressions that need desugaring like 'try' + tester().assertThat(new InlineProjections(getFunctionManager())) + .on(p -> { + p.variable("symbol"); + p.variable("complex"); + p.variable("literal"); + p.variable("complex_2"); + p.variable("x"); + return p.project( + Assignments.builder() + .put(p.variable("identity"), p.rowExpression("symbol")) // identity + .put(p.variable("multi_complex_1"), p.rowExpression("complex + 1")) // complex expression referenced multiple times + .put(p.variable("multi_complex_2"), p.rowExpression("complex + 2")) // complex expression referenced multiple times + .put(p.variable("multi_literal_1"), p.rowExpression("literal + 1")) // literal referenced multiple times + .put(p.variable("multi_literal_2"), p.rowExpression("literal + 2")) // literal referenced multiple times + .put(p.variable("single_complex"), p.rowExpression("complex_2 + 2")) // complex expression reference only once + .build(), + p.project(Assignments.builder() + .put(p.variable("symbol"), p.rowExpression("x")) + .put(p.variable("complex"), p.rowExpression("x * 2")) + .put(p.variable("literal"), p.rowExpression("1")) + .put(p.variable("complex_2"), p.rowExpression("x - 1")) + .build(), + p.values(p.variable("x")))); + }) + .matches( + project( + ImmutableMap.builder() + .put("out1", PlanMatchPattern.expression("x")) + .put("out2", PlanMatchPattern.expression("y + 1")) + .put("out3", PlanMatchPattern.expression("y + 2")) + .put("out4", PlanMatchPattern.expression("1 + 1")) + .put("out5", PlanMatchPattern.expression("1 + 2")) + .put("out6", PlanMatchPattern.expression("x - 1 + 2")) + .build(), + project( + ImmutableMap.of( + "x", PlanMatchPattern.expression("x"), + "y", PlanMatchPattern.expression("x * 2")), + values(ImmutableMap.of("x", 0))))); + } + @Test public void testIdentityProjections() { - tester().assertThat(new InlineProjections()) + tester().assertThat(new InlineProjections(getFunctionManager())) .on(p -> p.project( assignment(p.variable("output"), expression("value")), @@ -86,7 +131,7 @@ public void testIdentityProjections() @Test public void testSubqueryProjections() { - tester().assertThat(new InlineProjections()) + tester().assertThat(new InlineProjections(getFunctionManager())) .on(p -> p.project( identityAssignmentsAsSymbolReferences(p.variable("fromOuterScope"), p.variable("value")), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinEnumerator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinEnumerator.java index 4d269d1843979..96fed554a7eeb 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinEnumerator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinEnumerator.java @@ -41,12 +41,12 @@ import java.util.LinkedHashSet; import java.util.Optional; +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.sql.planner.iterative.Lookup.noLookup; import static com.facebook.presto.sql.planner.iterative.rule.ReorderJoins.JoinEnumerator.generatePartitions; import static com.facebook.presto.sql.tree.BooleanLiteral.TRUE_LITERAL; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.testing.Closeables.closeAllRuntimeException; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -121,8 +121,7 @@ private Rule.Context createContext() queryRunner.getCostCalculator(), statsProvider, Optional.empty(), - queryRunner.getDefaultSession(), - variableAllocator.getTypes()); + queryRunner.getDefaultSession()); return new Rule.Context() { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinNodeFlattener.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinNodeFlattener.java index f5d6af405a93d..24ef576812c9b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinNodeFlattener.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestJoinNodeFlattener.java @@ -35,6 +35,7 @@ import java.util.LinkedHashSet; import java.util.Optional; +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.sql.ExpressionUtils.and; import static com.facebook.presto.sql.planner.iterative.Lookup.noLookup; @@ -49,7 +50,6 @@ import static com.facebook.presto.sql.tree.ComparisonExpression.Operator.LESS_THAN; import static com.facebook.presto.sql.tree.ComparisonExpression.Operator.NOT_EQUAL; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.testing.Closeables.closeAllRuntimeException; import static org.testng.Assert.assertEquals; public class TestJoinNodeFlattener diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMergeAdjacentWindows.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMergeAdjacentWindows.java index a3cff98e70d29..862bd03d25b05 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMergeAdjacentWindows.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMergeAdjacentWindows.java @@ -14,12 +14,12 @@ package com.facebook.presto.sql.planner.iterative.rule; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.assertions.ExpectedValueProvider; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.WindowNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -228,6 +228,7 @@ private WindowNode.Function newWindowNodeFunction(String name, FunctionHandle fu functionHandle, BIGINT, Arrays.stream(symbols).map(symbol -> new VariableReferenceExpression(symbol, BIGINT)).collect(Collectors.toList())), - frame); + frame, + false); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPickTableLayout.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPickTableLayout.java index 4f6257829d15a..2b8aedf2260c9 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPickTableLayout.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPickTableLayout.java @@ -41,6 +41,7 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; +import static com.facebook.presto.sql.relational.Expressions.variable; import static io.airlift.slice.Slices.utf8Slice; public class TestPickTableLayout @@ -104,6 +105,17 @@ public void eliminateTableScanWhenNoLayoutExist() ImmutableList.of(p.variable("orderstatus", createVarcharType(1))), ImmutableMap.of(p.variable("orderstatus", createVarcharType(1)), new TpchColumnHandle("orderstatus", createVarcharType(1)))))) .matches(values("A")); + + tester().assertThat(pickTableLayout.pickTableLayoutForPredicate()) + .on(p -> { + p.variable("orderstatus", createVarcharType(1)); + return p.filter(p.rowExpression("orderstatus = 'G'"), + p.tableScan( + ordersTableHandle, + ImmutableList.of(variable("orderstatus", createVarcharType(1))), + ImmutableMap.of(variable("orderstatus", createVarcharType(1)), new TpchColumnHandle("orderstatus", createVarcharType(1))))); + }) + .matches(values("A")); } @Test @@ -119,6 +131,19 @@ public void replaceWithExistsWhenNoLayoutExist() TupleDomain.none(), TupleDomain.none()))) .matches(values("A")); + + tester().assertThat(pickTableLayout.pickTableLayoutForPredicate()) + .on(p -> { + p.variable("nationkey"); + return p.filter(p.rowExpression("nationkey = BIGINT '44'"), + p.tableScan( + nationTableHandle, + ImmutableList.of(variable("nationkey", BIGINT)), + ImmutableMap.of(variable("nationkey", BIGINT), columnHandle), + TupleDomain.none(), + TupleDomain.none())); + }) + .matches(values("A")); } @Test @@ -133,6 +158,19 @@ public void doesNotFireIfRuleNotChangePlan() TupleDomain.all(), TupleDomain.all()))) .doesNotFire(); + + tester().assertThat(pickTableLayout.pickTableLayoutForPredicate()) + .on(p -> { + p.variable("nationkey"); + return p.filter(p.rowExpression("nationkey % 17 = BIGINT '44' AND nationkey % 15 = BIGINT '43'"), + p.tableScan( + nationTableHandle, + ImmutableList.of(variable("nationkey", BIGINT)), + ImmutableMap.of(variable("nationkey", BIGINT), new TpchColumnHandle("nationkey", BIGINT)), + TupleDomain.all(), + TupleDomain.all())); + }) + .doesNotFire(); } @Test @@ -165,6 +203,18 @@ public void ruleAddedTableLayoutToFilterTableScan() ImmutableMap.of(p.variable("orderstatus", createVarcharType(1)), new TpchColumnHandle("orderstatus", createVarcharType(1)))))) .matches( constrainedTableScanWithTableLayout("orders", filterConstraint, ImmutableMap.of("orderstatus", "orderstatus"))); + + tester().assertThat(pickTableLayout.pickTableLayoutForPredicate()) + .on(p -> { + p.variable("orderstatus", createVarcharType(1)); + return p.filter(p.rowExpression("orderstatus = CAST ('F' AS VARCHAR(1))"), + p.tableScan( + ordersTableHandle, + ImmutableList.of(variable("orderstatus", createVarcharType(1))), + ImmutableMap.of(variable("orderstatus", createVarcharType(1)), new TpchColumnHandle("orderstatus", createVarcharType(1))))); + }) + .matches( + constrainedTableScanWithTableLayout("orders", filterConstraint, ImmutableMap.of("orderstatus", "orderstatus"))); } @Test @@ -181,6 +231,21 @@ public void ruleAddedNewTableLayoutIfTableScanHasEmptyConstraint() "orders", ImmutableMap.of("orderstatus", singleValue(createVarcharType(1), utf8Slice("F"))), ImmutableMap.of("orderstatus", "orderstatus"))); + + tester().assertThat(pickTableLayout.pickTableLayoutForPredicate()) + .on(p -> { + p.variable("orderstatus", createVarcharType(1)); + return p.filter(expression("orderstatus = 'F'"), + p.tableScan( + ordersTableHandle, + ImmutableList.of(variable("orderstatus", createVarcharType(1))), + ImmutableMap.of(variable("orderstatus", createVarcharType(1)), new TpchColumnHandle("orderstatus", createVarcharType(1))))); + }) + .matches( + constrainedTableScanWithTableLayout( + "orders", + ImmutableMap.of("orderstatus", singleValue(createVarcharType(1), utf8Slice("F"))), + ImmutableMap.of("orderstatus", "orderstatus"))); } @Test @@ -197,6 +262,20 @@ public void ruleWithPushdownableToTableLayoutPredicate() "orders", ImmutableMap.of("orderstatus", singleValue(orderStatusType, utf8Slice("O"))), ImmutableMap.of("orderstatus", "orderstatus"))); + + tester().assertThat(pickTableLayout.pickTableLayoutForPredicate()) + .on(p -> { + p.variable("orderstatus", orderStatusType); + return p.filter(p.rowExpression("orderstatus = 'O'"), + p.tableScan( + ordersTableHandle, + ImmutableList.of(variable("orderstatus", orderStatusType)), + ImmutableMap.of(variable("orderstatus", orderStatusType), new TpchColumnHandle("orderstatus", orderStatusType)))); + }) + .matches(constrainedTableScanWithTableLayout( + "orders", + ImmutableMap.of("orderstatus", singleValue(orderStatusType, utf8Slice("O"))), + ImmutableMap.of("orderstatus", "orderstatus"))); } @Test @@ -215,5 +294,21 @@ public void nonDeterministicPredicate() "orders", ImmutableMap.of("orderstatus", singleValue(orderStatusType, utf8Slice("O"))), ImmutableMap.of("orderstatus", "orderstatus")))); + + tester().assertThat(pickTableLayout.pickTableLayoutForPredicate()) + .on(p -> { + p.variable("orderstatus", orderStatusType); + return p.filter(p.rowExpression("orderstatus = 'O' AND rand() = 0"), + p.tableScan( + ordersTableHandle, + ImmutableList.of(variable("orderstatus", orderStatusType)), + ImmutableMap.of(variable("orderstatus", orderStatusType), new TpchColumnHandle("orderstatus", orderStatusType)))); + }) + .matches( + filter("rand() = 0", + constrainedTableScanWithTableLayout( + "orders", + ImmutableMap.of("orderstatus", singleValue(orderStatusType, utf8Slice("O"))), + ImmutableMap.of("orderstatus", "orderstatus")))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneAggregationColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneAggregationColumns.java index 757143997c1a3..4c0122b1f364a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneAggregationColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneAggregationColumns.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -24,13 +24,13 @@ import java.util.Optional; import java.util.function.Predicate; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.functionCall; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.singleGroupingSet; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.strictProject; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignmentsAsSymbolReferences; import static com.google.common.base.Predicates.alwaysTrue; import static com.google.common.collect.ImmutableSet.toImmutableSet; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneAggregationSourceColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneAggregationSourceColumns.java index 8e4c083091d1d..c9b44311b92d9 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneAggregationSourceColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneAggregationSourceColumns.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -25,6 +25,7 @@ import java.util.Optional; import java.util.function.Predicate; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; @@ -32,7 +33,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.singleGroupingSet; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.strictProject; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; import static com.google.common.base.Predicates.alwaysTrue; import static com.google.common.collect.ImmutableList.toImmutableList; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneCountAggregationOverScalar.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneCountAggregationOverScalar.java index a3426c71c3bdf..73f993a42bef5 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneCountAggregationOverScalar.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneCountAggregationOverScalar.java @@ -15,9 +15,9 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.SymbolReference; @@ -30,12 +30,12 @@ import java.util.Optional; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions; -import static com.facebook.presto.sql.planner.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCALE_FACTOR; public class TestPruneCountAggregationOverScalar @@ -90,7 +90,7 @@ public void testFiresOnCountAggregateOverValues() .globalGrouping() .source(p.values( ImmutableList.of(p.variable("orderkey")), - ImmutableList.of(constantExpressions(BIGINT, 1)))))) + ImmutableList.of(constantExpressions(BIGINT, 1L)))))) .matches(values(ImmutableMap.of("count_1", 0))); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneFilterColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneFilterColumns.java index 449122de058c1..3439bdd8b2730 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneFilterColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneFilterColumns.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneJoinColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneJoinColumns.java index ffda63829de7f..4b0e6dc493243 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneJoinColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneJoinColumns.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.JoinNode; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneLimitColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneLimitColumns.java index 29de47e139380..744c111dd13c7 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneLimitColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneLimitColumns.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneOrderByInAggregation.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneOrderByInAggregation.java index aa47fe869bf13..f78b4e1845266 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneOrderByInAggregation.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneOrderByInAggregation.java @@ -15,10 +15,10 @@ import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.tree.SortItem; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -27,13 +27,13 @@ import java.util.List; import java.util.Optional; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.functionCall; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.singleGroupingSet; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.sort; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; public class TestPruneOrderByInAggregation extends BaseRuleTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneTopNColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneTopNColumns.java index 861be3b10bc78..d6fa4d5e4c963 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneTopNColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneTopNColumns.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneValuesColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneValuesColumns.java index df6e6ead59f40..1f0bb2c1b39c2 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneValuesColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneValuesColumns.java @@ -39,8 +39,8 @@ public void testNotAllOutputsReferenced() p.values( ImmutableList.of(p.variable("unused"), p.variable("x")), ImmutableList.of( - constantExpressions(BIGINT, 1, 2), - constantExpressions(BIGINT, 3, 4))))) + constantExpressions(BIGINT, 1L, 2L), + constantExpressions(BIGINT, 3L, 4L))))) .matches( project( ImmutableMap.of("y", PlanMatchPattern.expression("x")), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java index b2e0aa50fa867..396470cf36c7e 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneWindowColumns.java @@ -15,9 +15,10 @@ import com.facebook.presto.spi.block.SortOrder; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.assertions.ExpectedValueProvider; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; @@ -213,8 +214,7 @@ private static PlanNode buildProjectedWindow( new WindowNode.Specification( ImmutableList.of(partitionKey), Optional.of(new OrderingScheme( - ImmutableList.of(orderKey), - ImmutableMap.of(orderKey, SortOrder.ASC_NULLS_FIRST)))), + ImmutableList.of(new Ordering(orderKey, SortOrder.ASC_NULLS_FIRST))))), ImmutableMap.of( output1, new WindowNode.Function( @@ -226,7 +226,8 @@ private static PlanNode buildProjectedWindow( CURRENT_ROW, Optional.of(endValue1), Optional.of(new SymbolReference(startValue1.getName())).map(Expression::toString), - Optional.of(new SymbolReference(endValue2.getName())).map(Expression::toString))), + Optional.of(new SymbolReference(endValue2.getName())).map(Expression::toString)), + false), output2, new WindowNode.Function( call(FUNCTION_NAME, FUNCTION_HANDLE, BIGINT, input2), @@ -237,7 +238,8 @@ private static PlanNode buildProjectedWindow( CURRENT_ROW, Optional.of(endValue2), Optional.of(new SymbolReference(startValue2.getName())).map(Expression::toString), - Optional.of(new SymbolReference(endValue2.getName())).map(Expression::toString)))), + Optional.of(new SymbolReference(endValue2.getName())).map(Expression::toString)), + false)), hash, p.values( filteredInputs, diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushAggregationThroughOuterJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushAggregationThroughOuterJoin.java index 67d606b4ddfe5..8778475796ed3 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushAggregationThroughOuterJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushAggregationThroughOuterJoin.java @@ -23,6 +23,7 @@ import java.util.Optional; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; @@ -36,7 +37,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.sort; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignmentsAsSymbolReferences; import static com.facebook.presto.sql.tree.SortItem.NullOrdering.LAST; import static com.facebook.presto.sql.tree.SortItem.Ordering.ASCENDING; @@ -52,7 +52,7 @@ public void testPushesAggregationThroughLeftJoin() .source( p.join( JoinNode.Type.LEFT, - p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10))), + p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10L))), p.values(p.variable("COL2")), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), @@ -94,7 +94,7 @@ public void testPushesAggregationThroughLeftJoinWithOrderByFromRightSideColumn() JoinNode.Type.LEFT, p.values( ImmutableList.of(p.variable("COL1"), p.variable("COL3")), - ImmutableList.of(constantExpressions(BIGINT, 10, 20))), + ImmutableList.of(constantExpressions(BIGINT, 10L, 20L))), p.values(p.variable("COL2"), p.variable("COL4")), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), @@ -143,7 +143,7 @@ public void testPushesAggregationThroughRightJoin() .source(p.join( JoinNode.Type.RIGHT, p.values(p.variable("COL2")), - p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10))), + p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10L))), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL2"), p.variable("COL1"))), ImmutableList.of(p.variable("COL2"), p.variable("COL1")), Optional.empty(), @@ -184,7 +184,7 @@ public void testDoesNotFireWhenNotDistinct() JoinNode.Type.LEFT, p.values( ImmutableList.of(p.variable("COL1")), - ImmutableList.of(constantExpressions(BIGINT, 10), constantExpressions(BIGINT, 11))), + ImmutableList.of(constantExpressions(BIGINT, 10L), constantExpressions(BIGINT, 11L))), p.values(p.variable("COL2")), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), @@ -207,7 +207,7 @@ public void testDoesNotFireWhenNotDistinct() .source( p.values( ImmutableList.of(p.variable("COL1"), p.variable("unused")), - ImmutableList.of(constantExpressions(BIGINT, 10, 1), constantExpressions(BIGINT, 10, 2)))))), + ImmutableList.of(constantExpressions(BIGINT, 10L, 1L), constantExpressions(BIGINT, 10L, 2L)))))), p.values(p.variable("COL2")), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), @@ -225,7 +225,7 @@ public void testDoesNotFireWhenGroupingOnInner() tester().assertThat(new PushAggregationThroughOuterJoin(getFunctionManager())) .on(p -> p.aggregation(ab -> ab .source(p.join(JoinNode.Type.LEFT, - p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10))), + p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10L))), p.values(p.variable("COL2"), p.variable("COL3")), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), @@ -244,8 +244,8 @@ public void testDoesNotFireWhenAggregationDoesNotHaveSymbols() .on(p -> p.aggregation(ab -> ab .source(p.join( JoinNode.Type.LEFT, - p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10))), - p.values(ImmutableList.of(p.variable("COL2")), ImmutableList.of(constantExpressions(BIGINT, 20))), + p.values(ImmutableList.of(p.variable("COL1")), ImmutableList.of(constantExpressions(BIGINT, 10L))), + p.values(ImmutableList.of(p.variable("COL2")), ImmutableList.of(constantExpressions(BIGINT, 20L))), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("COL1"), p.variable("COL2"))), ImmutableList.of(p.variable("COL1"), p.variable("COL2")), Optional.empty(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushLimitThroughMarkDistinct.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushLimitThroughMarkDistinct.java index 6cedb71cb4892..19561f1e8c5cc 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushLimitThroughMarkDistinct.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushLimitThroughMarkDistinct.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushPartialAggregationThroughJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushPartialAggregationThroughJoin.java index 430ac683868a9..185ce9709019a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushPartialAggregationThroughJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushPartialAggregationThroughJoin.java @@ -23,6 +23,7 @@ import java.util.Optional; import static com.facebook.presto.SystemSessionProperties.PUSH_PARTIAL_AGGREGATION_THROUGH_JOIN; +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; @@ -32,7 +33,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.singleGroupingSet; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER; public class TestPushPartialAggregationThroughJoin diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushProjectionThroughExchange.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushProjectionThroughExchange.java index d00acd6191a67..dce2704f55139 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushProjectionThroughExchange.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushProjectionThroughExchange.java @@ -14,17 +14,16 @@ package com.facebook.presto.sql.planner.iterative.rule; import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.Assignments; -import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; -import com.facebook.presto.sql.tree.LongLiteral; -import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; +import static com.facebook.presto.spi.function.OperatorType.MULTIPLY; +import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.exchange; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; @@ -33,8 +32,7 @@ import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; -import static com.facebook.presto.sql.relational.OriginalExpressionUtils.asSymbolReference; -import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; +import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.tree.SortItem.NullOrdering.FIRST; import static com.facebook.presto.sql.tree.SortItem.Ordering.ASCENDING; @@ -47,7 +45,7 @@ public void testDoesNotFireNoExchange() tester().assertThat(new PushProjectionThroughExchange()) .on(p -> p.project( - assignment(p.variable("x"), new LongLiteral("3")), + assignment(p.variable("x"), constant(3L, BIGINT)), p.values(p.variable("a")))) .doesNotFire(); } @@ -63,8 +61,8 @@ public void testDoesNotFireNarrowingProjection() return p.project( Assignments.builder() - .put(a, castToRowExpression(asSymbolReference(a))) - .put(b, castToRowExpression(asSymbolReference(b))) + .put(a, a) + .put(b, b) .build(), p.exchange(e -> e .addSource(p.values(a, b, c)) @@ -86,8 +84,8 @@ public void testSimpleMultipleInputs() VariableReferenceExpression x = p.variable("x"); return p.project( assignment( - x, new LongLiteral("3"), - c2, new SymbolReference("c")), + x, constant(3L, BIGINT), + c2, c), p.exchange(e -> e .addSource( p.values(a)) @@ -123,9 +121,9 @@ public void testPartitioningColumnAndHashWithoutIdentityMappingInProjection() VariableReferenceExpression hTimes5 = p.variable("h_times_5"); return p.project( Assignments.builder() - .put(aTimes5, castToRowExpression(new ArithmeticBinaryExpression(ArithmeticBinaryExpression.Operator.MULTIPLY, new SymbolReference("a"), new LongLiteral("5")))) - .put(bTimes5, castToRowExpression(new ArithmeticBinaryExpression(ArithmeticBinaryExpression.Operator.MULTIPLY, new SymbolReference("b"), new LongLiteral("5")))) - .put(hTimes5, castToRowExpression(new ArithmeticBinaryExpression(ArithmeticBinaryExpression.Operator.MULTIPLY, new SymbolReference("h"), new LongLiteral("5")))) + .put(aTimes5, p.binaryOperation(MULTIPLY, a, constant(5L, BIGINT))) + .put(bTimes5, p.binaryOperation(MULTIPLY, b, constant(5L, BIGINT))) + .put(hTimes5, p.binaryOperation(MULTIPLY, h, constant(5L, BIGINT))) .build(), p.exchange(e -> e .addSource( @@ -164,12 +162,12 @@ public void testOrderingColumnsArePreserved() VariableReferenceExpression bTimes5 = p.variable("b_times_5"); VariableReferenceExpression hTimes5 = p.variable("h_times_5"); VariableReferenceExpression sortVariable = p.variable("sortVariable"); - OrderingScheme orderingScheme = new OrderingScheme(ImmutableList.of(sortVariable), ImmutableMap.of(sortVariable, SortOrder.ASC_NULLS_FIRST)); + OrderingScheme orderingScheme = new OrderingScheme(ImmutableList.of(new Ordering(sortVariable, SortOrder.ASC_NULLS_FIRST))); return p.project( Assignments.builder() - .put(aTimes5, castToRowExpression(new ArithmeticBinaryExpression(ArithmeticBinaryExpression.Operator.MULTIPLY, new SymbolReference("a"), new LongLiteral("5")))) - .put(bTimes5, castToRowExpression(new ArithmeticBinaryExpression(ArithmeticBinaryExpression.Operator.MULTIPLY, new SymbolReference("b"), new LongLiteral("5")))) - .put(hTimes5, castToRowExpression(new ArithmeticBinaryExpression(ArithmeticBinaryExpression.Operator.MULTIPLY, new SymbolReference("h"), new LongLiteral("5")))) + .put(aTimes5, p.binaryOperation(MULTIPLY, a, constant(5L, BIGINT))) + .put(bTimes5, p.binaryOperation(MULTIPLY, b, constant(5L, BIGINT))) + .put(hTimes5, p.binaryOperation(MULTIPLY, h, constant(5L, BIGINT))) .build(), p.exchange(e -> e .addSource( @@ -177,6 +175,7 @@ public void testOrderingColumnsArePreserved() .addInputsSet(a, b, h, sortVariable) .singleDistributionPartitioningScheme( ImmutableList.of(a, b, h, sortVariable)) + .setEnsureSourceOrdering(true) .orderingScheme(orderingScheme))); }) .matches( diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveFullSample.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveFullSample.java index 1f15c5da6b938..cc7971810981f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveFullSample.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveFullSample.java @@ -53,8 +53,8 @@ public void test() p.values( ImmutableList.of(p.variable("a"), p.variable("b")), ImmutableList.of( - constantExpressions(BIGINT, 1, 10), - constantExpressions(BIGINT, 2, 11)))))) + constantExpressions(BIGINT, 1L, 10L), + constantExpressions(BIGINT, 2L, 11L)))))) // TODO: verify contents .matches(filter("b > 5", values(ImmutableMap.of("a", 0, "b", 1)))); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveTrivialFilters.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveTrivialFilters.java index 1c0a4fc7dfd69..627594230652d 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveTrivialFilters.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveTrivialFilters.java @@ -48,7 +48,7 @@ public void testRemovesFalseFilter() p.expression("FALSE"), p.values( ImmutableList.of(p.variable("a")), - ImmutableList.of(constantExpressions(BIGINT, 1))))) + ImmutableList.of(constantExpressions(BIGINT, 1L))))) .matches(values("a")); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveUnreferencedScalarApplyNodes.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveUnreferencedScalarApplyNodes.java index 7f8ce39c68c65..ffda9e31e958a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveUnreferencedScalarApplyNodes.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveUnreferencedScalarApplyNodes.java @@ -14,8 +14,8 @@ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.Assignments; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestReorderJoins.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestReorderJoins.java index d9dc39c2cdcc4..b54ec63a9c368 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestReorderJoins.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestReorderJoins.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Optional; +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; import static com.facebook.presto.SystemSessionProperties.JOIN_MAX_BROADCAST_TABLE_SIZE; import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; @@ -53,7 +54,6 @@ import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; import static com.facebook.presto.sql.tree.ComparisonExpression.Operator.EQUAL; import static com.facebook.presto.sql.tree.ComparisonExpression.Operator.LESS_THAN; -import static io.airlift.testing.Closeables.closeAllRuntimeException; public class TestReorderJoins { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyExpressions.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyExpressions.java index de67228fc7b70..3b8a0551857b8 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyExpressions.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyExpressions.java @@ -14,20 +14,30 @@ package com.facebook.presto.sql.planner.iterative.rule; import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.LiteralEncoder; import com.facebook.presto.sql.planner.PlanVariableAllocator; +import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.VariablesExtractor; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.ExpressionRewriter; import com.facebook.presto.sql.tree.ExpressionTreeRewriter; import com.facebook.presto.sql.tree.LogicalBinaryExpression; +import com.google.common.collect.Streams; import org.testng.annotations.Test; import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.function.Function; +import java.util.stream.IntStream; +import java.util.stream.Stream; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; @@ -37,7 +47,9 @@ import static com.facebook.presto.sql.ExpressionUtils.rewriteIdentifiersToSymbolReferences; import static com.facebook.presto.sql.planner.iterative.rule.SimplifyExpressions.rewrite; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static java.lang.String.format; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; import static org.testng.Assert.assertEquals; public class TestSimplifyExpressions @@ -45,6 +57,10 @@ public class TestSimplifyExpressions private static final SqlParser SQL_PARSER = new SqlParser(); private static final MetadataManager METADATA = createTestMetadataManager(); private static final LiteralEncoder LITERAL_ENCODER = new LiteralEncoder(METADATA.getBlockEncodingSerde()); + private static final Map TYPES = Streams.concat( + Stream.of("A", "B", "C", "D", "E", "F", "I", "V", "X", "Y", "Z"), + IntStream.range(1, 61).boxed().map(i -> format("A%s", i))) + .collect(toMap(Function.identity(), string -> BOOLEAN)); @Test public void testPushesDownNegations() @@ -67,6 +83,9 @@ public void testPushesDownNegations() @Test public void testExtractCommonPredicates() { + assertSimplifies("TRUE", "TRUE"); + assertSimplifies("IF(X, X, Y)", "IF(X, X, Y)"); + assertSimplifies("X AND Y", "X AND Y"); assertSimplifies("X OR Y", "X OR Y"); assertSimplifies("X AND X", "X"); @@ -86,7 +105,7 @@ public void testExtractCommonPredicates() assertSimplifies("((X OR V) AND X) OR ((X OR V) AND V)", "X OR V"); assertSimplifies("((X OR V) AND Z) OR ((X OR V) AND V)", "(X OR V) AND (Z OR V)"); - assertSimplifies("X AND ((Y AND Z) OR (Y AND V) OR (Y AND X))", "X AND Y AND (Z OR V OR X)"); + assertSimplifies("X AND ((Y AND Z) OR (Y AND V) OR (Y AND X))", "X AND Y AND (Z OR V OR X)", "X AND Y"); assertSimplifies("(A AND B AND C AND D) OR (A AND B AND E) OR (A AND F)", "A AND ((B AND C AND D) OR (B AND E) OR F)"); assertSimplifies("((A AND B) OR (A AND C)) AND D", "A AND (B OR C) AND D"); @@ -114,6 +133,11 @@ public void testExtractCommonPredicates() } private static void assertSimplifies(String expression, String expected) + { + assertSimplifies(expression, expected, null); + } + + private static void assertSimplifies(String expression, String expected, String rowExpressionExpected) { Expression actualExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(expression)); Expression expectedExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(expected)); @@ -121,6 +145,12 @@ private static void assertSimplifies(String expression, String expected) assertEquals( normalize(rewritten), normalize(expectedExpression)); + TestingRowExpressionTranslator translator = new TestingRowExpressionTranslator(METADATA); + RowExpression actualRowExpression = translator.translate(actualExpression, TypeProvider.viewOf(TYPES)); + RowExpression simplifiedRowExpression = SimplifyRowExpressions.rewrite(actualRowExpression, METADATA, TEST_SESSION.toConnectorSession()); + Expression expectedByRowExpression = Optional.ofNullable(rowExpressionExpected).map(expr -> rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(expr))).orElse(rewritten); + RowExpression simplifiedByExpression = translator.translate(expectedByRowExpression, TypeProvider.viewOf(TYPES)); + assertEquals(simplifiedRowExpression, simplifiedByExpression); } private static Set booleanVariablesFor(Expression expression) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSingleDistinctAggregationToGroupBy.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSingleDistinctAggregationToGroupBy.java index 8d20223a9a52d..4e5496aef422c 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSingleDistinctAggregationToGroupBy.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSingleDistinctAggregationToGroupBy.java @@ -22,6 +22,7 @@ import java.util.Optional; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.RealType.REAL; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; @@ -30,7 +31,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.singleGroupingSet; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; public class TestSingleDistinctAggregationToGroupBy extends BaseRuleTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSwapAdjacentWindowsBySpecifications.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSwapAdjacentWindowsBySpecifications.java index da708cd4e5da2..c43a78e2a360b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSwapAdjacentWindowsBySpecifications.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSwapAdjacentWindowsBySpecifications.java @@ -78,7 +78,7 @@ public void doesNotFireOnPlanWithSingleWindowNode() ImmutableList.of(p.variable("a")), Optional.empty()), ImmutableMap.of(p.variable("avg_1"), - new WindowNode.Function(call("avg", functionHandle, DOUBLE, ImmutableList.of()), frame)), + new WindowNode.Function(call("avg", functionHandle, DOUBLE, ImmutableList.of()), frame, false)), p.values(p.variable("a")))) .doesNotFire(); } @@ -139,6 +139,7 @@ private WindowNode.Function newWindowNodeFunction(List symbols) { return new WindowNode.Function( call("avg", functionHandle, BIGINT, symbols.stream().map(symbol -> new VariableReferenceExpression(symbol.getName(), BIGINT)).collect(Collectors.toList())), - frame); + frame, + false); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformCorrelatedScalarSubquery.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformCorrelatedScalarSubquery.java index e0f6fe8b1e0e6..f23150114bfa9 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformCorrelatedScalarSubquery.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformCorrelatedScalarSubquery.java @@ -51,8 +51,8 @@ public class TestTransformCorrelatedScalarSubquery extends BaseRuleTest { - private static final ImmutableList> ONE_ROW = ImmutableList.of(ImmutableList.of(constant(1, BIGINT))); - private static final ImmutableList> TWO_ROWS = ImmutableList.of(ImmutableList.of(constant(1, BIGINT)), ImmutableList.of(constant(2, BIGINT))); + private static final ImmutableList> ONE_ROW = ImmutableList.of(ImmutableList.of(constant(1L, BIGINT))); + private static final ImmutableList> TWO_ROWS = ImmutableList.of(ImmutableList.of(constant(1L, BIGINT)), ImmutableList.of(constant(2L, BIGINT))); private Rule rule = new TransformCorrelatedScalarSubquery(); @@ -82,7 +82,7 @@ public void doesNotFireOnUncorrelated() .on(p -> p.lateral( ImmutableList.of(), p.values(p.variable("a")), - p.values(ImmutableList.of(p.variable("b")), ImmutableList.of(constantExpressions(BIGINT, 1))))) + p.values(ImmutableList.of(p.variable("b")), ImmutableList.of(constantExpressions(BIGINT, 1L))))) .doesNotFire(); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformExistsApplyToLateralJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformExistsApplyToLateralJoin.java index 714f944807257..259536dc5d779 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformExistsApplyToLateralJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformExistsApplyToLateralJoin.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.Assignments; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedInPredicateSubqueryToSemiJoin.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedInPredicateSubqueryToSemiJoin.java index 73e277482bfcd..cfe98df346cba 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedInPredicateSubqueryToSemiJoin.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTransformUncorrelatedInPredicateSubqueryToSemiJoin.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.tree.ExistsPredicate; import com.facebook.presto.sql.tree.InPredicate; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestTranslateExpressions.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTranslateExpressions.java similarity index 64% rename from presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestTranslateExpressions.java rename to presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTranslateExpressions.java index 843895e35304d..5a7f40d8194c7 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestTranslateExpressions.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestTranslateExpressions.java @@ -11,42 +11,41 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.optimizations; +package com.facebook.presto.sql.planner.iterative.rule; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.type.FunctionType; import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; import java.util.Optional; -import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.variable; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; -import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; public class TestTranslateExpressions + extends BaseRuleTest { private static final Metadata METADATA = createTestMetadataManager(); private static final FunctionManager FUNCTION_MANAGER = METADATA.getFunctionManager(); @@ -62,23 +61,27 @@ public class TestTranslateExpressions @Test public void testTranslateAggregationWithLambda() { - TranslateExpressions translator = new TranslateExpressions(METADATA, new SqlParser()); - Aggregation aggregation = new Aggregation( - new CallExpression( - "reduce_agg", - REDUCE_AGG, - INTEGER, - ImmutableList.of( - castToRowExpression(expression("input")), - castToRowExpression(expression("0")), - castToRowExpression(expression("(x,y) -> x*y")), - castToRowExpression(expression("(a,b) -> a*b")))), - Optional.of(castToRowExpression(expression("input > 10"))), - Optional.empty(), - false, - Optional.empty()); - Aggregation translated = translator.translateAggregation(aggregation, TEST_SESSION, TypeProvider.viewOf(ImmutableMap.of("input", INTEGER))); - assertEquals(translated, new Aggregation( + PlanNode result = tester().assertThat(new TranslateExpressions(METADATA, new SqlParser()).aggregationRowExpressionRewriteRule()) + .on(p -> p.aggregation(builder -> builder.globalGrouping() + .addAggregation(variable("reduce_agg", INTEGER), new AggregationNode.Aggregation( + new CallExpression( + "reduce_agg", + REDUCE_AGG, + INTEGER, + ImmutableList.of( + castToRowExpression(expression("input")), + castToRowExpression(expression("0")), + castToRowExpression(expression("(x,y) -> x*y")), + castToRowExpression(expression("(a,b) -> a*b")))), + Optional.of(castToRowExpression(expression("input > 10"))), + Optional.empty(), + false, + Optional.empty())) + .source(p.values(p.variable("input", INTEGER))))) + .get(); + // TODO migrate this to RowExpressionMatcher + AggregationNode.Aggregation translated = ((AggregationNode) result).getAggregations().get(variable("reduce_agg", INTEGER)); + assertEquals(translated, new AggregationNode.Aggregation( new CallExpression( "reduce_agg", REDUCE_AGG, @@ -104,22 +107,25 @@ public void testTranslateAggregationWithLambda() @Test public void testTranslateIntermediateAggregationWithLambda() { - TranslateExpressions translator = new TranslateExpressions(METADATA, new SqlParser()); - Aggregation aggregation = new Aggregation( - new CallExpression( - "reduce_agg", - REDUCE_AGG, - INTEGER, - ImmutableList.of( - castToRowExpression(expression("input")), - castToRowExpression(expression("(x,y) -> x*y")), - castToRowExpression(expression("(a,b) -> a*b")))), - Optional.of(castToRowExpression(expression("input > 10"))), - Optional.empty(), - false, - Optional.empty()); - Aggregation translated = translator.translateAggregation(aggregation, TEST_SESSION, TypeProvider.viewOf(ImmutableMap.of("input", INTEGER))); - assertEquals(translated, new Aggregation( + PlanNode result = tester().assertThat(new TranslateExpressions(METADATA, new SqlParser()).aggregationRowExpressionRewriteRule()) + .on(p -> p.aggregation(builder -> builder.globalGrouping() + .addAggregation(variable("reduce_agg", INTEGER), new AggregationNode.Aggregation( + new CallExpression( + "reduce_agg", + REDUCE_AGG, + INTEGER, + ImmutableList.of( + castToRowExpression(expression("input")), + castToRowExpression(expression("(x,y) -> x*y")), + castToRowExpression(expression("(a,b) -> a*b")))), + Optional.of(castToRowExpression(expression("input > 10"))), + Optional.empty(), + false, + Optional.empty())) + .source(p.values(p.variable("input", INTEGER))))) + .get(); + AggregationNode.Aggregation translated = ((AggregationNode) result).getAggregations().get(variable("reduce_agg", INTEGER)); + assertEquals(translated, new AggregationNode.Aggregation( new CallExpression( "reduce_agg", REDUCE_AGG, diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/BaseRuleTest.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/BaseRuleTest.java index 51d20b0b75bd0..a5cb6360e1fd6 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/BaseRuleTest.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/BaseRuleTest.java @@ -21,7 +21,7 @@ import java.util.List; -import static io.airlift.testing.Closeables.closeAllRuntimeException; +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; public abstract class BaseRuleTest { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java index b84680bef5481..bee2f60d83eaa 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java @@ -23,11 +23,22 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.block.SortOrder; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.AggregationNode.Step; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.CallExpression; @@ -38,19 +49,14 @@ import com.facebook.presto.sql.analyzer.TypeSignatureProvider; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.Partitioning; import com.facebook.presto.sql.planner.PartitioningScheme; import com.facebook.presto.sql.planner.TestingConnectorIndexHandle; import com.facebook.presto.sql.planner.TestingConnectorTransactionHandle; import com.facebook.presto.sql.planner.TestingWriterTarget; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.AggregationNode; -import com.facebook.presto.sql.planner.plan.AggregationNode.Aggregation; -import com.facebook.presto.sql.planner.plan.AggregationNode.Step; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; @@ -58,19 +64,16 @@ import com.facebook.presto.sql.planner.plan.IndexSourceNode; import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; -import com.facebook.presto.sql.planner.plan.LimitNode; import com.facebook.presto.sql.planner.plan.MarkDistinctNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TopNNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.WindowNode; +import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.OriginalExpressionUtils; import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; import com.facebook.presto.sql.tree.Expression; @@ -78,12 +81,10 @@ import com.facebook.presto.sql.tree.NodeRef; import com.facebook.presto.testing.TestingMetadata.TestingTableHandle; import com.facebook.presto.testing.TestingTransactionHandle; -import com.google.common.base.Functions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ListMultimap; -import com.google.common.collect.Maps; import java.util.ArrayList; import java.util.Arrays; @@ -95,6 +96,7 @@ import java.util.function.Consumer; import java.util.stream.Stream; +import static com.facebook.presto.spi.plan.LimitNode.Step.FINAL; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypes; @@ -102,6 +104,8 @@ import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; import static com.facebook.presto.sql.planner.optimizations.ApplyNodeUtil.verifySubquerySupported; +import static com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils.fromListMultimap; +import static com.facebook.presto.sql.relational.Expressions.call; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.constantNull; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; @@ -132,11 +136,21 @@ public static Assignments assignment(VariableReferenceExpression variable, Expre return Assignments.builder().put(variable, OriginalExpressionUtils.castToRowExpression(expression)).build(); } + public static Assignments assignment(VariableReferenceExpression variable, RowExpression expression) + { + return Assignments.builder().put(variable, expression).build(); + } + public static Assignments assignment(VariableReferenceExpression variable1, Expression expression1, VariableReferenceExpression variable2, Expression expression2) { return Assignments.builder().put(variable1, OriginalExpressionUtils.castToRowExpression(expression1)).put(variable2, OriginalExpressionUtils.castToRowExpression(expression2)).build(); } + public static Assignments assignment(VariableReferenceExpression variable1, RowExpression expression1, VariableReferenceExpression variable2, RowExpression expression2) + { + return Assignments.builder().put(variable1, expression1).put(variable2, expression2).build(); + } + public OutputNode output(List columnNames, List variables, PlanNode source) { return new OutputNode( @@ -224,7 +238,7 @@ public EnforceSingleRowNode enforceSingleRow(PlanNode source) public LimitNode limit(long limit, PlanNode source) { - return new LimitNode(idAllocator.getNextId(), source, limit, false); + return new LimitNode(idAllocator.getNextId(), source, limit, FINAL); } public TopNNode topN(long count, List orderBy, PlanNode source) @@ -233,9 +247,7 @@ public TopNNode topN(long count, List orderBy, Plan idAllocator.getNextId(), source, count, - new OrderingScheme( - orderBy, - Maps.toMap(orderBy, Functions.constant(SortOrder.ASC_NULLS_FIRST))), + new OrderingScheme(orderBy.stream().map(variable -> new Ordering(variable, SortOrder.ASC_NULLS_FIRST)).collect(toImmutableList())), TopNNode.Step.SINGLE); } @@ -276,6 +288,18 @@ public AggregationNode aggregation(Consumer aggregationBuild return aggregationBuilder.build(); } + public CallExpression binaryOperation(OperatorType operatorType, RowExpression left, RowExpression right) + { + FunctionHandle functionHandle = new FunctionResolution(metadata.getFunctionManager()).arithmeticFunction(operatorType, left.getType(), right.getType()); + return call(operatorType.getOperator(), functionHandle, left.getType(), left, right); + } + + public CallExpression comparison(OperatorType operatorType, RowExpression left, RowExpression right) + { + FunctionHandle functionHandle = new FunctionResolution(metadata.getFunctionManager()).comparisonFunction(operatorType, left.getType(), right.getType()); + return call(operatorType.getOperator(), functionHandle, left.getType(), left, right); + } + public class AggregationBuilder { private final TypeProvider types; @@ -313,7 +337,7 @@ private AggregationBuilder addAggregation(VariableReferenceExpression output, Ex { checkArgument(expression instanceof FunctionCall); FunctionCall call = (FunctionCall) expression; - FunctionHandle functionHandle = metadata.getFunctionManager().resolveFunction(session, call.getName(), TypeSignatureProvider.fromTypes(inputTypes)); + FunctionHandle functionHandle = metadata.getFunctionManager().resolveFunction(session.getTransactionId(), call.getName(), TypeSignatureProvider.fromTypes(inputTypes)); return addAggregation(output, new Aggregation( new CallExpression( call.getName().getSuffix(), @@ -481,12 +505,11 @@ public TableFinishNode tableDelete(SchemaTableName schemaTableName, PlanNode del .addSource(new DeleteNode( idAllocator.getNextId(), deleteSource, - deleteHandle, deleteRowId, ImmutableList.of(deleteRowId))) .addInputsSet(deleteRowId) .singleDistributionPartitioningScheme(deleteRowId)), - deleteHandle, + Optional.of(deleteHandle), deleteRowId, Optional.empty(), Optional.empty()); @@ -575,6 +598,7 @@ public class ExchangeBuilder private ExchangeNode.Type type = ExchangeNode.Type.GATHER; private ExchangeNode.Scope scope = ExchangeNode.Scope.REMOTE_STREAMING; private PartitioningScheme partitioningScheme; + private boolean ensureSourceOrdering; private OrderingScheme orderingScheme; private List sources = new ArrayList<>(); private List> inputs = new ArrayList<>(); @@ -641,6 +665,12 @@ public ExchangeBuilder addInputsSet(List inputs) return this; } + public ExchangeBuilder setEnsureSourceOrdering(boolean ensureSourceOrdering) + { + this.ensureSourceOrdering = ensureSourceOrdering; + return this; + } + public ExchangeBuilder orderingScheme(OrderingScheme orderingScheme) { this.orderingScheme = orderingScheme; @@ -649,7 +679,7 @@ public ExchangeBuilder orderingScheme(OrderingScheme orderingScheme) protected ExchangeNode build() { - return new ExchangeNode(idAllocator.getNextId(), type, scope, partitioningScheme, sources, inputs, Optional.ofNullable(orderingScheme)); + return new ExchangeNode(idAllocator.getNextId(), type, scope, partitioningScheme, sources, inputs, ensureSourceOrdering, Optional.ofNullable(orderingScheme)); } } @@ -725,7 +755,8 @@ public PlanNode indexJoin(IndexJoinNode.Type type, TableScanNode probe, TableSca public UnionNode union(ListMultimap outputsToInputs, List sources) { - return new UnionNode(idAllocator.getNextId(), sources, outputsToInputs); + Map> mapping = fromListMultimap(outputsToInputs); + return new UnionNode(idAllocator.getNextId(), sources, ImmutableList.copyOf(mapping.keySet()), mapping); } public TableWriterNode tableWriter(List columns, List columnNames, PlanNode source) @@ -733,14 +764,13 @@ public TableWriterNode tableWriter(List columns, Li return new TableWriterNode( idAllocator.getNextId(), source, - new TestingWriterTarget(), + Optional.of(new TestingWriterTarget()), variable("partialrows", BIGINT), variable("fragment", VARBINARY), variable("tablecommitcontext", VARBINARY), columns, columnNames, Optional.empty(), - Optional.empty(), Optional.empty()); } @@ -834,8 +864,7 @@ public RowExpression rowExpression(String sql) ImmutableMap.of(), metadata.getFunctionManager(), metadata.getTypeManager(), - session, - false); + session); } public static RowExpression castToRowExpression(String sql) @@ -866,4 +895,9 @@ public TypeProvider getTypes() { return TypeProvider.viewOf(variables); } + + public PlanNodeIdAllocator getIdAllocator() + { + return idAllocator; + } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java index 47ea136528914..103848b5ec812 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java @@ -40,7 +40,7 @@ import com.facebook.presto.sql.planner.iterative.Memo; import com.facebook.presto.sql.planner.iterative.PlanNodeMatcher; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.optimizations.TranslateExpressions; +import com.facebook.presto.sql.planner.iterative.rule.TranslateExpressions; import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableSet; @@ -111,6 +111,21 @@ public RuleAssert on(Function planProvider) return this; } + public PlanNode get() + { + RuleApplication ruleApplication = applyRule(); + TypeProvider types = ruleApplication.types; + + if (!ruleApplication.wasRuleApplied()) { + fail(String.format( + "%s did not fire for:\n%s", + rule.getClass().getName(), + formatPlan(plan, types))); + } + + return ruleApplication.getTransformedPlan(); + } + public void doesNotFire() { RuleApplication ruleApplication = applyRule(); @@ -190,7 +205,7 @@ private static RuleApplication applyRule(Rule rule, PlanNode planNode, Ru private String formatPlan(PlanNode plan, TypeProvider types) { StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, types); - CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, session, types); + CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, session); return inTransaction(session -> textLogicalPlan(translateExpressions(plan, types), types, metadata.getFunctionManager(), StatsAndCosts.create(plan, statsProvider, costProvider), session, 2, false)); } @@ -214,7 +229,7 @@ private PlanNode translateExpressions(PlanNode node, TypeProvider typeProvider) private Rule.Context ruleContext(StatsCalculator statsCalculator, CostCalculator costCalculator, PlanVariableAllocator variableAllocator, Memo memo, Lookup lookup, Session session) { StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, Optional.of(memo), lookup, session, variableAllocator.getTypes()); - CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.of(memo), session, variableAllocator.getTypes()); + CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.of(memo), session); return new Rule.Context() { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/TestRuleTester.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/TestRuleTester.java index cd6b70fe5c6ea..d478c64fcb36a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/TestRuleTester.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/TestRuleTester.java @@ -15,9 +15,9 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.Assignments; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -39,7 +39,7 @@ public void testReportWrongMatch() Assignments.of(p.variable("y"), variable("x", BIGINT)), p.values( ImmutableList.of(p.variable("x")), - ImmutableList.of(constantExpressions(BIGINT, 1))))) + ImmutableList.of(constantExpressions(BIGINT, 1L))))) .matches( values(ImmutableList.of("different"), ImmutableList.of())); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestConnectorOptimization.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestConnectorOptimization.java index 2777b4857fe87..0865e73195c73 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestConnectorOptimization.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestConnectorOptimization.java @@ -30,6 +30,7 @@ import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.PlanVisitor; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.relation.RowExpression; @@ -44,7 +45,6 @@ import com.facebook.presto.sql.planner.assertions.SymbolAliases; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.UnionNode; import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; @@ -60,9 +60,9 @@ import java.util.Set; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.TRUE_CONSTANT; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.and; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.or; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.and; +import static com.facebook.presto.expressions.LogicalRowExpressions.or; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; import static com.google.common.base.MoreObjects.toStringHelper; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java index e74d9b5a93244..36fbf2e8f3d31 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java @@ -21,6 +21,7 @@ import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.IterativeOptimizer; import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantIdentityProjections; +import com.facebook.presto.sql.planner.iterative.rule.TranslateExpressions; import com.facebook.presto.sql.planner.plan.WindowNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -87,16 +88,37 @@ public void testNotEliminateSorts() public void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern pattern) { - List optimizers = ImmutableList.of( - new UnaliasSymbolReferences(), + List optimizersBeforeTranslation = ImmutableList.of( + new UnaliasSymbolReferences(getMetadata().getFunctionManager()), new AddExchanges(getQueryRunner().getMetadata(), new SqlParser()), new PruneUnreferencedOutputs(), + new IterativeOptimizer( + new RuleStatsRecorder(), + getQueryRunner().getStatsCalculator(), + getQueryRunner().getCostCalculator(), + new TranslateExpressions(getQueryRunner().getMetadata(), new SqlParser()).rules()), + new IterativeOptimizer( + new RuleStatsRecorder(), + getQueryRunner().getStatsCalculator(), + getQueryRunner().getCostCalculator(), + ImmutableSet.of(new RemoveRedundantIdentityProjections()))); + + List optimizersAfterTranslation = ImmutableList.of( + new AddExchanges(getQueryRunner().getMetadata(), new SqlParser()), + new IterativeOptimizer( + new RuleStatsRecorder(), + getQueryRunner().getStatsCalculator(), + getQueryRunner().getCostCalculator(), + new TranslateExpressions(getMetadata(), new SqlParser()).rules()), + new UnaliasSymbolReferences(getMetadata().getFunctionManager()), + new PruneUnreferencedOutputs(), new IterativeOptimizer( new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), getQueryRunner().getCostCalculator(), ImmutableSet.of(new RemoveRedundantIdentityProjections()))); - assertPlan(sql, pattern, optimizers); + assertPlan(sql, pattern, optimizersBeforeTranslation); + assertPlan(sql, pattern, optimizersAfterTranslation); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestForceSingleNodeOutput.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestForceSingleNodeOutput.java index c91d79e9a898a..6192f1d9d583c 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestForceSingleNodeOutput.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestForceSingleNodeOutput.java @@ -14,8 +14,8 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.Session; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.sql.planner.assertions.BasePlanTest; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestFullOuterJoinWithCoalesce.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestFullOuterJoinWithCoalesce.java new file mode 100644 index 0000000000000..a3cd96bb8fbd2 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestFullOuterJoinWithCoalesce.java @@ -0,0 +1,184 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.optimizations; + +import com.facebook.presto.sql.planner.assertions.BasePlanTest; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.exchange; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.LOCAL; +import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; +import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; +import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; +import static com.facebook.presto.sql.planner.plan.JoinNode.Type.FULL; + +public class TestFullOuterJoinWithCoalesce + extends BasePlanTest +{ + @Test + public void testFullOuterJoinWithCoalesce() + { + assertDistributedPlan("SELECT coalesce(r.a, ts.a) " + + "FROM (" + + " SELECT coalesce(t.a, s.a) AS a " + + " FROM (VALUES (1), (2), (3)) t(a) " + + " FULL OUTER JOIN (VALUES (1), (4)) s(a)" + + " ON t.a = s.a) ts " + + "FULL OUTER JOIN (VALUES (2), (5)) r(a) " + + "ON ts.a = r.a", + anyTree( + project( + ImmutableMap.of("expr", expression("coalesce(r, ts)")), + join( + FULL, + ImmutableList.of(equiJoinClause("ts", "r")), + project( + project( + ImmutableMap.of("ts", expression("coalesce(t, s)")), + join( + FULL, + ImmutableList.of(equiJoinClause("t", "s")), + exchange(REMOTE_STREAMING, REPARTITION, anyTree(values(ImmutableList.of("t")))), + exchange(LOCAL, GATHER, anyTree(values(ImmutableList.of("s"))))))), + exchange(LOCAL, GATHER, anyTree(values(ImmutableList.of("r")))))))); + } + + @Test + public void testDuplicateJoinClause() + { + assertDistributedPlan("SELECT coalesce(r.a, ts.a) " + + "FROM (" + + " SELECT coalesce(t.a, s.a) AS a " + + " FROM (VALUES (1), (2), (3)) t(a) " + + " FULL OUTER JOIN (VALUES (1), (4)) s(a)" + + " ON t.a = s.a AND t.a = s.a) ts " + + "FULL OUTER JOIN (VALUES (2), (5)) r(a) " + + "ON ts.a = r.a", + anyTree( + project( + ImmutableMap.of("expr", expression("coalesce(r, ts)")), + join( + FULL, + ImmutableList.of(equiJoinClause("ts", "r")), + project( + project( + ImmutableMap.of("ts", expression("coalesce(t, s)")), + join( + FULL, + ImmutableList.of(equiJoinClause("t", "s"), equiJoinClause("t", "s")), + exchange(REMOTE_STREAMING, REPARTITION, anyTree(values(ImmutableList.of("t")))), + exchange(LOCAL, GATHER, anyTree(values(ImmutableList.of("s"))))))), + exchange(LOCAL, GATHER, anyTree(values(ImmutableList.of("r")))))))); + } + + @Test + public void testDuplicatePartitionColumn() + { + assertDistributedPlan("SELECT coalesce(r.a, ts.a), coalesce(ts.b, r.b) " + + "FROM (" + + " SELECT coalesce(t.a, s.a) AS a, coalesce(t.a, s.b) AS b" + + " FROM (VALUES (1), (2), (3)) t(a) " + + " FULL OUTER JOIN (VALUES (1, 1), (4, 4)) s(a, b)" + + " ON t.a = s.a AND t.a = s.b) ts " + + "FULL OUTER JOIN (VALUES (2, 2), (5, 5)) r(a, b) " + + "ON ts.a = r.a and ts.b = r.b", + anyTree( + project( + ImmutableMap.of("tsra", expression("coalesce(ra, tsa)"), "tsrb", expression("coalesce(tsb, rb)")), + join( + FULL, + ImmutableList.of(equiJoinClause("tsa", "ra"), equiJoinClause("tsb", "rb")), + project( + project( + ImmutableMap.of("tsa", expression("coalesce(ta, sa)"), "tsb", expression("coalesce(ta, sb)")), + join( + FULL, + ImmutableList.of(equiJoinClause("ta", "sa"), equiJoinClause("ta", "sb")), + exchange(REMOTE_STREAMING, REPARTITION, anyTree(values(ImmutableList.of("ta")))), + exchange(LOCAL, GATHER, anyTree(values(ImmutableList.of("sa", "sb"))))))), + exchange(LOCAL, GATHER, anyTree(values(ImmutableList.of("ra", "rb")))))))); + } + + @Test + public void testCoalesceWithManyArgumentsAndGroupBy() + { + assertDistributedPlan("SELECT coalesce(t.a, s.a, r.a) " + + "FROM (VALUES (1), (2), (3)) t(a) " + + "FULL OUTER JOIN (VALUES (1), (4)) s(a) " + + "ON t.a = s.a " + + "FULL OUTER JOIN (VALUES (2), (5)) r(a) " + + "ON t.a = r.a " + + "GROUP BY 1", + anyTree(exchange( + REMOTE_STREAMING, + REPARTITION, + aggregation( + ImmutableMap.of(), + PARTIAL, + project( + project( + ImmutableMap.of("expr", expression("coalesce(t, s, r)")), + join( + FULL, + ImmutableList.of(equiJoinClause("t", "r")), + anyTree( + join( + FULL, + ImmutableList.of(equiJoinClause("t", "s")), + exchange(REMOTE_STREAMING, REPARTITION, anyTree(values(ImmutableList.of("t")))), + exchange(LOCAL, GATHER, anyTree(values(ImmutableList.of("s")))))), + exchange(LOCAL, GATHER, anyTree(values(ImmutableList.of("r"))))))))))); + } + + @Test + public void testCoalesceWithNonSymbolArguments() + { + assertDistributedPlan("SELECT coalesce(t.a, s.a + 1, r.a) " + + "FROM (VALUES (1), (2), (3)) t(a) " + + "FULL OUTER JOIN (VALUES (1), (4)) s(a) " + + "ON t.a = s.a " + + "FULL OUTER JOIN (VALUES (2), (5)) r(a) " + + "ON t.a = r.a " + + "GROUP BY 1", + anyTree(exchange( + REMOTE_STREAMING, + REPARTITION, + aggregation( + ImmutableMap.of(), + PARTIAL, + project( + project( + ImmutableMap.of("expr", expression("coalesce(t, s + 1, r)")), + join( + FULL, + ImmutableList.of(equiJoinClause("t", "r")), + anyTree( + join( + FULL, + ImmutableList.of(equiJoinClause("t", "s")), + exchange(REMOTE_STREAMING, REPARTITION, anyTree(values(ImmutableList.of("t")))), + exchange(LOCAL, GATHER, anyTree(values(ImmutableList.of("s")))))), + exchange(LOCAL, GATHER, anyTree(values(ImmutableList.of("r"))))))))))); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestLocalProperties.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestLocalProperties.java index e25021c69c2d1..7d82a9094dabb 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestLocalProperties.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestLocalProperties.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.planner.optimizations; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConstantProperty; import com.facebook.presto.spi.GroupingProperty; @@ -29,7 +30,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.json.ObjectMapperProvider; import org.testng.annotations.Test; import java.io.IOException; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestMergeWindows.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestMergeWindows.java index d09916d524caa..6bf931b8071a3 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestMergeWindows.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestMergeWindows.java @@ -553,7 +553,7 @@ public void testNotMergeDifferentNullOrdering() private void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern pattern) { List optimizers = ImmutableList.of( - new UnaliasSymbolReferences(), + new UnaliasSymbolReferences(getMetadata().getFunctionManager()), new IterativeOptimizer( new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestOptimizeMixedDistinctAggregations.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestOptimizeMixedDistinctAggregations.java index 355e68a321d9a..28b320cff7016 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestOptimizeMixedDistinctAggregations.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestOptimizeMixedDistinctAggregations.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anySymbol; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; @@ -42,7 +43,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.singleGroupingSet; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; public class TestOptimizeMixedDistinctAggregations extends BasePlanTest @@ -114,7 +114,7 @@ public void testNestedType() private void assertUnitPlan(String sql, PlanMatchPattern pattern) { List optimizers = ImmutableList.of( - new UnaliasSymbolReferences(), + new UnaliasSymbolReferences(getMetadata().getFunctionManager()), new IterativeOptimizer( new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java index a91bae8be13a4..f2c7c49d209ad 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java @@ -321,8 +321,8 @@ public void testReorderBDAC() private void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern pattern) { List optimizers = ImmutableList.of( - new UnaliasSymbolReferences(), - new PredicatePushDown(getQueryRunner().getMetadata(), getQueryRunner().getSqlParser()), + new UnaliasSymbolReferences(getMetadata().getFunctionManager()), + new PredicatePushDown(getMetadata(), getQueryRunner().getSqlParser()), new IterativeOptimizer( new RuleStatsRecorder(), getQueryRunner().getStatsCalculator(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestSetFlatteningOptimizer.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestSetFlatteningOptimizer.java index 82e4d9b5082c7..1a32f93c84fd8 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestSetFlatteningOptimizer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestSetFlatteningOptimizer.java @@ -126,7 +126,7 @@ public void testDoesNotFlattenDifferentSetOperations() public void assertPlan(String sql, PlanMatchPattern pattern) { List optimizers = ImmutableList.of( - new UnaliasSymbolReferences(), + new UnaliasSymbolReferences(getMetadata().getFunctionManager()), new PruneUnreferencedOutputs(), new IterativeOptimizer( new RuleStatsRecorder(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestUnion.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestUnion.java index 9b2a7992e71a1..d8c6875922252 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestUnion.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestUnion.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.sql.planner.optimizations; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.sql.planner.LogicalPlanner; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.assertions.BasePlanTest; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.TopNNode; import com.google.common.collect.Iterables; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestAssingments.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestAssignments.java similarity index 77% rename from presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestAssingments.java rename to presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestAssignments.java index 7278b6560dd43..ce90da24f7d69 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestAssingments.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestAssignments.java @@ -13,23 +13,27 @@ */ package com.facebook.presto.sql.planner.plan; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.google.common.collect.ImmutableCollection; import org.testng.annotations.Test; +import java.util.List; + import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; import static com.facebook.presto.sql.tree.BooleanLiteral.TRUE_LITERAL; import static org.testng.Assert.assertTrue; -public class TestAssingments +public class TestAssignments { private final Assignments assignments = assignment(new VariableReferenceExpression("test", BIGINT), TRUE_LITERAL); - @Test + @Test(expectedExceptions = {UnsupportedOperationException.class}) public void testOutputsImmutable() { - assertTrue(assignments.getOutputs() instanceof ImmutableCollection); + List outputs = assignments.getOutputs(); + // should throw as it is an unmodifiableList + outputs.add(new VariableReferenceExpression("test", BIGINT)); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestStatisticsWriterNode.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestStatisticsWriterNode.java index af82c111cd2e0..d61c235fd763f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestStatisticsWriterNode.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestStatisticsWriterNode.java @@ -13,7 +13,13 @@ */ package com.facebook.presto.sql.planner.plan; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.metadata.HandleJsonModule; +import com.facebook.presto.metadata.HandleResolver; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -23,25 +29,26 @@ import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.PlanVariableAllocator; +import com.facebook.presto.testing.TestingHandleResolver; +import com.facebook.presto.testing.TestingMetadata.TestingTableHandle; +import com.facebook.presto.testing.TestingTransactionHandle; import com.facebook.presto.type.TypeDeserializer; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonModule; import org.testng.annotations.Test; +import java.util.Optional; import java.util.UUID; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.inject.multibindings.Multibinder.newSetBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static org.testng.Assert.assertEquals; public class TestStatisticsWriterNode @@ -55,7 +62,7 @@ public void testJsonCodec() JsonCodec jsonCodec = getJsonCodec(); StatisticsWriterNode expected = createStatisticsWriterNode(); StatisticsWriterNode deserialized = jsonCodec.fromJson(jsonCodec.toJson(expected)); - assertEquals(deserialized.getTarget(), expected.getTarget()); + assertEquals(deserialized.getTableHandle(), expected.getTableHandle()); assertEquals(deserialized.getRowCountVariable(), expected.getRowCountVariable()); assertEquals(deserialized.isRowCountEnabled(), expected.isRowCountEnabled()); assertEquals(deserialized.getDescriptor(), expected.getDescriptor()); @@ -92,7 +99,7 @@ private StatisticsWriterNode createStatisticsWriterNode() return new StatisticsWriterNode( newId(), new ValuesNode(newId(), COLUMNS.stream().map(column -> new VariableReferenceExpression(column, BIGINT)).collect(toImmutableList()), ImmutableList.of()), - new StatisticsWriterNode.TestWriteStatisticsHandle(), + new TableHandle(new ConnectorId("test"), new TestingTableHandle(), TestingTransactionHandle.create(), Optional.empty()), variableAllocator.newVariable("count", BIGINT), true, createTestDescriptor()); @@ -118,6 +125,8 @@ private JsonCodec getJsonCodec() .doNotInitializeLogging() .quiet() .initialize(); + HandleResolver handleResolver = injector.getInstance(HandleResolver.class); + handleResolver.addConnectorName("test", new TestingHandleResolver()); return injector.getInstance(new Key>() {}); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java index a2a0000c41600..4e93e30d1c3b8 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java @@ -13,11 +13,16 @@ */ package com.facebook.presto.sql.planner.plan; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.metadata.HandleJsonModule; import com.facebook.presto.server.SliceDeserializer; import com.facebook.presto.server.SliceSerializer; import com.facebook.presto.spi.block.SortOrder; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.CallExpression; @@ -27,7 +32,6 @@ import com.facebook.presto.sql.Serialization; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.PlanVariableAllocator; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; @@ -39,9 +43,6 @@ import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonModule; import io.airlift.slice.Slice; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -51,6 +52,9 @@ import java.util.Set; import java.util.UUID; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; @@ -59,9 +63,6 @@ import static com.facebook.presto.sql.planner.plan.WindowNode.Frame.WindowType.RANGE; import static com.facebook.presto.sql.relational.Expressions.call; import static com.google.inject.multibindings.Multibinder.newSetBinder; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static org.testng.Assert.assertEquals; public class TestWindowNode @@ -111,11 +112,9 @@ public void testSerializationRoundtrip() PlanNodeId id = newId(); WindowNode.Specification specification = new WindowNode.Specification( ImmutableList.of(columnA), - Optional.of(new OrderingScheme( - ImmutableList.of(columnB), - ImmutableMap.of(columnB, SortOrder.ASC_NULLS_FIRST)))); + Optional.of(new OrderingScheme(ImmutableList.of(new Ordering(columnB, SortOrder.ASC_NULLS_FIRST))))); CallExpression call = call("sum", functionHandle, BIGINT, new VariableReferenceExpression(columnC.getName(), BIGINT)); - Map functions = ImmutableMap.of(windowVariable, new WindowNode.Function(call, frame)); + Map functions = ImmutableMap.of(windowVariable, new WindowNode.Function(call, frame, false)); Optional hashVariable = Optional.of(columnB); Set prePartitionedInputs = ImmutableSet.of(columnA); WindowNode windowNode = new WindowNode( diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateAggregationsWithDefaultValues.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateAggregationsWithDefaultValues.java index e4099ae45f35b..28c72762251fa 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateAggregationsWithDefaultValues.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateAggregationsWithDefaultValues.java @@ -38,10 +38,10 @@ import java.util.Optional; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.spi.plan.AggregationNode.groupingSets; import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.FINAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.PARTIAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.groupingSets; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.LOCAL; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingAggregations.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingAggregations.java index c89f1ed4768a4..88329b22a6698 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingAggregations.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingAggregations.java @@ -37,8 +37,8 @@ import java.util.function.Function; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; public class TestValidateStreamingAggregations extends BasePlanTest diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestVerifyNoOriginalExpression.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestVerifyNoOriginalExpression.java index 449566b6455ad..425628bdfccc8 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestVerifyNoOriginalExpression.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestVerifyNoOriginalExpression.java @@ -15,30 +15,27 @@ import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.Metadata; -import com.facebook.presto.metadata.StaticFunctionHandle; import com.facebook.presto.spi.block.SortOrder; -import com.facebook.presto.spi.function.FunctionKind; -import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.type.BooleanType; -import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.planner.OrderingScheme; import com.facebook.presto.sql.planner.TestingWriterTarget; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.ApplyNode; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.facebook.presto.sql.planner.plan.SpatialJoinNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; @@ -57,22 +54,17 @@ import java.util.Optional; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.spi.function.OperatorType.LESS_THAN; import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression; +import static com.google.common.collect.ImmutableList.toImmutableList; public class TestVerifyNoOriginalExpression extends BasePlanTest { private static final SqlParser SQL_PARSER = new SqlParser(); private static final VariableReferenceExpression VARIABLE_REFERENCE_EXPRESSION = new VariableReferenceExpression("expr", BIGINT); - private static final CallExpression COMPARISON_CALL_EXPRESSION = new CallExpression( - "LESS_THAN", - new StaticFunctionHandle(new Signature( - "LESS_THAN", - FunctionKind.SCALAR, - new TypeSignature("boolean"))), - BooleanType.BOOLEAN, - ImmutableList.of(VARIABLE_REFERENCE_EXPRESSION, VARIABLE_REFERENCE_EXPRESSION)); private static final ComparisonExpression COMPARISON_EXPRESSION = new ComparisonExpression( ComparisonExpression.Operator.EQUAL, new SymbolReference("count"), @@ -81,6 +73,7 @@ public class TestVerifyNoOriginalExpression private Metadata metadata; private PlanBuilder builder; private ValuesNode valuesNode; + private CallExpression comparisonCallExpression; @BeforeClass public void setup() @@ -88,12 +81,17 @@ public void setup() metadata = getQueryRunner().getMetadata(); builder = new PlanBuilder(TEST_SESSION, new PlanNodeIdAllocator(), metadata); valuesNode = builder.values(); + comparisonCallExpression = new CallExpression( + "LESS_THAN", + metadata.getFunctionManager().resolveOperator(LESS_THAN, fromTypes(BIGINT, BIGINT)), + BooleanType.BOOLEAN, + ImmutableList.of(VARIABLE_REFERENCE_EXPRESSION, VARIABLE_REFERENCE_EXPRESSION)); } @Test public void testValidateForJoin() { - RowExpression predicate = COMPARISON_CALL_EXPRESSION; + RowExpression predicate = comparisonCallExpression; validateJoin(predicate, null, true); } @@ -120,8 +118,9 @@ public void testValidateForWindow() originalStartValue, originalEndValue); WindowNode.Function function = new WindowNode.Function( - COMPARISON_CALL_EXPRESSION, - frame); + comparisonCallExpression, + frame, + false); ImmutableList partitionBy = ImmutableList.of(VARIABLE_REFERENCE_EXPRESSION); Optional orderingScheme = Optional.empty(); ImmutableMap functions = ImmutableMap.of(VARIABLE_REFERENCE_EXPRESSION, function); @@ -135,7 +134,7 @@ public void testValidateForWindow() @Test public void testValidateSpatialJoin() { - RowExpression filter = COMPARISON_CALL_EXPRESSION; + RowExpression filter = comparisonCallExpression; validateSpatialJoinWithFilter(filter); } @@ -164,14 +163,12 @@ public void testAggregation() ImmutableList groupingKeys = ImmutableList.of(VARIABLE_REFERENCE_EXPRESSION); int groupingSetCount = 1; ImmutableMap orderings = ImmutableMap.of(VARIABLE_REFERENCE_EXPRESSION, SortOrder.ASC_NULLS_FIRST); - OrderingScheme orderingScheme = new OrderingScheme( - groupingKeys, - orderings); + OrderingScheme orderingScheme = new OrderingScheme(groupingKeys.stream().map(variable -> new Ordering(variable, orderings.get(variable))).collect(toImmutableList())); ImmutableMap aggregations = ImmutableMap.of( VARIABLE_REFERENCE_EXPRESSION, new AggregationNode.Aggregation( - COMPARISON_CALL_EXPRESSION, - Optional.of(COMPARISON_CALL_EXPRESSION), + comparisonCallExpression, + Optional.of(comparisonCallExpression), Optional.of(orderingScheme), false, Optional.of(new VariableReferenceExpression("orderkey", BIGINT)))); @@ -212,7 +209,7 @@ public void testTableFinish() TableFinishNode tableFinishNode = new TableFinishNode( new PlanNodeId("1"), valuesNode, - new TestingWriterTarget(), + Optional.of(new TestingWriterTarget()), VARIABLE_REFERENCE_EXPRESSION, Optional.empty(), Optional.empty()); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestVerifyOnlyOneOutputNode.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestVerifyOnlyOneOutputNode.java index a8fec0bc7fb97..2eccdb8146fb8 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestVerifyOnlyOneOutputNode.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestVerifyOnlyOneOutputNode.java @@ -14,14 +14,14 @@ package com.facebook.presto.sql.planner.sanity; import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.plan.Assignments; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; import com.facebook.presto.sql.planner.plan.OutputNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/query/QueryAssertions.java b/presto-main/src/test/java/com/facebook/presto/sql/query/QueryAssertions.java index 6d87df4b8719a..a4ba27211197e 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/query/QueryAssertions.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/query/QueryAssertions.java @@ -28,9 +28,9 @@ import java.util.List; import java.util.function.Consumer; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.google.common.base.Strings.nullToEmpty; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static java.lang.String.format; import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; @@ -53,6 +53,11 @@ public QueryAssertions(Session session) runner = new LocalQueryRunner(session); } + public QueryRunner getQueryRunner() + { + return runner; + } + public void assertFails(@Language("SQL") String sql, @Language("RegExp") String expectedMessageRegExp) { try { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/query/TestSessionFunctions.java b/presto-main/src/test/java/com/facebook/presto/sql/query/TestSessionFunctions.java index 608fbf4b08f7f..db5c29882fee3 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/query/TestSessionFunctions.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/query/TestSessionFunctions.java @@ -15,7 +15,6 @@ import com.facebook.presto.Session; import com.facebook.presto.spi.security.Identity; -import com.facebook.presto.sql.SqlPath; import org.testng.annotations.Test; import java.util.Optional; @@ -32,24 +31,4 @@ public void testCurrentUser() queryAssertions.assertQuery("SELECT CURRENT_USER", "SELECT CAST('" + session.getUser() + "' AS VARCHAR)"); } } - - @Test - public void testCurrentPath() - { - Session session = testSessionBuilder() - .setPath(new SqlPath(Optional.of("testPath"))) - .build(); - - try (QueryAssertions queryAssertions = new QueryAssertions(session)) { - queryAssertions.assertQuery("SELECT CURRENT_PATH", "SELECT CAST('" + session.getPath().toString() + "' AS VARCHAR)"); - } - - Session emptyPathSession = testSessionBuilder() - .setPath(new SqlPath(Optional.of("\"\""))) - .build(); - - try (QueryAssertions queryAssertions = new QueryAssertions(emptyPathSession)) { - queryAssertions.assertQuery("SELECT CURRENT_PATH", "SELECT CAST('" + emptyPathSession.getPath().toString() + "' AS VARCHAR)"); - } - } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java b/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java index 0d8df87e9afc9..d19f642063fdd 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java @@ -13,12 +13,14 @@ */ package com.facebook.presto.sql.query; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; -import com.facebook.presto.sql.planner.plan.AggregationNode; import com.facebook.presto.sql.planner.plan.JoinNode; -import com.facebook.presto.sql.planner.plan.ProjectNode; +import com.facebook.presto.testing.QueryRunner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.intellij.lang.annotations.Language; @@ -28,6 +30,9 @@ import java.util.function.Consumer; +import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.exchange; @@ -35,9 +40,6 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.functionCall; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.FINAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.PARTIAL; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.LOCAL; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; import static org.testng.Assert.assertEquals; @@ -61,6 +63,15 @@ public void teardown() assertions = null; } + @Test(expectedExceptions = PrestoException.class, expectedExceptionsMessageRegExp = UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG) + public void testCorrelatedSubqueriesWithDistinct() + { + QueryRunner runner = assertions.getQueryRunner(); + runner.execute( + runner.getDefaultSession(), + "select a from (values (1, 10), (2, 20)) t(a,b) where a in (select distinct c from (values 1) t2(c) where b in (10, 11))"); + } + @Test public void testCorrelatedExistsSubqueriesWithOrPredicateAndNull() { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestLogicalRowExpressions.java b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestLogicalRowExpressions.java index 2f6492d1b302e..3e96f30548d3b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestLogicalRowExpressions.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestLogicalRowExpressions.java @@ -14,9 +14,10 @@ package com.facebook.presto.sql.relational; import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.relation.CallExpression; -import com.facebook.presto.spi.relation.LogicalRowExpressions; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -27,7 +28,18 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import static com.facebook.presto.spi.relation.LogicalRowExpressions.extractPredicates; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static com.facebook.presto.expressions.LogicalRowExpressions.FALSE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; +import static com.facebook.presto.expressions.LogicalRowExpressions.extractPredicates; +import static com.facebook.presto.spi.function.OperatorType.EQUAL; +import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; +import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN_OR_EQUAL; +import static com.facebook.presto.spi.function.OperatorType.LESS_THAN; +import static com.facebook.presto.spi.function.OperatorType.LESS_THAN_OR_EQUAL; +import static com.facebook.presto.spi.function.OperatorType.NOT_EQUAL; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.OR; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -43,24 +55,26 @@ public class TestLogicalRowExpressions { private FunctionManager functionManager; private LogicalRowExpressions logicalRowExpressions; + private static final RowExpression a = name("a"); + private static final RowExpression b = name("b"); + private static final RowExpression c = name("c"); + private static final RowExpression d = name("d"); + private static final RowExpression e = name("e"); + private static final RowExpression f = name("f"); + private static final RowExpression g = name("g"); + private static final RowExpression h = name("h"); @BeforeClass public void setup() { TypeManager typeManager = new TypeRegistry(); functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(functionManager), new FunctionResolution(functionManager)); + logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(functionManager), new FunctionResolution(functionManager), functionManager); } @Test public void testAnd() { - RowExpression a = name("a"); - RowExpression b = name("b"); - RowExpression c = name("c"); - RowExpression d = name("d"); - RowExpression e = name("e"); - assertEquals( LogicalRowExpressions.and(a, b, c, d, e), and(and(and(a, b), and(c, d)), e)); @@ -85,12 +99,6 @@ public void testAnd() @Test public void testOr() { - RowExpression a = name("a"); - RowExpression b = name("b"); - RowExpression c = name("c"); - RowExpression d = name("d"); - RowExpression e = name("e"); - assertEquals( LogicalRowExpressions.or(a, b, c, d, e), or(or(or(a, b), or(c, d)), e)); @@ -111,8 +119,6 @@ public void testOr() @Test public void testDeterminism() { - RowExpression a = name("a"); - RowExpression b = name("b"); RowExpression nondeterministic = call("random", functionManager.lookupFunction("random", fromTypes()), DOUBLE); RowExpression deterministic = call("length", functionManager.lookupFunction("length", fromTypes(VARCHAR)), INTEGER); @@ -125,11 +131,6 @@ public void testDeterminism() @Test public void testPushNegationToLeaves() { - RowExpression a = name("a"); - RowExpression b = name("b"); - RowExpression c = name("c"); - RowExpression d = name("d"); - assertEquals(logicalRowExpressions.pushNegationToLeaves(not(and(a, b))), or(not(a), not(b))); assertEquals(logicalRowExpressions.pushNegationToLeaves(not(or(a, b))), and(not(a), not(b))); assertEquals(logicalRowExpressions.pushNegationToLeaves(not(or(not(a), not(b)))), and(a, b)); @@ -141,6 +142,126 @@ public void testPushNegationToLeaves() assertEquals(logicalRowExpressions.pushNegationToLeaves(not(or(not(and(a, b)), not(or(b, not(and(c, d))))))), and(and(a, b), or(b, or(not(c), not(d))))); assertEquals(logicalRowExpressions.pushNegationToLeaves(or(not(and(a, b)), not(or(b, not(and(c, d)))))), or(or(not(a), not(b)), and(not(b), and(c, d)))); assertEquals(logicalRowExpressions.pushNegationToLeaves(and(and(a, b), c)), and(and(a, b), c)); + assertEquals(logicalRowExpressions.pushNegationToLeaves(not(compare(a, EQUAL, b))), compare(a, NOT_EQUAL, b)); + assertEquals(logicalRowExpressions.pushNegationToLeaves(not(compare(a, NOT_EQUAL, b))), compare(a, EQUAL, b)); + assertEquals(logicalRowExpressions.pushNegationToLeaves(not(compare(a, GREATER_THAN, b))), compare(a, LESS_THAN_OR_EQUAL, b)); + assertEquals(logicalRowExpressions.pushNegationToLeaves(not(compare(a, GREATER_THAN_OR_EQUAL, b))), compare(a, LESS_THAN, b)); + assertEquals(logicalRowExpressions.pushNegationToLeaves(not(compare(a, GREATER_THAN_OR_EQUAL, not(not(b))))), compare(a, LESS_THAN, b)); + assertEquals(logicalRowExpressions.pushNegationToLeaves(not(compare(a, LESS_THAN, b))), compare(a, GREATER_THAN_OR_EQUAL, b)); + assertEquals(logicalRowExpressions.pushNegationToLeaves(not(compare(a, LESS_THAN_OR_EQUAL, b))), compare(a, GREATER_THAN, b)); + assertEquals(logicalRowExpressions.pushNegationToLeaves(not(compare(a, LESS_THAN_OR_EQUAL, not(not(b))))), compare(a, GREATER_THAN, b)); + } + + @Test + public void testEliminateConstant() + { + // Testing eliminate constant + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(or(and(TRUE_CONSTANT, a), and(FALSE_CONSTANT, b))), + a); + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(or(and(TRUE_CONSTANT, a), and(FALSE_CONSTANT, b))), + a); + + // Testing eliminate constant in nested tree + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(and(a, and(b, or(c, and(FALSE_CONSTANT, d))))), + and(and(a, b), c)); + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(and(a, and(b, or(c, and(e, or(f, and(FALSE_CONSTANT, d))))))), + and(and(a, b), or(c, and(e, f)))); + } + + @Test + public void testEliminateDuplicate() + { + RowExpression nd = call("random", functionManager.lookupFunction("random", fromTypes()), DOUBLE); + + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(or(and(TRUE_CONSTANT, a), and(b, b))), + or(a, b)); + + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(or(and(a, b), and(a, b))), + and(a, b)); + // we will prefer most simplified expression than correct conjunctive/disjunctive form + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(or(and(a, b), and(a, b))), + and(a, b)); + + // eliminate duplicated items with different order, prefers the ones appears first. + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(or(and(b, a), and(a, b))), + and(b, a)); + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(or(and(b, a), and(a, b))), + and(b, a)); + + // (b && a) || a + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(or(and(b, a), a)), + a); + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(or(and(b, a), a)), + a); + // (b || a) && a + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(and(a, or(b, a))), + a); + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(and(a, or(b, a))), + a); + + // (b && a) || (a && b && c) -> b && a (should keep b && a instead of a && b as it appears first) + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(or(and(b, a), and(and(a, b), c))), + and(b, a)); + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(or(and(b, a), and(and(a, b), c))), + and(b, a)); + + // (b || a) && (a || b) && (a || b || c || d) || (a || b || c) -> b || a (should keep b || a instead of a || b as it appears first) + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(and(or(b, a), and(or(a, b), and(or(or(c, d), or(a, b)), or(a, or(b, c)))))), + or(b, a)); + + // (b || a) && (a || b || c) && (a || b) && (a || b || nd) && e + // we cannot eliminate nd because it is non-deterministic + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(and(and(or(b, a), and(or(a, or(b, c)), and(or(a, b), or(or(a, b), nd)))), e)), + and(and(or(b, a), or(or(a, b), nd)), e)); + // we cannot convert to disjunctive form because nd is non-deterministic + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(and(and(or(b, a), and(or(a, or(b, c)), and(or(a, b), or(or(a, b), nd)))), e)), + and(and(or(b, a), or(or(a, b), nd)), e)); + + // (b || a) && (a || b || c) && (a || b) && (a || b || d) && e + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(and(and(or(b, a), and(or(a, or(b, c)), and(or(a, b), or(or(a, b), d)))), e)), + and(or(b, a), e)); + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(and(and(or(b, a), and(or(a, or(b, c)), and(or(a, b), or(or(a, b), d)))), e)), + or(and(b, e), and(a, e))); + + // (b || a || c) && (a || b || d) && (a || b || e) && (a || b || f) + // already conjunctive form + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(and(or(or(b, a), c), and(or(d, or(a, b)), and(or(or(a, b), e), or(or(a, b), f))))), + and(and(or(or(b, a), c), or(or(b, a), d)), and(or(or(b, a), e), or(or(b, a), f)))); + // (b || a || c) && (a || b || d) && (a || b || f) -> b || a || (c && d && e && f) + // can be simplified by extract common predicates + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(and(or(or(b, a), c), and(or(d, or(a, b)), and(or(or(a, b), e), or(or(a, b), f))))), + or(or(b, a), and(and(c, d), and(e, f)))); + + // de-duplicate nested expression + // ((a && b) || (a && c) || (a && d)) && ((b && c) || (c && a) || (c && d)) -> a && c + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(and(or(and(a, b), and(a, c), and(a, d)), or(and(c, b), and(c, a), and(c, d)))), + and(a, c)); + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(and(or(and(a, b), and(a, c), and(a, d)), or(and(c, b), and(c, a), and(c, d)))), + and(a, c)); } @Test @@ -160,21 +281,18 @@ public void testConvertToCNF() // left = 3, right = 3 --> (3) expand and distribute both // Now let us test each of these permutations - RowExpression a = name("a"); - RowExpression b = name("b"); - RowExpression c = name("c"); - RowExpression d = name("d"); - RowExpression e = name("e"); - RowExpression f = name("f"); - RowExpression g = name("g"); - RowExpression h = name("h"); - // 1, 1 assertEquals( logicalRowExpressions.convertToConjunctiveNormalForm(and(a, b)), and(a, b), "Failed 1,1"); + // Should not change shape + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(or(a, b)), + or(a, b), + "Failed to keep same form if cannot convert"); + // 1, 1 with not pushdown assertEquals( logicalRowExpressions.convertToConjunctiveNormalForm(not(and(a, b))), @@ -221,7 +339,7 @@ public void testConvertToCNF() // 2, 1 assertEquals( logicalRowExpressions.convertToConjunctiveNormalForm(or(c, or(a, b))), - or(c, or(a, b)), + or(or(c, a), b), "Failed 2,1"); // 3, 1 @@ -236,31 +354,68 @@ public void testConvertToCNF() and(or(or(e, f), or(a, b)), or(or(g, h), or(a, b))), "Failed 3,2"); - // 3, 3 large with NOT pushdown + // 3, 3 large with NOT push down + // (a && b && (e || g)) || !(a && b) || !(b || !(c && d)) ==> (a || !a || !b) && (b || !a || !b) && ((e || g) || !a || !b) + // notice that a || !a cannot be easily optimized away in SQL since we have to handle if a is unknown (null). assertEquals( logicalRowExpressions.convertToConjunctiveNormalForm(or(and(a, and(b, or(e, g))), or(not(and(a, b)), not(or(b, not(and(c, d))))))), - and(and(and(and(or(a, or(or(not(a), not(b)), not(b))), or(a, or(or(not(a), not(b)), c))), and(or(a, or(or(not(a), not(b)), d)), or(b, or(or(not(a), not(b)), not(b))))), and(and(or(b, or(or(not(a), not(b)), c)), or(b, or(or(not(a), not(b)), d))), and(or(or(e, g), or(or(not(a), not(b)), not(b))), or(or(e, g), or(or(not(a), not(b)), c))))), or(or(e, g), or(or(not(a), not(b)), d))), + and(and(or(or(a, not(a)), not(b)), or(or(b, not(a)), not(b))), or(or(e, g), or(not(a), not(b)))), "Failed 3,3 large with NOT pushdown"); + + // a || b || c || d || (d && e) || (e && f) ==> (a || b || c || d || e) && (a || b || c || d || f) + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm( + or(a, or(b, or(c, or(d, or(and(d, e), and(e, f))))))), + and(or(or(or(a, b), or(c, d)), e), or(or(or(a, b), or(c, d)), f))); + + // (a && b && c) || (d && e) can increase size significantly so do not expand. + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(or(and(a, and(b, c)), and(and(d, e), f))), + or(and(and(a, b), c), and(and(d, e), f))); } @Test - public void testConvertToDNF() + public void testBigExpressions() { - RowExpression a = name("a"); - RowExpression b = name("b"); - RowExpression c = name("c"); - RowExpression d = name("d"); - RowExpression e = name("e"); - RowExpression f = name("f"); - RowExpression g = name("g"); - RowExpression h = name("h"); + // Do not expand big list (a0 && b0) || (a1 && b1) || .... + RowExpression bigExpression = or(IntStream.range(0, 1000) + .boxed() + .map(i -> and(name("a" + i), name("b" + i))) + .toArray(RowExpression[]::new)); + assertEquals(logicalRowExpressions.convertToConjunctiveNormalForm(bigExpression), bigExpression); + + // extract common predicates on (a && b0) || (a && b1) || .... + RowExpression bigExpressionWithCommonPredicate = or(IntStream.range(0, 10001) + .boxed() + .map(i -> and(name("a"), name("b" + i))) + .toArray(RowExpression[]::new)); + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(bigExpressionWithCommonPredicate), + or(IntStream.range(0, 10001).boxed().map(i -> and(name("a"), name("b" + i))).toArray(RowExpression[]::new))); + // a || (a && b0) || (a && b1) || ... can be simplified to a but if conjunctive is very large, we will skip reduction. + assertEquals( + logicalRowExpressions.convertToConjunctiveNormalForm(or(a, bigExpressionWithCommonPredicate)), + or(Stream.concat(Stream.of(name("a")), IntStream.range(0, 10001).boxed().map(i -> and(name("a"), name("b" + i)))).toArray(RowExpression[]::new))); + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(or(a, bigExpressionWithCommonPredicate)), + or(Stream.concat(Stream.of(name("a")), IntStream.range(0, 10001).boxed().map(i -> and(name("a"), name("b" + i)))).toArray(RowExpression[]::new))); + } + @Test + public void testConvertToDNF() + { // 1, 1 assertEquals( logicalRowExpressions.convertToDisjunctiveNormalForm(or(a, b)), or(a, b), "Failed 1,1"); + // Should not change shape + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(and(a, b)), + and(a, b), + "Failed to keep same form if cannot convert"); + // 1, 1 with not pushdown assertEquals( logicalRowExpressions.convertToDisjunctiveNormalForm(not(or(a, b))), @@ -307,7 +462,7 @@ public void testConvertToDNF() // 2, 1 assertEquals( logicalRowExpressions.convertToDisjunctiveNormalForm(and(c, and(a, b))), - and(c, and(a, b)), + and(and(c, a), b), "Failed 2,1"); // 3, 1 @@ -325,8 +480,13 @@ public void testConvertToDNF() // 3, 3 large with NOT pushdown assertEquals( logicalRowExpressions.convertToDisjunctiveNormalForm(and(or(a, or(b, and(e, g))), and(not(or(a, b)), not(and(b, not(or(c, d))))))), - or(or(or(or(and(a, and(and(not(a), not(b)), not(b))), and(a, and(and(not(a), not(b)), c))), or(and(a, and(and(not(a), not(b)), d)), and(b, and(and(not(a), not(b)), not(b))))), or(or(and(b, and(and(not(a), not(b)), c)), and(b, and(and(not(a), not(b)), d))), or(and(and(e, g), and(and(not(a), not(b)), not(b))), and(and(e, g), and(and(not(a), not(b)), c))))), and(and(e, g), and(and(not(a), not(b)), d))), + or(or(and(and(a, not(a)), not(b)), and(and(b, not(a)), not(b))), and(and(e, g), and(not(a), not(b)))), "Failed 3,3 large with NOT pushdown"); + + // (a || b || c) && ( d || e) will expand to too big if we convert to disjunctive form. + assertEquals( + logicalRowExpressions.convertToDisjunctiveNormalForm(and(or(a, or(b, c)), or(or(d, e), f))), + and(or(or(a, b), c), or(or(d, e), f))); } private static RowExpression name(String name) @@ -334,11 +494,26 @@ private static RowExpression name(String name) return new VariableReferenceExpression(name, BOOLEAN); } + private RowExpression compare(RowExpression left, OperatorType operator, RowExpression right) + { + return call( + operator.getOperator(), + new FunctionResolution(functionManager).comparisonFunction(operator, left.getType(), right.getType()), + BOOLEAN, + left, + right); + } + private RowExpression and(RowExpression left, RowExpression right) { return new SpecialFormExpression(AND, BOOLEAN, left, right); } + private RowExpression or(RowExpression... expressions) + { + return logicalRowExpressions.or(expressions); + } + private RowExpression or(RowExpression left, RowExpression right) { return new SpecialFormExpression(OR, BOOLEAN, left, right); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestRowExpressionOptimizer.java similarity index 85% rename from presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java rename to presto-main/src/test/java/com/facebook/presto/sql/relational/TestRowExpressionOptimizer.java index ec1284645f325..0ad5089be2a8d 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionOptimizer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestRowExpressionOptimizer.java @@ -11,10 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql; +package com.facebook.presto.sql.relational; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.block.IntArrayBlock; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.relation.CallExpression; @@ -25,13 +26,13 @@ import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import com.facebook.presto.sql.relational.optimizer.ExpressionOptimizer; import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.block.BlockAssertions.toValues; import static com.facebook.presto.metadata.CastType.CAST; import static com.facebook.presto.metadata.CastType.JSON_TO_ARRAY_CAST; @@ -39,6 +40,7 @@ import static com.facebook.presto.metadata.CastType.JSON_TO_ROW_CAST; import static com.facebook.presto.spi.function.OperatorType.ADD; import static com.facebook.presto.spi.function.OperatorType.EQUAL; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IF; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -53,20 +55,19 @@ import static com.facebook.presto.type.JsonType.JSON; import static com.facebook.presto.util.StructuralTestUtil.mapType; import static io.airlift.slice.Slices.utf8Slice; -import static io.airlift.testing.Assertions.assertInstanceOf; import static org.testng.Assert.assertEquals; -public class TestExpressionOptimizer +public class TestRowExpressionOptimizer { private FunctionManager functionManager; - private ExpressionOptimizer optimizer; + private RowExpressionOptimizer optimizer; @BeforeClass public void setUp() { TypeManager typeManager = new TypeRegistry(); functionManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - optimizer = new ExpressionOptimizer(functionManager, SESSION); + optimizer = new RowExpressionOptimizer(MetadataManager.createTestMetadataManager()); } @AfterClass(alwaysRun = true) @@ -83,19 +84,19 @@ public void testPossibleExponentialOptimizationTime() FunctionHandle functionHandle = functionManager.resolveOperator(ADD, fromTypes(BIGINT, BIGINT)); expression = new CallExpression(ADD.name(), functionHandle, BIGINT, ImmutableList.of(expression, constant(1L, BIGINT))); } - optimizer.optimize(expression); + optimize(expression); } @Test public void testIfConstantOptimization() { - assertEquals(optimizer.optimize(ifExpression(constant(true, BOOLEAN), 1L, 2L)), constant(1L, BIGINT)); - assertEquals(optimizer.optimize(ifExpression(constant(false, BOOLEAN), 1L, 2L)), constant(2L, BIGINT)); - assertEquals(optimizer.optimize(ifExpression(constant(null, BOOLEAN), 1L, 2L)), constant(2L, BIGINT)); + assertEquals(optimize(ifExpression(constant(true, BOOLEAN), 1L, 2L)), constant(1L, BIGINT)); + assertEquals(optimize(ifExpression(constant(false, BOOLEAN), 1L, 2L)), constant(2L, BIGINT)); + assertEquals(optimize(ifExpression(constant(null, BOOLEAN), 1L, 2L)), constant(2L, BIGINT)); FunctionHandle bigintEquals = functionManager.resolveOperator(EQUAL, fromTypes(BIGINT, BIGINT)); RowExpression condition = new CallExpression(EQUAL.name(), bigintEquals, BOOLEAN, ImmutableList.of(constant(3L, BIGINT), constant(3L, BIGINT))); - assertEquals(optimizer.optimize(ifExpression(condition, 1L, 2L)), constant(1L, BIGINT)); + assertEquals(optimize(ifExpression(condition, 1L, 2L)), constant(1L, BIGINT)); } @Test @@ -106,7 +107,7 @@ public void testCastWithJsonParseOptimization() // constant FunctionHandle jsonCastFunctionHandle = functionManager.lookupCast(CAST, JSON.getTypeSignature(), parseTypeSignature("array(integer)")); RowExpression jsonCastExpression = new CallExpression(CAST.name(), jsonCastFunctionHandle, new ArrayType(INTEGER), ImmutableList.of(call("json_parse", jsonParseFunctionHandle, JSON, constant(utf8Slice("[1, 2]"), VARCHAR)))); - RowExpression resultExpression = optimizer.optimize(jsonCastExpression); + RowExpression resultExpression = optimize(jsonCastExpression); assertInstanceOf(resultExpression, ConstantExpression.class); Object resultValue = ((ConstantExpression) resultExpression).getValue(); assertInstanceOf(resultValue, IntArrayBlock.class); @@ -115,7 +116,7 @@ public void testCastWithJsonParseOptimization() // varchar to array jsonCastFunctionHandle = functionManager.lookupCast(CAST, JSON.getTypeSignature(), parseTypeSignature("array(varchar)")); jsonCastExpression = call(CAST.name(), jsonCastFunctionHandle, new ArrayType(VARCHAR), ImmutableList.of(call("json_parse", jsonParseFunctionHandle, JSON, field(1, VARCHAR)))); - resultExpression = optimizer.optimize(jsonCastExpression); + resultExpression = optimize(jsonCastExpression); assertEquals( resultExpression, call(JSON_TO_ARRAY_CAST.name(), functionManager.lookupCast(JSON_TO_ARRAY_CAST, VARCHAR.getTypeSignature(), parseTypeSignature("array(varchar)")), new ArrayType(VARCHAR), field(1, VARCHAR))); @@ -123,7 +124,7 @@ public void testCastWithJsonParseOptimization() // varchar to map jsonCastFunctionHandle = functionManager.lookupCast(CAST, JSON.getTypeSignature(), parseTypeSignature("map(integer,varchar)")); jsonCastExpression = call(CAST.name(), jsonCastFunctionHandle, mapType(INTEGER, VARCHAR), ImmutableList.of(call("json_parse", jsonParseFunctionHandle, JSON, field(1, VARCHAR)))); - resultExpression = optimizer.optimize(jsonCastExpression); + resultExpression = optimize(jsonCastExpression); assertEquals( resultExpression, call(JSON_TO_MAP_CAST.name(), functionManager.lookupCast(JSON_TO_MAP_CAST, VARCHAR.getTypeSignature(), parseTypeSignature("map(integer, varchar)")), mapType(INTEGER, VARCHAR), field(1, VARCHAR))); @@ -131,7 +132,7 @@ public void testCastWithJsonParseOptimization() // varchar to row jsonCastFunctionHandle = functionManager.lookupCast(CAST, JSON.getTypeSignature(), parseTypeSignature("row(varchar,bigint)")); jsonCastExpression = call(CAST.name(), jsonCastFunctionHandle, RowType.anonymous(ImmutableList.of(VARCHAR, BIGINT)), ImmutableList.of(call("json_parse", jsonParseFunctionHandle, JSON, field(1, VARCHAR)))); - resultExpression = optimizer.optimize(jsonCastExpression); + resultExpression = optimize(jsonCastExpression); assertEquals( resultExpression, call(JSON_TO_ROW_CAST.name(), functionManager.lookupCast(JSON_TO_ROW_CAST, VARCHAR.getTypeSignature(), parseTypeSignature("row(varchar,bigint)")), RowType.anonymous(ImmutableList.of(VARCHAR, BIGINT)), field(1, VARCHAR))); @@ -141,4 +142,9 @@ private static RowExpression ifExpression(RowExpression condition, long trueValu { return new SpecialFormExpression(IF, BIGINT, ImmutableList.of(condition, constant(trueValue, BIGINT), constant(falseValue, BIGINT))); } + + private RowExpression optimize(RowExpression expression) + { + return optimizer.optimize(expression, OPTIMIZED, SESSION); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestRowExpressionTranslator.java b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestRowExpressionTranslator.java new file mode 100644 index 0000000000000..00191d0166d53 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestRowExpressionTranslator.java @@ -0,0 +1,295 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.relational; + +import com.facebook.presto.expressions.translator.FunctionTranslator; +import com.facebook.presto.expressions.translator.RowExpressionTranslator; +import com.facebook.presto.expressions.translator.RowExpressionTreeTranslator; +import com.facebook.presto.expressions.translator.TranslatedExpression; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.ScalarOperator; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.sql.TestingRowExpressionTranslator; +import com.facebook.presto.sql.planner.TypeProvider; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.facebook.presto.expressions.translator.FunctionTranslator.buildFunctionTranslator; +import static com.facebook.presto.expressions.translator.RowExpressionTreeTranslator.translateWith; +import static com.facebook.presto.expressions.translator.TranslatedExpression.untranslated; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Objects.requireNonNull; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestRowExpressionTranslator +{ + private static final Metadata METADATA = MetadataManager.createTestMetadataManager(); + + private final FunctionManager functionManager; + private final TestingRowExpressionTranslator sqlToRowExpressionTranslator; + + public TestRowExpressionTranslator() + { + this.functionManager = METADATA.getFunctionManager(); + this.sqlToRowExpressionTranslator = new TestingRowExpressionTranslator(METADATA); + } + + @Test + public void testEndToEndFunctionTranslation() + { + String untranslated = "LN(bitwise_and(1, col1))"; + TypeProvider typeProvider = TypeProvider.copyOf(ImmutableMap.of("col1", BIGINT)); + CallExpression callExpression = (CallExpression) sqlToRowExpressionTranslator.translate(expression(untranslated), typeProvider); + + TranslatedExpression translatedExpression = translateWith( + callExpression, + new TestFunctionTranslator(functionManager, buildFunctionTranslator(ImmutableSet.of(TestFunctions.class))), + emptyMap()); + assertTrue(translatedExpression.getTranslated().isPresent()); + assertEquals(translatedExpression.getTranslated().get(), "LNof(1 BITWISE_AND col1)"); + } + + @Test + public void testEndToEndSpecialFormTranslation() + { + String untranslated = "col1 AND col2"; + TypeProvider typeProvider = TypeProvider.copyOf(ImmutableMap.of("col1", BOOLEAN, "col2", BOOLEAN)); + + RowExpression specialForm = sqlToRowExpressionTranslator.translate(expression(untranslated), typeProvider); + + TranslatedExpression translatedExpression = translateWith( + specialForm, + new TestFunctionTranslator(functionManager, buildFunctionTranslator(ImmutableSet.of(TestFunctions.class))), + emptyMap()); + assertTrue(translatedExpression.getTranslated().isPresent()); + assertEquals(translatedExpression.getTranslated().get(), "col1 TEST_AND col2"); + } + + @Test + public void testMissingFunctionTranslator() + { + String untranslated = "ABS(col1)"; + TypeProvider typeProvider = TypeProvider.copyOf(ImmutableMap.of("col1", DOUBLE)); + + RowExpression specialForm = sqlToRowExpressionTranslator.translate(expression(untranslated), typeProvider); + + TranslatedExpression translatedExpression = translateWith( + specialForm, + new TestFunctionTranslator(functionManager, buildFunctionTranslator(ImmutableSet.of(TestFunctions.class))), + emptyMap()); + assertFalse(translatedExpression.getTranslated().isPresent()); + } + + @Test + public void testIncorrectFunctionSignatureInDefinition() + { + String untranslated = "CEIL(col1)"; + TypeProvider typeProvider = TypeProvider.copyOf(ImmutableMap.of("col1", DOUBLE)); + + RowExpression specialForm = sqlToRowExpressionTranslator.translate(expression(untranslated), typeProvider); + + TranslatedExpression translatedExpression = translateWith( + specialForm, + new TestFunctionTranslator(functionManager, buildFunctionTranslator(ImmutableSet.of(TestFunctions.class))), + emptyMap()); + assertFalse(translatedExpression.getTranslated().isPresent()); + } + + @Test + public void testHiddenFunctionNot() + { + String untranslated = "NOT true"; + RowExpression specialForm = sqlToRowExpressionTranslator.translate(expression(untranslated), TypeProvider.empty()); + + TranslatedExpression translatedExpression = translateWith( + specialForm, + new TestFunctionTranslator(functionManager, buildFunctionTranslator(ImmutableSet.of(TestFunctions.class))), + emptyMap()); + assertTrue(translatedExpression.getTranslated().isPresent()); + assertEquals(translatedExpression.getTranslated().get(), "NOT_2 true"); + } + + @Test + public void testBasicOperator() + { + String untranslated = "col1 + col2"; + TypeProvider typeProvider = TypeProvider.copyOf(ImmutableMap.of("col1", BIGINT, "col2", BIGINT)); + RowExpression specialForm = sqlToRowExpressionTranslator.translate(expression(untranslated), typeProvider); + + TranslatedExpression translatedExpression = translateWith( + specialForm, + new TestFunctionTranslator(functionManager, buildFunctionTranslator(ImmutableSet.of(TestFunctions.class))), + emptyMap()); + assertTrue(translatedExpression.getTranslated().isPresent()); + assertEquals(translatedExpression.getTranslated().get(), "col1 -|- col2"); + } + + @Test + public void testLessThanOperator() + { + String untranslated = "col1 < col2"; + TypeProvider typeProvider = TypeProvider.copyOf(ImmutableMap.of("col1", BIGINT, "col2", BIGINT)); + RowExpression specialForm = sqlToRowExpressionTranslator.translate(expression(untranslated), typeProvider); + + TranslatedExpression translatedExpression = translateWith( + specialForm, + new TestFunctionTranslator(functionManager, buildFunctionTranslator(ImmutableSet.of(TestFunctions.class))), + emptyMap()); + assertTrue(translatedExpression.getTranslated().isPresent()); + assertEquals(translatedExpression.getTranslated().get(), "col1 LT col2"); + } + + @Test + public void testUntranslatableSpecialForm() + { + String untranslated = "col1 OR col2"; + TypeProvider typeProvider = TypeProvider.copyOf(ImmutableMap.of("col1", BOOLEAN, "col2", BOOLEAN)); + RowExpression specialForm = sqlToRowExpressionTranslator.translate(expression(untranslated), typeProvider); + + TranslatedExpression translatedExpression = translateWith( + specialForm, + new TestFunctionTranslator(functionManager, buildFunctionTranslator(ImmutableSet.of(TestFunctions.class))), + emptyMap()); + assertFalse(translatedExpression.getTranslated().isPresent()); + } + + private class TestFunctionTranslator + extends RowExpressionTranslator> + { + private final FunctionManager functionManager; + private final FunctionTranslator functionTranslator; + + TestFunctionTranslator(FunctionManager functionManager, FunctionTranslator functionTranslator) + { + this.functionTranslator = requireNonNull(functionTranslator); + this.functionManager = requireNonNull(functionManager); + } + + @Override + public TranslatedExpression translateConstant(ConstantExpression literal, Map context, RowExpressionTreeTranslator> rowExpressionTreeTranslator) + { + return new TranslatedExpression<>(Optional.of(literal.toString()), literal, emptyList()); + } + + @Override + public TranslatedExpression translateCall(CallExpression callExpression, Map context, RowExpressionTreeTranslator> rowExpressionTreeTranslator) + { + List> translatedExpressions = callExpression.getArguments().stream() + .map(expression -> rowExpressionTreeTranslator.rewrite(expression, context)) + .collect(Collectors.toList()); + FunctionMetadata functionMetadata = functionManager.getFunctionMetadata(callExpression.getFunctionHandle()); + try { + return functionTranslator.translate(functionMetadata, callExpression, translatedExpressions); + } + catch (Throwable t) { + return untranslated(callExpression, translatedExpressions); + } + } + + @Override + public TranslatedExpression translateSpecialForm(SpecialFormExpression specialFormExpression, Map context, RowExpressionTreeTranslator> rowExpressionTreeTranslator) + { + if (!specialFormExpression.getForm().equals(SpecialFormExpression.Form.AND)) { + return untranslated(specialFormExpression); + } + + List> translatedExpressions = specialFormExpression.getArguments().stream() + .map(expression -> rowExpressionTreeTranslator.rewrite(expression, context)) + .collect(Collectors.toList()); + + assertTrue(translatedExpressions.get(0).getTranslated().isPresent()); + assertTrue(translatedExpressions.get(1).getTranslated().isPresent()); + return new TranslatedExpression<>( + Optional.of(translatedExpressions.get(0).getTranslated().get() + " TEST_AND " + translatedExpressions.get(1).getTranslated().get()), + specialFormExpression, + translatedExpressions); + } + + @Override + public TranslatedExpression translateVariable(VariableReferenceExpression variable, Map context, RowExpressionTreeTranslator> rowExpressionTreeTranslator) + { + return new TranslatedExpression<>(Optional.of(variable.getName()), variable, emptyList()); + } + } + + public static class TestFunctions + { + @ScalarFunction + @SqlType(StandardTypes.BIGINT) + public static String bitwiseAnd(@SqlType(StandardTypes.BIGINT) String left, @SqlType(StandardTypes.BIGINT) String right) + { + return left + " BITWISE_AND " + right; + } + + @ScalarFunction("ln") + @SqlType(StandardTypes.DOUBLE) + public static String ln(@SqlType(StandardTypes.DOUBLE) String sql) + { + return "LNof(" + sql + ")"; + } + + @ScalarFunction("ceil") + @SqlType(StandardTypes.DOUBLE) + public static String ceil(@SqlType(StandardTypes.BOOLEAN) String sql) + { + return "CEILof(" + sql + ")"; + } + + @ScalarFunction("not") + @SqlType(StandardTypes.BOOLEAN) + public static String not(@SqlType(StandardTypes.BOOLEAN) String sql) + { + return "NOT_2 " + sql; + } + + @ScalarOperator(OperatorType.ADD) + @SqlType(StandardTypes.BIGINT) + public static String plus(@SqlType(StandardTypes.BIGINT) String left, @SqlType(StandardTypes.BIGINT) String right) + { + return left + " -|- " + right; + } + + @ScalarOperator(OperatorType.LESS_THAN) + @SqlType(StandardTypes.BOOLEAN) + public static String lessThan(@SqlType(StandardTypes.BIGINT) String left, @SqlType(StandardTypes.BIGINT) String right) + { + return left + " LT " + right; + } + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestSubExpressions.java b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestSubExpressions.java new file mode 100644 index 0000000000000..cc0259a1d6b56 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestSubExpressions.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.relational; + +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.type.FunctionType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.testng.annotations.Test; + +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; +import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; +import static com.facebook.presto.spi.function.OperatorType.ADD; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.BIND; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; +import static com.facebook.presto.sql.relational.Expressions.constant; +import static com.facebook.presto.sql.relational.Expressions.specialForm; +import static com.facebook.presto.sql.relational.Expressions.subExpressions; +import static com.facebook.presto.sql.relational.Expressions.uniqueSubExpressions; +import static com.facebook.presto.sql.relational.Expressions.variable; +import static org.testng.Assert.assertEquals; + +public class TestSubExpressions +{ + private static final FunctionManager FUNCTION_MANAGER = createTestMetadataManager().getFunctionManager(); + + @Test + void testExtract() + { + RowExpression a = variable("a", BIGINT); + RowExpression b = constant(1L, BIGINT); + RowExpression c = call(ADD, a, b); + RowExpression d = new LambdaDefinitionExpression(ImmutableList.of(BIGINT), ImmutableList.of("a"), c); + RowExpression e = constant(1L, BIGINT); + RowExpression f = specialForm(BIND, new FunctionType(ImmutableList.of(BIGINT), BIGINT), e, d); + assertEquals(subExpressions(a), ImmutableList.of(a)); + assertEquals(subExpressions(b), ImmutableList.of(b)); + assertEquals(subExpressions(c), ImmutableList.of(c, a, b)); + assertEquals(subExpressions(d), ImmutableList.of(d, c, a, b)); + assertEquals(subExpressions(f), ImmutableList.of(f, e, d, c, a, b)); + assertEqualsIgnoreOrder(uniqueSubExpressions(f), ImmutableSet.of(a, b, c, d, f)); + } + + private RowExpression call(OperatorType operator, RowExpression left, RowExpression right) + { + FunctionHandle functionHandle = FUNCTION_MANAGER.resolveOperator(operator, fromTypes(left.getType(), right.getType())); + return Expressions.call(operator.getOperator(), functionHandle, left.getType(), left, right); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/tests/LogTestDurationListener.java b/presto-main/src/test/java/com/facebook/presto/tests/LogTestDurationListener.java new file mode 100644 index 0000000000000..b7c0c076d4774 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/tests/LogTestDurationListener.java @@ -0,0 +1,190 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests; + +import com.facebook.airlift.log.Logger; +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; +import org.testng.IClassListener; +import org.testng.IExecutionListener; +import org.testng.IInvokedMethod; +import org.testng.IInvokedMethodListener; +import org.testng.ITestClass; +import org.testng.ITestResult; + +import javax.annotation.concurrent.GuardedBy; + +import java.lang.management.ThreadInfo; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.google.common.base.Preconditions.checkState; +import static io.airlift.units.Duration.nanosSince; +import static java.lang.String.format; +import static java.lang.management.ManagementFactory.getThreadMXBean; +import static java.util.stream.Collectors.joining; + +public class LogTestDurationListener + implements IExecutionListener, IClassListener, IInvokedMethodListener +{ + private static final Logger LOG = Logger.get(LogTestDurationListener.class); + + private static final Duration SINGLE_TEST_LOGGING_THRESHOLD = Duration.valueOf("30s"); + private static final Duration CLASS_LOGGING_THRESHOLD = Duration.valueOf("1m"); + private static final Duration GLOBAL_IDLE_LOGGING_THRESHOLD = Duration.valueOf("8m"); + + private final ScheduledExecutorService scheduledExecutorService; + + private final Map started = new ConcurrentHashMap<>(); + private final AtomicLong lastChange = new AtomicLong(System.nanoTime()); + private final AtomicBoolean hangLogged = new AtomicBoolean(); + private final AtomicBoolean finished = new AtomicBoolean(); + @GuardedBy("this") + private ScheduledFuture monitorHangTask; + + public LogTestDurationListener() + { + scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(daemonThreadsNamed("TestHangMonitor")); + } + + @Override + public synchronized void onExecutionStart() + { + resetHangMonitor(); + finished.set(false); + if (monitorHangTask == null) { + monitorHangTask = scheduledExecutorService.scheduleWithFixedDelay(this::checkForTestHang, 5, 5, TimeUnit.SECONDS); + } + } + + @Override + public synchronized void onExecutionFinish() + { + resetHangMonitor(); + finished.set(true); + // do not stop hang task so notification of hung test JVM will fire + // Note: since the monitor uses daemon threads it will not prevent JVM shutdown + } + + private void checkForTestHang() + { + if (hangLogged.get()) { + return; + } + + Duration duration = nanosSince(lastChange.get()); + if (duration.compareTo(GLOBAL_IDLE_LOGGING_THRESHOLD) < 0) { + return; + } + + if (!hangLogged.compareAndSet(false, true)) { + return; + } + + Map runningTests = ImmutableMap.copyOf(started); + if (!runningTests.isEmpty()) { + String testDetails = runningTests.entrySet().stream() + .map(entry -> String.format("%s running for %s", entry.getKey(), nanosSince(entry.getValue()))) + .collect(joining("\n\t", "\n\t", "")); + dumpAllThreads(format("No test started or completed in %s. Running tests:%s.", GLOBAL_IDLE_LOGGING_THRESHOLD, testDetails)); + } + else if (finished.get()) { + dumpAllThreads(format("Tests finished, but JVM did not shutdown in %s.", GLOBAL_IDLE_LOGGING_THRESHOLD)); + } + else { + dumpAllThreads(format("No test started in %s", GLOBAL_IDLE_LOGGING_THRESHOLD)); + } + } + + private static void dumpAllThreads(String message) + { + LOG.warn("%s\n\nFull Thread Dump:\n%s", message, + Arrays.stream(getThreadMXBean().dumpAllThreads(true, true)) + .map(ThreadInfo::toString) + .collect(joining("\n"))); + } + + private void resetHangMonitor() + { + lastChange.set(System.nanoTime()); + hangLogged.set(false); + } + + @Override + public void onBeforeClass(ITestClass testClass) + { + beginExecution(getName(testClass)); + } + + @Override + public void onAfterClass(ITestClass testClass) + { + String name = getName(testClass); + Duration duration = endExecution(name); + if (duration.compareTo(CLASS_LOGGING_THRESHOLD) > 0) { + LOG.warn("Tests from %s took %s", name, duration); + } + } + + @Override + public void beforeInvocation(IInvokedMethod method, ITestResult testResult) + { + beginExecution(getName(method)); + } + + @Override + public void afterInvocation(IInvokedMethod method, ITestResult testResult) + { + String name = getName(method); + Duration duration = endExecution(name); + if (duration.compareTo(SINGLE_TEST_LOGGING_THRESHOLD) > 0) { + LOG.info("Test %s took %s", name, duration); + } + } + + private void beginExecution(String name) + { + resetHangMonitor(); + Long existingEntry = started.putIfAbsent(name, System.nanoTime()); + // You can get concurrent tests with the same name when using @Factory. Instead of adding complex support for + // having multiple running tests with the same name, we simply don't use @Factory. + checkState(existingEntry == null, "There already is a start record for test: %s", name); + } + + private Duration endExecution(String name) + { + resetHangMonitor(); + Long startTime = started.remove(name); + checkState(startTime != null, "There is no start record for test: %s", name); + return nanosSince(startTime); + } + + private static String getName(ITestClass testClass) + { + return testClass.getName(); + } + + private static String getName(IInvokedMethod method) + { + return format("%s::%s", method.getTestMethod().getTestClass().getName(), method.getTestMethod().getMethodName()); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/transaction/TestTransactionManager.java b/presto-main/src/test/java/com/facebook/presto/transaction/TestTransactionManager.java index 70105a7e57f3d..97abf684311a6 100644 --- a/presto-main/src/test/java/com/facebook/presto/transaction/TestTransactionManager.java +++ b/presto-main/src/test/java/com/facebook/presto/transaction/TestTransactionManager.java @@ -38,12 +38,12 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.spi.ConnectorId.createInformationSchemaConnectorId; import static com.facebook.presto.spi.ConnectorId.createSystemTablesConnectorId; import static com.facebook.presto.spi.StandardErrorCode.TRANSACTION_ALREADY_ABORTED; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; import static org.testng.Assert.assertEquals; @@ -172,7 +172,7 @@ public void testExpiration() throws Exception { try (IdleCheckExecutor executor = new IdleCheckExecutor()) { - TransactionManager transactionManager = InMemoryTransactionManager.create( + InMemoryTransactionManager inMemoryTransactionManager = (InMemoryTransactionManager) InMemoryTransactionManager.create( new TransactionManagerConfig() .setIdleTimeout(new Duration(1, TimeUnit.MILLISECONDS)) .setIdleCheckInterval(new Duration(5, TimeUnit.MILLISECONDS)), @@ -180,18 +180,19 @@ public void testExpiration() new CatalogManager(), finishingExecutor); - TransactionId transactionId = transactionManager.beginTransaction(false); + TransactionId transactionId = inMemoryTransactionManager.beginTransaction(false); - assertEquals(transactionManager.getAllTransactionInfos().size(), 1); - TransactionInfo transactionInfo = transactionManager.getTransactionInfo(transactionId); + assertEquals(inMemoryTransactionManager.getAllTransactionInfos().size(), 1); + TransactionInfo transactionInfo = inMemoryTransactionManager.getTransactionInfo(transactionId); assertFalse(transactionInfo.isAutoCommitContext()); assertTrue(transactionInfo.getConnectorIds().isEmpty()); assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); - transactionManager.trySetInactive(transactionId); + inMemoryTransactionManager.trySetInactive(transactionId); TimeUnit.MILLISECONDS.sleep(100); - - assertTrue(transactionManager.getAllTransactionInfos().isEmpty()); + // make sure it is cleaned up + inMemoryTransactionManager.cleanUpExpiredTransactions(); + assertTrue(inMemoryTransactionManager.getAllTransactionInfos().isEmpty()); } } diff --git a/presto-main/src/test/java/com/facebook/presto/transaction/TestTransactionManagerConfig.java b/presto-main/src/test/java/com/facebook/presto/transaction/TestTransactionManagerConfig.java index 2f0840ab22a25..b6c3f5a8b1f80 100644 --- a/presto-main/src/test/java/com/facebook/presto/transaction/TestTransactionManagerConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/transaction/TestTransactionManagerConfig.java @@ -20,9 +20,9 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestTransactionManagerConfig { @@ -32,6 +32,7 @@ public void testDefaults() assertRecordedDefaults(recordDefaults(TransactionManagerConfig.class) .setIdleCheckInterval(new Duration(1, TimeUnit.MINUTES)) .setIdleTimeout(new Duration(5, TimeUnit.MINUTES)) + .setCompanionCatalogs("") .setMaxFinishingConcurrency(1)); } @@ -42,11 +43,13 @@ public void testExplicitPropertyMappings() .put("transaction.idle-check-interval", "1s") .put("transaction.idle-timeout", "10s") .put("transaction.max-finishing-concurrency", "100") + .put("transaction.companion-catalogs", "cat1=cat2,cat2=cat3") .build(); TransactionManagerConfig expected = new TransactionManagerConfig() .setIdleCheckInterval(new Duration(1, TimeUnit.SECONDS)) .setIdleTimeout(new Duration(10, TimeUnit.SECONDS)) + .setCompanionCatalogs("cat1=cat2,cat2=cat3") .setMaxFinishingConcurrency(100); assertFullMapping(properties, expected); diff --git a/presto-main/src/test/java/com/facebook/presto/type/AbstractTestType.java b/presto-main/src/test/java/com/facebook/presto/type/AbstractTestType.java index fe43321a1024c..74067332516b0 100644 --- a/presto-main/src/test/java/com/facebook/presto/type/AbstractTestType.java +++ b/presto-main/src/test/java/com/facebook/presto/type/AbstractTestType.java @@ -34,6 +34,7 @@ import java.util.SortedMap; import java.util.TreeMap; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.block.BlockSerdeUtil.writeBlock; import static com.facebook.presto.operator.OperatorAssertion.toRow; import static com.facebook.presto.spi.block.SortOrder.ASC_NULLS_FIRST; @@ -46,7 +47,6 @@ import static com.facebook.presto.util.StructuralTestUtil.arrayBlockOf; import static com.facebook.presto.util.StructuralTestUtil.mapBlockOf; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.testing.Assertions.assertInstanceOf; import static java.util.Collections.unmodifiableSortedMap; import static java.util.Objects.requireNonNull; import static org.testng.Assert.assertEquals; diff --git a/presto-main/src/test/java/com/facebook/presto/type/BenchmarkDecimalOperators.java b/presto-main/src/test/java/com/facebook/presto/type/BenchmarkDecimalOperators.java index 0f65118f020d7..8a3466015059a 100644 --- a/presto-main/src/test/java/com/facebook/presto/type/BenchmarkDecimalOperators.java +++ b/presto-main/src/test/java/com/facebook/presto/type/BenchmarkDecimalOperators.java @@ -30,6 +30,7 @@ import com.facebook.presto.sql.gen.PageFunctionCompiler; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.NodeRef; @@ -64,6 +65,7 @@ import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.operator.scalar.FunctionAssertions.createExpression; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.DecimalType.createDecimalType; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; @@ -594,7 +596,7 @@ protected void generateInputPage(int... initialValues) protected void generateProcessor(String expression) { - processor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)).compilePageProcessor(Optional.empty(), ImmutableList.of(rowExpression(expression))).get(); + processor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)).compilePageProcessor(SESSION.getSqlFunctionProperties(), Optional.empty(), ImmutableList.of(rowExpression(expression))).get(); } protected void setDoubleMaxValue(double doubleMaxValue) @@ -607,7 +609,9 @@ private RowExpression rowExpression(String value) Expression expression = createExpression(value, metadata, TypeProvider.copyOf(symbolTypes)); Map, Type> expressionTypes = getExpressionTypes(TEST_SESSION, metadata, SQL_PARSER, TypeProvider.copyOf(symbolTypes), expression, emptyList(), WarningCollector.NOOP); - return SqlToRowExpressionTranslator.translate(expression, expressionTypes, sourceLayout, metadata.getFunctionManager(), metadata.getTypeManager(), TEST_SESSION, true); + RowExpression rowExpression = SqlToRowExpressionTranslator.translate(expression, expressionTypes, sourceLayout, metadata.getFunctionManager(), metadata.getTypeManager(), TEST_SESSION); + RowExpressionOptimizer optimizer = new RowExpressionOptimizer(metadata); + return optimizer.optimize(rowExpression, OPTIMIZED, TEST_SESSION.toConnectorSession()); } private Object generateRandomValue(Type type) diff --git a/presto-main/src/test/java/com/facebook/presto/type/TestIpPrefixOperators.java b/presto-main/src/test/java/com/facebook/presto/type/TestIpPrefixOperators.java new file mode 100644 index 0000000000000..5c8d635886dbd --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/type/TestIpPrefixOperators.java @@ -0,0 +1,200 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.type; + +import com.facebook.presto.operator.scalar.AbstractTestFunctions; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.google.common.net.InetAddresses; +import io.airlift.slice.Slices; +import org.testng.annotations.Test; + +import static com.facebook.presto.spi.function.OperatorType.HASH_CODE; +import static com.facebook.presto.spi.function.OperatorType.INDETERMINATE; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.type.IpPrefixType.IPPREFIX; +import static java.lang.System.arraycopy; + +public class TestIpPrefixOperators + extends AbstractTestFunctions +{ + @Test + public void testVarcharToIpPrefixCast() + { + assertFunction("CAST('::ffff:1.2.3.4/24' AS IPPREFIX)", IPPREFIX, "1.2.3.0/24"); + assertFunction("CAST('192.168.0.0/24' AS IPPREFIX)", IPPREFIX, "192.168.0.0/24"); + assertFunction("CAST('255.2.3.4/0' AS IPPREFIX)", IPPREFIX, "0.0.0.0/0"); + assertFunction("CAST('255.2.3.4/1' AS IPPREFIX)", IPPREFIX, "128.0.0.0/1"); + assertFunction("CAST('255.2.3.4/2' AS IPPREFIX)", IPPREFIX, "192.0.0.0/2"); + assertFunction("CAST('255.2.3.4/4' AS IPPREFIX)", IPPREFIX, "240.0.0.0/4"); + assertFunction("CAST('1.2.3.4/8' AS IPPREFIX)", IPPREFIX, "1.0.0.0/8"); + assertFunction("CAST('1.2.3.4/16' AS IPPREFIX)", IPPREFIX, "1.2.0.0/16"); + assertFunction("CAST('1.2.3.4/24' AS IPPREFIX)", IPPREFIX, "1.2.3.0/24"); + assertFunction("CAST('1.2.3.255/25' AS IPPREFIX)", IPPREFIX, "1.2.3.128/25"); + assertFunction("CAST('1.2.3.255/26' AS IPPREFIX)", IPPREFIX, "1.2.3.192/26"); + assertFunction("CAST('1.2.3.255/28' AS IPPREFIX)", IPPREFIX, "1.2.3.240/28"); + assertFunction("CAST('1.2.3.255/30' AS IPPREFIX)", IPPREFIX, "1.2.3.252/30"); + assertFunction("CAST('1.2.3.255/32' AS IPPREFIX)", IPPREFIX, "1.2.3.255/32"); + assertFunction("CAST('2001:0db8:0000:0000:0000:ff00:0042:8329/128' AS IPPREFIX)", IPPREFIX, "2001:db8::ff00:42:8329/128"); + assertFunction("CAST('2001:db8::ff00:42:8329/128' AS IPPREFIX)", IPPREFIX, "2001:db8::ff00:42:8329/128"); + assertFunction("CAST('2001:db8:0:0:1:0:0:1/128' AS IPPREFIX)", IPPREFIX, "2001:db8::1:0:0:1/128"); + assertFunction("CAST('2001:db8:0:0:1::1/128' AS IPPREFIX)", IPPREFIX, "2001:db8::1:0:0:1/128"); + assertFunction("CAST('2001:db8::1:0:0:1/128' AS IPPREFIX)", IPPREFIX, "2001:db8::1:0:0:1/128"); + assertFunction("CAST('2001:DB8::FF00:ABCD:12EF/128' AS IPPREFIX)", IPPREFIX, "2001:db8::ff00:abcd:12ef/128"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/0' AS IPPREFIX)", IPPREFIX, "::/0"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/1' AS IPPREFIX)", IPPREFIX, "8000::/1"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/2' AS IPPREFIX)", IPPREFIX, "c000::/2"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/4' AS IPPREFIX)", IPPREFIX, "f000::/4"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/8' AS IPPREFIX)", IPPREFIX, "ff00::/8"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/16' AS IPPREFIX)", IPPREFIX, "ffff::/16"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/32' AS IPPREFIX)", IPPREFIX, "ffff:ffff::/32"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/48' AS IPPREFIX)", IPPREFIX, "ffff:ffff:ffff::/48"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/64' AS IPPREFIX)", IPPREFIX, "ffff:ffff:ffff:ffff::/64"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/80' AS IPPREFIX)", IPPREFIX, "ffff:ffff:ffff:ffff:ffff::/80"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/96' AS IPPREFIX)", IPPREFIX, "ffff:ffff:ffff:ffff:ffff:ffff::/96"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/112' AS IPPREFIX)", IPPREFIX, "ffff:ffff:ffff:ffff:ffff:ffff:ffff:0/112"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/120' AS IPPREFIX)", IPPREFIX, "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ff00/120"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/124' AS IPPREFIX)", IPPREFIX, "ffff:ffff:ffff:ffff:ffff:ffff:ffff:fff0/124"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/126' AS IPPREFIX)", IPPREFIX, "ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffc/126"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/127' AS IPPREFIX)", IPPREFIX, "ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe/127"); + assertFunction("CAST('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128' AS IPPREFIX)", IPPREFIX, "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128"); + assertFunction("IPPREFIX '10.0.0.0/32'", IPPREFIX, "10.0.0.0/32"); + assertFunction("IPPREFIX '64:ff9b::10.0.0.0/128'", IPPREFIX, "64:ff9b::a00:0/128"); + assertInvalidCast("CAST('facebook.com/32' AS IPPREFIX)", "Cannot cast value to IPPREFIX: facebook.com/32"); + assertInvalidCast("CAST('localhost/32' AS IPPREFIX)", "Cannot cast value to IPPREFIX: localhost/32"); + assertInvalidCast("CAST('2001:db8::1::1/128' AS IPPREFIX)", "Cannot cast value to IPPREFIX: 2001:db8::1::1/128"); + assertInvalidCast("CAST('2001:zxy::1::1/128' AS IPPREFIX)", "Cannot cast value to IPPREFIX: 2001:zxy::1::1/128"); + assertInvalidCast("CAST('789.1.1.1/32' AS IPPREFIX)", "Cannot cast value to IPPREFIX: 789.1.1.1/32"); + assertInvalidCast("CAST('192.1.1.1' AS IPPREFIX)", "Cannot cast value to IPPREFIX: 192.1.1.1"); + assertInvalidCast("CAST('192.1.1.1/128' AS IPPREFIX)", "Cannot cast value to IPPREFIX: 192.1.1.1/128"); + } + + @Test + public void testIpPrefixToVarcharCast() + { + assertFunction("CAST(IPPREFIX '::ffff:1.2.3.4/32' AS VARCHAR)", VARCHAR, "1.2.3.4/32"); + assertFunction("CAST(IPPREFIX '::ffff:102:304/32' AS VARCHAR)", VARCHAR, "1.2.3.4/32"); + assertFunction("CAST(IPPREFIX '2001:0db8:0000:0000:0000:ff00:0042:8329/128' AS VARCHAR)", VARCHAR, "2001:db8::ff00:42:8329/128"); + assertFunction("CAST(IPPREFIX '2001:db8::ff00:42:8329/128' AS VARCHAR)", VARCHAR, "2001:db8::ff00:42:8329/128"); + assertFunction("CAST(IPPREFIX '2001:db8:0:0:1:0:0:1/128' AS VARCHAR)", VARCHAR, "2001:db8::1:0:0:1/128"); + assertFunction("CAST(CAST('1.2.3.4/32' AS IPPREFIX) AS VARCHAR)", VARCHAR, "1.2.3.4/32"); + assertFunction("CAST(CAST('2001:db8:0:0:1::1/128' AS IPPREFIX) AS VARCHAR)", VARCHAR, "2001:db8::1:0:0:1/128"); + assertFunction("CAST(CAST('64:ff9b::10.0.0.0/128' AS IPPREFIX) AS VARCHAR)", VARCHAR, "64:ff9b::a00:0/128"); + } + + @Test + public void testIpPrefixToIpAddressCast() + { + assertFunction("CAST(IPPREFIX '1.2.3.4/32' AS IPADDRESS)", IPADDRESS, "1.2.3.4"); + assertFunction("CAST(IPPREFIX '1.2.3.4/24' AS IPADDRESS)", IPADDRESS, "1.2.3.0"); + assertFunction("CAST(IPPREFIX '::1/128' AS IPADDRESS)", IPADDRESS, "::1"); + assertFunction("CAST(IPPREFIX '2001:db8::ff00:42:8329/128' AS IPADDRESS)", IPADDRESS, "2001:db8::ff00:42:8329"); + assertFunction("CAST(IPPREFIX '2001:db8::ff00:42:8329/64' AS IPADDRESS)", IPADDRESS, "2001:db8::"); + } + + @Test + public void testIpAddressToIpPrefixCast() + { + assertFunction("CAST(IPADDRESS '1.2.3.4' AS IPPREFIX)", IPPREFIX, "1.2.3.4/32"); + assertFunction("CAST(IPADDRESS '::ffff:102:304' AS IPPREFIX)", IPPREFIX, "1.2.3.4/32"); + assertFunction("CAST(IPADDRESS '::1' AS IPPREFIX)", IPPREFIX, "::1/128"); + assertFunction("CAST(IPADDRESS '2001:db8::ff00:42:8329' AS IPPREFIX)", IPPREFIX, "2001:db8::ff00:42:8329/128"); + } + + @Test + public void testEquals() + { + assertFunction("IPPREFIX '2001:0db8:0000:0000:0000:ff00:0042:8329/128' = IPPREFIX '2001:db8::ff00:42:8329/128'", BOOLEAN, true); + assertFunction("CAST('1.2.3.4/32' AS IPPREFIX) = CAST('::ffff:1.2.3.4/32' AS IPPREFIX)", BOOLEAN, true); + assertFunction("IPPREFIX '192.168.0.0/32' = IPPREFIX '::ffff:192.168.0.0/32'", BOOLEAN, true); + assertFunction("IPPREFIX '10.0.0.0/32' = IPPREFIX '::ffff:a00:0/32'", BOOLEAN, true); + assertFunction("CAST('1.2.3.4/24' AS IPPREFIX) = IPPREFIX '1.2.3.5/24'", BOOLEAN, true); + assertFunction("IPPREFIX '2001:db8::ff00:42:8329/128' = IPPREFIX '2001:db8::ff00:42:8300/128'", BOOLEAN, false); + assertFunction("CAST('1.2.3.4/32' AS IPPREFIX) = IPPREFIX '1.2.3.5/32'", BOOLEAN, false); + assertFunction("CAST('1.2.0.0/24' AS IPPREFIX) = IPPREFIX '1.2.0.0/25'", BOOLEAN, false); + } + + @Test + public void testDistinctFrom() + { + assertFunction("IPPREFIX '2001:0db8:0000:0000:0000:ff00:0042:8329/128' IS DISTINCT FROM IPPREFIX '2001:db8::ff00:42:8329/128'", BOOLEAN, false); + assertFunction("CAST(NULL AS IPPREFIX) IS DISTINCT FROM CAST(NULL AS IPPREFIX)", BOOLEAN, false); + assertFunction("IPPREFIX '2001:0db8:0000:0000:0000:ff00:0042:8329/128' IS DISTINCT FROM IPPREFIX '2001:db8::ff00:42:8328/128'", BOOLEAN, true); + assertFunction("IPPREFIX '2001:0db8:0000:0000:0000:ff00:0042:8329/128' IS DISTINCT FROM CAST(NULL AS IPPREFIX)", BOOLEAN, true); + assertFunction("CAST(NULL AS IPPREFIX) IS DISTINCT FROM IPPREFIX '2001:db8::ff00:42:8328/128'", BOOLEAN, true); + } + + @Test + public void testNotEquals() + { + assertFunction("IPPREFIX '2001:0db8:0000:0000:0000:ff00:0042:8329/128' != IPPREFIX '1.2.3.4/32'", BOOLEAN, true); + assertFunction("CAST('1.2.3.4/32' AS IPPREFIX) <> CAST('1.2.3.5/32' AS IPPREFIX)", BOOLEAN, true); + assertFunction("CAST('1.2.3.4/32' AS IPPREFIX) != IPPREFIX '1.2.3.4/32'", BOOLEAN, false); + assertFunction("IPPREFIX '2001:0db8:0000:0000:0000:ff00:0042:8329/128' <> IPPREFIX '2001:db8::ff00:42:8329/128'", BOOLEAN, false); + assertFunction("CAST('1.2.3.4/32' AS IPPREFIX) <> CAST('::ffff:1.2.3.4/32' AS IPPREFIX)", BOOLEAN, false); + } + + @Test + public void testOrderOperators() + { + assertFunction("IPPREFIX '2001:0db8:0000:0000:0000:ff00:0042:8329/128' > IPPREFIX '1.2.3.4/32'", BOOLEAN, true); + assertFunction("IPPREFIX '1.2.3.4/32' > IPPREFIX '2001:0db8:0000:0000:0000:ff00:0042:8329/128'", BOOLEAN, false); + + assertFunction("CAST('1.2.3.4/32' AS IPPREFIX) < CAST('1.2.3.5/32' AS IPPREFIX)", BOOLEAN, true); + assertFunction("CAST('1.2.3.5/32' AS IPPREFIX) < CAST('1.2.3.4/32' AS IPPREFIX)", BOOLEAN, false); + + assertFunction("CAST('1.2.0.0/24' AS IPPREFIX) < CAST('1.2.0.0/25' AS IPPREFIX)", BOOLEAN, true); + + assertFunction("IPPREFIX '::1/128' <= CAST('1.2.3.5/32' AS IPPREFIX)", BOOLEAN, true); + assertFunction("IPPREFIX '1.2.3.5/32' <= CAST('1.2.3.5/32' AS IPPREFIX)", BOOLEAN, true); + assertFunction("IPPREFIX '1.2.3.6/32' <= CAST('1.2.3.5/32' AS IPPREFIX)", BOOLEAN, false); + + assertFunction("IPPREFIX '::1/128' >= IPPREFIX '::/128'", BOOLEAN, true); + assertFunction("IPPREFIX '::1/128' >= IPPREFIX '::1/128'", BOOLEAN, true); + assertFunction("IPPREFIX '::/128' >= IPPREFIX '::1/128'", BOOLEAN, false); + + assertFunction("IPPREFIX '::1/128' BETWEEN IPPREFIX '::/128' AND IPPREFIX '::1234/128'", BOOLEAN, true); + assertFunction("IPPREFIX '::2222/128' BETWEEN IPPREFIX '::/128' AND IPPREFIX '::1234/128'", BOOLEAN, false); + } + + @Test + public void testIndeterminate() + { + assertOperator(INDETERMINATE, "CAST(null AS IPPREFIX)", BOOLEAN, true); + assertOperator(INDETERMINATE, "IPPREFIX '::2222/128'", BOOLEAN, false); + } + + @Test + public void testHash() + { + assertOperator(HASH_CODE, "CAST(null AS IPPREFIX)", BIGINT, null); + assertOperator(HASH_CODE, "IPPREFIX '::2222/128'", BIGINT, hashFromType("::2222/128")); + } + + private static long hashFromType(String address) + { + BlockBuilder blockBuilder = IPPREFIX.createBlockBuilder(null, 1); + String[] parts = address.split("/"); + byte[] bytes = new byte[IPPREFIX.getFixedSize()]; + byte[] addressBytes = InetAddresses.forString(parts[0]).getAddress(); + arraycopy(addressBytes, 0, bytes, 0, 16); + bytes[IPPREFIX.getFixedSize() - 1] = (byte) Integer.parseInt(parts[1]); + IPPREFIX.writeSlice(blockBuilder, Slices.wrappedBuffer(bytes)); + Block block = blockBuilder.build(); + return IPPREFIX.hash(block, 0); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/type/TestIpPrefixType.java b/presto-main/src/test/java/com/facebook/presto/type/TestIpPrefixType.java new file mode 100644 index 0000000000000..f85217ee77c28 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/type/TestIpPrefixType.java @@ -0,0 +1,85 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.type; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.google.common.net.InetAddresses; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import org.testng.annotations.Test; + +import static com.facebook.presto.type.IpPrefixType.IPPREFIX; +import static com.google.common.base.Preconditions.checkState; +import static java.lang.System.arraycopy; +import static org.testng.Assert.assertEquals; + +public class TestIpPrefixType + extends AbstractTestType +{ + public TestIpPrefixType() + { + super(IPPREFIX, String.class, createTestBlock()); + } + + public static Block createTestBlock() + { + BlockBuilder blockBuilder = IPPREFIX.createBlockBuilder(null, 1); + IPPREFIX.writeSlice(blockBuilder, getSliceForPrefix("2001:db8::ff00:42:8320/48")); + IPPREFIX.writeSlice(blockBuilder, getSliceForPrefix("2001:db8::ff00:42:8321/48")); + IPPREFIX.writeSlice(blockBuilder, getSliceForPrefix("2001:db8::ff00:42:8322/48")); + IPPREFIX.writeSlice(blockBuilder, getSliceForPrefix("2001:db8::ff00:42:8323/48")); + IPPREFIX.writeSlice(blockBuilder, getSliceForPrefix("2001:db8::ff00:42:8324/48")); + IPPREFIX.writeSlice(blockBuilder, getSliceForPrefix("2001:db8::ff00:42:8325/48")); + IPPREFIX.writeSlice(blockBuilder, getSliceForPrefix("2001:db8::ff00:42:8326/48")); + IPPREFIX.writeSlice(blockBuilder, getSliceForPrefix("2001:db8::ff00:42:8327/48")); + IPPREFIX.writeSlice(blockBuilder, getSliceForPrefix("2001:db8::ff00:42:8328/48")); + IPPREFIX.writeSlice(blockBuilder, getSliceForPrefix("2001:db8::ff00:42:8329/48")); + return blockBuilder.build(); + } + + @Override + protected Object getGreaterValue(Object value) + { + byte[] prefix = ((Slice) value).getBytes(); + checkState(++prefix[prefix.length - 2] != 0, "Last byte of address is 0xff"); + return Slices.wrappedBuffer(prefix); + } + + @Override + protected Object getNonNullValue() + { + byte[] bytes = new byte[IPPREFIX.getFixedSize()]; + byte[] ipAddress = InetAddresses.forString("::").getAddress(); + arraycopy(ipAddress, 0, bytes, 0, 16); + bytes[IPPREFIX.getFixedSize() - 1] = (byte) 0x20; + return Slices.wrappedBuffer(bytes); + } + + @Test + public void testDisplayName() + { + assertEquals((IPPREFIX).getDisplayName(), "ipprefix"); + } + + private static Slice getSliceForPrefix(String prefix) + { + String[] parts = prefix.split("/"); + byte[] bytes = new byte[IPPREFIX.getFixedSize()]; + byte[] ipAddress = InetAddresses.forString(parts[0]).getAddress(); + arraycopy(ipAddress, 0, bytes, 0, 16); + bytes[IPPREFIX.getFixedSize() - 1] = (byte) Integer.parseInt(parts[1]); + return Slices.wrappedBuffer(bytes); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/type/TestSmallintOperators.java b/presto-main/src/test/java/com/facebook/presto/type/TestSmallintOperators.java index ecbe766c87e80..6e9a172ac9c13 100644 --- a/presto-main/src/test/java/com/facebook/presto/type/TestSmallintOperators.java +++ b/presto-main/src/test/java/com/facebook/presto/type/TestSmallintOperators.java @@ -17,7 +17,6 @@ import org.testng.annotations.Test; import static com.facebook.presto.spi.StandardErrorCode.DIVISION_BY_ZERO; -import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; import static com.facebook.presto.spi.function.OperatorType.INDETERMINATE; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -27,6 +26,7 @@ import static com.facebook.presto.spi.type.SmallintType.SMALLINT; import static com.facebook.presto.spi.type.TinyintType.TINYINT; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_LITERAL; import static java.lang.String.format; public class TestSmallintOperators @@ -37,7 +37,7 @@ public void testLiteral() { assertFunction("SMALLINT'37'", SMALLINT, (short) 37); assertFunction("SMALLINT'17'", SMALLINT, (short) 17); - assertInvalidCast("SMALLINT'" + ((long) Short.MAX_VALUE + 1L) + "'"); + assertInvalidFunction("SMALLINT'" + ((long) Short.MAX_VALUE + 1L) + "'", INVALID_LITERAL); } @Test @@ -52,7 +52,7 @@ public void testUnaryMinus() { assertFunction("SMALLINT'-37'", SMALLINT, (short) -37); assertFunction("SMALLINT'-17'", SMALLINT, (short) -17); - assertInvalidFunction("SMALLINT'-" + Short.MIN_VALUE + "'", INVALID_CAST_ARGUMENT); + assertInvalidFunction("SMALLINT'-" + Short.MIN_VALUE + "'", INVALID_LITERAL); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/type/TestTinyintOperators.java b/presto-main/src/test/java/com/facebook/presto/type/TestTinyintOperators.java index 9f1ab8c0b67ef..5ce6ec257cfdb 100644 --- a/presto-main/src/test/java/com/facebook/presto/type/TestTinyintOperators.java +++ b/presto-main/src/test/java/com/facebook/presto/type/TestTinyintOperators.java @@ -17,7 +17,6 @@ import org.testng.annotations.Test; import static com.facebook.presto.spi.StandardErrorCode.DIVISION_BY_ZERO; -import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; import static com.facebook.presto.spi.function.OperatorType.INDETERMINATE; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -27,6 +26,7 @@ import static com.facebook.presto.spi.type.SmallintType.SMALLINT; import static com.facebook.presto.spi.type.TinyintType.TINYINT; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_LITERAL; import static java.lang.String.format; public class TestTinyintOperators @@ -37,7 +37,7 @@ public void testLiteral() { assertFunction("TINYINT'37'", TINYINT, (byte) 37); assertFunction("TINYINT'17'", TINYINT, (byte) 17); - assertInvalidCast("TINYINT'" + ((long) Byte.MAX_VALUE + 1L) + "'"); + assertInvalidFunction("TINYINT'" + ((long) Byte.MAX_VALUE + 1L) + "'", INVALID_LITERAL); } @Test @@ -52,7 +52,7 @@ public void testUnaryMinus() { assertFunction("TINYINT'-37'", TINYINT, (byte) -37); assertFunction("TINYINT'-17'", TINYINT, (byte) -17); - assertInvalidFunction("TINYINT'-" + Byte.MIN_VALUE + "'", INVALID_CAST_ARGUMENT); + assertInvalidFunction("TINYINT'-" + Byte.MIN_VALUE + "'", INVALID_LITERAL); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/type/setdigest/TestSetDigest.java b/presto-main/src/test/java/com/facebook/presto/type/setdigest/TestSetDigest.java index 3f7ff4e304c31..670fbf9607f64 100644 --- a/presto-main/src/test/java/com/facebook/presto/type/setdigest/TestSetDigest.java +++ b/presto-main/src/test/java/com/facebook/presto/type/setdigest/TestSetDigest.java @@ -14,10 +14,10 @@ package com.facebook.presto.type.setdigest; +import com.facebook.airlift.json.ObjectMapperProvider; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableSet; -import io.airlift.json.ObjectMapperProvider; import io.airlift.slice.Slice; import org.testng.annotations.Test; diff --git a/presto-main/src/test/java/com/facebook/presto/util/TestPowerOfTwoValidator.java b/presto-main/src/test/java/com/facebook/presto/util/TestPowerOfTwoValidator.java index 227133ced1ca6..1146075a0da02 100644 --- a/presto-main/src/test/java/com/facebook/presto/util/TestPowerOfTwoValidator.java +++ b/presto-main/src/test/java/com/facebook/presto/util/TestPowerOfTwoValidator.java @@ -13,94 +13,43 @@ */ package com.facebook.presto.util; -import org.apache.bval.jsr.ApacheValidationProvider; import org.testng.annotations.Test; -import javax.validation.ConstraintValidatorContext; -import javax.validation.ConstraintViolation; -import javax.validation.Validation; -import javax.validation.Validator; - -import java.util.Set; - -import static io.airlift.testing.Assertions.assertInstanceOf; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; +import static com.facebook.airlift.testing.ValidationAssertions.assertFailsValidation; +import static com.facebook.airlift.testing.ValidationAssertions.assertValidates; public class TestPowerOfTwoValidator { - private static final Validator VALIDATOR = Validation.byProvider(ApacheValidationProvider.class).configure().buildValidatorFactory().getValidator(); - @Test public void testValidator() { - PowerOfTwoValidator validator = new PowerOfTwoValidator(); - validator.initialize(new MockPowerOfTwo()); - - assertTrue(validator.isValid(1, new MockContext())); - assertTrue(validator.isValid(2, new MockContext())); - assertTrue(validator.isValid(64, new MockContext())); - assertFalse(validator.isValid(0, new MockContext())); - assertFalse(validator.isValid(3, new MockContext())); - assertFalse(validator.isValid(99, new MockContext())); - assertFalse(validator.isValid(-1, new MockContext())); - assertFalse(validator.isValid(-2, new MockContext())); - assertFalse(validator.isValid(-4, new MockContext())); + assertValid(1); + assertValid(2); + assertValid(64); + assertInvalid(0); + assertInvalid(3); + assertInvalid(99); + assertInvalid(-1); + assertInvalid(-2); + assertInvalid(-4); } @Test public void testAllowsNullPowerOfTwoAnnotation() { - VALIDATOR.validate(new NullPowerOfTwoAnnotation()); + assertValidates(new NullPowerOfTwoAnnotation()); } - @Test - public void testPassesValidation() + private static void assertValid(int value) { - ConstrainedPowerOfTwo object = new ConstrainedPowerOfTwo(128); - Set> violations = VALIDATOR.validate(object); - assertTrue(violations.isEmpty()); - } - - @Test - public void testFailsValidation() - { - ConstrainedPowerOfTwo object = new ConstrainedPowerOfTwo(11); - Set> violations = VALIDATOR.validate(object); - assertEquals(violations.size(), 2); - - for (ConstraintViolation violation : violations) { - assertInstanceOf(violation.getConstraintDescriptor().getAnnotation(), PowerOfTwo.class); - } + assertValidates(new ConstrainedPowerOfTwo(value)); } - private static class MockContext - implements ConstraintValidatorContext + private static void assertInvalid(int value) { - @Override - public void disableDefaultConstraintViolation() - { - throw new UnsupportedOperationException(); - } - - @Override - public String getDefaultConstraintMessageTemplate() - { - throw new UnsupportedOperationException(); - } - - @Override - public ConstraintViolationBuilder buildConstraintViolationWithTemplate(String s) - { - throw new UnsupportedOperationException(); - } - - @Override - public T unwrap(Class type) - { - throw new UnsupportedOperationException(); - } + Object object = new ConstrainedPowerOfTwo(value); + assertFailsValidation(object, "unboxed", "is not a power of two", PowerOfTwo.class); + assertFailsValidation(object, "boxed", "is not a power of two", PowerOfTwo.class); } @SuppressWarnings("UnusedDeclaration") @@ -114,13 +63,13 @@ public ConstrainedPowerOfTwo(int value) } @PowerOfTwo - public int getConstrainedByPowerOfTwoUnboxed() + public int getUnboxed() { return value; } @PowerOfTwo - public Integer getConstrainedByPowerOfTwoBoxed() + public Integer getBoxed() { return value; } @@ -130,7 +79,7 @@ public Integer getConstrainedByPowerOfTwoBoxed() public static class NullPowerOfTwoAnnotation { @PowerOfTwo - public static Integer getConstrainedByPowerOfTwo() + public Integer getNull() { return null; } diff --git a/presto-main/src/test/java/com/facebook/presto/util/TestStringTableUtils.java b/presto-main/src/test/java/com/facebook/presto/util/TestStringTableUtils.java new file mode 100644 index 0000000000000..301a508f88b29 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/util/TestStringTableUtils.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.util; + +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.List; + +import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static org.testng.Assert.assertThrows; + +public class TestStringTableUtils +{ + @Test + public void testGetShortestTableStringFormatSimpleSuccess() + { + List> table = Arrays.asList( + Arrays.asList("Header1", "Header2", "Headr3"), + Arrays.asList("Value1", "Value2", "Value3"), + Arrays.asList("LongValue1", "SVal2", "SVal3")); + + assertEquals( + StringTableUtils.getShortestTableStringFormat(table), + "| %-10s | %-7s | %-6s |"); + } + + @Test + public void testGetShortestTableStringFormatBadInput() + { + List> table = Arrays.asList( + Arrays.asList("Header1", "Header2", "Headr3"), + Arrays.asList("Value1", "Value2"), + Arrays.asList("LongValue1", "SVal2", "SVal3")); + + assertThrows( + IllegalArgumentException.class, + () -> StringTableUtils.getShortestTableStringFormat(table)); + } + + @Test + public void testGetTableStringsSimpleSuccess() + { + List> table = Arrays.asList( + Arrays.asList("Header1", "Header2", "Headr3"), + Arrays.asList("Value1", "Value2", "Value3"), + Arrays.asList("LongValue1", "SVal2", "SVal3")); + + assertEquals( + StringTableUtils.getTableStrings(table), + Arrays.asList( + "| Header1 | Header2 | Headr3 |", + "| Value1 | Value2 | Value3 |", + "| LongValue1 | SVal2 | SVal3 |")); + } +} diff --git a/presto-main/src/test/resources/META-INF/services/org.testng.ITestNGListener b/presto-main/src/test/resources/META-INF/services/org.testng.ITestNGListener new file mode 100644 index 0000000000000..ed81df7bba0bb --- /dev/null +++ b/presto-main/src/test/resources/META-INF/services/org.testng.ITestNGListener @@ -0,0 +1 @@ +com.facebook.presto.tests.LogTestDurationListener diff --git a/presto-matching/pom.xml b/presto-matching/pom.xml index f489f7c5c757f..6a64a6a1566b8 100644 --- a/presto-matching/pom.xml +++ b/presto-matching/pom.xml @@ -18,7 +18,7 @@ presto-root com.facebook.presto - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-matching diff --git a/presto-memory-context/pom.xml b/presto-memory-context/pom.xml index d1ff1f6eb5725..c96317186ba93 100644 --- a/presto-memory-context/pom.xml +++ b/presto-memory-context/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-memory-context @@ -41,7 +41,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-memory-context/src/main/java/com/facebook/presto/memory/context/AbstractAggregatedMemoryContext.java b/presto-memory-context/src/main/java/com/facebook/presto/memory/context/AbstractAggregatedMemoryContext.java index fb8f4c4b87716..f2b49e844bb41 100644 --- a/presto-memory-context/src/main/java/com/facebook/presto/memory/context/AbstractAggregatedMemoryContext.java +++ b/presto-memory-context/src/main/java/com/facebook/presto/memory/context/AbstractAggregatedMemoryContext.java @@ -24,7 +24,7 @@ import static java.lang.String.format; @ThreadSafe -abstract class AbstractAggregatedMemoryContext +public abstract class AbstractAggregatedMemoryContext implements AggregatedMemoryContext { static final ListenableFuture NOT_BLOCKED = Futures.immediateFuture(null); @@ -32,7 +32,7 @@ abstract class AbstractAggregatedMemoryContext // When an aggregated memory context is closed, it force-frees the memory allocated by its // children local memory contexts. Since the memory pool API enforces a tag to be used for // reserve/free operations, we define this special tag to use with such free operations. - protected static final String FORCE_FREE_TAG = "FORCE_FREE_OPERATION"; + public static final String FORCE_FREE_TAG = "FORCE_FREE_OPERATION"; @GuardedBy("this") private long usedBytes; diff --git a/presto-memory/pom.xml b/presto-memory/pom.xml index 67715f958eb95..838b15573a0b4 100644 --- a/presto-memory/pom.xml +++ b/presto-memory/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-memory @@ -23,17 +23,17 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift configuration @@ -85,13 +85,13 @@ - io.airlift + com.facebook.airlift log runtime - io.airlift + com.facebook.airlift log-manager runtime @@ -134,7 +134,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryConfig.java b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryConfig.java index 4d59f1fee191c..f9ad6dbefc3c2 100644 --- a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryConfig.java +++ b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.plugin.memory; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import io.airlift.units.DataSize; import javax.validation.constraints.NotNull; diff --git a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryConnectorFactory.java b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryConnectorFactory.java index 407b495a01226..d30f6c53cd4e2 100644 --- a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryConnectorFactory.java +++ b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryConnectorFactory.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.plugin.memory; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.connector.ConnectorFactory; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; diff --git a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryDataFragment.java b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryDataFragment.java index cadb164b39cc2..cbcbd33219013 100644 --- a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryDataFragment.java +++ b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryDataFragment.java @@ -13,15 +13,15 @@ */ package com.facebook.presto.plugin.memory; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.HostAddress; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; import io.airlift.slice.Slices; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.json.JsonCodec.jsonCodec; import static java.util.Objects.requireNonNull; public class MemoryDataFragment diff --git a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryModule.java b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryModule.java index bb2eba37468a5..01303f2a0de78 100644 --- a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryModule.java +++ b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryModule.java @@ -19,7 +19,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; public class MemoryModule diff --git a/presto-memory/src/test/java/com/facebook/presto/plugin/memory/MemoryQueryRunner.java b/presto-memory/src/test/java/com/facebook/presto/plugin/memory/MemoryQueryRunner.java index 7019c8ec3b536..9acbf62d00d7c 100644 --- a/presto-memory/src/test/java/com/facebook/presto/plugin/memory/MemoryQueryRunner.java +++ b/presto-memory/src/test/java/com/facebook/presto/plugin/memory/MemoryQueryRunner.java @@ -13,20 +13,20 @@ */ package com.facebook.presto.plugin.memory; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.log.Logging; import com.facebook.presto.Session; import com.facebook.presto.tests.DistributedQueryRunner; import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import io.airlift.log.Logging; import io.airlift.tpch.TpchTable; import java.util.Map; +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tests.QueryAssertions.copyTpchTables; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static io.airlift.testing.Closeables.closeAllSuppress; public final class MemoryQueryRunner { diff --git a/presto-memory/src/test/java/com/facebook/presto/plugin/memory/TestMemoryMetadata.java b/presto-memory/src/test/java/com/facebook/presto/plugin/memory/TestMemoryMetadata.java index c03aa0278ff7e..b67ce011a09b4 100644 --- a/presto-memory/src/test/java/com/facebook/presto/plugin/memory/TestMemoryMetadata.java +++ b/presto-memory/src/test/java/com/facebook/presto/plugin/memory/TestMemoryMetadata.java @@ -36,10 +36,10 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS; import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; import static com.facebook.presto.testing.TestingConnectorSession.SESSION; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; diff --git a/presto-memory/src/test/java/com/facebook/presto/plugin/memory/TestMemoryWorkerCrash.java b/presto-memory/src/test/java/com/facebook/presto/plugin/memory/TestMemoryWorkerCrash.java index 0a0fc65a0f7fe..d63536f95282e 100644 --- a/presto-memory/src/test/java/com/facebook/presto/plugin/memory/TestMemoryWorkerCrash.java +++ b/presto-memory/src/test/java/com/facebook/presto/plugin/memory/TestMemoryWorkerCrash.java @@ -19,7 +19,7 @@ import io.airlift.units.Duration; import org.testng.annotations.Test; -import static io.airlift.testing.Assertions.assertLessThan; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static io.airlift.units.Duration.nanosSince; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index d9fe7cdf7d92d..f4cc55304ea97 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-ml @@ -32,12 +32,12 @@ - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift concurrent diff --git a/presto-ml/src/main/java/com/facebook/presto/ml/AbstractSvmModel.java b/presto-ml/src/main/java/com/facebook/presto/ml/AbstractSvmModel.java index b18f21f0787b1..e805a1fa47e2f 100644 --- a/presto-ml/src/main/java/com/facebook/presto/ml/AbstractSvmModel.java +++ b/presto-ml/src/main/java/com/facebook/presto/ml/AbstractSvmModel.java @@ -32,8 +32,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static com.google.common.base.Throwables.throwIfUnchecked; -import static io.airlift.concurrent.Threads.threadsNamed; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; diff --git a/presto-ml/src/main/java/com/facebook/presto/ml/EvaluateClassifierPredictionsStateSerializer.java b/presto-ml/src/main/java/com/facebook/presto/ml/EvaluateClassifierPredictionsStateSerializer.java index 39dfae4519232..9da2b1b4b785a 100644 --- a/presto-ml/src/main/java/com/facebook/presto/ml/EvaluateClassifierPredictionsStateSerializer.java +++ b/presto-ml/src/main/java/com/facebook/presto/ml/EvaluateClassifierPredictionsStateSerializer.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.ml; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AccumulatorStateSerializer; @@ -21,7 +22,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import io.airlift.json.ObjectMapperProvider; import io.airlift.slice.Slice; import io.airlift.slice.Slices; diff --git a/presto-ml/src/main/java/com/facebook/presto/ml/MLFeaturesFunctions.java b/presto-ml/src/main/java/com/facebook/presto/ml/MLFeaturesFunctions.java index 41b77aa18ae90..1dcdb9bf94b9a 100644 --- a/presto-ml/src/main/java/com/facebook/presto/ml/MLFeaturesFunctions.java +++ b/presto-ml/src/main/java/com/facebook/presto/ml/MLFeaturesFunctions.java @@ -221,6 +221,6 @@ private static Block featuresHelper(PageBuilder pageBuilder, double... features) mapBlockBuilder.closeEntry(); pageBuilder.declarePosition(); - return mapBlockBuilder.getObject(mapBlockBuilder.getPositionCount() - 1, Block.class); + return mapBlockBuilder.getBlock(mapBlockBuilder.getPositionCount() - 1); } } diff --git a/presto-ml/src/test/java/com/facebook/presto/ml/AbstractTestMLFunctions.java b/presto-ml/src/test/java/com/facebook/presto/ml/AbstractTestMLFunctions.java index 1206bd70f9da0..dffbefd9189aa 100644 --- a/presto-ml/src/test/java/com/facebook/presto/ml/AbstractTestMLFunctions.java +++ b/presto-ml/src/test/java/com/facebook/presto/ml/AbstractTestMLFunctions.java @@ -25,7 +25,7 @@ abstract class AbstractTestMLFunctions @BeforeClass protected void registerFunctions() { - functionAssertions.getMetadata().addFunctions( + functionAssertions.getMetadata().registerBuiltInFunctions( extractFunctions(new MLPlugin().getFunctions())); } } diff --git a/presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java b/presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java index 4f8b007a901e6..cb59c7300ae4c 100644 --- a/presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java +++ b/presto-ml/src/test/java/com/facebook/presto/ml/TestEvaluateClassifierPredictions.java @@ -42,7 +42,7 @@ public class TestEvaluateClassifierPredictions @Test public void testEvaluateClassifierPredictions() { - metadata.addFunctions(extractFunctions(new MLPlugin().getFunctions())); + metadata.registerBuiltInFunctions(extractFunctions(new MLPlugin().getFunctions())); InternalAggregationFunction aggregation = functionManager.getAggregateFunctionImplementation( functionManager.lookupFunction("evaluate_classifier_predictions", fromTypes(BIGINT, BIGINT))); Accumulator accumulator = aggregation.bind(ImmutableList.of(0, 1), Optional.empty()).createAccumulator(); diff --git a/presto-ml/src/test/java/com/facebook/presto/ml/TestMLQueries.java b/presto-ml/src/test/java/com/facebook/presto/ml/TestMLQueries.java index 6162e9a981e35..6fa261f686a83 100644 --- a/presto-ml/src/test/java/com/facebook/presto/ml/TestMLQueries.java +++ b/presto-ml/src/test/java/com/facebook/presto/ml/TestMLQueries.java @@ -71,7 +71,7 @@ private static LocalQueryRunner createLocalQueryRunner() for (ParametricType parametricType : plugin.getParametricTypes()) { localQueryRunner.getTypeManager().addParametricType(parametricType); } - localQueryRunner.getMetadata().addFunctions(extractFunctions(new MLPlugin().getFunctions())); + localQueryRunner.getMetadata().registerBuiltInFunctions(extractFunctions(new MLPlugin().getFunctions())); return localQueryRunner; } diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 6b40d6fbc310d..c8a8fbc5ecf36 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-mongodb @@ -42,22 +42,22 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift configuration @@ -137,7 +137,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientConfig.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientConfig.java index 9d16b73ebe162..ebbb1833b7f88 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientConfig.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientConfig.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.mongodb; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.DefunctConfig; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; import com.mongodb.Tag; import com.mongodb.TagSet; -import io.airlift.configuration.Config; -import io.airlift.configuration.DefunctConfig; import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; @@ -37,7 +37,6 @@ public class MongoClientConfig { private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); private static final Splitter PORT_SPLITTER = Splitter.on(':').trimResults().omitEmptyStrings(); - private static final Splitter USER_SPLITTER = Splitter.onPattern("[:@]").trimResults().omitEmptyStrings(); private static final Splitter TAGSET_SPLITTER = Splitter.on('&').trimResults().omitEmptyStrings(); private static final Splitter TAG_SPLITTER = Splitter.on(':').trimResults().omitEmptyStrings(); @@ -135,9 +134,17 @@ private List buildCredentials(Iterable userPasses) { ImmutableList.Builder builder = ImmutableList.builder(); for (String userPass : userPasses) { - List values = USER_SPLITTER.splitToList(userPass); - checkArgument(values.size() == 3, "Invalid Credential format. Requires user:password@collection"); - builder.add(createCredential(values.get(0), values.get(2), values.get(1).toCharArray())); + int atPos = userPass.lastIndexOf('@'); + checkArgument(atPos > 0, "Invalid credential format. Required user:password@collection"); + String userAndPassword = userPass.substring(0, atPos); + String collection = userPass.substring(atPos + 1); + + int colonPos = userAndPassword.indexOf(':'); + checkArgument(colonPos > 0, "Invalid credential format. Required user:password@collection"); + String user = userAndPassword.substring(0, colonPos); + String password = userAndPassword.substring(colonPos + 1); + + builder.add(createCredential(user, collection, password.toCharArray())); } return builder.build(); } diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientModule.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientModule.java index 58c7cae22da45..5285f3eb355b3 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientModule.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoClientModule.java @@ -23,7 +23,7 @@ import javax.inject.Singleton; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; public class MongoClientModule diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoConnectorFactory.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoConnectorFactory.java index a567e9fc2f7c5..3825e5129b42b 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoConnectorFactory.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoConnectorFactory.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.mongodb; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorContext; import com.facebook.presto.spi.connector.ConnectorFactory; import com.facebook.presto.spi.type.TypeManager; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoMetadata.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoMetadata.java index e6d9d90b26542..fcf4a7ae4f8ea 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoMetadata.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoMetadata.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.mongodb; +import com.facebook.airlift.log.Logger; import com.facebook.presto.mongodb.MongoIndex.MongodbIndexKey; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; @@ -38,7 +39,6 @@ import com.facebook.presto.spi.statistics.ComputedStatistics; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import java.util.Collection; diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoPageSink.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoPageSink.java index f30d6b8255169..2dc61fc23ff6f 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoPageSink.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoPageSink.java @@ -178,7 +178,7 @@ private Object getObjectValue(Type type, Block block, int position) if (isArrayType(type)) { Type elementType = type.getTypeParameters().get(0); - Block arrayBlock = block.getObject(position, Block.class); + Block arrayBlock = block.getBlock(position); List list = new ArrayList<>(arrayBlock.getPositionCount()); for (int i = 0; i < arrayBlock.getPositionCount(); i++) { @@ -192,7 +192,7 @@ private Object getObjectValue(Type type, Block block, int position) Type keyType = type.getTypeParameters().get(0); Type valueType = type.getTypeParameters().get(1); - Block mapBlock = block.getObject(position, Block.class); + Block mapBlock = block.getBlock(position); // map type is converted into list of fixed keys document List values = new ArrayList<>(mapBlock.getPositionCount() / 2); @@ -206,7 +206,7 @@ private Object getObjectValue(Type type, Block block, int position) return unmodifiableList(values); } if (isRowType(type)) { - Block rowBlock = block.getObject(position, Block.class); + Block rowBlock = block.getBlock(position); List fieldTypes = type.getTypeParameters(); if (fieldTypes.size() != rowBlock.getPositionCount()) { diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoPageSource.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoPageSource.java index 8e98723187cbd..9e11375484d78 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoPageSource.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoPageSource.java @@ -63,7 +63,8 @@ public class MongoPageSource private final List columnNames; private final List columnTypes; private Document currentDoc; - private long count; + private long completedBytes; + private long completedPositions; private boolean finished; private final PageBuilder pageBuilder; @@ -84,7 +85,13 @@ public MongoPageSource( @Override public long getCompletedBytes() { - return count; + return completedBytes; + } + + @Override + public long getCompletedPositions() + { + return completedPositions; } @Override @@ -109,14 +116,13 @@ public long getSystemMemoryUsage() public Page getNextPage() { verify(pageBuilder.isEmpty()); - count = 0; + for (int i = 0; i < ROWS_PER_REQUEST; i++) { if (!cursor.hasNext()) { finished = true; break; } currentDoc = cursor.next(); - count++; pageBuilder.declarePosition(); for (int column = 0; column < columnTypes.size(); column++) { @@ -127,6 +133,10 @@ public Page getNextPage() Page page = pageBuilder.build(); pageBuilder.reset(); + + completedBytes += page.getSizeInBytes(); + completedPositions += page.getPositionCount(); + return page; } diff --git a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoSession.java b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoSession.java index d1f1915754186..09af37298b960 100644 --- a/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoSession.java +++ b/presto-mongodb/src/main/java/com/facebook/presto/mongodb/MongoSession.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.mongodb; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaNotFoundException; @@ -42,7 +43,6 @@ import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.IndexOptions; import com.mongodb.client.result.DeleteResult; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import org.bson.Document; import org.bson.types.ObjectId; diff --git a/presto-mongodb/src/test/java/com/facebook/presto/mongodb/MongoQueryRunner.java b/presto-mongodb/src/test/java/com/facebook/presto/mongodb/MongoQueryRunner.java index bfba7bc110f1f..9c07bed90d15a 100644 --- a/presto-mongodb/src/test/java/com/facebook/presto/mongodb/MongoQueryRunner.java +++ b/presto-mongodb/src/test/java/com/facebook/presto/mongodb/MongoQueryRunner.java @@ -26,10 +26,10 @@ import java.net.InetSocketAddress; import java.util.Map; +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tests.QueryAssertions.copyTpchTables; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static io.airlift.testing.Closeables.closeAllSuppress; public class MongoQueryRunner extends DistributedQueryRunner diff --git a/presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoClientConfig.java b/presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoClientConfig.java new file mode 100644 index 0000000000000..42e4c35c2edfe --- /dev/null +++ b/presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoClientConfig.java @@ -0,0 +1,102 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.mongodb; + +import com.facebook.airlift.configuration.testing.ConfigAssertions; +import com.google.common.collect.ImmutableMap; +import com.mongodb.MongoCredential; +import org.testng.annotations.Test; + +import java.util.Map; + +import static org.testng.Assert.assertEquals; + +public class TestMongoClientConfig +{ + @Test + public void testDefaults() + { + ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(MongoClientConfig.class) + .setSchemaCollection("_schema") + .setSeeds("") + .setCredentials("") + .setMinConnectionsPerHost(0) + .setConnectionsPerHost(100) + .setMaxWaitTime(120_000) + .setConnectionTimeout(10_000) + .setSocketTimeout(0) + .setSocketKeepAlive(false) + .setSslEnabled(false) + .setCursorBatchSize(0) + .setReadPreference(ReadPreferenceType.PRIMARY) + .setReadPreferenceTags("") + .setWriteConcern(WriteConcernType.ACKNOWLEDGED) + .setRequiredReplicaSetName(null) + .setImplicitRowFieldPrefix("_pos")); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("mongodb.schema-collection", "_my_schema") + .put("mongodb.seeds", "host1,host2:27016") + .put("mongodb.credentials", "username:password@collection") + .put("mongodb.min-connections-per-host", "1") + .put("mongodb.connections-per-host", "99") + .put("mongodb.max-wait-time", "120001") + .put("mongodb.connection-timeout", "9999") + .put("mongodb.socket-timeout", "1") + .put("mongodb.socket-keep-alive", "true") + .put("mongodb.ssl.enabled", "true") + .put("mongodb.cursor-batch-size", "1") + .put("mongodb.read-preference", "NEAREST") + .put("mongodb.read-preference-tags", "tag_name:tag_value") + .put("mongodb.write-concern", "UNACKNOWLEDGED") + .put("mongodb.required-replica-set", "replica_set") + .put("mongodb.implicit-row-field-prefix", "_prefix") + .build(); + + MongoClientConfig expected = new MongoClientConfig() + .setSchemaCollection("_my_schema") + .setSeeds("host1", "host2:27016") + .setCredentials("username:password@collection") + .setMinConnectionsPerHost(1) + .setConnectionsPerHost(99) + .setMaxWaitTime(120_001) + .setConnectionTimeout(9_999) + .setSocketTimeout(1) + .setSocketKeepAlive(true) + .setSslEnabled(true) + .setCursorBatchSize(1) + .setReadPreference(ReadPreferenceType.NEAREST) + .setReadPreferenceTags("tag_name:tag_value") + .setWriteConcern(WriteConcernType.UNACKNOWLEDGED) + .setRequiredReplicaSetName("replica_set") + .setImplicitRowFieldPrefix("_prefix"); + + ConfigAssertions.assertFullMapping(properties, expected); + } + + @Test + public void testSpecialCharacterCredential() + { + MongoClientConfig config = new MongoClientConfig() + .setCredentials("username:P@ss:w0rd@database"); + + MongoCredential credential = config.getCredentials().get(0); + MongoCredential expected = MongoCredential.createCredential("username", "database", "P@ss:w0rd".toCharArray()); + assertEquals(credential, expected); + } +} diff --git a/presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoSplit.java b/presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoSplit.java index 0ee4b11a67f44..828c73cf326c5 100644 --- a/presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoSplit.java +++ b/presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoSplit.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.mongodb; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.predicate.TupleDomain; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; diff --git a/presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoTableHandle.java b/presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoTableHandle.java index 9c44a830b5ead..db26db8f29352 100644 --- a/presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoTableHandle.java +++ b/presto-mongodb/src/test/java/com/facebook/presto/mongodb/TestMongoTableHandle.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.mongodb; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.SchemaTableName; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 79d5c59878e8e..4272cd18cd348 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-mysql @@ -23,7 +23,7 @@ - io.airlift + com.facebook.airlift configuration @@ -78,6 +78,18 @@ + + org.assertj + assertj-core + test + + + + com.facebook.presto + presto-testing-docker + test + + org.testng testng @@ -85,13 +97,13 @@ - io.airlift + com.facebook.airlift testing test - io.airlift + com.facebook.airlift json test @@ -121,8 +133,14 @@ - io.airlift - testing-mysql-server + com.facebook.presto + testing-mysql-server-5 + test + + + + com.facebook.presto + testing-mysql-server-base test diff --git a/presto-mysql/src/main/java/com/facebook/presto/plugin/mysql/MySqlClient.java b/presto-mysql/src/main/java/com/facebook/presto/plugin/mysql/MySqlClient.java index dd8e77e8bf241..5fd16e471c6da 100644 --- a/presto-mysql/src/main/java/com/facebook/presto/plugin/mysql/MySqlClient.java +++ b/presto-mysql/src/main/java/com/facebook/presto/plugin/mysql/MySqlClient.java @@ -19,7 +19,9 @@ import com.facebook.presto.plugin.jdbc.DriverConnectionFactory; import com.facebook.presto.plugin.jdbc.JdbcColumnHandle; import com.facebook.presto.plugin.jdbc.JdbcConnectorId; +import com.facebook.presto.plugin.jdbc.JdbcIdentity; import com.facebook.presto.plugin.jdbc.JdbcTableHandle; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; @@ -36,8 +38,9 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collection; +import java.util.Optional; import java.util.Properties; -import java.util.Set; import static com.facebook.presto.plugin.jdbc.DriverConnectionFactory.basicConnectionProperties; import static com.facebook.presto.plugin.jdbc.JdbcErrorCode.JDBC_ERROR; @@ -86,16 +89,15 @@ private static ConnectionFactory connectionFactory(BaseJdbcConfig config, MySqlC } @Override - public Set getSchemaNames() + protected Collection listSchemas(Connection connection) { // for MySQL, we need to list catalogs instead of schemas - try (Connection connection = connectionFactory.openConnection(); - ResultSet resultSet = connection.getMetaData().getCatalogs()) { + try (ResultSet resultSet = connection.getMetaData().getCatalogs()) { ImmutableSet.Builder schemaNames = ImmutableSet.builder(); while (resultSet.next()) { - String schemaName = resultSet.getString("TABLE_CAT").toLowerCase(ENGLISH); + String schemaName = resultSet.getString("TABLE_CAT"); // skip internal schemas - if (!schemaName.equals("information_schema") && !schemaName.equals("mysql")) { + if (!schemaName.equalsIgnoreCase("information_schema") && !schemaName.equalsIgnoreCase("mysql")) { schemaNames.add(schemaName); } } @@ -127,27 +129,25 @@ public PreparedStatement getPreparedStatement(Connection connection, String sql) } @Override - protected ResultSet getTables(Connection connection, String schemaName, String tableName) + protected ResultSet getTables(Connection connection, Optional schemaName, Optional tableName) throws SQLException { // MySQL maps their "database" to SQL catalogs and does not have schemas DatabaseMetaData metadata = connection.getMetaData(); - String escape = metadata.getSearchStringEscape(); + Optional escape = Optional.ofNullable(metadata.getSearchStringEscape()); return metadata.getTables( - schemaName, + schemaName.orElse(null), null, - escapeNamePattern(tableName, escape), + escapeNamePattern(tableName, escape).orElse(null), new String[] {"TABLE", "VIEW"}); } @Override - protected SchemaTableName getSchemaTableName(ResultSet resultSet) + protected String getTableSchemaName(ResultSet resultSet) throws SQLException { // MySQL uses catalogs instead of schemas - return new SchemaTableName( - resultSet.getString("TABLE_CAT").toLowerCase(ENGLISH), - resultSet.getString("TABLE_NAME").toLowerCase(ENGLISH)); + return resultSet.getString("TABLE_CAT"); } @Override @@ -186,10 +186,10 @@ protected String toSqlType(Type type) } @Override - public void createTable(ConnectorTableMetadata tableMetadata) + public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata) { try { - createTable(tableMetadata, null, tableMetadata.getTable().getTableName()); + createTable(tableMetadata, session, tableMetadata.getTable().getTableName()); } catch (SQLException e) { if (SQL_STATE_ER_TABLE_EXISTS_ERROR.equals(e.getSQLState())) { @@ -200,9 +200,9 @@ public void createTable(ConnectorTableMetadata tableMetadata) } @Override - public void renameColumn(JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName) + public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName) { - try (Connection connection = connectionFactory.openConnection()) { + try (Connection connection = connectionFactory.openConnection(identity)) { DatabaseMetaData metadata = connection.getMetaData(); if (metadata.storesUpperCaseIdentifiers()) { newColumnName = newColumnName.toUpperCase(ENGLISH); @@ -224,10 +224,10 @@ public void renameColumn(JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, St } @Override - protected void renameTable(String catalogName, SchemaTableName oldTable, SchemaTableName newTable) + protected void renameTable(JdbcIdentity identity, String catalogName, SchemaTableName oldTable, SchemaTableName newTable) { // MySQL doesn't support specifying the catalog name in a rename; by setting the // catalogName parameter to null it will be omitted in the alter table statement. - super.renameTable(null, oldTable, newTable); + super.renameTable(identity, null, oldTable, newTable); } } diff --git a/presto-mysql/src/main/java/com/facebook/presto/plugin/mysql/MySqlClientModule.java b/presto-mysql/src/main/java/com/facebook/presto/plugin/mysql/MySqlClientModule.java index 0ae1425dc4d3f..24166377cf309 100644 --- a/presto-mysql/src/main/java/com/facebook/presto/plugin/mysql/MySqlClientModule.java +++ b/presto-mysql/src/main/java/com/facebook/presto/plugin/mysql/MySqlClientModule.java @@ -13,18 +13,18 @@ */ package com.facebook.presto.plugin.mysql; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.facebook.presto.plugin.jdbc.BaseJdbcConfig; import com.facebook.presto.plugin.jdbc.JdbcClient; import com.google.inject.Binder; import com.google.inject.Scopes; import com.mysql.jdbc.Driver; -import io.airlift.configuration.AbstractConfigurationAwareModule; import java.sql.SQLException; import java.util.Properties; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.configuration.ConfigBinder.configBinder; public class MySqlClientModule extends AbstractConfigurationAwareModule diff --git a/presto-mysql/src/main/java/com/facebook/presto/plugin/mysql/MySqlConfig.java b/presto-mysql/src/main/java/com/facebook/presto/plugin/mysql/MySqlConfig.java index 223cb52645fb5..6425d3f57abc7 100644 --- a/presto-mysql/src/main/java/com/facebook/presto/plugin/mysql/MySqlConfig.java +++ b/presto-mysql/src/main/java/com/facebook/presto/plugin/mysql/MySqlConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.plugin.mysql; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import io.airlift.units.Duration; import javax.validation.constraints.Min; diff --git a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/DockerizedMySqlServer.java b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/DockerizedMySqlServer.java new file mode 100644 index 0000000000000..852639bad4c92 --- /dev/null +++ b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/DockerizedMySqlServer.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.mysql; + +import com.facebook.presto.testing.docker.DockerContainer; +import com.facebook.presto.testing.docker.DockerContainer.HostPortProvider; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.io.Closeable; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import static java.lang.String.format; + +public class DockerizedMySqlServer + implements Closeable +{ + private static final int MYSQL_PORT = 3306; + + private static final String MYSQL_ROOT_USER = "root"; + private static final String MYSQL_ROOT_PASSWORD = "mysqlrootpassword"; + private static final String MYSQL_USER = "testuser"; + private static final String MYSQL_PASSWORD = "testpassword"; + + private final DockerContainer dockerContainer; + + public DockerizedMySqlServer() + { + this.dockerContainer = new DockerContainer( + "mysql:8.0.15", + ImmutableList.of(MYSQL_PORT), + ImmutableMap.of( + "MYSQL_ROOT_PASSWORD", MYSQL_ROOT_PASSWORD, + "MYSQL_USER", MYSQL_USER, + "MYSQL_PASSWORD", MYSQL_PASSWORD, + "MYSQL_DATABASE", "tpch"), + DockerizedMySqlServer::healthCheck); + } + + private static void healthCheck(HostPortProvider hostPortProvider) + throws SQLException + { + String jdbcUrl = getJdbcUrl(hostPortProvider, MYSQL_ROOT_USER, MYSQL_ROOT_PASSWORD); + try (Connection conn = DriverManager.getConnection(jdbcUrl); + Statement stmt = conn.createStatement()) { + stmt.execute("SELECT 1"); + } + } + + public void init() + throws SQLException + { + String jdbcUrl = getJdbcUrl(dockerContainer::getHostPort, MYSQL_ROOT_USER, MYSQL_ROOT_PASSWORD); + try (Connection conn = DriverManager.getConnection(jdbcUrl); + Statement stmt = conn.createStatement()) { + stmt.execute(format("GRANT ALL PRIVILEGES ON *.* TO '%s'", MYSQL_USER)); + } + } + + public String getJdbcUrl() + { + return getJdbcUrl(dockerContainer::getHostPort, MYSQL_USER, MYSQL_PASSWORD); + } + + private static String getJdbcUrl(HostPortProvider hostPortProvider, String user, String password) + { + return format("jdbc:mysql://localhost:%s?user=%s&password=%s&useSSL=false&allowPublicKeyRetrieval=true", hostPortProvider.getHostPort(MYSQL_PORT), user, password); + } + + @Override + public void close() + { + dockerContainer.close(); + } +} diff --git a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/MySqlQueryRunner.java b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/MySqlQueryRunner.java index f07428402c2c2..ecc0ad9d19095 100644 --- a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/MySqlQueryRunner.java +++ b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/MySqlQueryRunner.java @@ -15,19 +15,20 @@ import com.facebook.presto.Session; import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.testing.mysql.TestingMySqlServer; import com.facebook.presto.tests.DistributedQueryRunner; import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.testing.mysql.TestingMySqlServer; import io.airlift.tpch.TpchTable; +import java.util.HashMap; import java.util.Map; +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tests.QueryAssertions.copyTpchTables; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static io.airlift.testing.Closeables.closeAllSuppress; public final class MySqlQueryRunner { @@ -40,10 +41,22 @@ private MySqlQueryRunner() public static QueryRunner createMySqlQueryRunner(TestingMySqlServer server, TpchTable... tables) throws Exception { - return createMySqlQueryRunner(server, ImmutableList.copyOf(tables)); + return createMySqlQueryRunner(server, ImmutableMap.of(), ImmutableList.copyOf(tables)); } - public static QueryRunner createMySqlQueryRunner(TestingMySqlServer server, Iterable> tables) + public static QueryRunner createMySqlQueryRunner(TestingMySqlServer server, Map connectorProperties, Iterable> tables) + throws Exception + { + try { + return createMySqlQueryRunner(server.getJdbcUrl(), connectorProperties, tables); + } + catch (Throwable e) { + closeAllSuppress(e, server); + throw e; + } + } + + public static QueryRunner createMySqlQueryRunner(String jdbcUrl, Map connectorProperties, Iterable> tables) throws Exception { DistributedQueryRunner queryRunner = null; @@ -53,20 +66,19 @@ public static QueryRunner createMySqlQueryRunner(TestingMySqlServer server, Iter queryRunner.installPlugin(new TpchPlugin()); queryRunner.createCatalog("tpch", "tpch"); - Map properties = ImmutableMap.builder() - .put("connection-url", server.getJdbcUrl()) - .put("allow-drop-table", "true") - .build(); + connectorProperties = new HashMap<>(ImmutableMap.copyOf(connectorProperties)); + connectorProperties.putIfAbsent("connection-url", jdbcUrl); + connectorProperties.putIfAbsent("allow-drop-table", "true"); queryRunner.installPlugin(new MySqlPlugin()); - queryRunner.createCatalog("mysql", "mysql", properties); + queryRunner.createCatalog("mysql", "mysql", connectorProperties); copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), tables); return queryRunner; } catch (Throwable e) { - closeAllSuppress(e, queryRunner, server); + closeAllSuppress(e, queryRunner); throw e; } } diff --git a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlConfig.java b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlConfig.java index 14a9914939795..543145fe12efb 100644 --- a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlConfig.java +++ b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlConfig.java @@ -20,9 +20,9 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestMySqlConfig { diff --git a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlDistributedQueries.java b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlDistributedQueries.java index f85928cab9d59..9110cacf380f9 100644 --- a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlDistributedQueries.java +++ b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlDistributedQueries.java @@ -14,15 +14,18 @@ package com.facebook.presto.plugin.mysql; import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.mysql.MySqlOptions; +import com.facebook.presto.testing.mysql.TestingMySqlServer; import com.facebook.presto.tests.AbstractTestDistributedQueries; import com.google.common.collect.ImmutableList; -import io.airlift.testing.mysql.MySqlOptions; -import io.airlift.testing.mysql.TestingMySqlServer; +import com.google.common.collect.ImmutableMap; import io.airlift.tpch.TpchTable; import io.airlift.units.Duration; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; +import java.io.IOException; + import static com.facebook.presto.plugin.mysql.MySqlQueryRunner.createMySqlQueryRunner; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; @@ -47,7 +50,7 @@ public TestMySqlDistributedQueries() public TestMySqlDistributedQueries(TestingMySqlServer mysqlServer) { - super(() -> createMySqlQueryRunner(mysqlServer, TpchTable.getTables())); + super(() -> createMySqlQueryRunner(mysqlServer, ImmutableMap.of(), TpchTable.getTables())); this.mysqlServer = mysqlServer; } @@ -59,6 +62,7 @@ protected boolean supportsViews() @AfterClass(alwaysRun = true) public final void destroy() + throws IOException { mysqlServer.close(); } diff --git a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlIntegrationSmokeTest.java b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlIntegrationSmokeTest.java index baac57074ed68..68a3afdaca469 100644 --- a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlIntegrationSmokeTest.java +++ b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlIntegrationSmokeTest.java @@ -17,16 +17,17 @@ import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.MaterializedRow; import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.testing.mysql.MySqlOptions; +import com.facebook.presto.testing.mysql.TestingMySqlServer; import com.facebook.presto.tests.AbstractTestIntegrationSmokeTest; import com.facebook.presto.tests.DistributedQueryRunner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.testing.mysql.MySqlOptions; -import io.airlift.testing.mysql.TestingMySqlServer; import io.airlift.units.Duration; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; +import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -67,6 +68,7 @@ public TestMySqlIntegrationSmokeTest(TestingMySqlServer mysqlServer) @AfterClass(alwaysRun = true) public final void destroy() + throws IOException { mysqlServer.close(); } diff --git a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlTypeMapping.java b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlTypeMapping.java index 95028c7aafac4..7520c36d4923a 100644 --- a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlTypeMapping.java +++ b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlTypeMapping.java @@ -16,6 +16,8 @@ import com.facebook.presto.Session; import com.facebook.presto.spi.type.TimeZoneKey; import com.facebook.presto.spi.type.VarcharType; +import com.facebook.presto.testing.mysql.MySqlOptions; +import com.facebook.presto.testing.mysql.TestingMySqlServer; import com.facebook.presto.tests.AbstractTestQueryFramework; import com.facebook.presto.tests.datatype.CreateAndInsertDataSetup; import com.facebook.presto.tests.datatype.CreateAsSelectDataSetup; @@ -24,12 +26,12 @@ import com.facebook.presto.tests.sql.JdbcSqlExecutor; import com.facebook.presto.tests.sql.PrestoSqlExecutor; import com.google.common.collect.ImmutableList; -import io.airlift.testing.mysql.MySqlOptions; -import io.airlift.testing.mysql.TestingMySqlServer; +import com.google.common.collect.ImmutableMap; import io.airlift.units.Duration; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; +import java.io.IOException; import java.math.BigDecimal; import java.time.LocalDate; import java.time.ZoneId; @@ -53,7 +55,6 @@ import static com.google.common.base.Strings.repeat; import static com.google.common.base.Verify.verify; import static java.lang.String.format; -import static java.util.Collections.emptyList; import static java.util.concurrent.TimeUnit.SECONDS; @Test @@ -75,12 +76,13 @@ public TestMySqlTypeMapping() private TestMySqlTypeMapping(TestingMySqlServer mysqlServer) { - super(() -> createMySqlQueryRunner(mysqlServer, emptyList())); + super(() -> createMySqlQueryRunner(mysqlServer, ImmutableMap.of(), ImmutableList.of())); this.mysqlServer = mysqlServer; } @AfterClass(alwaysRun = true) public final void destroy() + throws IOException { mysqlServer.close(); } diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 122d1a0e49e82..adb442f8fdbdb 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-orc @@ -52,7 +52,7 @@ - io.airlift + com.facebook.airlift log @@ -62,10 +62,15 @@ - io.airlift + com.facebook.airlift stats + + com.facebook.airlift + configuration + + joda-time joda-time @@ -148,7 +153,7 @@ - io.airlift + com.facebook.airlift testing test @@ -166,7 +171,7 @@ - io.airlift + com.facebook.airlift json test diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/AbstractOrcRecordReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/AbstractOrcRecordReader.java index 6b5cc42e10631..a81592ff6a7a4 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/AbstractOrcRecordReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/AbstractOrcRecordReader.java @@ -131,7 +131,8 @@ public AbstractOrcRecordReader( Map userMetadata, AggregatedMemoryContext systemMemoryUsage, Optional writeValidation, - int initialBatchSize) + int initialBatchSize, + StripeMetadataSource stripeMetadataSource) { requireNonNull(includedColumns, "includedColumns is null"); requireNonNull(predicate, "predicate is null"); @@ -224,7 +225,8 @@ public AbstractOrcRecordReader( predicate, hiveWriterVersion, metadataReader, - writeValidation); + writeValidation, + stripeMetadataSource); this.streamReaders = requireNonNull(streamReaders, "streamReaders is null"); for (int columnId = 0; columnId < root.getFieldCount(); columnId++) { @@ -346,6 +348,11 @@ public void close() { try (Closer closer = Closer.create()) { closer.register(orcDataSource); + for (StreamReader column : streamReaders) { + if (column != null) { + closer.register(column::close); + } + } } if (writeChecksumBuilder.isPresent()) { @@ -602,7 +609,7 @@ else if (type.getOrcTypeKind() == OrcType.OrcTypeKind.MAP) { nestedStreams.add(createStreamDescriptor(parentStreamName, "key", type.getFieldTypeIndex(0), types, dataSource)); nestedStreams.add(createStreamDescriptor(parentStreamName, "value", type.getFieldTypeIndex(1), types, dataSource)); } - return new StreamDescriptor(parentStreamName, typeId, fieldName, type.getOrcTypeKind(), dataSource, nestedStreams.build()); + return new StreamDescriptor(parentStreamName, typeId, fieldName, type, dataSource, nestedStreams.build()); } protected boolean shouldValidateWritePageChecksum() diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/ByteArrayUtils.java b/presto-orc/src/main/java/com/facebook/presto/orc/ByteArrayUtils.java index bb4387acda35c..45d2c3144563b 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/ByteArrayUtils.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/ByteArrayUtils.java @@ -48,35 +48,41 @@ private ByteArrayUtils() {} // Adapted from io.airlift.slice.Slice public static int compareRanges(byte[] left, int leftOffset, int leftLength, byte[] right, int rightOffset, int rightLength) { - int minLength = min(leftLength, rightLength); - int i = 0; - if (minLength >= SIZE_OF_LONG) { - while (i < minLength) { - long leftLong = unsafe.getLong(left, (long) ARRAY_BYTE_BASE_OFFSET + leftOffset + i); - long rightLong = unsafe.getLong(right, (long) ARRAY_BYTE_BASE_OFFSET + rightOffset + i); + long leftAddress = ARRAY_BYTE_BASE_OFFSET + leftOffset; + long rightAddress = ARRAY_BYTE_BASE_OFFSET + rightOffset; - if (leftLong != rightLong) { - return longBytesToLong(leftLong) < longBytesToLong(rightLong) ? -1 : 1; - } + int lengthToCompare = min(leftLength, rightLength); + while (lengthToCompare >= SIZE_OF_LONG) { + long leftLong = unsafe.getLong(left, leftAddress); + long rightLong = unsafe.getLong(right, rightAddress); - i += SIZE_OF_LONG; + if (leftLong != rightLong) { + return longBytesToLong(leftLong) < longBytesToLong(rightLong) ? -1 : 1; } - i -= SIZE_OF_LONG; - } - while (i < minLength) { - int leftByte = Byte.toUnsignedInt(left[leftOffset + i]); - int rightByte = Byte.toUnsignedInt(right[rightOffset + i]); + leftAddress += SIZE_OF_LONG; + rightAddress += SIZE_OF_LONG; + lengthToCompare -= SIZE_OF_LONG; + } - if (leftByte != rightByte) { - return leftByte - rightByte; + while (lengthToCompare > 0) { + int compareResult = compareUnsignedBytes(unsafe.getByte(left, leftAddress), unsafe.getByte(right, rightAddress)); + if (compareResult != 0) { + return compareResult; } - i++; + leftAddress++; + rightAddress++; + lengthToCompare--; } return Integer.compare(leftLength, rightLength); } + private static int compareUnsignedBytes(byte thisByte, byte thatByte) + { + return Byte.toUnsignedInt(thisByte) - Byte.toUnsignedInt(thatByte); + } + /** * Turns a long representing a sequence of 8 bytes read in little-endian order * into a number that when compared produces the same effect as comparing the diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/CacheStatsMBean.java b/presto-orc/src/main/java/com/facebook/presto/orc/CacheStatsMBean.java new file mode 100644 index 0000000000000..68ca070a1fe29 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/CacheStatsMBean.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.google.common.cache.Cache; +import org.weakref.jmx.Managed; + +import static java.util.Objects.requireNonNull; + +public class CacheStatsMBean +{ + private final Cache cache; + + public CacheStatsMBean(Cache cache) + { + this.cache = requireNonNull(cache, "cache is null"); + } + + @Managed + public long getSize() + { + return cache.size(); + } + + @Managed + public long getHitCount() + { + return cache.stats().hitCount(); + } + + @Managed + public long getMissCount() + { + return cache.stats().missCount(); + } + + @Managed + public double getHitRate() + { + return cache.stats().hitRate(); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/CachingStripeMetadataSource.java b/presto-orc/src/main/java/com/facebook/presto/orc/CachingStripeMetadataSource.java new file mode 100644 index 0000000000000..c669864e86f76 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/CachingStripeMetadataSource.java @@ -0,0 +1,111 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.facebook.presto.orc.StripeReader.StripeId; +import com.facebook.presto.orc.StripeReader.StripeStreamId; +import com.facebook.presto.orc.metadata.Stream.StreamKind; +import com.google.common.cache.Cache; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.UncheckedExecutionException; +import io.airlift.slice.BasicSliceInput; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; + +import java.io.IOException; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ExecutionException; + +import static com.facebook.presto.orc.metadata.Stream.StreamKind.BLOOM_FILTER; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.ROW_INDEX; +import static com.google.common.base.Throwables.throwIfInstanceOf; +import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; + +public class CachingStripeMetadataSource + implements StripeMetadataSource +{ + private final StripeMetadataSource delegate; + private final Cache footerSliceCache; + private final Cache stripeStreamCache; + + public CachingStripeMetadataSource(StripeMetadataSource delegate, Cache footerSliceCache, Cache stripeStreamCache) + { + this.delegate = requireNonNull(delegate, "delegate is null"); + this.footerSliceCache = requireNonNull(footerSliceCache, "footerSliceCache is null"); + this.stripeStreamCache = requireNonNull(stripeStreamCache, "rowIndexSliceCache is null"); + } + + @Override + public Slice getStripeFooterSlice(OrcDataSource orcDataSource, StripeId stripeId, long footerOffset, int footerLength) + throws IOException + { + try { + return footerSliceCache.get(stripeId, () -> delegate.getStripeFooterSlice(orcDataSource, stripeId, footerOffset, footerLength)); + } + catch (ExecutionException | UncheckedExecutionException e) { + throwIfInstanceOf(e.getCause(), IOException.class); + throw new IOException("Unexpected error in stripe footer reading after footerSliceCache miss", e.getCause()); + } + } + + @Override + public Map getInputs(OrcDataSource orcDataSource, StripeId stripeId, Map diskRanges) + throws IOException + { + // + // Note: this code does not use the Java 8 stream APIs to avoid any extra object allocation + // + + // Fetch existing stream slice from cache + ImmutableMap.Builder inputsBuilder = ImmutableMap.builder(); + ImmutableMap.Builder uncachedDiskRangesBuilder = ImmutableMap.builder(); + for (Entry entry : diskRanges.entrySet()) { + if (isCachedStream(entry.getKey().getStreamKind())) { + Slice streamSlice = stripeStreamCache.getIfPresent(new StripeStreamId(stripeId, entry.getKey())); + if (streamSlice != null) { + inputsBuilder.put(entry.getKey(), new OrcDataSourceInput(new BasicSliceInput(streamSlice), streamSlice.length())); + } + else { + uncachedDiskRangesBuilder.put(entry); + } + } + else { + uncachedDiskRangesBuilder.put(entry); + } + } + + // read ranges and update cache + Map uncachedInputs = delegate.getInputs(orcDataSource, stripeId, uncachedDiskRangesBuilder.build()); + for (Entry entry : uncachedInputs.entrySet()) { + if (isCachedStream(entry.getKey().getStreamKind())) { + // We need to rewind the input after eagerly reading the slice. + Slice streamSlice = Slices.wrappedBuffer(entry.getValue().getInput().readSlice(toIntExact(entry.getValue().getInput().length())).getBytes()); + stripeStreamCache.put(new StripeStreamId(stripeId, entry.getKey()), streamSlice); + inputsBuilder.put(entry.getKey(), new OrcDataSourceInput(new BasicSliceInput(streamSlice), toIntExact(streamSlice.getRetainedSize()))); + } + else { + inputsBuilder.put(entry.getKey(), entry.getValue()); + } + } + return inputsBuilder.build(); + } + + private static boolean isCachedStream(StreamKind streamKind) + { + // BLOOM_FILTER and ROW_INDEX are on the critical path to generate a stripe. Other stream kinds could be lazily read. + return streamKind == BLOOM_FILTER || streamKind == ROW_INDEX; + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/FilterFunction.java b/presto-orc/src/main/java/com/facebook/presto/orc/FilterFunction.java index d8a568ee9f3d4..2655b770b2619 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/FilterFunction.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/FilterFunction.java @@ -15,18 +15,33 @@ import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.DictionaryBlock; import com.facebook.presto.spi.relation.Predicate; +import static com.facebook.presto.array.Arrays.ensureCapacity; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Verify.verify; +import static java.util.Arrays.fill; import static java.util.Objects.requireNonNull; public class FilterFunction { + private static final byte FILTER_NOT_EVALUATED = 0; + private static final byte FILTER_PASSED = 1; + private static final byte FILTER_FAILED = 2; + private final ConnectorSession session; private final Predicate predicate; private final boolean deterministic; private final int[] inputChannels; + // If the function has a single argument and this is a DictionaryBlock, we can cache results. The cache is valid as long + // as the dictionary inside the block is physically the same. + private byte[] dictionaryResults; + private Block previousDictionary; + private Page dictionaryPage; + public FilterFunction(ConnectorSession session, boolean deterministic, Predicate predicate) { this.session = requireNonNull(session, "session is null"); @@ -53,6 +68,10 @@ public int filter(Page page, int[] positions, int positionCount, RuntimeExceptio checkArgument(positionCount <= positions.length); checkArgument(positionCount <= errors.length); + if (deterministic && inputChannels.length == 1 && page.getBlock(0) instanceof DictionaryBlock) { + return filterWithDictionary(page, positions, positionCount, errors); + } + int outputCount = 0; for (int i = 0; i < positionCount; i++) { int position = positions[i]; @@ -73,6 +92,56 @@ public int filter(Page page, int[] positions, int positionCount, RuntimeExceptio return outputCount; } + private int filterWithDictionary(Page page, int[] positions, int positionCount, RuntimeException[] errors) + { + int outputCount = 0; + DictionaryBlock block = (DictionaryBlock) page.getBlock(0); + Block dictionary = block.getDictionary(); + if (dictionary != previousDictionary) { + previousDictionary = dictionary; + int numEntries = dictionary.getPositionCount(); + dictionaryPage = new Page(numEntries, dictionary); + dictionaryResults = ensureCapacity(dictionaryResults, numEntries); + fill(dictionaryResults, 0, numEntries, FILTER_NOT_EVALUATED); + } + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + int dictionaryPosition = block.getId(position); + byte result = dictionaryResults[dictionaryPosition]; + switch (result) { + case FILTER_FAILED: + continue; + case FILTER_PASSED: + positions[outputCount] = position; + errors[outputCount] = errors[i]; + outputCount++; + continue; + case FILTER_NOT_EVALUATED: + try { + if (predicate.evaluate(session, dictionaryPage, dictionaryPosition)) { + positions[outputCount] = position; + errors[outputCount] = errors[i]; + outputCount++; + dictionaryResults[dictionaryPosition] = FILTER_PASSED; + } + else { + dictionaryResults[dictionaryPosition] = FILTER_FAILED; + } + } + catch (RuntimeException e) { + // We do not record errors in the dictionary results. + positions[outputCount] = position; + errors[outputCount] = e; // keep last error + outputCount++; + } + break; + default: + verify(false, "Unexpected filter result: " + result); + } + } + return outputCount; + } + public int[] getInputChannels() { return inputChannels; diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcBatchRecordReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcBatchRecordReader.java index cdf7d52dfa0cd..6302b934344b7 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcBatchRecordReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcBatchRecordReader.java @@ -65,7 +65,10 @@ public OrcBatchRecordReader( Map userMetadata, AggregatedMemoryContext systemMemoryUsage, Optional writeValidation, - int initialBatchSize) + int initialBatchSize, + StripeMetadataSource stripeMetadataSource) + throws OrcCorruptionException + { super(includedColumns, // The streamReadersSystemMemoryContext covers the StreamReader local buffer sizes, plus leaf node StreamReaders' @@ -94,7 +97,8 @@ public OrcBatchRecordReader( userMetadata, systemMemoryUsage, writeValidation, - initialBatchSize); + initialBatchSize, + stripeMetadataSource); this.includedColumns = includedColumns; } @@ -118,10 +122,10 @@ public int nextBatch() return batchSize; } - public Block readBlock(Type type, int columnIndex) + public Block readBlock(int columnIndex) throws IOException { - Block block = getStreamReaders()[columnIndex].readBlock(type); + Block block = getStreamReaders()[columnIndex].readBlock(); updateMaxCombinedBytesPerRow(columnIndex, block); return block; } @@ -142,7 +146,7 @@ private void validateWritePageChecksum(int batchSize) if (shouldValidateWritePageChecksum()) { Block[] blocks = new Block[getStreamReaders().length]; for (int columnIndex = 0; columnIndex < getStreamReaders().length; columnIndex++) { - blocks[columnIndex] = readBlock(includedColumns.get(columnIndex), columnIndex); + blocks[columnIndex] = readBlock(columnIndex); } Page page = new Page(batchSize, blocks); validateWritePageChecksum(page); @@ -155,6 +159,7 @@ private static BatchStreamReader[] createStreamReaders( DateTimeZone hiveStorageTimeZone, Map includedColumns, AggregatedMemoryContext systemMemoryContext) + throws OrcCorruptionException { List streamDescriptors = createStreamDescriptor("", "", 0, types, orcDataSource).getNestedStreams(); @@ -162,8 +167,11 @@ private static BatchStreamReader[] createStreamReaders( BatchStreamReader[] streamReaders = new BatchStreamReader[rowType.getFieldCount()]; for (int columnId = 0; columnId < rowType.getFieldCount(); columnId++) { if (includedColumns.containsKey(columnId)) { - StreamDescriptor streamDescriptor = streamDescriptors.get(columnId); - streamReaders[columnId] = BatchStreamReaders.createStreamReader(streamDescriptor, hiveStorageTimeZone); + Type type = includedColumns.get(columnId); + if (type != null) { + StreamDescriptor streamDescriptor = streamDescriptors.get(columnId); + streamReaders[columnId] = BatchStreamReaders.createStreamReader(type, streamDescriptor, hiveStorageTimeZone, systemMemoryContext); + } } } return streamReaders; diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcDecompressor.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcDecompressor.java index 773d21f3cab9c..1c046553b36b4 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcDecompressor.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcDecompressor.java @@ -28,6 +28,12 @@ public interface OrcDecompressor static Optional createOrcDecompressor(OrcDataSourceId orcDataSourceId, CompressionKind compression, int bufferSize) throws OrcCorruptionException + { + return createOrcDecompressor(orcDataSourceId, compression, bufferSize, false); + } + + static Optional createOrcDecompressor(OrcDataSourceId orcDataSourceId, CompressionKind compression, int bufferSize, boolean zstdJniDecompressionEnabled) + throws OrcCorruptionException { if ((compression != NONE) && ((bufferSize <= 0) || (bufferSize > MAX_BUFFER_SIZE))) { throw new OrcCorruptionException(orcDataSourceId, "Invalid compression block size: " + bufferSize); @@ -42,7 +48,7 @@ static Optional createOrcDecompressor(OrcDataSourceId orcDataSo case LZ4: return Optional.of(new OrcLz4Decompressor(orcDataSourceId, bufferSize)); case ZSTD: - return Optional.of(new OrcZstdDecompressor(orcDataSourceId, bufferSize)); + return Optional.of(new OrcZstdDecompressor(orcDataSourceId, bufferSize, zstdJniDecompressionEnabled)); default: throw new OrcCorruptionException(orcDataSourceId, "Unknown compression type: " + compression); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcReader.java index b001a6a1740f6..c96052f00599a 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcReader.java @@ -14,20 +14,19 @@ package com.facebook.presto.orc; import com.facebook.presto.memory.context.AggregatedMemoryContext; +import com.facebook.presto.orc.cache.OrcFileTailSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.orc.metadata.ExceptionWrappingMetadataReader; import com.facebook.presto.orc.metadata.Footer; import com.facebook.presto.orc.metadata.Metadata; -import com.facebook.presto.orc.metadata.PostScript; +import com.facebook.presto.orc.metadata.OrcFileTail; import com.facebook.presto.orc.metadata.PostScript.HiveWriterVersion; import com.facebook.presto.orc.stream.OrcInputStream; import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.Type; -import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; import io.airlift.units.DataSize; import org.joda.time.DateTimeZone; @@ -36,14 +35,11 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.function.Predicate; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.OrcDecompressor.createOrcDecompressor; -import static com.facebook.presto.orc.metadata.PostScript.MAGIC; -import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; -import static io.airlift.units.DataSize.Unit.MEGABYTE; -import static java.lang.Math.min; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; @@ -53,18 +49,8 @@ public class OrcReader public static final int INITIAL_BATCH_SIZE = 1; public static final int BATCH_SIZE_GROWTH_FACTOR = 2; - private static final Logger log = Logger.get(OrcReader.class); - - private static final int CURRENT_MAJOR_VERSION = 0; - private static final int CURRENT_MINOR_VERSION = 12; - private static final int EXPECTED_FOOTER_SIZE = 16 * 1024; - private final OrcDataSource orcDataSource; private final ExceptionWrappingMetadataReader metadataReader; - private final DataSize maxMergeDistance; - private final DataSize maxReadSize; - private final DataSize tinyStripeThreshold; - private final DataSize maxBlockSize; private final HiveWriterVersion hiveWriterVersion; private final int bufferSize; private final CompressionKind compressionKind; @@ -74,123 +60,59 @@ public class OrcReader private final Optional writeValidation; + private final StripeMetadataSource stripeMetadataSource; + private final OrcReaderOptions orcReaderOptions; + // This is based on the Apache Hive ORC code - public OrcReader(OrcDataSource orcDataSource, OrcEncoding orcEncoding, DataSize maxMergeDistance, DataSize maxReadSize, DataSize tinyStripeThreshold, DataSize maxBlockSize) + public OrcReader( + OrcDataSource orcDataSource, + OrcEncoding orcEncoding, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource, + OrcReaderOptions orcReaderOptions) throws IOException { - this(orcDataSource, orcEncoding, maxMergeDistance, maxReadSize, tinyStripeThreshold, maxBlockSize, Optional.empty()); + this(orcDataSource, orcEncoding, orcFileTailSource, stripeMetadataSource, Optional.empty(), orcReaderOptions); } OrcReader( OrcDataSource orcDataSource, OrcEncoding orcEncoding, - DataSize maxMergeDistance, - DataSize maxReadSize, - DataSize tinyStripeThreshold, - DataSize maxBlockSize, - Optional writeValidation) + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource, + Optional writeValidation, + OrcReaderOptions orcReaderOptions) throws IOException { - orcDataSource = wrapWithCacheIfTiny(orcDataSource, tinyStripeThreshold); + this.orcReaderOptions = requireNonNull(orcReaderOptions, "orcReaderOptions is null"); + orcDataSource = wrapWithCacheIfTiny(orcDataSource, orcReaderOptions.getTinyStripeThreshold()); this.orcDataSource = orcDataSource; requireNonNull(orcEncoding, "orcEncoding is null"); this.metadataReader = new ExceptionWrappingMetadataReader(orcDataSource.getId(), orcEncoding.createMetadataReader()); - this.maxMergeDistance = requireNonNull(maxMergeDistance, "maxMergeDistance is null"); - this.maxReadSize = requireNonNull(maxReadSize, "maxReadSize is null"); - this.tinyStripeThreshold = requireNonNull(tinyStripeThreshold, "tinyStripeThreshold is null"); - this.maxBlockSize = requireNonNull(maxBlockSize, "maxBlockSize is null"); this.writeValidation = requireNonNull(writeValidation, "writeValidation is null"); - // - // Read the file tail: - // - // variable: Footer - // variable: Metadata - // variable: PostScript - contains length of footer and metadata - // 1 byte: postScriptSize - - // figure out the size of the file using the option or filesystem - long size = orcDataSource.getSize(); - if (size <= MAGIC.length()) { - throw new OrcCorruptionException(orcDataSource.getId(), "Invalid file size %s", size); - } - - // Read the tail of the file - byte[] buffer = new byte[toIntExact(min(size, EXPECTED_FOOTER_SIZE))]; - orcDataSource.readFully(size - buffer.length, buffer); - - // get length of PostScript - last byte of the file - int postScriptSize = buffer[buffer.length - SIZE_OF_BYTE] & 0xff; - if (postScriptSize >= buffer.length) { - throw new OrcCorruptionException(orcDataSource.getId(), "Invalid postscript length %s", postScriptSize); - } - - // decode the post script - PostScript postScript; - try { - postScript = metadataReader.readPostScript(buffer, buffer.length - SIZE_OF_BYTE - postScriptSize, postScriptSize); - } - catch (OrcCorruptionException e) { - // check if this is an ORC file and not an RCFile or something else - if (!isValidHeaderMagic(orcDataSource)) { - throw new OrcCorruptionException(orcDataSource.getId(), "Not an ORC file"); - } - throw e; - } - - // verify this is a supported version - checkOrcVersion(orcDataSource, postScript.getVersion()); - validateWrite(validation -> validation.getVersion().equals(postScript.getVersion()), "Unexpected version"); - - this.bufferSize = toIntExact(postScript.getCompressionBlockSize()); - - // check compression codec is supported - this.compressionKind = postScript.getCompression(); - this.decompressor = createOrcDecompressor(orcDataSource.getId(), compressionKind, bufferSize); - validateWrite(validation -> validation.getCompression() == compressionKind, "Unexpected compression"); + this.stripeMetadataSource = requireNonNull(stripeMetadataSource, "stripeMetadataSource is null"); - this.hiveWriterVersion = postScript.getHiveWriterVersion(); + OrcFileTail orcFileTail = orcFileTailSource.getOrcFileTail(orcDataSource, metadataReader, writeValidation); + this.bufferSize = orcFileTail.getBufferSize(); + this.compressionKind = orcFileTail.getCompressionKind(); + this.decompressor = createOrcDecompressor(orcDataSource.getId(), compressionKind, bufferSize, orcReaderOptions.isOrcZstdJniDecompressionEnabled()); + this.hiveWriterVersion = orcFileTail.getHiveWriterVersion(); - int footerSize = toIntExact(postScript.getFooterLength()); - int metadataSize = toIntExact(postScript.getMetadataLength()); - - // check if extra bytes need to be read - Slice completeFooterSlice; - int completeFooterSize = footerSize + metadataSize + postScriptSize + SIZE_OF_BYTE; - if (completeFooterSize > buffer.length) { - // allocate a new buffer large enough for the complete footer - byte[] newBuffer = new byte[completeFooterSize]; - completeFooterSlice = Slices.wrappedBuffer(newBuffer); - - // initial read was not large enough, so read missing section - orcDataSource.readFully(size - completeFooterSize, newBuffer, 0, completeFooterSize - buffer.length); - - // copy already read bytes into the new buffer - completeFooterSlice.setBytes(completeFooterSize - buffer.length, buffer); + try (InputStream footerInputStream = new OrcInputStream(orcDataSource.getId(), orcFileTail.getFooterSlice().getInput(), decompressor, newSimpleAggregatedMemoryContext(), orcFileTail.getFooterSize())) { + this.footer = metadataReader.readFooter(hiveWriterVersion, footerInputStream); } - else { - // footer is already in the bytes in buffer, just adjust position, length - completeFooterSlice = Slices.wrappedBuffer(buffer, buffer.length - completeFooterSize, completeFooterSize); + if (this.footer.getTypes().size() == 0) { + throw new OrcCorruptionException(orcDataSource.getId(), "File has no columns"); } - // read metadata - Slice metadataSlice = completeFooterSlice.slice(0, metadataSize); - try (InputStream metadataInputStream = new OrcInputStream(orcDataSource.getId(), metadataSlice.getInput(), decompressor, newSimpleAggregatedMemoryContext(), metadataSize)) { + try (InputStream metadataInputStream = new OrcInputStream(orcDataSource.getId(), orcFileTail.getMetadataSlice().getInput(), decompressor, newSimpleAggregatedMemoryContext(), orcFileTail.getMetadataSize())) { this.metadata = metadataReader.readMetadata(hiveWriterVersion, metadataInputStream); } - // read footer - Slice footerSlice = completeFooterSlice.slice(metadataSize, footerSize); - try (InputStream footerInputStream = new OrcInputStream(orcDataSource.getId(), footerSlice.getInput(), decompressor, newSimpleAggregatedMemoryContext(), footerSize)) { - this.footer = metadataReader.readFooter(hiveWriterVersion, footerInputStream); - } - if (footer.getTypes().size() == 0) { - throw new OrcCorruptionException(orcDataSource.getId(), "File has no columns"); - } - - validateWrite(validation -> validation.getColumnNames().equals(getColumnNames()), "Unexpected column names"); - validateWrite(validation -> validation.getRowGroupMaxRowCount() == footer.getRowsInRowGroup(), "Unexpected rows in group"); + validateWrite(writeValidation, orcDataSource, validation -> validation.getColumnNames().equals(footer.getTypes().get(0).getFieldNames()), "Unexpected column names"); + validateWrite(writeValidation, orcDataSource, validation -> validation.getRowGroupMaxRowCount() == footer.getRowsInRowGroup(), "Unexpected rows in group"); if (writeValidation.isPresent()) { writeValidation.get().validateMetadata(orcDataSource.getId(), footer.getUserMetadata()); writeValidation.get().validateFileStatistics(orcDataSource.getId(), footer.getFileStats()); @@ -224,6 +146,7 @@ public CompressionKind getCompressionKind() } public OrcBatchRecordReader createBatchRecordReader(Map includedColumns, OrcPredicate predicate, DateTimeZone hiveStorageTimeZone, AggregatedMemoryContext systemMemoryUsage, int initialBatchSize) + throws OrcCorruptionException { return createBatchRecordReader(includedColumns, predicate, 0, orcDataSource.getSize(), hiveStorageTimeZone, systemMemoryUsage, initialBatchSize); } @@ -236,6 +159,7 @@ public OrcBatchRecordReader createBatchRecordReader( DateTimeZone hiveStorageTimeZone, AggregatedMemoryContext systemMemoryUsage, int initialBatchSize) + throws OrcCorruptionException { return new OrcBatchRecordReader( requireNonNull(includedColumns, "includedColumns is null"), @@ -253,23 +177,25 @@ public OrcBatchRecordReader createBatchRecordReader( requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null"), hiveWriterVersion, metadataReader, - maxMergeDistance, - tinyStripeThreshold, - maxBlockSize, + orcReaderOptions.getMaxMergeDistance(), + orcReaderOptions.getTinyStripeThreshold(), + orcReaderOptions.getMaxBlockSize(), footer.getUserMetadata(), systemMemoryUsage.newAggregatedMemoryContext(), writeValidation, - initialBatchSize); + initialBatchSize, + stripeMetadataSource); } public OrcSelectiveRecordReader createSelectiveRecordReader( Map includedColumns, List outputColumns, - Map filters, + Map> filters, List filterFunctions, Map filterFunctionInputs, Map> requiredSubfields, Map constantValues, + Map> coercers, OrcPredicate predicate, long offset, long length, @@ -286,6 +212,7 @@ public OrcSelectiveRecordReader createSelectiveRecordReader( filterFunctionInputs, requiredSubfields, constantValues, + coercers, predicate, footer.getNumberOfRows(), footer.getStripes(), @@ -300,13 +227,14 @@ public OrcSelectiveRecordReader createSelectiveRecordReader( hiveStorageTimeZone, hiveWriterVersion, metadataReader, - maxMergeDistance, - tinyStripeThreshold, - maxBlockSize, + orcReaderOptions.getMaxMergeDistance(), + orcReaderOptions.getTinyStripeThreshold(), + orcReaderOptions.getMaxBlockSize(), footer.getUserMetadata(), systemMemoryUsage.newAggregatedMemoryContext(), writeValidation, - initialBatchSize); + initialBatchSize, + stripeMetadataSource); } private static OrcDataSource wrapWithCacheIfTiny(OrcDataSource dataSource, DataSize maxCacheSize) @@ -321,56 +249,13 @@ private static OrcDataSource wrapWithCacheIfTiny(OrcDataSource dataSource, DataS return new CachingOrcDataSource(dataSource, desiredOffset -> diskRange); } - /** - * Does the file start with the ORC magic bytes? - */ - private static boolean isValidHeaderMagic(OrcDataSource source) - throws IOException - { - byte[] headerMagic = new byte[MAGIC.length()]; - source.readFully(0, headerMagic); - - return MAGIC.equals(Slices.wrappedBuffer(headerMagic)); - } - - /** - * Check to see if this ORC file is from a future version and if so, - * warn the user that we may not be able to read all of the column encodings. - */ - // This is based on the Apache Hive ORC code - private static void checkOrcVersion(OrcDataSource orcDataSource, List version) - { - if (version.size() >= 1) { - int major = version.get(0); - int minor = 0; - if (version.size() > 1) { - minor = version.get(1); - } - - if (major > CURRENT_MAJOR_VERSION || (major == CURRENT_MAJOR_VERSION && minor > CURRENT_MINOR_VERSION)) { - log.warn("ORC file %s was written by a newer Hive version %s. This file may not be readable by this version of Hive (%s.%s).", - orcDataSource, - Joiner.on('.').join(version), - CURRENT_MAJOR_VERSION, - CURRENT_MINOR_VERSION); - } - } - } - - private void validateWrite(Predicate test, String messageFormat, Object... args) - throws OrcCorruptionException - { - if (writeValidation.isPresent() && !test.test(writeValidation.get())) { - throw new OrcCorruptionException(orcDataSource.getId(), "Write validation failed: " + messageFormat, args); - } - } - static void validateFile( OrcWriteValidation writeValidation, OrcDataSource input, List types, DateTimeZone hiveStorageTimeZone, - OrcEncoding orcEncoding) + OrcEncoding orcEncoding, + OrcReaderOptions orcReaderOptions) throws OrcCorruptionException { ImmutableMap.Builder readTypes = ImmutableMap.builder(); @@ -378,7 +263,13 @@ static void validateFile( readTypes.put(columnIndex, types.get(columnIndex)); } try { - OrcReader orcReader = new OrcReader(input, orcEncoding, new DataSize(1, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(16, MEGABYTE), Optional.of(writeValidation)); + OrcReader orcReader = new OrcReader( + input, + orcEncoding, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + Optional.of(writeValidation), + orcReaderOptions); try (OrcBatchRecordReader orcRecordReader = orcReader.createBatchRecordReader(readTypes.build(), OrcPredicate.TRUE, hiveStorageTimeZone, newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE)) { while (orcRecordReader.nextBatch() >= 0) { // ignored @@ -389,4 +280,12 @@ static void validateFile( throw new OrcCorruptionException(e, input.getId(), "Validation failed"); } } + + public static void validateWrite(Optional writeValidation, OrcDataSource orcDataSource, Predicate test, String messageFormat, Object... args) + throws OrcCorruptionException + { + if (writeValidation.isPresent() && !test.test(writeValidation.get())) { + throw new OrcCorruptionException(orcDataSource.getId(), "Write validation failed: " + messageFormat, args); + } + } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcReaderOptions.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcReaderOptions.java new file mode 100644 index 0000000000000..64aa9c011bf34 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcReaderOptions.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import io.airlift.units.DataSize; + +import static java.util.Objects.requireNonNull; + +public class OrcReaderOptions +{ + private final DataSize maxMergeDistance; + private final DataSize tinyStripeThreshold; + private final DataSize maxBlockSize; + private final boolean zstdJniDecompressionEnabled; + + public OrcReaderOptions(DataSize maxMergeDistance, DataSize tinyStripeThreshold, DataSize maxBlockSize, boolean zstdJniDecompressionEnabled) + { + this.maxMergeDistance = requireNonNull(maxMergeDistance, "maxMergeDistance is null"); + this.maxBlockSize = requireNonNull(maxBlockSize, "maxBlockSize is null"); + this.tinyStripeThreshold = requireNonNull(tinyStripeThreshold, "tinyStripeThreshold is null"); + this.zstdJniDecompressionEnabled = zstdJniDecompressionEnabled; + } + + public DataSize getMaxMergeDistance() + { + return maxMergeDistance; + } + + public DataSize getMaxBlockSize() + { + return maxBlockSize; + } + + public boolean isOrcZstdJniDecompressionEnabled() + { + return zstdJniDecompressionEnabled; + } + + public DataSize getTinyStripeThreshold() + { + return tinyStripeThreshold; + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcSelectiveRecordReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcSelectiveRecordReader.java index 32ce0937ebeff..c0bc297c530b7 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcSelectiveRecordReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcSelectiveRecordReader.java @@ -14,6 +14,9 @@ package com.facebook.presto.orc; import com.facebook.presto.memory.context.AggregatedMemoryContext; +import com.facebook.presto.orc.TupleDomainFilter.BigintMultiRange; +import com.facebook.presto.orc.TupleDomainFilter.BigintRange; +import com.facebook.presto.orc.TupleDomainFilter.BigintValues; import com.facebook.presto.orc.metadata.MetadataReader; import com.facebook.presto.orc.metadata.OrcType; import com.facebook.presto.orc.metadata.PostScript; @@ -25,62 +28,126 @@ import com.facebook.presto.spi.Subfield; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.block.LazyBlock; +import com.facebook.presto.spi.block.LazyBlockLoader; import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.CharType; +import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.io.Closer; +import com.google.common.primitives.Ints; import io.airlift.slice.Slice; import io.airlift.units.DataSize; import org.joda.time.DateTimeZone; +import javax.annotation.Nullable; + import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.IntStream; +import static com.facebook.presto.orc.OrcReader.MAX_BATCH_SIZE; import static com.facebook.presto.orc.reader.SelectiveStreamReaders.createStreamReader; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DateType.DATE; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RealType.REAL; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.facebook.presto.spi.type.Varchars.isVarcharType; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Predicates.not; +import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static java.lang.Math.max; +import static java.lang.Math.min; import static java.util.Objects.requireNonNull; public class OrcSelectiveRecordReader extends AbstractOrcRecordReader { + // Marks a SQL null when occurring in constantValues. + private static final byte[] NULL_MARKER = new byte[0]; + private static final Page EMPTY_PAGE = new Page(0); + private final int[] hiveColumnIndices; // elements are hive column indices private final List outputColumns; // elements are hive column indices private final Map columnTypes; // key: index into hiveColumnIndices array private final Object[] constantValues; // aligned with hiveColumnIndices array - private final List filterFunctions; + private final Function[] coercers; // aligned with hiveColumnIndices array + + // non-deterministic filter function with no inputs (rand() < 0.1); evaluated before any column is read + private final Optional filterFunctionWithoutInput; private final Map filterFunctionInputMapping; // channel-to-index-into-hiveColumnIndices-array mapping - private final Set filterFunctionInputs; // channels - private final Set columnsWithFilters; // elements are indices into hiveColumnIndices array + private final Map columnsWithFilterScores; // keys are indices into hiveColumnIndices array; values are filter scores // Optimal order of stream readers private int[] streamReaderOrder; // elements are indices into hiveColumnIndices array + private List[] filterFunctionsOrder; // aligned with streamReaderOrder order; each filter function is placed + // into a list positioned at the last necessary input + private Set[] filterFunctionInputs; // aligned with filterFunctionsOrder + private boolean reorderFilters; + private int[] reorderableColumns; // values are hiveColumnIndices + + // non-deterministic filter functions with only constant inputs; evaluated before any column is read + private List filterFunctionsWithConstantInputs; + private Set filterFunctionConstantInputs; + // An immutable list of initial positions; includes all positions: 0,1,2,3,4,.. // This array may grow, but cannot shrink. The values don't change. private int[] positions; // Used in applyFilterFunctions; mutable private int[] outputPositions; + + // errors encountered while evaluating filter functions; indices are positions in the batch + // of rows being processed by getNextPage (errors[outputPositions[i]] is valid) private RuntimeException[] errors; + // temporary array to be used in applyFilterFunctions only; exists solely for the purpose of re-using memory + // indices are positions in a page provided to the filter filters (it contains a subset of rows that passed earlier filters) + private RuntimeException[] tmpErrors; + + // flag indicating whether range filter on a constant column is false; no data is read in that case + private boolean constantFilterIsFalse; + + // an error occurred while evaluating deterministic filter function with only constant + // inputs; thrown unless other filters eliminate all rows + @Nullable + private RuntimeException constantFilterError; + + private int readPositions; + public OrcSelectiveRecordReader( Map includedColumns, // key: hiveColumnIndex List outputColumns, // elements are hive column indices - Map filters, // key: hiveColumnIndex + Map> filters, // key: hiveColumnIndex List filterFunctions, Map filterFunctionInputMapping, // channel-to-hiveColumnIndex mapping for all filter function inputs Map> requiredSubfields, // key: hiveColumnIndex Map constantValues, // key: hiveColumnIndex + Map> coercers, // key: hiveColumnIndex OrcPredicate predicate, long numberOfRows, List fileStripes, @@ -101,7 +168,8 @@ public OrcSelectiveRecordReader( Map userMetadata, AggregatedMemoryContext systemMemoryUsage, Optional writeValidation, - int initialBatchSize) + int initialBatchSize, + StripeMetadataSource stripeMetadataSource) { super(includedColumns, createStreamReaders( @@ -135,7 +203,8 @@ public OrcSelectiveRecordReader( userMetadata, systemMemoryUsage, writeValidation, - initialBatchSize); + initialBatchSize, + stripeMetadataSource); // Hive column indices can't be used to index into arrays because they are negative // for partition and hidden columns. Hence, we create synthetic zero-based indices. @@ -148,44 +217,330 @@ public OrcSelectiveRecordReader( this.hiveColumnIndices = hiveColumnIndices.stream().mapToInt(i -> i).toArray(); this.outputColumns = outputColumns.stream().map(zeroBasedIndices::get).collect(toImmutableList()); this.columnTypes = includedColumns.entrySet().stream().collect(toImmutableMap(entry -> zeroBasedIndices.get(entry.getKey()), Map.Entry::getValue)); - this.filterFunctions = filterFunctions; - this.filterFunctionInputMapping = Maps.transformValues(filterFunctionInputMapping, zeroBasedIndices::get); - filterFunctionInputs = filterFunctions.stream() + this.filterFunctionWithoutInput = getFilterFunctionWithoutInputs(filterFunctions); + + Set usedInputChannels = filterFunctions.stream() .flatMapToInt(function -> Arrays.stream(function.getInputChannels())) .boxed() - .map(this.filterFunctionInputMapping::get) .collect(toImmutableSet()); - columnsWithFilters = filters.keySet().stream().map(zeroBasedIndices::get).collect(toImmutableSet()); + this.filterFunctionInputMapping = Maps.transformValues(Maps.filterKeys(filterFunctionInputMapping, usedInputChannels::contains), zeroBasedIndices::get); + this.columnsWithFilterScores = filters + .entrySet() + .stream() + .collect(toImmutableMap(entry -> zeroBasedIndices.get(entry.getKey()), entry -> scoreFilter(entry.getValue()))); + + requireNonNull(coercers, "coercers is null"); + this.coercers = new Function[this.hiveColumnIndices.length]; + for (Map.Entry> entry : coercers.entrySet()) { + this.coercers[zeroBasedIndices.get(entry.getKey())] = entry.getValue(); + } requireNonNull(constantValues, "constantValues is null"); this.constantValues = new Object[this.hiveColumnIndices.length]; + for (int columnIndex : includedColumns.keySet()) { + if (!isColumnPresent(columnIndex)) { + // Any filter not true of null on a missing column + // fails the whole split. Filters on prefilled columns + // are already evaluated, hence we only check filters + // for missing columns here. + if (columnIndex >= 0 && containsNonNullFilter(filters.get(columnIndex))) { + constantFilterIsFalse = true; + // No further initialization needed. + return; + } + this.constantValues[zeroBasedIndices.get(columnIndex)] = NULL_MARKER; + } + } + for (Map.Entry entry : constantValues.entrySet()) { - this.constantValues[zeroBasedIndices.get(entry.getKey())] = entry.getValue(); + // all included columns will be null, the constant columns should have a valid predicate or null marker so that there is no streamReader created below + if (entry.getValue() != null) { + this.constantValues[zeroBasedIndices.get(entry.getKey())] = entry.getValue(); + } + } + + if (!evaluateDeterministicFilterFunctionsWithConstantInputs(filterFunctions)) { + constantFilterIsFalse = true; + // No further initialization needed. + return; } // Initial order of stream readers is: - // - readers with simple filters + // - readers with integer equality + // - readers with integer range / multivalues / inequality + // - readers with filters // - followed by readers for columns that provide input to filter functions // - followed by readers for columns that doesn't have any filtering - streamReaderOrder = orderStreamReaders(columnTypes.keySet().stream().filter(index -> this.constantValues[index] == null).collect(toImmutableSet()), columnsWithFilters, filterFunctionInputs); + streamReaderOrder = orderStreamReaders(columnTypes.keySet().stream().filter(index -> this.constantValues[index] == null).collect(toImmutableSet()), columnsWithFilterScores, this.filterFunctionInputMapping.keySet(), columnTypes); + + List filterFunctionsWithInputs = filterFunctions.stream() + .filter(OrcSelectiveRecordReader::hasInputs) + .filter(not(this::allConstantInputs)) + .collect(toImmutableList()); + + // figure out when to evaluate filter functions; a function is ready for evaluation as soon as the last input has been read + List filterFunctionsWithStats = filterFunctionsWithInputs.stream() + .map(function -> new FilterFunctionWithStats(function, new FilterStats())) + .collect(toImmutableList()); + filterFunctionsOrder = orderFilterFunctionsWithInputs(streamReaderOrder, filterFunctionsWithStats, this.filterFunctionInputMapping); + filterFunctionInputs = collectFilterFunctionInputs(filterFunctionsOrder, this.filterFunctionInputMapping); + reorderableColumns = Arrays.stream(streamReaderOrder) + .filter(columnIndex -> !columnsWithFilterScores.containsKey(columnIndex)) + .filter(this.filterFunctionInputMapping::containsKey) + .toArray(); + reorderFilters = filterFunctionsWithStats.size() > 1 && reorderableColumns.length > 1; + + filterFunctionsWithConstantInputs = filterFunctions.stream() + .filter(not(FilterFunction::isDeterministic)) + .filter(OrcSelectiveRecordReader::hasInputs) + .filter(this::allConstantInputs) + .map(function -> new FilterFunctionWithStats(function, new FilterStats())) + .collect(toImmutableList()); + filterFunctionConstantInputs = filterFunctionsWithConstantInputs.stream() + .flatMapToInt(function -> Arrays.stream(function.getFunction().getInputChannels())) + .boxed() + .map(this.filterFunctionInputMapping::get) + .collect(toImmutableSet()); + } + + private boolean evaluateDeterministicFilterFunctionsWithConstantInputs(List filterFunctions) + { + for (FilterFunction function : filterFunctions) { + if (function.isDeterministic() && hasInputs(function) && allConstantInputs(function) && !evaluateDeterministicFilterFunctionWithConstantInputs(function)) { + return false; + } + } + return true; + } + + private boolean evaluateDeterministicFilterFunctionWithConstantInputs(FilterFunction function) + { + int[] inputs = function.getInputChannels(); + Block[] blocks = new Block[inputs.length]; + for (int i = 0; i < inputs.length; i++) { + int columnIndex = filterFunctionInputMapping.get(inputs[i]); + Object constantValue = constantValues[columnIndex]; + blocks[i] = RunLengthEncodedBlock.create(columnTypes.get(columnIndex), constantValue == NULL_MARKER ? null : constantValue, 1); + } + + initializeTmpErrors(1); + int positionCount = function.filter(new Page(blocks), new int[] {0}, 1, tmpErrors); + + if (tmpErrors[0] != null) { + constantFilterError = tmpErrors[0]; + } + return positionCount == 1; + } + + private static boolean hasInputs(FilterFunction function) + { + return function.getInputChannels().length > 0; } - private static int[] orderStreamReaders(Collection columnIndices, Set columnsWithFilters, Set filterFunctionInputs) + private boolean allConstantInputs(FilterFunction function) { + return Arrays.stream(function.getInputChannels()) + .map(filterFunctionInputMapping::get) + .allMatch(columnIndex -> constantValues[columnIndex] != null); + } + + private void reorderFiltersIfNeeded() + { + List filters = Arrays.stream(filterFunctionsOrder) + .filter(Objects::nonNull) + .flatMap(functions -> functions.stream()) + .sorted(Comparator.comparingDouble(function -> function.getStats().getElapsedNanonsPerDroppedPosition())) + .collect(toImmutableList()); + + assert filters.size() > 1; + + Map columnScore = new HashMap<>(); + for (int i = 0; i < filters.size(); i++) { + int score = i; + Arrays.stream(filters.get(i).getFunction().getInputChannels()) + .map(filterFunctionInputMapping::get) + // exclude columns with range filters + .filter(columnIndex -> !columnsWithFilterScores.containsKey(columnIndex)) + // exclude constant columns + .filter(columnIndex -> constantValues[columnIndex] == null) + .forEach(columnIndex -> columnScore.compute(columnIndex, (k, v) -> v == null ? score : min(score, v))); + } + + int[] newColumnOrder = columnScore.entrySet().stream() + .sorted(Comparator.comparing(Map.Entry::getValue)) + .mapToInt(Map.Entry::getKey) + .toArray(); + + // Update streamReaderOrder, + // filterFunctionsOrder (aligned with streamReaderOrder), + // filterFunctionInputs (aligned with filterFunctionsOrder) + boolean sameOrder = true; + for (int i = 0; i < streamReaderOrder.length; i++) { + if (!columnsWithFilterScores.containsKey(streamReaderOrder[i])) { + for (int j = 0; j < newColumnOrder.length; j++) { + if (streamReaderOrder[i] != newColumnOrder[j]) { + sameOrder = false; + } + streamReaderOrder[i++] = newColumnOrder[j]; + } + break; + } + } + + if (!sameOrder) { + filterFunctionsOrder = orderFilterFunctionsWithInputs(streamReaderOrder, filters, this.filterFunctionInputMapping); + filterFunctionInputs = collectFilterFunctionInputs(filterFunctionsOrder, this.filterFunctionInputMapping); + } + } + + private static List[] orderFilterFunctionsWithInputs(int[] streamReaderOrder, List filterFunctions, Map inputMapping) + { + List[] order = new List[streamReaderOrder.length]; + for (FilterFunctionWithStats function : filterFunctions) { + int[] inputs = function.getFunction().getInputChannels(); + int lastIndex = -1; + for (int input : inputs) { + int columnIndex = inputMapping.get(input); + lastIndex = max(lastIndex, Ints.indexOf(streamReaderOrder, columnIndex)); + } + + verify(lastIndex >= 0); + if (order[lastIndex] == null) { + order[lastIndex] = new ArrayList<>(); + } + order[lastIndex].add(function); + } + + return order; + } + + private static Set[] collectFilterFunctionInputs(List[] functionsOrder, Map inputMapping) + { + Set[] inputs = new Set[functionsOrder.length]; + for (int i = 0; i < functionsOrder.length; i++) { + List functions = functionsOrder[i]; + if (functions != null) { + inputs[i] = functions.stream() + .flatMapToInt(function -> Arrays.stream(function.getFunction().getInputChannels())) + .boxed() + .map(inputMapping::get) + .collect(toImmutableSet()); + } + } + + return inputs; + } + + private static Optional getFilterFunctionWithoutInputs(List filterFunctions) + { + List functions = filterFunctions.stream() + .filter(not(OrcSelectiveRecordReader::hasInputs)) + .collect(toImmutableList()); + if (functions.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(Iterables.getOnlyElement(functions)); + } + + private static boolean containsNonNullFilter(Map columnFilters) + { + return columnFilters != null && !columnFilters.values().stream().allMatch(TupleDomainFilter::testNull); + } + + private static int scoreFilter(Map filters) + { + checkArgument(!filters.isEmpty()); + + if (filters.size() > 1) { + // Complex type column. Complex types are expensive! + return 1000; + } + + Map.Entry filterEntry = Iterables.getOnlyElement(filters.entrySet()); + if (!filterEntry.getKey().getPath().isEmpty()) { + // Complex type column. Complex types are expensive! + return 1000; + } + + TupleDomainFilter filter = filterEntry.getValue(); + if (filter instanceof BigintRange) { + if (((BigintRange) filter).isSingleValue()) { + // Integer equality. Generally cheap. + return 10; + } + return 50; + } + + if (filter instanceof BigintValues || filter instanceof BigintMultiRange) { + return 50; + } + + return 100; + } + + private static int scoreType(Type type) + { + if (type == BOOLEAN) { + return 10; + } + + if (type == TINYINT || type == SMALLINT || type == INTEGER || type == BIGINT || type == TIMESTAMP || type == DATE) { + return 20; + } + + if (type == REAL || type == DOUBLE) { + return 30; + } + + if (type instanceof DecimalType) { + return 40; + } + + if (isVarcharType(type) || type instanceof CharType) { + return 50; + } + + return 100; + } + + private static int[] orderStreamReaders( + Collection columnIndices, + Map columnToScore, + Set filterFunctionInputs, + Map columnTypes) + { + List sortedColumnsByFilterScore = columnToScore.entrySet() + .stream() + .sorted(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .collect(toImmutableList()); + int[] order = new int[columnIndices.size()]; int i = 0; - for (int columnIndex : columnsWithFilters) { + for (int columnIndex : sortedColumnsByFilterScore) { if (columnIndices.contains(columnIndex)) { order[i++] = columnIndex; } } - for (int columnIndex : filterFunctionInputs) { - if (columnIndices.contains(columnIndex) && !columnsWithFilters.contains(columnIndex)) { + + // read primitive types first + List sortedFilterFunctionInputs = filterFunctionInputs.stream() + .collect(toImmutableMap(Function.identity(), columnIndex -> scoreType(columnTypes.get(columnIndex)))) + .entrySet() + .stream() + .sorted(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .collect(toImmutableList()); + + for (int columnIndex : sortedFilterFunctionInputs) { + if (columnIndices.contains(columnIndex) && !sortedColumnsByFilterScore.contains(columnIndex)) { order[i++] = columnIndex; } } + for (int columnIndex : columnIndices) { - if (!columnsWithFilters.contains(columnIndex) && !filterFunctionInputs.contains(columnIndex)) { + if (!sortedColumnsByFilterScore.contains(columnIndex) && !filterFunctionInputs.contains(columnIndex)) { order[i++] = columnIndex; } } @@ -199,7 +554,7 @@ private static SelectiveStreamReader[] createStreamReaders( DateTimeZone hiveStorageTimeZone, Map includedColumns, List outputColumns, - Map filters, + Map> filters, List filterFunctions, Map filterFunctionInputMapping, Map> requiredSubfields, @@ -224,7 +579,7 @@ private static SelectiveStreamReader[] createStreamReaders( boolean outputRequired = outputColumns.contains(columnId) || filterFunctionInputColumns.contains(columnId); streamReaders[columnId] = createStreamReader( streamDescriptor, - Optional.ofNullable(filters.get(columnId)), + Optional.ofNullable(filters.get(columnId)).orElse(ImmutableMap.of()), outputRequired ? Optional.of(includedColumns.get(columnId)) : Optional.empty(), Optional.ofNullable(requiredSubfields.get(columnId)).orElse(ImmutableList.of()), hiveStorageTimeZone, @@ -234,59 +589,121 @@ private static SelectiveStreamReader[] createStreamReaders( return streamReaders; } + public int getReadPositions() + { + return readPositions; + } + public Page getNextPage() throws IOException { + if (constantFilterIsFalse) { + return null; + } int batchSize = prepareNextBatch(); if (batchSize < 0) { return null; } + readPositions += batchSize; initializePositions(batchSize); int[] positionsToRead = this.positions; int positionCount = batchSize; - boolean filterFunctionsApplied = filterFunctions.isEmpty(); - for (int columnIndex : streamReaderOrder) { - if (!filterFunctionsApplied && !hasAnyFilter(columnIndex)) { - positionCount = applyFilterFunctions(positionsToRead, positionCount); - if (positionCount == 0) { - break; - } - positionsToRead = outputPositions; - filterFunctionsApplied = true; + if (filterFunctionWithoutInput.isPresent()) { + positionCount = applyFilterFunctionWithNoInputs(positionCount); + + if (positionCount == 0) { + batchRead(batchSize); + return EMPTY_PAGE; + } + + positionsToRead = outputPositions; + } + + if (!filterFunctionsWithConstantInputs.isEmpty()) { + positionCount = applyFilterFunctions(filterFunctionsWithConstantInputs, filterFunctionConstantInputs, positionsToRead, positionCount); + + if (positionCount == 0) { + batchRead(batchSize); + return EMPTY_PAGE; + } + + positionsToRead = outputPositions; + } + + int offset = getNextRowInGroup(); + + if (reorderFilters && offset >= MAX_BATCH_SIZE) { + reorderFiltersIfNeeded(); + } + + for (int i = 0; i < streamReaderOrder.length; i++) { + int columnIndex = streamReaderOrder[i]; + + if (!hasAnyFilter(columnIndex)) { + break; } SelectiveStreamReader streamReader = getStreamReader(columnIndex); - positionCount = streamReader.read(getNextRowInGroup(), positionsToRead, positionCount); + positionCount = streamReader.read(offset, positionsToRead, positionCount); if (positionCount == 0) { break; } positionsToRead = streamReader.getReadPositions(); - } + verify(positionCount == 1 || positionsToRead[positionCount - 1] - positionsToRead[0] >= positionCount - 1, "positions must monotonically increase"); - if (positionCount > 0 && !filterFunctionsApplied) { - positionCount = applyFilterFunctions(positionsToRead, positionCount); - positionsToRead = outputPositions; + if (filterFunctionsOrder[i] != null) { + positionCount = applyFilterFunctions(filterFunctionsOrder[i], filterFunctionInputs[i], positionsToRead, positionCount); + if (positionCount == 0) { + break; + } + + positionsToRead = outputPositions; + } } batchRead(batchSize); if (positionCount == 0) { - return new Page(0); + return EMPTY_PAGE; + } + + if (constantFilterError != null) { + throw constantFilterError; + } + + for (int i = 0; i < positionCount; i++) { + if (errors[positionsToRead[i]] != null) { + throw errors[positionsToRead[i]]; + } + } + + for (SelectiveStreamReader reader : getStreamReaders()) { + if (reader != null) { + reader.throwAnyError(positionsToRead, positionCount); + } } Block[] blocks = new Block[outputColumns.size()]; for (int i = 0; i < outputColumns.size(); i++) { int columnIndex = outputColumns.get(i); if (constantValues[columnIndex] != null) { - blocks[i] = RunLengthEncodedBlock.create(columnTypes.get(columnIndex), constantValues[columnIndex], batchSize); + blocks[i] = RunLengthEncodedBlock.create(columnTypes.get(columnIndex), constantValues[columnIndex] == NULL_MARKER ? null : constantValues[columnIndex], positionCount); + } + else if (!hasAnyFilter(columnIndex)) { + blocks[i] = new LazyBlock(positionCount, new OrcBlockLoader(getStreamReader(columnIndex), coercers[columnIndex], offset, positionsToRead, positionCount)); } else { Block block = getStreamReader(columnIndex).getBlock(positionsToRead, positionCount); updateMaxCombinedBytesPerRow(columnIndex, block); + + if (coercers[columnIndex] != null) { + block = coercers[columnIndex].apply(block); + } + blocks[i] = block; } } @@ -305,7 +722,7 @@ private SelectiveStreamReader getStreamReader(int columnIndex) private boolean hasAnyFilter(int columnIndex) { - return columnsWithFilters.contains(columnIndex) || filterFunctionInputs.contains(columnIndex); + return columnsWithFilterScores.containsKey(columnIndex) || filterFunctionInputMapping.containsKey(columnIndex); } private void initializePositions(int batchSize) @@ -316,51 +733,78 @@ private void initializePositions(int batchSize) positions[i] = i; } } + + if (errors == null || errors.length < batchSize) { + errors = new RuntimeException[batchSize]; + } + else { + Arrays.fill(errors, null); + } } - private int applyFilterFunctions(int[] positions, int positionCount) + private int applyFilterFunctionWithNoInputs(int positionCount) + { + initializeOutputPositions(positionCount); + Page page = new Page(positionCount); + return filterFunctionWithoutInput.get().filter(page, outputPositions, positionCount, errors); + } + + private int applyFilterFunctions(List filterFunctions, Set filterFunctionInputs, int[] positions, int positionCount) { BlockLease[] blockLeases = new BlockLease[hiveColumnIndices.length]; Block[] blocks = new Block[hiveColumnIndices.length]; for (int columnIndex : filterFunctionInputs) { if (constantValues[columnIndex] != null) { - blocks[columnIndex] = RunLengthEncodedBlock.create(columnTypes.get(columnIndex), constantValues[columnIndex], positionCount); + blocks[columnIndex] = RunLengthEncodedBlock.create(columnTypes.get(columnIndex), constantValues[columnIndex] == NULL_MARKER ? null : constantValues[columnIndex], positionCount); } else { blockLeases[columnIndex] = getStreamReader(columnIndex).getBlockView(positions, positionCount); - blocks[columnIndex] = blockLeases[columnIndex].get(); + Block block = blockLeases[columnIndex].get(); + if (coercers[columnIndex] != null) { + block = coercers[columnIndex].apply(block); + } + blocks[columnIndex] = block; } } + initializeTmpErrors(positionCount); + for (int i = 0; i < positionCount; i++) { + tmpErrors[i] = errors[positions[i]]; + } + + Arrays.fill(errors, null); + try { initializeOutputPositions(positionCount); - for (FilterFunction function : filterFunctions) { + for (int i = 0; i < filterFunctions.size(); i++) { + FilterFunctionWithStats functionWithStats = filterFunctions.get(i); + + FilterFunction function = functionWithStats.getFunction(); int[] inputs = function.getInputChannels(); Block[] inputBlocks = new Block[inputs.length]; - for (int i = 0; i < inputs.length; i++) { - inputBlocks[i] = blocks[filterFunctionInputMapping.get(inputs[i])]; + for (int j = 0; j < inputs.length; j++) { + inputBlocks[j] = blocks[filterFunctionInputMapping.get(inputs[j])]; } Page page = new Page(positionCount, inputBlocks); - positionCount = function.filter(page, outputPositions, positionCount, errors); + long startTime = System.nanoTime(); + int inputPositionCount = positionCount; + positionCount = function.filter(page, outputPositions, positionCount, tmpErrors); + functionWithStats.getStats().update(inputPositionCount, positionCount, System.nanoTime() - startTime); + if (positionCount == 0) { break; } } - for (int i = 0; i < positionCount; i++) { - if (errors[i] != null) { - throw errors[i]; - } - } - // at this point outputPositions are relative to page, e.g. they are indices into positions array // translate outputPositions to positions relative to the start of the row group, // e.g. make outputPositions a subset of positions array for (int i = 0; i < positionCount; i++) { outputPositions[i] = positions[outputPositions[i]]; + errors[outputPositions[i]] = tmpErrors[i]; } return positionCount; } @@ -373,6 +817,16 @@ private int applyFilterFunctions(int[] positions, int positionCount) } } + private void initializeTmpErrors(int positionCount) + { + if (tmpErrors == null || tmpErrors.length < positionCount) { + tmpErrors = new RuntimeException[positionCount]; + } + else { + Arrays.fill(tmpErrors, null); + } + } + private void initializeOutputPositions(int positionCount) { if (outputPositions == null || outputPositions.length < positionCount) { @@ -382,13 +836,6 @@ private void initializeOutputPositions(int positionCount) for (int i = 0; i < positionCount; i++) { outputPositions[i] = i; } - - if (errors == null || errors.length < positionCount) { - errors = new RuntimeException[positionCount]; - } - else { - Arrays.fill(errors, null); - } } @Override @@ -405,4 +852,89 @@ public void close() super.close(); } + + private static final class OrcBlockLoader + implements LazyBlockLoader + { + private final SelectiveStreamReader reader; + @Nullable + private final Function coercer; + private final int offset; + private final int[] positions; + private final int positionCount; + private boolean loaded; + + public OrcBlockLoader(SelectiveStreamReader reader, @Nullable Function coercer, int offset, int[] positions, int positionCount) + { + this.reader = requireNonNull(reader, "reader is null"); + this.coercer = coercer; // can be null + this.offset = offset; + this.positions = requireNonNull(positions, "positions is null"); + this.positionCount = positionCount; + } + + @Override + public final void load(LazyBlock lazyBlock) + { + if (loaded) { + return; + } + + try { + reader.read(offset, positions, positionCount); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + + Block block = reader.getBlock(positions, positionCount); + if (coercer != null) { + block = coercer.apply(block); + } + lazyBlock.setBlock(block); + + loaded = true; + } + } + + private static final class FilterFunctionWithStats + { + private final FilterFunction function; + private final FilterStats stats; + + private FilterFunctionWithStats(FilterFunction function, FilterStats stats) + { + this.function = function; + this.stats = stats; + } + + public FilterFunction getFunction() + { + return function; + } + + public FilterStats getStats() + { + return stats; + } + } + + private static final class FilterStats + { + private long inputPositions; + private long outputPositions; + private long elapsedNanos; + + public void update(int inputPositions, int outputPositions, long elapsedNanos) + { + this.inputPositions += inputPositions; + this.outputPositions += outputPositions; + this.elapsedNanos += elapsedNanos; + } + + public double getElapsedNanonsPerDroppedPosition() + { + return (double) elapsedNanos / (1 + inputPositions - outputPositions); + } + } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriter.java index 2a0191d6e1e2d..a8e401cfb3a59 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriter.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc; +import com.facebook.airlift.log.Logger; import com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationBuilder; import com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode; import com.facebook.presto.orc.OrcWriterStats.FlushReason; @@ -36,9 +37,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import io.airlift.slice.Slices; +import io.airlift.units.DataSize; import org.joda.time.DateTimeZone; import org.openjdk.jol.info.ClassLayout; @@ -68,6 +69,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static io.airlift.slice.Slices.utf8Slice; +import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.lang.Integer.max; import static java.lang.Integer.min; import static java.lang.Math.toIntExact; @@ -328,22 +330,26 @@ private void flushStripe(FlushReason flushReason) outputData.add(createDataOutput(MAGIC)); stripeStartOffset += MAGIC.length(); } - // add stripe data - outputData.addAll(bufferStripeData(stripeStartOffset, flushReason)); - // if the file is being closed, add the file footer - if (flushReason == CLOSED) { - outputData.addAll(bufferFileFooter()); - } - // write all data - orcDataSink.write(outputData); + try { + // add stripe data + outputData.addAll(bufferStripeData(stripeStartOffset, flushReason)); + // if the file is being closed, add the file footer + if (flushReason == CLOSED) { + outputData.addAll(bufferFileFooter()); + } - // open next stripe - columnWriters.forEach(ColumnWriter::reset); - dictionaryCompressionOptimizer.reset(); - rowGroupRowCount = 0; - stripeRowCount = 0; - bufferedBytes = toIntExact(columnWriters.stream().mapToLong(ColumnWriter::getBufferedBytes).sum()); + // write all data + orcDataSink.write(outputData); + } + finally { + // open next stripe + columnWriters.forEach(ColumnWriter::reset); + dictionaryCompressionOptimizer.reset(); + rowGroupRowCount = 0; + stripeRowCount = 0; + bufferedBytes = toIntExact(columnWriters.stream().mapToLong(ColumnWriter::getBufferedBytes).sum()); + } } /** @@ -515,7 +521,8 @@ public void validate(OrcDataSource input) input, types, hiveStorageTimeZone, - orcEncoding); + orcEncoding, + new OrcReaderOptions(new DataSize(1, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(16, MEGABYTE), false)); } private int estimateAverageLogicalSizePerRow(Page page) diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriterFlushStats.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriterFlushStats.java index 72b7cc3bcd04a..6dcd62d2dfaf1 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriterFlushStats.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcWriterFlushStats.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.orc; -import io.airlift.stats.DistributionStat; +import com.facebook.airlift.stats.DistributionStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/OrcZstdDecompressor.java b/presto-orc/src/main/java/com/facebook/presto/orc/OrcZstdDecompressor.java index 4497c0cf36b26..6a1f5cea3ac8d 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/OrcZstdDecompressor.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/OrcZstdDecompressor.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc; +import com.github.luben.zstd.Zstd; import io.airlift.compress.MalformedInputException; import io.airlift.compress.zstd.ZstdDecompressor; @@ -24,12 +25,26 @@ class OrcZstdDecompressor { private final OrcDataSourceId orcDataSourceId; private final int maxBufferSize; - private final ZstdDecompressor decompressor = new ZstdDecompressor(); + private final Decompressor decompressor; - public OrcZstdDecompressor(OrcDataSourceId orcDataSourceId, int maxBufferSize) + public OrcZstdDecompressor(OrcDataSourceId orcDataSourceId, int maxBufferSize, boolean zstdJniDecompressionEnabled) { this.orcDataSourceId = requireNonNull(orcDataSourceId, "orcDataSourceId is null"); this.maxBufferSize = maxBufferSize; + if (zstdJniDecompressionEnabled) { + this.decompressor = (input, inputOffset, inputLength, output, outputOffset, maxOutputLength) -> { + long size = Zstd.decompressByteArray(output, 0, maxOutputLength, input, inputOffset, inputLength); + if (Zstd.isError(size)) { + String errorName = Zstd.getErrorName(size); + throw new MalformedInputException(inputOffset, "Zstd JNI decompressor failed with " + errorName); + } + return toIntExact(size); + }; + } + else { + ZstdDecompressor zstdDecompressor = new ZstdDecompressor(); + this.decompressor = zstdDecompressor::decompress; + } } @Override @@ -55,4 +70,10 @@ public String toString() { return "zstd"; } + + interface Decompressor + { + int decompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset, int maxOutputLength) + throws MalformedInputException; + } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/StorageStripeMetadataSource.java b/presto-orc/src/main/java/com/facebook/presto/orc/StorageStripeMetadataSource.java new file mode 100644 index 0000000000000..db6ee0dcdb6c9 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/StorageStripeMetadataSource.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.facebook.presto.orc.StripeReader.StripeId; +import com.google.common.collect.ImmutableMap; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; + +import java.io.IOException; +import java.util.Map; + +public class StorageStripeMetadataSource + implements StripeMetadataSource +{ + @Override + public Slice getStripeFooterSlice(OrcDataSource orcDataSource, StripeId stripeId, long footerOffset, int footerLength) + throws IOException + { + byte[] tailBuffer = new byte[footerLength]; + orcDataSource.readFully(footerOffset, tailBuffer); + return Slices.wrappedBuffer(tailBuffer); + } + + @Override + public Map getInputs(OrcDataSource orcDataSource, StripeId stripeId, Map diskRanges) + throws IOException + { + // + // Note: this code does not use the Java 8 stream APIs to avoid any extra object allocation + // + + // transform ranges to have an absolute offset in file + ImmutableMap.Builder diskRangesBuilder = ImmutableMap.builder(); + for (Map.Entry entry : diskRanges.entrySet()) { + DiskRange diskRange = entry.getValue(); + diskRangesBuilder.put(entry.getKey(), new DiskRange(stripeId.getOffset() + diskRange.getOffset(), diskRange.getLength())); + } + diskRanges = diskRangesBuilder.build(); + + // read ranges + return orcDataSource.readFully(diskRanges); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/StreamDescriptor.java b/presto-orc/src/main/java/com/facebook/presto/orc/StreamDescriptor.java index 9d01b3bb67bc1..805f29f518d87 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/StreamDescriptor.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/StreamDescriptor.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc; +import com.facebook.presto.orc.metadata.OrcType; import com.facebook.presto.orc.metadata.OrcType.OrcTypeKind; import com.google.common.collect.ImmutableList; @@ -27,23 +28,23 @@ public final class StreamDescriptor private final String streamName; private final int streamId; private final int sequence; - private final OrcTypeKind streamType; + private final OrcType orcType; private final String fieldName; private final OrcDataSource orcDataSource; private final List nestedStreams; - public StreamDescriptor(String streamName, int streamId, String fieldName, OrcTypeKind streamType, OrcDataSource orcDataSource, List nestedStreams) + public StreamDescriptor(String streamName, int streamId, String fieldName, OrcType orcType, OrcDataSource orcDataSource, List nestedStreams) { - this(streamName, streamId, fieldName, streamType, orcDataSource, nestedStreams, DEFAULT_SEQUENCE_ID); + this(streamName, streamId, fieldName, orcType, orcDataSource, nestedStreams, DEFAULT_SEQUENCE_ID); } - public StreamDescriptor(String streamName, int streamId, String fieldName, OrcTypeKind streamType, OrcDataSource orcDataSource, List nestedStreams, int sequence) + public StreamDescriptor(String streamName, int streamId, String fieldName, OrcType orcType, OrcDataSource orcDataSource, List nestedStreams, int sequence) { this.streamName = requireNonNull(streamName, "streamName is null"); this.streamId = streamId; this.sequence = sequence; this.fieldName = requireNonNull(fieldName, "fieldName is null"); - this.streamType = requireNonNull(streamType, "type is null"); + this.orcType = requireNonNull(orcType, "orcType is null"); this.orcDataSource = requireNonNull(orcDataSource, "orcDataSource is null"); this.nestedStreams = ImmutableList.copyOf(requireNonNull(nestedStreams, "nestedStreams is null")); } @@ -63,9 +64,14 @@ public int getSequence() return sequence; } - public OrcTypeKind getStreamType() + public OrcTypeKind getOrcTypeKind() { - return streamType; + return orcType.getOrcTypeKind(); + } + + public OrcType getOrcType() + { + return this.orcType; } public String getFieldName() @@ -95,7 +101,7 @@ public String toString() .add("streamName", streamName) .add("streamId", streamId) .add("sequence", sequence) - .add("streamType", streamType) + .add("orcType", orcType) .add("dataSource", orcDataSource.getId()) .toString(); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/StripeMetadataSource.java b/presto-orc/src/main/java/com/facebook/presto/orc/StripeMetadataSource.java new file mode 100644 index 0000000000000..dc2706bb532cc --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/StripeMetadataSource.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.facebook.presto.orc.StripeReader.StripeId; +import io.airlift.slice.Slice; + +import java.io.IOException; +import java.util.Map; + +public interface StripeMetadataSource +{ + Slice getStripeFooterSlice(OrcDataSource orcDataSource, StripeId stripeId, long footerOffset, int footerLength) + throws IOException; + + Map getInputs( + OrcDataSource orcDataSource, + StripeId stripeId, + Map diskRanges) + throws IOException; +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/StripeReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/StripeReader.java index 493009f4ac143..14b9ace23ce3b 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/StripeReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/StripeReader.java @@ -41,7 +41,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; -import io.airlift.slice.Slices; +import io.airlift.slice.Slice; import java.io.IOException; import java.io.InputStream; @@ -51,6 +51,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.SortedMap; @@ -67,6 +68,7 @@ import static com.facebook.presto.orc.metadata.Stream.StreamKind.ROW_INDEX; import static com.facebook.presto.orc.metadata.statistics.ColumnStatistics.mergeColumnStatistics; import static com.facebook.presto.orc.stream.CheckpointInputStreamSource.createCheckpointStreamSource; +import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Iterables.getOnlyElement; @@ -84,6 +86,7 @@ public class StripeReader private final OrcPredicate predicate; private final MetadataReader metadataReader; private final Optional writeValidation; + private final StripeMetadataSource stripeMetadataSource; public StripeReader(OrcDataSource orcDataSource, Optional decompressor, @@ -93,7 +96,8 @@ public StripeReader(OrcDataSource orcDataSource, OrcPredicate predicate, HiveWriterVersion hiveWriterVersion, MetadataReader metadataReader, - Optional writeValidation) + Optional writeValidation, + StripeMetadataSource stripeMetadataSource) { this.orcDataSource = requireNonNull(orcDataSource, "orcDataSource is null"); this.decompressor = requireNonNull(decompressor, "decompressor is null"); @@ -104,13 +108,16 @@ public StripeReader(OrcDataSource orcDataSource, this.hiveWriterVersion = requireNonNull(hiveWriterVersion, "hiveWriterVersion is null"); this.metadataReader = requireNonNull(metadataReader, "metadataReader is null"); this.writeValidation = requireNonNull(writeValidation, "writeValidation is null"); + this.stripeMetadataSource = requireNonNull(stripeMetadataSource, "stripeMetadataSource is null"); } public Stripe readStripe(StripeInformation stripe, AggregatedMemoryContext systemMemoryUsage) throws IOException { + StripeId stripeId = new StripeId(orcDataSource.getId(), stripe.getOffset()); + // read the stripe footer - StripeFooter stripeFooter = readStripeFooter(stripe, systemMemoryUsage); + StripeFooter stripeFooter = readStripeFooter(stripeId, stripe, systemMemoryUsage); List columnEncodings = stripeFooter.getColumnEncodings(); // get streams for selected columns @@ -146,10 +153,10 @@ public Stripe readStripe(StripeInformation stripe, AggregatedMemoryContext syste diskRanges = Maps.filterKeys(diskRanges, Predicates.in(streams.keySet())); // read the file regions - Map streamsData = readDiskRanges(stripe.getOffset(), diskRanges, systemMemoryUsage); + Map streamsData = readDiskRanges(stripeId, diskRanges, systemMemoryUsage); // read the bloom filter for each column - Map> bloomFilterIndexes = readBloomFilterIndexes(streams, streamsData); + Map> bloomFilterIndexes = readBloomFilterIndexes(streams, streamsData); // read the row index for each column Map> columnIndexes = readColumnIndexes(streams, streamsData, bloomFilterIndexes); @@ -207,7 +214,7 @@ public Stripe readStripe(StripeInformation stripe, AggregatedMemoryContext syste ImmutableMap diskRanges = diskRangesBuilder.build(); // read the file regions - Map streamsData = readDiskRanges(stripe.getOffset(), diskRanges, systemMemoryUsage); + Map streamsData = readDiskRanges(stripeId, diskRanges, systemMemoryUsage); long minAverageRowBytes = 0; for (Entry entry : streams.entrySet()) { @@ -245,23 +252,15 @@ public Stripe readStripe(StripeInformation stripe, AggregatedMemoryContext syste return new Stripe(stripe.getNumberOfRows(), columnEncodings, ImmutableList.of(rowGroup), dictionaryStreamSources); } - public Map readDiskRanges(long stripeOffset, Map diskRanges, AggregatedMemoryContext systemMemoryUsage) + private Map readDiskRanges(StripeId stripeId, Map diskRanges, AggregatedMemoryContext systemMemoryUsage) throws IOException { // // Note: this code does not use the Java 8 stream APIs to avoid any extra object allocation // - // transform ranges to have an absolute offset in file - ImmutableMap.Builder diskRangesBuilder = ImmutableMap.builder(); - for (Entry entry : diskRanges.entrySet()) { - DiskRange diskRange = entry.getValue(); - diskRangesBuilder.put(entry.getKey(), new DiskRange(stripeOffset + diskRange.getOffset(), diskRange.getLength())); - } - diskRanges = diskRangesBuilder.build(); - // read ranges - Map streamsData = orcDataSource.readFully(diskRanges); + Map streamsData = stripeMetadataSource.getInputs(orcDataSource, stripeId, diskRanges); // transform streams to OrcInputStream ImmutableMap.Builder streamsBuilder = ImmutableMap.builder(); @@ -374,16 +373,15 @@ public static RowGroup createRowGroup(int groupId, int rowOffset, int rowCount, return new RowGroup(groupId, rowOffset, rowCount, minAverageRowBytes, rowGroupStreams); } - public StripeFooter readStripeFooter(StripeInformation stripe, AggregatedMemoryContext systemMemoryUsage) + public StripeFooter readStripeFooter(StripeId stripeId, StripeInformation stripe, AggregatedMemoryContext systemMemoryUsage) throws IOException { - long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(); - int tailLength = toIntExact(stripe.getFooterLength()); + long footerOffset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(); + int footerLength = toIntExact(stripe.getFooterLength()); // read the footer - byte[] tailBuffer = new byte[tailLength]; - orcDataSource.readFully(offset, tailBuffer); - try (InputStream inputStream = new OrcInputStream(orcDataSource.getId(), Slices.wrappedBuffer(tailBuffer).getInput(), decompressor, systemMemoryUsage, tailLength)) { + Slice footerSlice = stripeMetadataSource.getStripeFooterSlice(orcDataSource, stripeId, footerOffset, footerLength); + try (InputStream inputStream = new OrcInputStream(orcDataSource.getId(), footerSlice.getInput(), decompressor, systemMemoryUsage, footerLength)) { return metadataReader.readStripeFooter(types, inputStream); } } @@ -393,22 +391,22 @@ static boolean isIndexStream(Stream stream) return stream.getStreamKind() == ROW_INDEX || stream.getStreamKind() == DICTIONARY_COUNT || stream.getStreamKind() == BLOOM_FILTER || stream.getStreamKind() == BLOOM_FILTER_UTF8; } - private Map> readBloomFilterIndexes(Map streams, Map streamsData) + private Map> readBloomFilterIndexes(Map streams, Map streamsData) throws IOException { - ImmutableMap.Builder> bloomFilters = ImmutableMap.builder(); + ImmutableMap.Builder> bloomFilters = ImmutableMap.builder(); for (Entry entry : streams.entrySet()) { Stream stream = entry.getValue(); if (stream.getStreamKind() == BLOOM_FILTER) { OrcInputStream inputStream = streamsData.get(entry.getKey()); - bloomFilters.put(entry.getKey(), metadataReader.readBloomFilterIndexes(inputStream)); + bloomFilters.put(entry.getKey().getColumn(), metadataReader.readBloomFilterIndexes(inputStream)); } // TODO: add support for BLOOM_FILTER_UTF8 } return bloomFilters.build(); } - private Map> readColumnIndexes(Map streams, Map streamsData, Map> bloomFilterIndexes) + private Map> readColumnIndexes(Map streams, Map streamsData, Map> bloomFilterIndexes) throws IOException { ImmutableMap.Builder> columnIndexes = ImmutableMap.builder(); @@ -416,7 +414,7 @@ private Map> readColumnIndexes(Map bloomFilters = bloomFilterIndexes.get(entry.getKey()); + List bloomFilters = bloomFilterIndexes.get(entry.getKey().getColumn()); List rowGroupIndexes = metadataReader.readRowIndexes(hiveWriterVersion, inputStream); if (bloomFilters != null && !bloomFilters.isEmpty()) { ImmutableList.Builder newRowGroupIndexes = ImmutableList.builder(); @@ -531,4 +529,101 @@ private static int ceil(int dividend, int divisor) { return ((dividend + divisor) - 1) / divisor; } + + public static class StripeId + { + private final OrcDataSourceId sourceId; + private final long offset; + + public StripeId(OrcDataSourceId sourceId, long offset) + { + this.sourceId = requireNonNull(sourceId, "sourceId is null"); + this.offset = offset; + } + + public OrcDataSourceId getSourceId() + { + return sourceId; + } + + public long getOffset() + { + return offset; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StripeId stripeId = (StripeId) o; + return offset == stripeId.offset && + Objects.equals(sourceId, stripeId.sourceId); + } + + @Override + public int hashCode() + { + return Objects.hash(sourceId, offset); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("sourceId", sourceId) + .add("offset", offset) + .toString(); + } + } + + public static class StripeStreamId + { + private final StripeId stripeId; + private final StreamId streamId; + + public StripeStreamId(StripeId stripeId, StreamId streamId) + { + this.stripeId = requireNonNull(stripeId, "stripeId is null"); + this.streamId = requireNonNull(streamId, "streamId is null"); + } + + public StreamId getStreamId() + { + return streamId; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StripeStreamId that = (StripeStreamId) o; + return Objects.equals(stripeId, that.stripeId) && + Objects.equals(streamId, that.streamId); + } + + @Override + public int hashCode() + { + return Objects.hash(stripeId, streamId); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("stripeId", stripeId) + .add("streamId", streamId) + .toString(); + } + } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainFilter.java b/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainFilter.java index 981fa761f1cda..f1b69ab5263ef 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainFilter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainFilter.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.orc; +import com.google.common.annotations.VisibleForTesting; + import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -34,8 +36,27 @@ public interface TupleDomainFilter TupleDomainFilter IS_NULL = new IsNull(); TupleDomainFilter IS_NOT_NULL = new IsNotNull(); + /** + * A filter becomes non-deterministic when applies to nested column, + * e.g. a[1] > 10 is non-deterministic because > 10 filter applies only to some + * positions, e.g. first entry in a set of entries that correspond to a single + * top-level position. + */ + boolean isDeterministic(); + boolean testNull(); + /** + * Used to apply is [not] null filters to complex types, e.g. + * a[1] is null AND a[3] is not null, where a is an array(array(T)). + * + * In these case, the exact values are not known, but it is known whether they are + * null or not. Furthermore, for some positions only nulls are allowed (a[1] is null), + * for others only non-nulls (a[3] is not null), and for the rest both are allowed + * (a[2] and a[N], where N > 3). + */ + boolean testNonNull(); + boolean testLong(long value); boolean testDouble(double value); @@ -48,50 +69,111 @@ public interface TupleDomainFilter boolean testBytes(byte[] buffer, int offset, int length); + /** + * Filters like string equality and IN, as well as conditions on cardinality of lists and maps can be at least partly + * decided by looking at lengths alone. If this is false, then no further checks are needed. If true, eventual filters on the + * data itself need to be evaluated. + */ + boolean testLength(int length); + + /** + * When a filter applied to a nested column fails, the whole top-level position should + * fail. To enable this functionality, the filter keeps track of the boundaries of + * top-level positions and allows the caller to find out where the current top-level + * position started and how far it continues. + * @return number of positions from the start of the current top-level position up to + * the current position (excluding current position) + */ + int getPrecedingPositionsToFail(); + + /** + * @return number of positions remaining until the end of the current top-level position + */ + int getSucceedingPositionsToFail(); + abstract class AbstractTupleDomainFilter implements TupleDomainFilter { protected final boolean nullAllowed; + private final boolean deterministic; - private AbstractTupleDomainFilter(boolean nullAllowed) + protected AbstractTupleDomainFilter(boolean deterministic, boolean nullAllowed) { this.nullAllowed = nullAllowed; + this.deterministic = deterministic; + } + + @Override + public boolean isDeterministic() + { + return deterministic; } + @Override public boolean testNull() { return nullAllowed; } + @Override + public boolean testNonNull() + { + throw new UnsupportedOperationException(); + } + + @Override public boolean testLong(long value) { throw new UnsupportedOperationException(); } + @Override public boolean testDouble(double value) { throw new UnsupportedOperationException(); } + @Override public boolean testFloat(float value) { throw new UnsupportedOperationException(); } + @Override public boolean testDecimal(long low, long high) { throw new UnsupportedOperationException(); } + @Override public boolean testBoolean(boolean value) { throw new UnsupportedOperationException(); } + @Override public boolean testBytes(byte[] buffer, int offset, int length) { throw new UnsupportedOperationException(); } + + @Override + public boolean testLength(int length) + { + throw new UnsupportedOperationException(); + } + + @Override + public int getPrecedingPositionsToFail() + { + return 0; + } + + @Override + public int getSucceedingPositionsToFail() + { + return 0; + } } class AlwaysFalse @@ -99,7 +181,13 @@ class AlwaysFalse { private AlwaysFalse() { - super(false); + super(true, false); + } + + @Override + public boolean testNonNull() + { + return false; } @Override @@ -138,6 +226,12 @@ public boolean testBytes(byte[] buffer, int offset, int length) return false; } + @Override + public boolean testLength(int length) + { + return false; + } + @Override public String toString() { @@ -150,7 +244,13 @@ class IsNull { private IsNull() { - super(true); + super(true, true); + } + + @Override + public boolean testNonNull() + { + return false; } @Override @@ -189,6 +289,12 @@ public boolean testBytes(byte[] buffer, int offset, int length) return false; } + @Override + public boolean testLength(int length) + { + return false; + } + @Override public String toString() { @@ -201,7 +307,13 @@ class IsNotNull { private IsNotNull() { - super(false); + super(true, false); + } + + @Override + public boolean testNonNull() + { + return true; } @Override @@ -240,6 +352,12 @@ public boolean testBytes(byte[] buffer, int offset, int length) return true; } + @Override + public boolean testLength(int length) + { + return true; + } + @Override public String toString() { @@ -254,7 +372,7 @@ class BooleanValue private BooleanValue(boolean value, boolean nullAllowed) { - super(nullAllowed); + super(true, nullAllowed); this.value = value; } @@ -307,9 +425,10 @@ class BigintRange private final long lower; private final long upper; - private BigintRange(long lower, long upper, boolean nullAllowed) + @VisibleForTesting + protected BigintRange(long lower, long upper, boolean nullAllowed) { - super(nullAllowed); + super(true, nullAllowed); checkArgument(lower <= upper, "lower must be less than or equal to upper"); this.lower = lower; this.upper = upper; @@ -389,7 +508,7 @@ class BigintValues private BigintValues(long[] values, boolean nullAllowed) { - super(nullAllowed); + super(true, nullAllowed); requireNonNull(values, "values is null"); checkArgument(values.length > 1, "values must contain at least 2 entries"); @@ -484,7 +603,7 @@ class AbstractRange private AbstractRange(boolean lowerUnbounded, boolean lowerExclusive, boolean upperUnbounded, boolean upperExclusive, boolean nullAllowed) { - super(nullAllowed); + super(true, nullAllowed); this.lowerUnbounded = lowerUnbounded; this.lowerExclusive = lowerExclusive; this.upperUnbounded = upperUnbounded; @@ -498,9 +617,12 @@ class DoubleRange private final double lower; private final double upper; - private DoubleRange(double lower, boolean lowerUnbounded, boolean lowerExclusive, double upper, boolean upperUnbounded, boolean upperExclusive, boolean nullAllowed) + @VisibleForTesting + protected DoubleRange(double lower, boolean lowerUnbounded, boolean lowerExclusive, double upper, boolean upperUnbounded, boolean upperExclusive, boolean nullAllowed) { super(lowerUnbounded, lowerExclusive, upperUnbounded, upperExclusive, nullAllowed); + checkArgument(lowerUnbounded || !Double.isNaN(lower)); + checkArgument(upperUnbounded || !Double.isNaN(upper)); this.lower = lower; this.upper = upper; } @@ -510,9 +632,22 @@ public static DoubleRange of(double lower, boolean lowerUnbounded, boolean lower return new DoubleRange(lower, lowerUnbounded, lowerExclusive, upper, upperUnbounded, upperExclusive, nullAllowed); } + public double getLower() + { + return lower; + } + + public double getUpper() + { + return upper; + } + @Override public boolean testDouble(double value) { + if (Double.isNaN(value)) { + return false; + } if (!lowerUnbounded) { if (value < lower) { return false; @@ -583,6 +718,8 @@ class FloatRange private FloatRange(float lower, boolean lowerUnbounded, boolean lowerExclusive, float upper, boolean upperUnbounded, boolean upperExclusive, boolean nullAllowed) { super(lowerUnbounded, lowerExclusive, upperUnbounded, upperExclusive, nullAllowed); + checkArgument(lowerUnbounded || !Float.isNaN(lower)); + checkArgument(upperUnbounded || !Float.isNaN(upper)); this.lower = lower; this.upper = upper; } @@ -595,6 +732,9 @@ public static FloatRange of(float lower, boolean lowerUnbounded, boolean lowerEx @Override public boolean testFloat(float value) { + if (Float.isNaN(value)) { + return false; + } if (!lowerUnbounded) { if (value < lower) { return false; @@ -759,7 +899,7 @@ class BytesRange private BytesRange(byte[] lower, boolean lowerExclusive, byte[] upper, boolean upperExclusive, boolean nullAllowed) { - super(nullAllowed); + super(true, nullAllowed); this.lower = lower; this.upper = upper; this.lowerExclusive = lowerExclusive; @@ -784,8 +924,8 @@ public boolean testBytes(byte[] buffer, int offset, int length) if (buffer[i + offset] != lower[i]) { return false; } - return true; } + return true; } if (lower != null) { @@ -802,6 +942,12 @@ public boolean testBytes(byte[] buffer, int offset, int length) return true; } + @Override + public boolean testLength(int length) + { + return !singleValue || lower.length == length; + } + @Override public int hashCode() { @@ -858,15 +1004,18 @@ class BytesValues private final int hashTableSizeMask; private final long[] bloom; private final int bloomSize; + // Contains true in position i if at least one of the values has length i. + private final boolean[] lengthExists; private BytesValues(byte[][] values, boolean nullAllowed) { - super(nullAllowed); + super(true, nullAllowed); requireNonNull(values, "values is null"); checkArgument(values.length > 1, "values must contain at least 2 entries"); this.values = values; + lengthExists = new boolean[Arrays.stream(values).mapToInt(value -> value.length).max().getAsInt() + 1]; // Linear hash table size is the highest power of two less than or equal to number of values * 4. This means that the // table is under half full, e.g. 127 elements gets 256 slots. int hashTableSize = Integer.highestOneBit(values.length * 4); @@ -876,6 +1025,7 @@ private BytesValues(byte[][] values, boolean nullAllowed) bloomSize = Math.max(1, hashTableSize / 8); bloom = new long[bloomSize]; for (byte[] value : values) { + lengthExists[value.length] = true; long hashCode = hash(value, 0, value.length); bloom[bloomIndex(hashCode)] |= bloomMask(hashCode); int position = (int) (hashCode & hashTableSizeMask); @@ -918,6 +1068,12 @@ public boolean testBytes(byte[] value, int offset, int length) return false; } + @Override + public boolean testLength(int length) + { + return length < lengthExists.length && lengthExists[length]; + } + private static long bloomMask(long hashCode) { return (1L << ((hashCode >> 20) & 63)) | (1L << ((hashCode >> 26) & 63)) | (1L << ((hashCode >> 32) & 63)); @@ -975,7 +1131,7 @@ class BigintMultiRange private BigintMultiRange(List ranges, boolean nullAllowed) { - super(nullAllowed); + super(true, nullAllowed); requireNonNull(ranges, "ranges is null"); checkArgument(!ranges.isEmpty(), "ranges is empty"); @@ -1049,7 +1205,7 @@ class MultiRange private MultiRange(List filters, boolean nullAllowed) { - super(nullAllowed); + super(true, nullAllowed); requireNonNull(filters, "filters is null"); checkArgument(filters.size() > 1, "filters must contain at least 2 entries"); @@ -1105,6 +1261,17 @@ public boolean testBytes(byte[] buffer, int offset, int length) return false; } + @Override + public boolean testLength(int length) + { + for (TupleDomainFilter filter : filters) { + if (filter.testLength(length)) { + return true; + } + } + return false; + } + @Override public boolean equals(Object o) { @@ -1136,4 +1303,266 @@ public String toString() .toString(); } } + + abstract class BasePositionalFilter + implements TupleDomainFilter + { + // Index into filters array pointing to the next filter to apply + protected int filterIndex; + + // Indices into filters array identifying boundaries of the outer rows + protected int[] offsets; + + // Index into offsets array pointing to the next offset + private int offsetIndex; + + private boolean[] failed; + + // Number of positions from the start of the top-level position to fail; + // populated on first failure within a top-level position + private int precedingPositionsToFail; + + // Number of positions to the end of the top-level position to fail; + // populated on first failure within a top-level position + private int succeedingPositionsToFail; + + @Override + public boolean isDeterministic() + { + return false; + } + + public boolean[] getFailed() + { + return failed; + } + + @Override + public int getPrecedingPositionsToFail() + { + return precedingPositionsToFail; + } + + @Override + public int getSucceedingPositionsToFail() + { + return succeedingPositionsToFail; + } + + protected void reset() + { + filterIndex = 0; + offsetIndex = 1; + + if (failed == null || failed.length < offsets.length) { + failed = new boolean[offsets.length]; + } + else { + Arrays.fill(failed, false); + } + failed[0] = false; + precedingPositionsToFail = 0; + succeedingPositionsToFail = 0; + } + + protected void advance() + { + while (filterIndex == offsets[offsetIndex]) { + // start of the next top-level position + precedingPositionsToFail = 0; + succeedingPositionsToFail = 0; + failed[offsetIndex] = false; + offsetIndex++; + } + } + + protected boolean recordTestResult(boolean result) + { + if (!result) { + failed[offsetIndex - 1] = true; + precedingPositionsToFail = filterIndex - offsets[offsetIndex - 1] - 1; + succeedingPositionsToFail = offsets[offsetIndex] - filterIndex; + filterIndex = offsets[offsetIndex]; + } + + return result; + } + } + + class PositionalFilter + extends BasePositionalFilter + { + // Filters for individual positions being read; some may be null + private TupleDomainFilter[] filters; + + public void setFilters(TupleDomainFilter[] filters, int[] offsets) + { + this.filters = requireNonNull(filters, "filters is null"); + this.offsets = requireNonNull(offsets, "offsets is null"); + reset(); + } + + @Override + public boolean testNull() + { + advance(); + TupleDomainFilter filter = filters[filterIndex++]; + if (filter == null) { + return true; + } + + return recordTestResult(filter.testNull()); + } + + @Override + public boolean testNonNull() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean testLong(long value) + { + advance(); + TupleDomainFilter filter = filters[filterIndex++]; + if (filter == null) { + return true; + } + + return recordTestResult(filter.testLong(value)); + } + + @Override + public boolean testDouble(double value) + { + advance(); + TupleDomainFilter filter = filters[filterIndex++]; + if (filter == null) { + return true; + } + + return recordTestResult(filter.testDouble(value)); + } + + @Override + public boolean testFloat(float value) + { + advance(); + TupleDomainFilter filter = filters[filterIndex++]; + if (filter == null) { + return true; + } + + return recordTestResult(filter.testFloat(value)); + } + + @Override + public boolean testDecimal(long low, long high) + { + advance(); + TupleDomainFilter filter = filters[filterIndex++]; + if (filter == null) { + return true; + } + + return recordTestResult(filter.testDecimal(low, high)); + } + + @Override + public boolean testBoolean(boolean value) + { + advance(); + TupleDomainFilter filter = filters[filterIndex++]; + if (filter == null) { + return true; + } + + return recordTestResult(filter.testBoolean(value)); + } + + @Override + public boolean testBytes(byte[] buffer, int offset, int length) + { + advance(); + TupleDomainFilter filter = filters[filterIndex++]; + if (filter == null) { + return true; + } + + return recordTestResult(filter.testBytes(buffer, offset, length)); + } + + public boolean testLength(int length) + { + // Returns true without advancing to the next filter because this is a pre-check followed by a test on the value, + // which will advance the state. TODO: We could advance the state on false and not advance on true. Consider the + // case where testLength is the only filter on a list/map inside another. This would imply exposing advancing as a + // separate operation. + return true; + } + } + + class NullsFilter + extends BasePositionalFilter + { + private boolean[] nullsAllowed; + private boolean[] nonNullsAllowed; + + public void setup(boolean[] nullsAllowed, boolean[] nonNullsAllowed, int[] offsets) + { + this.nullsAllowed = requireNonNull(nullsAllowed, "nullsAllowed is null"); + this.nonNullsAllowed = requireNonNull(nonNullsAllowed, "nonNullsAllowed is null"); + this.offsets = requireNonNull(offsets, "offsets is null"); + reset(); + } + + @Override + public boolean testNull() + { + advance(); + return recordTestResult(nullsAllowed[filterIndex++]); + } + + @Override + public boolean testNonNull() + { + advance(); + return recordTestResult(nonNullsAllowed[filterIndex++]); + } + + public boolean testLong(long value) + { + throw new UnsupportedOperationException(); + } + + public boolean testDouble(double value) + { + throw new UnsupportedOperationException(); + } + + public boolean testFloat(float value) + { + throw new UnsupportedOperationException(); + } + + public boolean testDecimal(long low, long high) + { + throw new UnsupportedOperationException(); + } + + public boolean testBoolean(boolean value) + { + throw new UnsupportedOperationException(); + } + + public boolean testBytes(byte[] buffer, int offset, int length) + { + throw new UnsupportedOperationException(); + } + + public boolean testLength(int length) + { + throw new UnsupportedOperationException(); + } + } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainFilterUtils.java b/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainFilterUtils.java index 6e135d6dcf13d..8a9db0cc18e5e 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainFilterUtils.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/TupleDomainFilterUtils.java @@ -62,10 +62,21 @@ private TupleDomainFilterUtils() {} public static TupleDomainFilter toFilter(Domain domain) { ValueSet values = domain.getValues(); + boolean nullAllowed = domain.isNullAllowed(); + + if (values.isAll()) { + checkArgument(!nullAllowed, "Unexpected allways-true filter"); + return IS_NOT_NULL; + } + + if (values.isNone()) { + checkArgument(nullAllowed, "Unexpected allways-false filter"); + return IS_NULL; + } + checkArgument(values instanceof SortedRangeSet, "Unexpected domain type: " + values.getClass().getSimpleName()); List ranges = ((SortedRangeSet) values).getOrderedRanges(); - boolean nullAllowed = domain.isNullAllowed(); if (ranges.isEmpty() && nullAllowed) { return IS_NULL; @@ -197,6 +208,12 @@ private static TupleDomainFilter doubleRangeToFilter(Range range, boolean nullAl Marker high = range.getHigh(); double lowerDouble = low.isLowerUnbounded() ? Double.MIN_VALUE : (double) low.getValue(); double upperDouble = high.isUpperUnbounded() ? Double.MAX_VALUE : (double) high.getValue(); + if (!low.isLowerUnbounded() && Double.isNaN(lowerDouble)) { + return ALWAYS_FALSE; + } + if (!high.isUpperUnbounded() && Double.isNaN(upperDouble)) { + return ALWAYS_FALSE; + } return DoubleRange.of( lowerDouble, low.isLowerUnbounded(), @@ -213,6 +230,12 @@ private static TupleDomainFilter floatRangeToFilter(Range range, boolean nullAll Marker high = range.getHigh(); float lowerFloat = low.isLowerUnbounded() ? Float.MIN_VALUE : intBitsToFloat(toIntExact((long) low.getValue())); float upperFloat = high.isUpperUnbounded() ? Float.MAX_VALUE : intBitsToFloat(toIntExact((long) high.getValue())); + if (!low.isLowerUnbounded() && Float.isNaN(lowerFloat)) { + return ALWAYS_FALSE; + } + if (!high.isUpperUnbounded() && Float.isNaN(upperFloat)) { + return ALWAYS_FALSE; + } return FloatRange.of( lowerFloat, low.isLowerUnbounded(), diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/cache/CachingOrcFileTailSource.java b/presto-orc/src/main/java/com/facebook/presto/orc/cache/CachingOrcFileTailSource.java new file mode 100644 index 0000000000000..1bcfdc5114e41 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/cache/CachingOrcFileTailSource.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.cache; + +import com.facebook.presto.orc.OrcDataSource; +import com.facebook.presto.orc.OrcDataSourceId; +import com.facebook.presto.orc.OrcWriteValidation; +import com.facebook.presto.orc.metadata.MetadataReader; +import com.facebook.presto.orc.metadata.OrcFileTail; +import com.google.common.cache.Cache; +import com.google.common.util.concurrent.UncheckedExecutionException; + +import java.io.IOException; +import java.util.Optional; +import java.util.concurrent.ExecutionException; + +import static com.google.common.base.Throwables.throwIfInstanceOf; +import static java.util.Objects.requireNonNull; + +public class CachingOrcFileTailSource + implements OrcFileTailSource +{ + private final Cache cache; + private final OrcFileTailSource delegate; + + public CachingOrcFileTailSource(OrcFileTailSource delegate, Cache cache) + { + this.cache = requireNonNull(cache, "cache is null"); + this.delegate = requireNonNull(delegate, "delegate is null"); + } + + @Override + public OrcFileTail getOrcFileTail(OrcDataSource orcDataSource, MetadataReader metadataReader, Optional writeValidation) + throws IOException + { + try { + return cache.get(orcDataSource.getId(), () -> delegate.getOrcFileTail(orcDataSource, metadataReader, writeValidation)); + } + catch (ExecutionException | UncheckedExecutionException e) { + throwIfInstanceOf(e.getCause(), IOException.class); + throw new IOException("Unexpected error in orc file tail reading after cache miss", e.getCause()); + } + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/cache/OrcCacheConfig.java b/presto-orc/src/main/java/com/facebook/presto/orc/cache/OrcCacheConfig.java new file mode 100644 index 0000000000000..e56baf3f03d25 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/cache/OrcCacheConfig.java @@ -0,0 +1,147 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.cache; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import io.airlift.units.DataSize; +import io.airlift.units.Duration; +import io.airlift.units.MinDataSize; +import io.airlift.units.MinDuration; + +import static io.airlift.units.DataSize.Unit.BYTE; +import static java.util.concurrent.TimeUnit.SECONDS; + +public class OrcCacheConfig +{ + private boolean fileTailCacheEnabled; + private DataSize fileTailCacheSize = new DataSize(0, BYTE); + private Duration fileTailCacheTtlSinceLastAccess = new Duration(0, SECONDS); + + private boolean stripeMetadataCacheEnabled; + private DataSize stripeFooterCacheSize = new DataSize(0, BYTE); + private Duration stripeFooterCacheTtlSinceLastAccess = new Duration(0, SECONDS); + private DataSize stripeStreamCacheSize = new DataSize(0, BYTE); + private Duration stripeStreamCacheTtlSinceLastAccess = new Duration(0, SECONDS); + + public boolean isFileTailCacheEnabled() + { + return fileTailCacheEnabled; + } + + @Config("orc.file-tail-cache-enabled") + @ConfigDescription("Enable cache for orc file tail") + public OrcCacheConfig setFileTailCacheEnabled(boolean fileTailCacheEnabled) + { + this.fileTailCacheEnabled = fileTailCacheEnabled; + return this; + } + + @MinDataSize("0B") + public DataSize getFileTailCacheSize() + { + return fileTailCacheSize; + } + + @Config("orc.file-tail-cache-size") + @ConfigDescription("Size of the orc file tail cache") + public OrcCacheConfig setFileTailCacheSize(DataSize fileTailCacheSize) + { + this.fileTailCacheSize = fileTailCacheSize; + return this; + } + + @MinDuration("0s") + public Duration getFileTailCacheTtlSinceLastAccess() + { + return fileTailCacheTtlSinceLastAccess; + } + + @Config("orc.file-tail-cache-ttl-since-last-access") + @ConfigDescription("Time-to-live for file tail cache entry after last access") + public OrcCacheConfig setFileTailCacheTtlSinceLastAccess(Duration fileTailCacheTtlSinceLastAccess) + { + this.fileTailCacheTtlSinceLastAccess = fileTailCacheTtlSinceLastAccess; + return this; + } + + public boolean isStripeMetadataCacheEnabled() + { + return stripeMetadataCacheEnabled; + } + + @Config("orc.stripe-metadata-cache-enabled") + @ConfigDescription("Enable cache for stripe metadata") + public OrcCacheConfig setStripeMetadataCacheEnabled(boolean stripeMetadataCacheEnabled) + { + this.stripeMetadataCacheEnabled = stripeMetadataCacheEnabled; + return this; + } + + @MinDataSize("0B") + public DataSize getStripeFooterCacheSize() + { + return stripeFooterCacheSize; + } + + @Config("orc.stripe-footer-cache-size") + @ConfigDescription("Size of the stripe footer cache") + public OrcCacheConfig setStripeFooterCacheSize(DataSize stripeFooterCacheSize) + { + this.stripeFooterCacheSize = stripeFooterCacheSize; + return this; + } + + @MinDuration("0s") + public Duration getStripeFooterCacheTtlSinceLastAccess() + { + return stripeFooterCacheTtlSinceLastAccess; + } + + @Config("orc.stripe-footer-cache-ttl-since-last-access") + @ConfigDescription("Time-to-live for stripe footer cache entry after last access") + public OrcCacheConfig setStripeFooterCacheTtlSinceLastAccess(Duration stripeFooterCacheTtlSinceLastAccess) + { + this.stripeFooterCacheTtlSinceLastAccess = stripeFooterCacheTtlSinceLastAccess; + return this; + } + + @MinDataSize("0B") + public DataSize getStripeStreamCacheSize() + { + return stripeStreamCacheSize; + } + + @Config("orc.stripe-stream-cache-size") + @ConfigDescription("Size of the stripe stream cache") + public OrcCacheConfig setStripeStreamCacheSize(DataSize stripeStreamCacheSize) + { + this.stripeStreamCacheSize = stripeStreamCacheSize; + return this; + } + + @MinDuration("0s") + public Duration getStripeStreamCacheTtlSinceLastAccess() + { + return stripeStreamCacheTtlSinceLastAccess; + } + + @Config("orc.stripe-stream-cache-ttl-since-last-access") + @ConfigDescription("Time-to-live for stripe stream cache entry after last access") + public OrcCacheConfig setStripeStreamCacheTtlSinceLastAccess(Duration stripeStreamCacheTtlSinceLastAccess) + { + this.stripeStreamCacheTtlSinceLastAccess = stripeStreamCacheTtlSinceLastAccess; + return this; + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/cache/OrcFileTailSource.java b/presto-orc/src/main/java/com/facebook/presto/orc/cache/OrcFileTailSource.java new file mode 100644 index 0000000000000..067a2d32d1f50 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/cache/OrcFileTailSource.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.cache; + +import com.facebook.presto.orc.OrcDataSource; +import com.facebook.presto.orc.OrcWriteValidation; +import com.facebook.presto.orc.metadata.MetadataReader; +import com.facebook.presto.orc.metadata.OrcFileTail; + +import java.io.IOException; +import java.util.Optional; + +public interface OrcFileTailSource +{ + OrcFileTail getOrcFileTail(OrcDataSource orcDataSource, MetadataReader metadataReader, Optional writeValidation) + throws IOException; +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/cache/StorageOrcFileTailSource.java b/presto-orc/src/main/java/com/facebook/presto/orc/cache/StorageOrcFileTailSource.java new file mode 100644 index 0000000000000..8ccaa32346014 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/cache/StorageOrcFileTailSource.java @@ -0,0 +1,155 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.cache; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.orc.OrcCorruptionException; +import com.facebook.presto.orc.OrcDataSource; +import com.facebook.presto.orc.OrcWriteValidation; +import com.facebook.presto.orc.metadata.CompressionKind; +import com.facebook.presto.orc.metadata.MetadataReader; +import com.facebook.presto.orc.metadata.OrcFileTail; +import com.facebook.presto.orc.metadata.PostScript; +import com.google.common.base.Joiner; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.orc.OrcReader.validateWrite; +import static com.facebook.presto.orc.metadata.PostScript.MAGIC; +import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; +import static java.lang.Math.min; +import static java.lang.Math.toIntExact; + +public class StorageOrcFileTailSource + implements OrcFileTailSource +{ + private static final Logger log = Logger.get(StorageOrcFileTailSource.class); + + private static final int EXPECTED_FOOTER_SIZE = 16 * 1024; + + private static final int CURRENT_MAJOR_VERSION = 0; + private static final int CURRENT_MINOR_VERSION = 12; + + @Override + public OrcFileTail getOrcFileTail(OrcDataSource orcDataSource, MetadataReader metadataReader, Optional writeValidation) + throws IOException + { + long size = orcDataSource.getSize(); + if (size <= MAGIC.length()) { + throw new OrcCorruptionException(orcDataSource.getId(), "Invalid file size %s", size); + } + + // Read the tail of the file + byte[] buffer = new byte[toIntExact(min(size, EXPECTED_FOOTER_SIZE))]; + orcDataSource.readFully(size - buffer.length, buffer); + + // get length of PostScript - last byte of the file + int postScriptSize = buffer[buffer.length - SIZE_OF_BYTE] & 0xff; + if (postScriptSize >= buffer.length) { + throw new OrcCorruptionException(orcDataSource.getId(), "Invalid postscript length %s", postScriptSize); + } + + // decode the post script + PostScript postScript; + try { + postScript = metadataReader.readPostScript(buffer, buffer.length - SIZE_OF_BYTE - postScriptSize, postScriptSize); + } + catch (OrcCorruptionException e) { + // check if this is an ORC file and not an RCFile or something else + if (!isValidHeaderMagic(orcDataSource)) { + throw new OrcCorruptionException(orcDataSource.getId(), "Not an ORC file"); + } + throw e; + } + + // verify this is a supported version + checkOrcVersion(orcDataSource, postScript.getVersion()); + validateWrite(writeValidation, orcDataSource, validation -> validation.getVersion().equals(postScript.getVersion()), "Unexpected version"); + + int bufferSize = toIntExact(postScript.getCompressionBlockSize()); + + // check compression codec is supported + CompressionKind compressionKind = postScript.getCompression(); + validateWrite(writeValidation, orcDataSource, validation -> validation.getCompression() == compressionKind, "Unexpected compression"); + + PostScript.HiveWriterVersion hiveWriterVersion = postScript.getHiveWriterVersion(); + + int footerSize = toIntExact(postScript.getFooterLength()); + int metadataSize = toIntExact(postScript.getMetadataLength()); + + // check if extra bytes need to be read + Slice completeFooterSlice; + int completeFooterSize = footerSize + metadataSize + postScriptSize + SIZE_OF_BYTE; + if (completeFooterSize > buffer.length) { + // allocate a new buffer large enough for the complete footer + byte[] newBuffer = new byte[completeFooterSize]; + completeFooterSlice = Slices.wrappedBuffer(newBuffer); + + // initial read was not large enough, so read missing section + orcDataSource.readFully(size - completeFooterSize, newBuffer, 0, completeFooterSize - buffer.length); + + // copy already read bytes into the new buffer + completeFooterSlice.setBytes(completeFooterSize - buffer.length, buffer); + } + else { + // footer is already in the bytes in buffer, just adjust position, length + completeFooterSlice = Slices.wrappedBuffer(buffer, buffer.length - completeFooterSize, completeFooterSize); + } + + Slice metadataSlice = completeFooterSlice.slice(0, metadataSize); + Slice footerSlice = completeFooterSlice.slice(metadataSize, footerSize); + + return new OrcFileTail(hiveWriterVersion, bufferSize, compressionKind, footerSlice, footerSize, metadataSlice, metadataSize); + } + + /** + * Does the file start with the ORC magic bytes? + */ + private static boolean isValidHeaderMagic(OrcDataSource source) + throws IOException + { + byte[] headerMagic = new byte[MAGIC.length()]; + source.readFully(0, headerMagic); + + return MAGIC.equals(Slices.wrappedBuffer(headerMagic)); + } + + /** + * Check to see if this ORC file is from a future version and if so, + * warn the user that we may not be able to read all of the column encodings. + */ + // This is based on the Apache Hive ORC code + private static void checkOrcVersion(OrcDataSource orcDataSource, List version) + { + if (version.size() >= 1) { + int major = version.get(0); + int minor = 0; + if (version.size() > 1) { + minor = version.get(1); + } + + if (major > CURRENT_MAJOR_VERSION || (major == CURRENT_MAJOR_VERSION && minor > CURRENT_MINOR_VERSION)) { + log.warn("ORC file %s was written by a newer Hive version %s. This file may not be readable by this version of Hive (%s.%s).", + orcDataSource, + Joiner.on('.').join(version), + CURRENT_MAJOR_VERSION, + CURRENT_MINOR_VERSION); + } + } + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcFileTail.java b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcFileTail.java new file mode 100644 index 0000000000000..0ef33dd3f64aa --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/metadata/OrcFileTail.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.metadata; + +import com.facebook.presto.orc.metadata.PostScript.HiveWriterVersion; +import io.airlift.slice.Slice; + +import static java.util.Objects.requireNonNull; + +public class OrcFileTail +{ + private final HiveWriterVersion hiveWriterVersion; + private final int bufferSize; + private final CompressionKind compressionKind; + private final Slice footerSlice; + private final int footerSize; + private final Slice metadataSlice; + private final int metadataSize; + + public OrcFileTail( + HiveWriterVersion hiveWriterVersion, + int bufferSize, + CompressionKind compressionKind, + Slice footerSlice, + int footerSize, + Slice metadataSlice, + int metadataSize) + { + this.hiveWriterVersion = requireNonNull(hiveWriterVersion, "hiveWriterVersion is null"); + this.bufferSize = bufferSize; + this.compressionKind = requireNonNull(compressionKind, "compressionKind is null"); + this.footerSlice = requireNonNull(footerSlice, "footerSlice is null"); + this.footerSize = footerSize; + this.metadataSlice = requireNonNull(metadataSlice, "metadataSlice is null"); + this.metadataSize = metadataSize; + } + + public HiveWriterVersion getHiveWriterVersion() + { + return hiveWriterVersion; + } + + public int getBufferSize() + { + return bufferSize; + } + + public CompressionKind getCompressionKind() + { + return compressionKind; + } + + public Slice getFooterSlice() + { + return footerSlice; + } + + public int getFooterSize() + { + return footerSize; + } + + public Slice getMetadataSlice() + { + return metadataSlice; + } + + public int getMetadataSize() + { + return metadataSize; + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/AbstractDecimalSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/AbstractDecimalSelectiveStreamReader.java new file mode 100644 index 0000000000000..598193ab0a68f --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/AbstractDecimalSelectiveStreamReader.java @@ -0,0 +1,335 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.metadata.ColumnEncoding; +import com.facebook.presto.orc.stream.BooleanInputStream; +import com.facebook.presto.orc.stream.DecimalInputStream; +import com.facebook.presto.orc.stream.InputStreamSource; +import com.facebook.presto.orc.stream.InputStreamSources; +import com.facebook.presto.orc.stream.LongInputStream; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.block.ClosingBlockLease; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.Type; +import org.openjdk.jol.info.ClassLayout; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.SECONDARY; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; +import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.util.Objects.requireNonNull; + +public abstract class AbstractDecimalSelectiveStreamReader + implements SelectiveStreamReader +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(AbstractDecimalSelectiveStreamReader.class).instanceSize(); + + protected final TupleDomainFilter filter; + protected final boolean nullsAllowed; + protected final boolean outputRequired; + protected final boolean nonDeterministicFilter; + protected final int scale; + + protected long[] values; + protected boolean[] nulls; + protected int[] outputPositions; + protected int outputPositionCount; + protected BooleanInputStream presentStream; + protected DecimalInputStream dataStream; + protected LongInputStream scaleStream; + + private final int valuesPerPosition; + private final Block nullBlock; + private final StreamDescriptor streamDescriptor; + private final LocalMemoryContext systemMemoryContext; + + private int readOffset; + private boolean rowGroupOpen; + private boolean allNulls; + private boolean valuesInUse; + private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); + private InputStreamSource dataStreamSource = missingStreamSource(DecimalInputStream.class); + private InputStreamSource scaleStreamSource = missingStreamSource(LongInputStream.class); + + public AbstractDecimalSelectiveStreamReader( + StreamDescriptor streamDescriptor, + Optional filter, + Optional outputType, + LocalMemoryContext systemMemoryContext, + int valuesPerPosition) + { + requireNonNull(filter, "filter is null"); + requireNonNull(outputType, "outputType is null"); + checkArgument(filter.isPresent() || outputType.isPresent(), "filter must be present if output is not required"); + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + this.filter = filter.orElse(null); + this.outputRequired = outputType.isPresent(); + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.nonDeterministicFilter = this.filter != null && !this.filter.isDeterministic(); + this.nullsAllowed = this.filter == null || this.nonDeterministicFilter || this.filter.testNull(); + this.scale = streamDescriptor.getOrcType().getScale().get(); + this.nullBlock = outputType.map(type -> type.createBlockBuilder(null, 1).appendNull().build()).orElse(null); + this.valuesPerPosition = valuesPerPosition; + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + { + presentStreamSource = missingStreamSource(BooleanInputStream.class); + dataStreamSource = missingStreamSource(DecimalInputStream.class); + scaleStreamSource = missingStreamSource(LongInputStream.class); + readOffset = 0; + presentStream = null; + dataStream = null; + rowGroupOpen = false; + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + { + presentStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, PRESENT, BooleanInputStream.class); + dataStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, DATA, DecimalInputStream.class); + scaleStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, SECONDARY, LongInputStream.class); + readOffset = 0; + presentStream = null; + dataStream = null; + scaleStream = null; + rowGroupOpen = false; + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + sizeOf(values) + sizeOf(nulls) + sizeOf(outputPositions); + } + + private void openRowGroup() + throws IOException + { + presentStream = presentStreamSource.openStream(); + dataStream = dataStreamSource.openStream(); + scaleStream = scaleStreamSource.openStream(); + rowGroupOpen = true; + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (!rowGroupOpen) { + openRowGroup(); + } + + allNulls = false; + + if (outputRequired) { + ensureValuesCapacity(positionCount, nullsAllowed && presentStream != null); + } + + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); + + // account memory used by values, nulls and outputPositions + systemMemoryContext.setBytes(getRetainedSizeInBytes()); + + if (readOffset < offset) { + skip(offset - readOffset); + } + + int streamPosition = 0; + outputPositionCount = 0; + if (dataStream == null && scaleStream == null && presentStream != null) { + streamPosition = readAllNulls(positions, positionCount); + } + else if (filter == null) { + streamPosition = readNoFilter(positions, positionCount); + } + else { + streamPosition = readWithFilter(positions, positionCount); + } + + readOffset = offset + streamPosition; + return outputPositionCount; + } + + private int readAllNulls(int[] positions, int positionCount) + throws IOException + { + presentStream.skip(positions[positionCount - 1]); + + if (nonDeterministicFilter) { + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + if (filter.testNull()) { + outputPositionCount++; + } + else { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + i += filter.getSucceedingPositionsToFail(); + } + } + } + else if (nullsAllowed) { + outputPositionCount = positionCount; + } + else { + outputPositionCount = 0; + } + + allNulls = true; + return positions[positionCount - 1] + 1; + } + + protected void skip(int items) + throws IOException + { + if (dataStream == null) { + presentStream.skip(items); + } + else if (presentStream != null) { + int dataToSkip = presentStream.countBitsSet(items); + dataStream.skip(dataToSkip); + scaleStream.skip(dataToSkip); + } + else { + dataStream.skip(items); + scaleStream.skip(items); + } + } + + @Override + public int[] getReadPositions() + { + return outputPositions; + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } + + private BlockLease newLease(Block block) + { + valuesInUse = true; + return ClosingBlockLease.newLease(block, () -> valuesInUse = false); + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return new RunLengthEncodedBlock(nullBlock, positionCount); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + + if (positionCount == outputPositionCount) { + Block block = makeBlock(positionCount, nullsAllowed, nulls, values); + nulls = null; + values = null; + return block; + } + + long[] valuesCopy = new long[valuesPerPosition * positionCount]; + boolean[] nullsCopy = null; + + if (includeNulls) { + nullsCopy = new boolean[positionCount]; + } + + copyValues(positions, positionCount, valuesCopy, nullsCopy); + + return makeBlock(positionCount, includeNulls, nullsCopy, valuesCopy); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return newLease(new RunLengthEncodedBlock(nullBlock, positionCount)); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount != outputPositionCount) { + compactValues(positions, positionCount, includeNulls); + } + + return newLease(makeBlock(positionCount, includeNulls, nulls, values)); + } + + private void ensureValuesCapacity(int capacity, boolean nullAllowed) + { + int valuesCapacity = valuesPerPosition * capacity; + if (values == null || values.length < valuesCapacity) { + values = new long[valuesCapacity]; + } + + if (nullAllowed) { + if (nulls == null || nulls.length < capacity) { + nulls = new boolean[capacity]; + } + } + } + + abstract void copyValues(int[] positions, int positionsCount, long[] valuesCopy, boolean[] nullsCopy); + + abstract Block makeBlock(int positionCount, boolean includeNulls, boolean[] nulls, long[] values); + + abstract void compactValues(int[] positions, int positionCount, boolean compactNulls); + + abstract int readNoFilter(int[] positions, int position) + throws IOException; + + abstract int readWithFilter(int[] positions, int position) + throws IOException; + + @Override + public void close() + { + systemMemoryContext.close(); + } + + @Override + public String toString() + { + return toStringHelper(this) + .addValue(streamDescriptor) + .toString(); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/AbstractLongSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/AbstractLongSelectiveStreamReader.java index 2bb3b5ebcb54e..7531d653056e4 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/AbstractLongSelectiveStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/AbstractLongSelectiveStreamReader.java @@ -25,6 +25,7 @@ import java.util.Optional; +import static com.facebook.presto.array.Arrays.ensureCapacity; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.DateType.DATE; import static com.facebook.presto.spi.type.IntegerType.INTEGER; @@ -58,8 +59,8 @@ abstract class AbstractLongSelectiveStreamReader protected AbstractLongSelectiveStreamReader(Optional outputType) { - this.outputRequired = outputType.isPresent(); - this.outputType = requireNonNull(outputType, "outputType is null").orElse(null); + this.outputRequired = requireNonNull(outputType, "outputType is null").isPresent(); + this.outputType = outputType.orElse(null); } protected void prepareNextRead(int positionCount, boolean withNulls) @@ -79,6 +80,11 @@ public int[] getReadPositions() return outputPositions; } + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } + protected BlockLease buildOutputBlockView(int[] positions, int positionCount, boolean includeNulls) { checkState(!valuesInUse, "BlockLease hasn't been closed yet"); @@ -202,7 +208,10 @@ private Block getLongArrayBlock(int[] positions, int positionCount, boolean incl private Block getIntArrayBlock(int[] positions, int positionCount, boolean includeNulls) { if (intValuesPopulated && positionCount == outputPositionCount) { - return new IntArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), intValues); + Block block = new IntArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), intValues); + intValues = null; + nulls = null; + return block; } int[] valuesCopy = new int[positionCount]; @@ -239,7 +248,10 @@ private Block getIntArrayBlock(int[] positions, int positionCount, boolean inclu private Block getShortArrayBlock(int[] positions, int positionCount, boolean includeNulls) { if (shortValuesPopulated && positionCount == outputPositionCount) { - return new ShortArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), shortValues); + Block block = new ShortArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), shortValues); + shortValues = null; + nulls = null; + return block; } short[] valuesCopy = new short[positionCount]; @@ -273,23 +285,12 @@ private Block getShortArrayBlock(int[] positions, int positionCount, boolean inc return new ShortArrayBlock(positionCount, Optional.ofNullable(nullsCopy), valuesCopy); } - protected void ensureValuesCapacity(int capacity, boolean recordNulls) + private void ensureValuesCapacity(int capacity, boolean recordNulls) { - if (values == null || values.length < capacity) { - values = new long[capacity]; - } + values = ensureCapacity(values, capacity); if (recordNulls) { - if (nulls == null || nulls.length < capacity) { - nulls = new boolean[capacity]; - } - } - } - - protected void ensureOutputPositionsCapacity(int capacity) - { - if (outputPositions == null || outputPositions.length < capacity) { - outputPositions = new int[capacity]; + nulls = ensureCapacity(nulls, capacity); } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/ApacheHiveTimestampDecoder.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ApacheHiveTimestampDecoder.java new file mode 100644 index 0000000000000..4b02acba04f5e --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ApacheHiveTimestampDecoder.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +final class ApacheHiveTimestampDecoder +{ + private ApacheHiveTimestampDecoder() {} + + // This comes from the Apache Hive ORC code + public static long decodeTimestamp(long seconds, long serializedNanos, long baseTimestampInSeconds) + { + long millis = (seconds + baseTimestampInSeconds) * 1000; + long nanos = parseNanos(serializedNanos); + if (nanos > 999999999 || nanos < 0) { + throw new IllegalArgumentException("nanos field of an encoded timestamp in ORC must be between 0 and 999999999 inclusive, got " + nanos); + } + + // the rounding error exists because java always rounds up when dividing integers + // -42001/1000 = -42; and -42001 % 1000 = -1 (+ 1000) + // to get the correct value we need + // (-42 - 1)*1000 + 999 = -42001 + // (42)*1000 + 1 = 42001 + if (millis < 0 && nanos != 0) { + millis -= 1000; + } + // Truncate nanos to millis and add to mills + return millis + (nanos / 1_000_000); + } + + // This comes from the Apache Hive ORC code + private static int parseNanos(long serialized) + { + int zeros = ((int) serialized) & 0b111; + int result = (int) (serialized >>> 3); + if (zeros != 0) { + for (int i = 0; i <= zeros; ++i) { + result *= 10; + } + } + return result; + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/BatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/BatchStreamReader.java index 9e3e61f552bdd..7d6e9653f9a87 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/BatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/BatchStreamReader.java @@ -14,14 +14,13 @@ package com.facebook.presto.orc.reader; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.type.Type; import java.io.IOException; public interface BatchStreamReader extends StreamReader { - Block readBlock(Type type) + Block readBlock() throws IOException; void prepareNextRead(int batchSize); diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/BatchStreamReaders.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/BatchStreamReaders.java index 0985778b9261f..0c3c44fd7217f 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/BatchStreamReaders.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/BatchStreamReaders.java @@ -13,7 +13,10 @@ */ package com.facebook.presto.orc.reader; +import com.facebook.presto.memory.context.AggregatedMemoryContext; +import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.spi.type.Type; import org.joda.time.DateTimeZone; public final class BatchStreamReaders @@ -22,42 +25,41 @@ private BatchStreamReaders() { } - public static BatchStreamReader createStreamReader( - StreamDescriptor streamDescriptor, - DateTimeZone hiveStorageTimeZone) + public static BatchStreamReader createStreamReader(Type type, StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone, AggregatedMemoryContext systemMemoryContext) + throws OrcCorruptionException { - switch (streamDescriptor.getStreamType()) { + switch (streamDescriptor.getOrcTypeKind()) { case BOOLEAN: - return new BooleanBatchStreamReader(streamDescriptor); + return new BooleanBatchStreamReader(type, streamDescriptor, systemMemoryContext.newLocalMemoryContext(BatchStreamReaders.class.getSimpleName())); case BYTE: - return new ByteBatchStreamReader(streamDescriptor); + return new ByteBatchStreamReader(type, streamDescriptor, systemMemoryContext.newLocalMemoryContext(BatchStreamReaders.class.getSimpleName())); case SHORT: case INT: case LONG: case DATE: - return new LongBatchStreamReader(streamDescriptor); + return new LongBatchStreamReader(type, streamDescriptor, systemMemoryContext); case FLOAT: - return new FloatBatchStreamReader(streamDescriptor); + return new FloatBatchStreamReader(type, streamDescriptor); case DOUBLE: - return new DoubleBatchStreamReader(streamDescriptor); + return new DoubleBatchStreamReader(type, streamDescriptor); case BINARY: case STRING: case VARCHAR: case CHAR: - return new SliceBatchStreamReader(streamDescriptor); + return new SliceBatchStreamReader(type, streamDescriptor, systemMemoryContext); case TIMESTAMP: - return new TimestampBatchStreamReader(streamDescriptor, hiveStorageTimeZone); + return new TimestampBatchStreamReader(type, streamDescriptor, hiveStorageTimeZone); case LIST: - return new ListBatchStreamReader(streamDescriptor, hiveStorageTimeZone); + return new ListBatchStreamReader(type, streamDescriptor, hiveStorageTimeZone, systemMemoryContext); case STRUCT: - return new StructBatchStreamReader(streamDescriptor, hiveStorageTimeZone); + return new StructBatchStreamReader(type, streamDescriptor, hiveStorageTimeZone, systemMemoryContext); case MAP: - return new MapBatchStreamReader(streamDescriptor, hiveStorageTimeZone); + return new MapBatchStreamReader(type, streamDescriptor, hiveStorageTimeZone, systemMemoryContext); case DECIMAL: - return new DecimalBatchStreamReader(streamDescriptor); + return new DecimalBatchStreamReader(type, streamDescriptor); case UNION: default: - throw new IllegalArgumentException("Unsupported type: " + streamDescriptor.getStreamType()); + throw new IllegalArgumentException("Unsupported type: " + streamDescriptor.getOrcTypeKind()); } } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/BooleanBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/BooleanBatchStreamReader.java index 2578c4f81c1ab..10183cf5f37eb 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/BooleanBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/BooleanBatchStreamReader.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc.reader; +import com.facebook.presto.memory.context.LocalMemoryContext; import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.metadata.ColumnEncoding; @@ -20,8 +21,9 @@ import com.facebook.presto.orc.stream.InputStreamSource; import com.facebook.presto.orc.stream.InputStreamSources; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.block.ByteArrayBlock; import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.BooleanType; import com.facebook.presto.spi.type.Type; import org.openjdk.jol.info.ClassLayout; @@ -29,11 +31,18 @@ import java.io.IOException; import java.util.List; +import java.util.Optional; import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.ReaderUtils.minNonNullValueSize; +import static com.facebook.presto.orc.reader.ReaderUtils.unpackByteNulls; +import static com.facebook.presto.orc.reader.ReaderUtils.verifyStreamType; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Verify.verify; +import static io.airlift.slice.SizeOf.sizeOf; import static java.util.Objects.requireNonNull; public class BooleanBatchStreamReader @@ -56,9 +65,17 @@ public class BooleanBatchStreamReader private boolean rowGroupOpen; - public BooleanBatchStreamReader(StreamDescriptor streamDescriptor) + private byte[] nonNullValueTemp = new byte[0]; + + private final LocalMemoryContext systemMemoryContext; + + public BooleanBatchStreamReader(Type type, StreamDescriptor streamDescriptor, LocalMemoryContext systemMemoryContext) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, BooleanType.class::isInstance); this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); } @Override @@ -69,7 +86,7 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -92,36 +109,65 @@ public Block readBlock(Type type) if (dataStream == null && presentStream != null) { presentStream.skip(nextBatchSize); - Block nullValueBlock = new RunLengthEncodedBlock( - type.createBlockBuilder(null, 1).appendNull().build(), - nextBatchSize); + Block nullValueBlock = RunLengthEncodedBlock.create(BOOLEAN, null, nextBatchSize); readOffset = 0; nextBatchSize = 0; return nullValueBlock; } - BlockBuilder builder = type.createBlockBuilder(null, nextBatchSize); - if (presentStream == null) { - if (dataStream == null) { - throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); + Block block; + if (dataStream == null) { + if (presentStream == null) { + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is null but present stream is missing"); } - dataStream.getSetBits(type, nextBatchSize, builder); + presentStream.skip(nextBatchSize); + block = RunLengthEncodedBlock.create(BOOLEAN, null, nextBatchSize); + } + else if (presentStream == null) { + block = readNonNullBlock(); } else { - for (int i = 0; i < nextBatchSize; i++) { - if (presentStream.nextBit()) { - type.writeBoolean(builder, dataStream.nextBit()); - } - else { - builder.appendNull(); - } + boolean[] isNull = new boolean[nextBatchSize]; + int nullCount = presentStream.getUnsetBits(nextBatchSize, isNull); + if (nullCount == 0) { + block = readNonNullBlock(); + } + else if (nullCount != nextBatchSize) { + block = readNullBlock(isNull, nextBatchSize - nullCount); + } + else { + block = RunLengthEncodedBlock.create(BOOLEAN, null, nextBatchSize); } } - readOffset = 0; nextBatchSize = 0; - return builder.build(); + return block; + } + + private Block readNonNullBlock() + throws IOException + { + verify(dataStream != null); + byte[] values = dataStream.getSetBits(nextBatchSize); + return new ByteArrayBlock(nextBatchSize, Optional.empty(), values); + } + + private Block readNullBlock(boolean[] isNull, int nonNullCount) + throws IOException + { + verify(dataStream != null); + int minNonNullValueSize = minNonNullValueSize(nonNullCount); + if (nonNullValueTemp.length < minNonNullValueSize) { + nonNullValueTemp = new byte[minNonNullValueSize]; + systemMemoryContext.setBytes(sizeOf(nonNullValueTemp)); + } + + dataStream.getSetBits(nonNullCount, nonNullValueTemp); + + byte[] result = unpackByteNulls(nonNullValueTemp, isNull); + + return new ByteArrayBlock(nextBatchSize, Optional.of(isNull), result); } private void openRowGroup() @@ -170,6 +216,13 @@ public String toString() .toString(); } + @Override + public void close() + { + systemMemoryContext.close(); + nonNullValueTemp = null; + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/BooleanSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/BooleanSelectiveStreamReader.java index 80303c7637c8c..aab27658898b5 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/BooleanSelectiveStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/BooleanSelectiveStreamReader.java @@ -33,8 +33,10 @@ import java.util.List; import java.util.Optional; +import static com.facebook.presto.array.Arrays.ensureCapacity; import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.google.common.base.MoreObjects.toStringHelper; @@ -52,6 +54,7 @@ public class BooleanSelectiveStreamReader private final StreamDescriptor streamDescriptor; @Nullable private final TupleDomainFilter filter; + private final boolean nonDeterministicFilter; private final boolean nullsAllowed; private final boolean outputRequired; @@ -90,7 +93,8 @@ public BooleanSelectiveStreamReader( this.outputRequired = outputRequired; this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); - nullsAllowed = this.filter == null || this.filter.testNull(); + nonDeterministicFilter = this.filter != null && !this.filter.isDeterministic(); + nullsAllowed = this.filter == null || nonDeterministicFilter || this.filter.testNull(); } @Override @@ -147,6 +151,7 @@ private void openRowGroup() public int read(int offset, int[] positions, int positionCount) throws IOException { + checkArgument(positionCount > 0, "positionCount must be greater than zero"); checkState(!valuesInUse, "BlockLease hasn't been closed yet"); if (!rowGroupOpen) { @@ -159,12 +164,7 @@ public int read(int offset, int[] positions, int positionCount) ensureValuesCapacity(positionCount, nullsAllowed && presentStream != null); } - if (filter != null) { - ensureOutputPositionsCapacity(positionCount); - } - else { - outputPositions = positions; - } + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); // account memory used by values, nulls and outputPositions systemMemoryContext.setBytes(getRetainedSizeInBytes()); @@ -196,7 +196,7 @@ else if (filter == null) { } if (presentStream != null && !presentStream.nextBit()) { - if (nullsAllowed) { + if ((nonDeterministicFilter && filter.testNull()) || nullsAllowed) { if (outputRequired) { nulls[outputPositionCount] = true; } @@ -217,7 +217,22 @@ else if (filter == null) { outputPositionCount++; } } + + outputPositionCount -= filter.getPrecedingPositionsToFail(); + streamPosition++; + + int succeedingPositionsToFail = filter.getSucceedingPositionsToFail(); + if (succeedingPositionsToFail > 0) { + int positionsToSkip = 0; + for (int j = 0; j < succeedingPositionsToFail; j++) { + i++; + int nextPosition = positions[i]; + positionsToSkip += 1 + nextPosition - streamPosition; + streamPosition = nextPosition + 1; + } + skip(positionsToSkip); + } } } @@ -230,14 +245,26 @@ private int readAllNulls(int[] positions, int positionCount) { presentStream.skip(positions[positionCount - 1]); - if (nullsAllowed) { + if (nonDeterministicFilter) { + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + if (filter.testNull()) { + outputPositionCount++; + } + else { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + i += filter.getSucceedingPositionsToFail(); + } + } + } + else if (nullsAllowed) { outputPositionCount = positionCount; - allNulls = true; } else { outputPositionCount = 0; } + allNulls = true; return positions[positionCount - 1] + 1; } @@ -285,21 +312,10 @@ else if (presentStream != null) { private void ensureValuesCapacity(int capacity, boolean recordNulls) { - if (values == null || values.length < capacity) { - values = new byte[capacity]; - } + values = ensureCapacity(values, capacity); if (recordNulls) { - if (nulls == null || nulls.length < capacity) { - nulls = new boolean[capacity]; - } - } - } - - private void ensureOutputPositionsCapacity(int capacity) - { - if (outputPositions == null || outputPositions.length < capacity) { - outputPositions = new int[capacity]; + nulls = ensureCapacity(nulls, capacity); } } @@ -318,7 +334,7 @@ public Block getBlock(int[] positions, int positionCount) checkState(!valuesInUse, "BlockLease hasn't been closed yet"); if (allNulls) { - return new RunLengthEncodedBlock(NULL_BLOCK, outputPositionCount); + return new RunLengthEncodedBlock(NULL_BLOCK, positionCount); } boolean includeNulls = nullsAllowed && presentStream != null; @@ -380,6 +396,11 @@ public BlockLease getBlockView(int[] positions, int positionCount) return newLease(new ByteArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values)); } + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } + private BlockLease newLease(Block block) { valuesInUse = true; diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/ByteBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ByteBatchStreamReader.java index f5ba1094156c7..c979cd5f5338a 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/ByteBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ByteBatchStreamReader.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc.reader; +import com.facebook.presto.memory.context.LocalMemoryContext; import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.metadata.ColumnEncoding; @@ -21,8 +22,9 @@ import com.facebook.presto.orc.stream.InputStreamSource; import com.facebook.presto.orc.stream.InputStreamSources; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.block.ByteArrayBlock; import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.TinyintType; import com.facebook.presto.spi.type.Type; import org.openjdk.jol.info.ClassLayout; @@ -30,11 +32,17 @@ import java.io.IOException; import java.util.List; +import java.util.Optional; import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.ReaderUtils.minNonNullValueSize; +import static com.facebook.presto.orc.reader.ReaderUtils.verifyStreamType; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Verify.verify; +import static io.airlift.slice.SizeOf.sizeOf; import static java.util.Objects.requireNonNull; public class ByteBatchStreamReader @@ -57,9 +65,17 @@ public class ByteBatchStreamReader private boolean rowGroupOpen; - public ByteBatchStreamReader(StreamDescriptor streamDescriptor) + private byte[] nonNullValueTemp = new byte[0]; + + private final LocalMemoryContext systemMemoryContext; + + public ByteBatchStreamReader(Type type, StreamDescriptor streamDescriptor, LocalMemoryContext systemMemoryContext) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, TinyintType.class::isInstance); this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); } @Override @@ -70,7 +86,7 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -85,44 +101,66 @@ public Block readBlock(Type type) } if (readOffset > 0) { if (dataStream == null) { - throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is missing"); } dataStream.skip(readOffset); } } - if (dataStream == null && presentStream != null) { + Block block; + if (dataStream == null) { + if (presentStream == null) { + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is null but present stream is missing"); + } presentStream.skip(nextBatchSize); - Block nullValueBlock = new RunLengthEncodedBlock( - type.createBlockBuilder(null, 1).appendNull().build(), - nextBatchSize); - readOffset = 0; - nextBatchSize = 0; - return nullValueBlock; + block = RunLengthEncodedBlock.create(TINYINT, null, nextBatchSize); } - - BlockBuilder builder = type.createBlockBuilder(null, nextBatchSize); - if (presentStream == null) { - if (dataStream == null) { - throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); - } - dataStream.nextVector(type, nextBatchSize, builder); + else if (presentStream == null) { + block = readNonNullBlock(); } else { - for (int i = 0; i < nextBatchSize; i++) { - if (presentStream.nextBit()) { - type.writeLong(builder, dataStream.next()); - } - else { - builder.appendNull(); - } + boolean[] isNull = new boolean[nextBatchSize]; + int nullCount = presentStream.getUnsetBits(nextBatchSize, isNull); + if (nullCount == 0) { + block = readNonNullBlock(); + } + else if (nullCount != nextBatchSize) { + block = readNullBlock(isNull, nextBatchSize - nullCount); + } + else { + block = RunLengthEncodedBlock.create(TINYINT, null, nextBatchSize); } } readOffset = 0; nextBatchSize = 0; - return builder.build(); + return block; + } + + private Block readNonNullBlock() + throws IOException + { + verify(dataStream != null); + byte[] values = dataStream.next(nextBatchSize); + return new ByteArrayBlock(nextBatchSize, Optional.empty(), values); + } + + private Block readNullBlock(boolean[] isNull, int nonNullCount) + throws IOException + { + verify(dataStream != null); + int minNonNullValueSize = minNonNullValueSize(nonNullCount); + if (nonNullValueTemp.length < minNonNullValueSize) { + nonNullValueTemp = new byte[minNonNullValueSize]; + systemMemoryContext.setBytes(sizeOf(nonNullValueTemp)); + } + + dataStream.next(nonNullValueTemp, nonNullCount); + + byte[] result = ReaderUtils.unpackByteNulls(nonNullValueTemp, isNull); + + return new ByteArrayBlock(nextBatchSize, Optional.of(isNull), result); } private void openRowGroup() @@ -172,6 +210,13 @@ public String toString() .toString(); } + @Override + public void close() + { + systemMemoryContext.close(); + nonNullValueTemp = null; + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/ByteSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ByteSelectiveStreamReader.java new file mode 100644 index 0000000000000..93aeb97e76eca --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ByteSelectiveStreamReader.java @@ -0,0 +1,442 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.metadata.ColumnEncoding; +import com.facebook.presto.orc.stream.BooleanInputStream; +import com.facebook.presto.orc.stream.ByteInputStream; +import com.facebook.presto.orc.stream.InputStreamSource; +import com.facebook.presto.orc.stream.InputStreamSources; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.block.ByteArrayBlock; +import com.facebook.presto.spi.block.ClosingBlockLease; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import org.openjdk.jol.info.ClassLayout; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; +import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.util.Objects.requireNonNull; + +public class ByteSelectiveStreamReader + implements SelectiveStreamReader +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(ByteSelectiveStreamReader.class).instanceSize(); + private static final Block NULL_BLOCK = TINYINT.createBlockBuilder(null, 1).appendNull().build(); + + private final StreamDescriptor streamDescriptor; + private final TupleDomainFilter filter; + private final boolean nullsAllowed; + private final boolean outputRequired; + private final LocalMemoryContext systemMemoryContext; + private final boolean nonDeterministicFilter; + + private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); + private InputStreamSource dataStreamSource = missingStreamSource(ByteInputStream.class); + + @Nullable + private BooleanInputStream presentStream; + @Nullable + private ByteInputStream dataStream; + private boolean rowGroupOpen; + + private int readOffset; + @Nullable + private byte[] values; + @Nullable + private boolean[] nulls; + @Nullable + private int[] outputPositions; + private int outputPositionCount; + private boolean allNulls; + private boolean valuesInUse; + + public ByteSelectiveStreamReader( + StreamDescriptor streamDescriptor, + Optional filter, + boolean outputRequired, + LocalMemoryContext systemMemoryContext) + { + requireNonNull(filter, "filter is null"); + checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false"); + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + this.filter = filter.orElse(null); + this.outputRequired = outputRequired; + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.nonDeterministicFilter = this.filter != null && !this.filter.isDeterministic(); + this.nullsAllowed = this.filter == null || nonDeterministicFilter || this.filter.testNull(); + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + { + presentStreamSource = missingStreamSource(BooleanInputStream.class); + dataStreamSource = missingStreamSource(ByteInputStream.class); + readOffset = 0; + presentStream = null; + dataStream = null; + rowGroupOpen = false; + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + { + presentStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, PRESENT, BooleanInputStream.class); + dataStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, DATA, ByteInputStream.class); + readOffset = 0; + presentStream = null; + dataStream = null; + rowGroupOpen = false; + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + sizeOf(values) + sizeOf(nulls) + sizeOf(outputPositions); + } + + private void openRowGroup() + throws IOException + { + presentStream = presentStreamSource.openStream(); + dataStream = dataStreamSource.openStream(); + rowGroupOpen = true; + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (!rowGroupOpen) { + openRowGroup(); + } + + allNulls = false; + + if (outputRequired) { + ensureValuesCapacity(positionCount, nullsAllowed && presentStream != null); + } + + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); + + // account memory used by values, nulls and outputPositions + systemMemoryContext.setBytes(getRetainedSizeInBytes()); + + if (readOffset < offset) { + skip(offset - readOffset); + } + + int streamPosition = 0; + if (dataStream == null && presentStream != null) { + streamPosition = readAllNulls(positions, positionCount); + } + else if (filter == null) { + streamPosition = readNoFilter(positions, positionCount); + } + else { + streamPosition = readWithFilter(positions, positionCount); + } + + readOffset = offset + streamPosition; + return outputPositionCount; + } + + private int readWithFilter(int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + if ((nonDeterministicFilter && filter.testNull()) || nullsAllowed) { + if (outputRequired) { + nulls[outputPositionCount] = true; + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + else { + byte value = dataStream.next(); + if (filter.testLong(value)) { + if (outputRequired) { + values[outputPositionCount] = value; + if (nullsAllowed && presentStream != null) { + nulls[outputPositionCount] = false; + } + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + streamPosition++; + + if (filter != null) { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + int succeedingPositionsToFail = filter.getSucceedingPositionsToFail(); + if (succeedingPositionsToFail > 0) { + int positionsToSkip = 0; + for (int j = 0; j < succeedingPositionsToFail; j++) { + i++; + int nextPosition = positions[i]; + positionsToSkip += 1 + nextPosition - streamPosition; + streamPosition = nextPosition + 1; + } + skip(positionsToSkip); + } + } + } + return streamPosition; + } + + private int readAllNulls(int[] positions, int positionCount) + throws IOException + { + presentStream.skip(positions[positionCount - 1]); + + if (nonDeterministicFilter) { + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + if (filter.testNull()) { + outputPositionCount++; + } + else { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + i += filter.getSucceedingPositionsToFail(); + } + } + } + else if (nullsAllowed) { + outputPositionCount = positionCount; + } + else { + outputPositionCount = 0; + } + + allNulls = true; + return positions[positionCount - 1] + 1; + } + + private int readNoFilter(int[] positions, int positionCount) + throws IOException + { + // filter == null implies outputRequired == true + if (presentStream == null && positions[positionCount - 1] == positionCount - 1) { + // contiguous chunk of rows, no nulls + dataStream.next(values, positionCount); + outputPositionCount = positionCount; + return positionCount; + } + + int streamPosition = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + nulls[i] = true; + } + else { + values[i] = dataStream.next(); + if (presentStream != null) { + nulls[i] = false; + } + } + streamPosition++; + } + outputPositionCount = positionCount; + return streamPosition; + } + + private void skip(int items) + throws IOException + { + if (dataStream == null) { + presentStream.skip(items); + } + else if (presentStream != null) { + int dataToSkip = presentStream.countBitsSet(items); + dataStream.skip(dataToSkip); + } + else { + dataStream.skip(items); + } + } + + private void ensureValuesCapacity(int capacity, boolean recordNulls) + { + values = ensureCapacity(values, capacity); + + if (recordNulls) { + nulls = ensureCapacity(nulls, capacity); + } + } + + @Override + public int[] getReadPositions() + { + return outputPositions; + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return new RunLengthEncodedBlock(NULL_BLOCK, positionCount); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount == outputPositionCount) { + Block block = new ByteArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values); + nulls = null; + values = null; + return block; + } + + byte[] valuesCopy = new byte[positionCount]; + boolean[] nullsCopy = null; + if (includeNulls) { + nullsCopy = new boolean[positionCount]; + } + + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + valuesCopy[positionIndex] = this.values[i]; + if (nullsCopy != null) { + nullsCopy[positionIndex] = this.nulls[i]; + } + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + + nextPosition = positions[positionIndex]; + } + + return new ByteArrayBlock(positionCount, Optional.ofNullable(nullsCopy), valuesCopy); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return newLease(new RunLengthEncodedBlock(NULL_BLOCK, outputPositionCount)); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount != outputPositionCount) { + compactValues(positions, positionCount, includeNulls); + } + + return newLease(new ByteArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values)); + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } + + private BlockLease newLease(Block block) + { + valuesInUse = true; + return ClosingBlockLease.newLease(block, () -> valuesInUse = false); + } + + private void compactValues(int[] positions, int positionCount, boolean compactNulls) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + values[positionIndex] = values[i]; + if (compactNulls) { + nulls[positionIndex] = nulls[i]; + } + outputPositions[positionIndex] = nextPosition; + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + nextPosition = positions[positionIndex]; + } + + outputPositionCount = positionCount; + } + + @Override + public String toString() + { + return toStringHelper(this) + .addValue(streamDescriptor) + .toString(); + } + + @Override + public void close() + { + systemMemoryContext.close(); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/DecimalBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/DecimalBatchStreamReader.java index bd92f68be2249..13ffc9bd90a46 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/DecimalBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/DecimalBatchStreamReader.java @@ -39,6 +39,7 @@ import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; import static com.facebook.presto.orc.metadata.Stream.StreamKind.SECONDARY; +import static com.facebook.presto.orc.reader.ReaderUtils.verifyStreamType; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; import static com.facebook.presto.spi.type.UnscaledDecimal128Arithmetic.rescale; import static com.google.common.base.MoreObjects.toStringHelper; @@ -50,6 +51,7 @@ public class DecimalBatchStreamReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(DecimalBatchStreamReader.class).instanceSize(); + private final DecimalType type; private final StreamDescriptor streamDescriptor; private int readOffset; @@ -69,8 +71,12 @@ public class DecimalBatchStreamReader private boolean rowGroupOpen; - public DecimalBatchStreamReader(StreamDescriptor streamDescriptor) + public DecimalBatchStreamReader(Type type, StreamDescriptor streamDescriptor) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, DecimalType.class::isInstance); + this.type = (DecimalType) type; this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); } @@ -82,12 +88,9 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { - DecimalType decimalType = (DecimalType) type; - int targetScale = decimalType.getScale(); - if (!rowGroupOpen) { openRowGroup(); } @@ -96,15 +99,13 @@ public Block readBlock(Type type) if (decimalStream == null && scaleStream == null && presentStream != null) { presentStream.skip(nextBatchSize); - Block nullValueBlock = new RunLengthEncodedBlock( - type.createBlockBuilder(null, 1).appendNull().build(), - nextBatchSize); + Block nullValueBlock = RunLengthEncodedBlock.create(type, null, nextBatchSize); readOffset = 0; nextBatchSize = 0; return nullValueBlock; } - BlockBuilder builder = decimalType.createBlockBuilder(null, nextBatchSize); + BlockBuilder builder = type.createBlockBuilder(null, nextBatchSize); if (presentStream == null) { if (decimalStream == null) { @@ -116,17 +117,17 @@ public Block readBlock(Type type) for (int i = 0; i < nextBatchSize; i++) { long sourceScale = scaleStream.next(); - if (decimalType.isShort()) { - long rescaledDecimal = Decimals.rescale(decimalStream.nextLong(), (int) sourceScale, decimalType.getScale()); - decimalType.writeLong(builder, rescaledDecimal); + if (type.isShort()) { + long rescaledDecimal = Decimals.rescale(decimalStream.nextLong(), (int) sourceScale, type.getScale()); + type.writeLong(builder, rescaledDecimal); } else { Slice decimal = UnscaledDecimal128Arithmetic.unscaledDecimal(); Slice rescaledDecimal = UnscaledDecimal128Arithmetic.unscaledDecimal(); decimalStream.nextLongDecimal(decimal); - rescale(decimal, (int) (decimalType.getScale() - sourceScale), rescaledDecimal); - decimalType.writeSlice(builder, rescaledDecimal); + rescale(decimal, (int) (type.getScale() - sourceScale), rescaledDecimal); + type.writeSlice(builder, rescaledDecimal); } } } @@ -137,17 +138,17 @@ public Block readBlock(Type type) if (presentStream.nextBit()) { // The current row is not null long sourceScale = scaleStream.next(); - if (decimalType.isShort()) { - long rescaledDecimal = Decimals.rescale(decimalStream.nextLong(), (int) sourceScale, decimalType.getScale()); - decimalType.writeLong(builder, rescaledDecimal); + if (type.isShort()) { + long rescaledDecimal = Decimals.rescale(decimalStream.nextLong(), (int) sourceScale, type.getScale()); + type.writeLong(builder, rescaledDecimal); } else { Slice decimal = UnscaledDecimal128Arithmetic.unscaledDecimal(); Slice rescaledDecimal = UnscaledDecimal128Arithmetic.unscaledDecimal(); decimalStream.nextLongDecimal(decimal); - rescale(decimal, (int) (decimalType.getScale() - sourceScale), rescaledDecimal); - decimalType.writeSlice(builder, rescaledDecimal); + rescale(decimal, (int) (type.getScale() - sourceScale), rescaledDecimal); + type.writeSlice(builder, rescaledDecimal); } } else { @@ -236,6 +237,11 @@ public String toString() .toString(); } + @Override + public void close() + { + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/DoubleBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/DoubleBatchStreamReader.java index 87b4c76ea13ff..db340abb3350d 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/DoubleBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/DoubleBatchStreamReader.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.DoubleType; import com.facebook.presto.spi.type.Type; import org.openjdk.jol.info.ClassLayout; @@ -33,7 +34,9 @@ import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.ReaderUtils.verifyStreamType; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; @@ -57,8 +60,11 @@ public class DoubleBatchStreamReader private boolean rowGroupOpen; - public DoubleBatchStreamReader(StreamDescriptor streamDescriptor) + public DoubleBatchStreamReader(Type type, StreamDescriptor streamDescriptor) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, DoubleType.class::isInstance); this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); } @@ -70,7 +76,7 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -93,25 +99,23 @@ public Block readBlock(Type type) if (dataStream == null && presentStream != null) { presentStream.skip(nextBatchSize); - Block nullValueBlock = new RunLengthEncodedBlock( - type.createBlockBuilder(null, 1).appendNull().build(), - nextBatchSize); + Block nullValueBlock = RunLengthEncodedBlock.create(DOUBLE, null, nextBatchSize); readOffset = 0; nextBatchSize = 0; return nullValueBlock; } - BlockBuilder builder = type.createBlockBuilder(null, nextBatchSize); + BlockBuilder builder = DOUBLE.createBlockBuilder(null, nextBatchSize); if (presentStream == null) { if (dataStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); } - dataStream.nextVector(type, nextBatchSize, builder); + dataStream.nextVector(DOUBLE, nextBatchSize, builder); } else { for (int i = 0; i < nextBatchSize; i++) { if (presentStream.nextBit()) { - type.writeDouble(builder, dataStream.next()); + DOUBLE.writeDouble(builder, dataStream.next()); } else { builder.appendNull(); @@ -172,6 +176,11 @@ public String toString() .toString(); } + @Override + public void close() + { + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/DoubleSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/DoubleSelectiveStreamReader.java new file mode 100644 index 0000000000000..29f2e07929508 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/DoubleSelectiveStreamReader.java @@ -0,0 +1,445 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.metadata.ColumnEncoding; +import com.facebook.presto.orc.stream.BooleanInputStream; +import com.facebook.presto.orc.stream.DoubleInputStream; +import com.facebook.presto.orc.stream.InputStreamSource; +import com.facebook.presto.orc.stream.InputStreamSources; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.block.ClosingBlockLease; +import com.facebook.presto.spi.block.LongArrayBlock; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import org.openjdk.jol.info.ClassLayout; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; +import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.lang.Double.doubleToLongBits; +import static java.util.Objects.requireNonNull; + +public class DoubleSelectiveStreamReader + implements SelectiveStreamReader +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(DoubleSelectiveStreamReader.class).instanceSize(); + + private static final Block NULL_BLOCK = DOUBLE.createBlockBuilder(null, 1).appendNull().build(); + + private final StreamDescriptor streamDescriptor; + @Nullable + private final TupleDomainFilter filter; + private final boolean nonDeterministicFilter; + private final boolean nullsAllowed; + private final boolean outputRequired; + private final LocalMemoryContext systemMemoryContext; + + private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); + @Nullable + private BooleanInputStream presentStream; + + private InputStreamSource dataStreamSource = missingStreamSource(DoubleInputStream.class); + @Nullable + private DoubleInputStream dataStream; + + private boolean rowGroupOpen; + private int readOffset; + @Nullable + private long[] values; + @Nullable + private boolean[] nulls; + @Nullable + private int[] outputPositions; + private int outputPositionCount; + private boolean allNulls; + private boolean valuesInUse; + + public DoubleSelectiveStreamReader( + StreamDescriptor streamDescriptor, + Optional filter, + boolean outputRequired, + LocalMemoryContext systemMemoryContext) + { + requireNonNull(filter, "filter is null"); + checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false"); + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + this.filter = filter.orElse(null); + this.outputRequired = outputRequired; + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + + nonDeterministicFilter = this.filter != null && !this.filter.isDeterministic(); + nullsAllowed = this.filter == null || nonDeterministicFilter || this.filter.testNull(); + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + { + presentStreamSource = missingStreamSource(BooleanInputStream.class); + dataStreamSource = missingStreamSource(DoubleInputStream.class); + + readOffset = 0; + + presentStream = null; + dataStream = null; + + rowGroupOpen = false; + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + { + presentStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, PRESENT, BooleanInputStream.class); + dataStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, DATA, DoubleInputStream.class); + + readOffset = 0; + + presentStream = null; + dataStream = null; + + rowGroupOpen = false; + } + + @Override + public String toString() + { + return toStringHelper(this) + .addValue(streamDescriptor) + .toString(); + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + sizeOf(values) + sizeOf(nulls) + sizeOf(outputPositions); + } + + private void openRowGroup() + throws IOException + { + presentStream = presentStreamSource.openStream(); + dataStream = dataStreamSource.openStream(); + rowGroupOpen = true; + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (!rowGroupOpen) { + openRowGroup(); + } + + allNulls = false; + + if (outputRequired) { + ensureValuesCapacity(positionCount, nullsAllowed && presentStream != null); + } + + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); + + // account memory used by values, nulls and outputPositions + systemMemoryContext.setBytes(getRetainedSizeInBytes()); + + if (readOffset < offset) { + skip(offset - readOffset); + } + + int streamPosition; + if (dataStream == null && presentStream != null) { + streamPosition = readAllNulls(positions, positionCount); + } + else if (filter == null) { + streamPosition = readNoFilter(positions, positionCount); + } + else { + streamPosition = readWithFilter(positions, positionCount); + } + + readOffset = offset + streamPosition; + return outputPositionCount; + } + + private int readAllNulls(int[] positions, int positionCount) + throws IOException + { + presentStream.skip(positions[positionCount - 1]); + + if (nonDeterministicFilter) { + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + if (filter.testNull()) { + outputPositionCount++; + } + else { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + i += filter.getSucceedingPositionsToFail(); + } + } + } + else if (nullsAllowed) { + outputPositionCount = positionCount; + } + else { + outputPositionCount = 0; + } + + allNulls = true; + return positions[positionCount - 1] + 1; + } + + private int readNoFilter(int[] positions, int positionCount) + throws IOException + { + // filter == null implies outputRequired == true + int streamPosition = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + nulls[i] = true; + } + else { + values[i] = doubleToLongBits(dataStream.next()); + if (presentStream != null) { + nulls[i] = false; + } + } + streamPosition++; + } + outputPositionCount = positionCount; + return streamPosition; + } + + private int readWithFilter(int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + if ((nonDeterministicFilter && filter.testNull()) || nullsAllowed) { + if (outputRequired) { + nulls[outputPositionCount] = true; + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + else { + double value = dataStream.next(); + if (filter.testDouble(value)) { + if (outputRequired) { + values[outputPositionCount] = doubleToLongBits(value); + if (nullsAllowed && presentStream != null) { + nulls[outputPositionCount] = false; + } + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + streamPosition++; + + if (filter != null) { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + int succeedingPositionsToFail = filter.getSucceedingPositionsToFail(); + if (succeedingPositionsToFail > 0) { + int positionsToSkip = 0; + for (int j = 0; j < succeedingPositionsToFail; j++) { + i++; + int nextPosition = positions[i]; + positionsToSkip += 1 + nextPosition - streamPosition; + streamPosition = nextPosition + 1; + } + skip(positionsToSkip); + } + } + } + + return streamPosition; + } + + private void skip(int items) + throws IOException + { + if (dataStream == null) { + presentStream.skip(items); + } + else if (presentStream != null) { + int dataToSkip = presentStream.countBitsSet(items); + dataStream.skip(dataToSkip); + } + else { + dataStream.skip(items); + } + } + + private void ensureValuesCapacity(int capacity, boolean recordNulls) + { + values = ensureCapacity(values, capacity); + if (recordNulls) { + nulls = ensureCapacity(nulls, capacity); + } + } + + @Override + public int[] getReadPositions() + { + return outputPositions; + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return new RunLengthEncodedBlock(NULL_BLOCK, positionCount); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount == outputPositionCount) { + Block block = new LongArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values); + nulls = null; + values = null; + return block; + } + + long[] valuesCopy = new long[positionCount]; + boolean[] nullsCopy = null; + if (includeNulls) { + nullsCopy = new boolean[positionCount]; + } + + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + valuesCopy[positionIndex] = this.values[i]; + if (nullsCopy != null) { + nullsCopy[positionIndex] = this.nulls[i]; + } + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + + nextPosition = positions[positionIndex]; + } + + return new LongArrayBlock(positionCount, Optional.ofNullable(nullsCopy), valuesCopy); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return newLease(new RunLengthEncodedBlock(NULL_BLOCK, outputPositionCount)); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount != outputPositionCount) { + compactValues(positions, positionCount, includeNulls); + } + + return newLease(new LongArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values)); + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } + + private BlockLease newLease(Block block) + { + valuesInUse = true; + return ClosingBlockLease.newLease(block, () -> valuesInUse = false); + } + + private void compactValues(int[] positions, int positionCount, boolean compactNulls) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + values[positionIndex] = values[i]; + if (compactNulls) { + nulls[positionIndex] = nulls[i]; + } + outputPositions[positionIndex] = nextPosition; + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + nextPosition = positions[positionIndex]; + } + + outputPositionCount = positionCount; + } + + @Override + public void close() + { + systemMemoryContext.close(); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/FloatBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/FloatBatchStreamReader.java index 5e16ca3499f2b..5b96070f5b067 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/FloatBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/FloatBatchStreamReader.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.RealType; import com.facebook.presto.spi.type.Type; import org.openjdk.jol.info.ClassLayout; @@ -33,7 +34,9 @@ import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.ReaderUtils.verifyStreamType; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.facebook.presto.spi.type.RealType.REAL; import static com.google.common.base.MoreObjects.toStringHelper; import static java.lang.Float.floatToRawIntBits; import static java.util.Objects.requireNonNull; @@ -58,8 +61,11 @@ public class FloatBatchStreamReader private boolean rowGroupOpen; - public FloatBatchStreamReader(StreamDescriptor streamDescriptor) + public FloatBatchStreamReader(Type type, StreamDescriptor streamDescriptor) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, RealType.class::isInstance); this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); } @@ -71,7 +77,7 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -94,25 +100,23 @@ public Block readBlock(Type type) if (dataStream == null && presentStream != null) { presentStream.skip(nextBatchSize); - Block nullValueBlock = new RunLengthEncodedBlock( - type.createBlockBuilder(null, 1).appendNull().build(), - nextBatchSize); + Block nullValueBlock = RunLengthEncodedBlock.create(REAL, null, nextBatchSize); readOffset = 0; nextBatchSize = 0; return nullValueBlock; } - BlockBuilder builder = type.createBlockBuilder(null, nextBatchSize); + BlockBuilder builder = REAL.createBlockBuilder(null, nextBatchSize); if (presentStream == null) { if (dataStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); } - dataStream.nextVector(type, nextBatchSize, builder); + dataStream.nextVector(REAL, nextBatchSize, builder); } else { for (int i = 0; i < nextBatchSize; i++) { if (presentStream.nextBit()) { - type.writeLong(builder, floatToRawIntBits(dataStream.next())); + REAL.writeLong(builder, floatToRawIntBits(dataStream.next())); } else { builder.appendNull(); @@ -173,6 +177,11 @@ public String toString() .toString(); } + @Override + public void close() + { + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/FloatSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/FloatSelectiveStreamReader.java new file mode 100644 index 0000000000000..f8c6e1c7eca14 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/FloatSelectiveStreamReader.java @@ -0,0 +1,428 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.metadata.ColumnEncoding; +import com.facebook.presto.orc.stream.BooleanInputStream; +import com.facebook.presto.orc.stream.FloatInputStream; +import com.facebook.presto.orc.stream.InputStreamSource; +import com.facebook.presto.orc.stream.InputStreamSources; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.block.ClosingBlockLease; +import com.facebook.presto.spi.block.IntArrayBlock; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import org.openjdk.jol.info.ClassLayout; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; +import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.facebook.presto.spi.type.RealType.REAL; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.lang.Float.floatToRawIntBits; +import static java.util.Objects.requireNonNull; + +public class FloatSelectiveStreamReader + implements SelectiveStreamReader +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(FloatSelectiveStreamReader.class).instanceSize(); + private static final Block NULL_BLOCK = REAL.createBlockBuilder(null, 1).appendNull().build(); + + private final StreamDescriptor streamDescriptor; + private final TupleDomainFilter filter; + private final boolean nullsAllowed; + private final boolean outputRequired; + private final LocalMemoryContext systemMemoryContext; + private final boolean nonDeterministicFilter; + + private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); + private InputStreamSource dataStreamSource = missingStreamSource(FloatInputStream.class); + private BooleanInputStream presentStream; + private FloatInputStream dataStream; + + private boolean rowGroupOpen; + private int readOffset; + private int[] values; + private boolean[] nulls; + private int[] outputPositions; + private int outputPositionCount; + private boolean allNulls; + private boolean valuesInUse; + + public FloatSelectiveStreamReader( + StreamDescriptor streamDescriptor, + Optional filter, + boolean outputRequired, + LocalMemoryContext systemMemoryContext) + { + requireNonNull(filter, "filter is null"); + checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false"); + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + this.filter = filter.orElse(null); + this.outputRequired = outputRequired; + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.nonDeterministicFilter = this.filter != null && !this.filter.isDeterministic(); + this.nullsAllowed = this.filter == null || nonDeterministicFilter || this.filter.testNull(); + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + { + presentStreamSource = missingStreamSource(BooleanInputStream.class); + dataStreamSource = missingStreamSource(FloatInputStream.class); + readOffset = 0; + presentStream = null; + dataStream = null; + rowGroupOpen = false; + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + { + presentStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, PRESENT, BooleanInputStream.class); + dataStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, DATA, FloatInputStream.class); + readOffset = 0; + presentStream = null; + dataStream = null; + rowGroupOpen = false; + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + sizeOf(values) + sizeOf(nulls) + sizeOf(outputPositions); + } + + private void openRowGroup() + throws IOException + { + presentStream = presentStreamSource.openStream(); + dataStream = dataStreamSource.openStream(); + rowGroupOpen = true; + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (!rowGroupOpen) { + openRowGroup(); + } + + allNulls = false; + + if (outputRequired) { + ensureValuesCapacity(positionCount, nullsAllowed && presentStream != null); + } + + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); + + // account memory used by values, nulls and outputPositions + systemMemoryContext.setBytes(getRetainedSizeInBytes()); + + if (readOffset < offset) { + skip(offset - readOffset); + } + + int streamPosition = 0; + if (dataStream == null && presentStream != null) { + streamPosition = readAllNulls(positions, positionCount); + } + else if (filter == null) { + streamPosition = readNoFilter(positions, positionCount); + } + else { + streamPosition = readWithFilter(positions, positionCount); + } + + readOffset = offset + streamPosition; + return outputPositionCount; + } + + private int readWithFilter(int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + if ((nonDeterministicFilter && filter.testNull()) || nullsAllowed) { + if (outputRequired) { + nulls[outputPositionCount] = true; + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + else { + float value = dataStream.next(); + if (filter.testFloat(value)) { + if (outputRequired) { + values[outputPositionCount] = floatToRawIntBits(value); + if (nullsAllowed && presentStream != null) { + nulls[outputPositionCount] = false; + } + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + streamPosition++; + + if (filter != null) { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + int succeedingPositionsToFail = filter.getSucceedingPositionsToFail(); + if (succeedingPositionsToFail > 0) { + int positionsToSkip = 0; + for (int j = 0; j < succeedingPositionsToFail; j++) { + i++; + int nextPosition = positions[i]; + positionsToSkip += 1 + nextPosition - streamPosition; + streamPosition = nextPosition + 1; + } + skip(positionsToSkip); + } + } + } + + return streamPosition; + } + + private int readAllNulls(int[] positions, int positionCount) + throws IOException + { + presentStream.skip(positions[positionCount - 1]); + + if (nonDeterministicFilter) { + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + if (filter.testNull()) { + outputPositionCount++; + } + else { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + i += filter.getSucceedingPositionsToFail(); + } + } + } + else if (nullsAllowed) { + outputPositionCount = positionCount; + } + else { + outputPositionCount = 0; + } + + allNulls = true; + return positions[positionCount - 1] + 1; + } + + private int readNoFilter(int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + nulls[i] = true; + } + else { + values[i] = floatToRawIntBits(dataStream.next()); + if (presentStream != null) { + nulls[i] = false; + } + } + streamPosition++; + } + outputPositionCount = positionCount; + return streamPosition; + } + + private void skip(int items) + throws IOException + { + if (dataStream == null) { + presentStream.skip(items); + } + else if (presentStream != null) { + int dataToSkip = presentStream.countBitsSet(items); + dataStream.skip(dataToSkip); + } + else { + dataStream.skip(items); + } + } + + private void ensureValuesCapacity(int capacity, boolean recordNulls) + { + values = ensureCapacity(values, capacity); + + if (recordNulls) { + nulls = ensureCapacity(nulls, capacity); + } + } + + @Override + public int[] getReadPositions() + { + return outputPositions; + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return new RunLengthEncodedBlock(NULL_BLOCK, positionCount); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount == outputPositionCount) { + Block block = new IntArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values); + nulls = null; + values = null; + return block; + } + + int[] valuesCopy = new int[positionCount]; + boolean[] nullsCopy = null; + if (includeNulls) { + nullsCopy = new boolean[positionCount]; + } + + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + valuesCopy[positionIndex] = this.values[i]; + if (nullsCopy != null) { + nullsCopy[positionIndex] = this.nulls[i]; + } + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + + nextPosition = positions[positionIndex]; + } + + return new IntArrayBlock(positionCount, Optional.ofNullable(nullsCopy), valuesCopy); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return newLease(new RunLengthEncodedBlock(NULL_BLOCK, positionCount)); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount != outputPositionCount) { + compactValues(positions, positionCount, includeNulls); + } + + return newLease(new IntArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values)); + } + + private BlockLease newLease(Block block) + { + valuesInUse = true; + return ClosingBlockLease.newLease(block, () -> valuesInUse = false); + } + + private void compactValues(int[] positions, int positionCount, boolean compactNulls) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + values[positionIndex] = values[i]; + if (compactNulls) { + nulls[positionIndex] = nulls[i]; + } + outputPositions[positionIndex] = nextPosition; + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + nextPosition = positions[positionIndex]; + } + + outputPositionCount = positionCount; + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } + + @Override + public void close() + { + systemMemoryContext.close(); + } + + @Override + public String toString() + { + return toStringHelper(this) + .addValue(streamDescriptor) + .toString(); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/HierarchicalFilter.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/HierarchicalFilter.java new file mode 100644 index 0000000000000..d4b87a6425a20 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/HierarchicalFilter.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.TupleDomainFilter.NullsFilter; +import com.facebook.presto.orc.TupleDomainFilter.PositionalFilter; +import com.facebook.presto.spi.Subfield; + +import java.util.Map; + +public interface HierarchicalFilter +{ + HierarchicalFilter getChild(); + + // Positional IS [NOT] NULL filters to apply at the next level + NullsFilter getNullsFilter(); + + // Positional range filters to apply at the very bottom level + PositionalFilter getPositionalFilter(); + + // Top-level offsets + int[] getTopLevelOffsets(); + + // Number of valid entries in the top-level offsets array + int getTopLevelOffsetCount(); + + // Filters per-position; positions with no filters are populated with nulls + long[] getElementFilters(); + + // Flags indicating top-level positions with at least one subfield with failed filter + boolean[] getTopLevelFailed(); + + // Flags indicating top-level positions missing elements to apply subfield filters to + boolean[] getTopLevelIndexOutOfBounds(); + + long getRetainedSizeInBytes(); + + static HierarchicalFilter createHierarchicalFilter(StreamDescriptor streamDescriptor, Map subfieldFilters, int level, HierarchicalFilter parent) + { + switch (streamDescriptor.getOrcTypeKind()) { + case LIST: + return new ListFilter(streamDescriptor, subfieldFilters, level, parent); + case MAP: + case STRUCT: + throw new UnsupportedOperationException(); + default: + return null; + } + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListBatchStreamReader.java index 39d09b286c85f..55f6351b4c4ec 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListBatchStreamReader.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc.reader; +import com.facebook.presto.memory.context.AggregatedMemoryContext; import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.metadata.ColumnEncoding; @@ -22,19 +23,25 @@ import com.facebook.presto.orc.stream.LongInputStream; import com.facebook.presto.spi.block.ArrayBlock; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; +import com.google.common.io.Closer; import org.joda.time.DateTimeZone; import org.openjdk.jol.info.ClassLayout; import javax.annotation.Nullable; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.List; import java.util.Optional; import static com.facebook.presto.orc.metadata.Stream.StreamKind.LENGTH; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; import static com.facebook.presto.orc.reader.BatchStreamReaders.createStreamReader; +import static com.facebook.presto.orc.reader.ReaderUtils.convertLengthVectorToOffsetVector; +import static com.facebook.presto.orc.reader.ReaderUtils.unpackLengthNulls; +import static com.facebook.presto.orc.reader.ReaderUtils.verifyStreamType; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; import static com.google.common.base.MoreObjects.toStringHelper; import static java.lang.Math.toIntExact; @@ -45,6 +52,7 @@ public class ListBatchStreamReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(ListBatchStreamReader.class).instanceSize(); + private final Type elementType; private final StreamDescriptor streamDescriptor; private final BatchStreamReader elementStreamReader; @@ -62,10 +70,14 @@ public class ListBatchStreamReader private boolean rowGroupOpen; - public ListBatchStreamReader(StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone) + public ListBatchStreamReader(Type type, StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone, AggregatedMemoryContext systemMemoryContext) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, ArrayType.class::isInstance); + elementType = ((ArrayType) type).getElementType(); this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); - this.elementStreamReader = createStreamReader(streamDescriptor.getNestedStreams().get(0), hiveStorageTimeZone); + this.elementStreamReader = createStreamReader(elementType, streamDescriptor.getNestedStreams().get(0), hiveStorageTimeZone, systemMemoryContext); } @Override @@ -76,7 +88,7 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -106,7 +118,7 @@ public Block readBlock(Type type) if (lengthStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); } - lengthStream.nextIntVector(nextBatchSize, offsetVector, 0); + lengthStream.next(offsetVector, nextBatchSize); } else { nullVector = new boolean[nextBatchSize]; @@ -115,26 +127,19 @@ public Block readBlock(Type type) if (lengthStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); } - lengthStream.nextIntVector(nextBatchSize, offsetVector, 0, nullVector); + lengthStream.next(offsetVector, nextBatchSize - nullValues); + unpackLengthNulls(offsetVector, nullVector, nextBatchSize - nullValues); } } - // Convert the length values in the offsetVector to offset values in place - int currentLength = offsetVector[0]; - offsetVector[0] = 0; - for (int i = 1; i < offsetVector.length; i++) { - int nextLength = offsetVector[i]; - offsetVector[i] = offsetVector[i - 1] + currentLength; - currentLength = nextLength; - } + convertLengthVectorToOffsetVector(offsetVector); - Type elementType = type.getTypeParameters().get(0); int elementCount = offsetVector[offsetVector.length - 1]; Block elements; if (elementCount > 0) { elementStreamReader.prepareNextRead(elementCount); - elements = elementStreamReader.readBlock(elementType); + elements = elementStreamReader.readBlock(); } else { elements = elementType.createBlockBuilder(null, 0).build(); @@ -200,6 +205,17 @@ public String toString() .toString(); } + @Override + public void close() + { + try (Closer closer = Closer.create()) { + closer.register(elementStreamReader::close); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListFilter.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListFilter.java new file mode 100644 index 0000000000000..913a86cabf6d6 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListFilter.java @@ -0,0 +1,436 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.TupleDomainFilter.NullsFilter; +import com.facebook.presto.orc.TupleDomainFilter.PositionalFilter; +import com.facebook.presto.spi.Subfield; +import org.openjdk.jol.info.ClassLayout; + +import javax.annotation.Nullable; + +import java.util.Arrays; +import java.util.Map; +import java.util.function.Predicate; + +import static com.facebook.presto.orc.TupleDomainFilter.IS_NOT_NULL; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NULL; +import static com.facebook.presto.orc.reader.HierarchicalFilter.createHierarchicalFilter; +import static com.google.common.base.Preconditions.checkArgument; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.lang.Long.numberOfTrailingZeros; +import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; + +public class ListFilter + implements HierarchicalFilter +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(ListFilter.class).instanceSize(); + + @Nullable + private final HierarchicalFilter parent; + @Nullable + private final HierarchicalFilter child; + + // Array of filters in a fixed order + private final TupleDomainFilter[] tupleDomainFilters; + + // Set only at the deepest level + @Nullable + private final PositionalFilter positionalFilter; + + // filter per position; populated only at the deepest level + // used to setup positionalFilter + private TupleDomainFilter[] elementTupleDomainFilters; + + // filters per subscript: subscriptFilters[subscript - 1] = {"> 3", "= 10"} + // individual subscripts may have multiple filters + // for example, consider this filter: a[1][2] > 3 and a[1][4] = 10 and a[3][2] < 100 + // - at level 0, subscript 1 has 2 filters (> 3, = 10) and subscript 3 has one filter (< 100) + // - at level 2, subscript 2 has 2 filters (> 3, < 100), subscript 4 has one filter (= 10) + // each value represents multiple filters - the long value has bits corresponding to filter + // indices in tupleDomainFilters array set + private final long[] subscriptFilters; + + // filters per position; positions with no filters are populated with nulls + private long[] elementFilters; + + // offsets into elementFilters array identifying ranges corresponding to individual top-level values + private int[] topLevelOffsets; + + // number of valid entries in topLevelOffsets array, e.g. number of top-level positions + private int topLevelOffsetCount; + + // IS [NOT] NULL filters that apply to this level + private final long nullFilters; + + // IS [NOT] NULL positional filters + private final NullsFilter nullsFilter; + + // positions with nulls allowed, e.g. positions with no filter or with IS NULL filter; used to setup nullsFilter + private boolean[] nullsAllowed; + + // positions with non-nulls allowed, e.g. positions with no filter or with IS NOT NULL filter; used to setup nullsFilter + private boolean[] nonNullsAllowed; + + // top-level positions with "Array index out of bounds" errors + private boolean[] indexOutOfBounds; + + public ListFilter(StreamDescriptor streamDescriptor, Map subfieldFilters) + { + this(streamDescriptor, subfieldFilters, 0, null); + } + + public ListFilter(StreamDescriptor streamDescriptor, Map subfieldFilters, int level, HierarchicalFilter parent) + { + requireNonNull(streamDescriptor, "streamDescriptor is null"); + requireNonNull(subfieldFilters, "subfieldFilters is null"); + checkArgument(!subfieldFilters.isEmpty(), "subfieldFilters is empty"); + checkArgument(subfieldFilters.size() <= 64, "Number of filters cannot exceed 64"); + checkArgument((level == 0 && parent == null) || (level > 0 && parent != null), "parent must be null for top-level filter and non-null otherwise"); + + this.parent = parent; + tupleDomainFilters = subfieldFilters.values().toArray(new TupleDomainFilter[0]); + subscriptFilters = extractFilters(subfieldFilters, level, subfield -> subfield.getPath().size() > level); + nullFilters = combineFilters(extractFilters(subfieldFilters, level, subfield -> subfield.getPath().size() == level + 1)); + if (nullFilters > 0) { + nullsFilter = new NullsFilter(); + } + else { + nullsFilter = null; + } + + StreamDescriptor elementStreamDescriptor = streamDescriptor.getNestedStreams().get(0); + child = createHierarchicalFilter(elementStreamDescriptor, subfieldFilters, level + 1, this); + if (child == null) { + positionalFilter = new PositionalFilter(); + } + else { + positionalFilter = null; + } + } + + public HierarchicalFilter getParent() + { + return parent; + } + + @Override + public HierarchicalFilter getChild() + { + return child; + } + + @Override + public NullsFilter getNullsFilter() + { + return nullsFilter; + } + + @Override + public int[] getTopLevelOffsets() + { + return topLevelOffsets; + } + + @Override + public int getTopLevelOffsetCount() + { + return topLevelOffsetCount; + } + + @Override + public long[] getElementFilters() + { + return elementFilters; + } + + @Override + public PositionalFilter getPositionalFilter() + { + return positionalFilter; + } + + public void populateElementFilters(int positionCount, boolean[] nulls, int[] lengths, int lengthSum) + { + elementFilters = ensureCapacity(elementFilters, lengthSum); + + if (parent == null) { + topLevelOffsetCount = positionCount + 1; + } + else { + topLevelOffsetCount = parent.getTopLevelOffsetCount(); + } + topLevelOffsets = ensureCapacity(topLevelOffsets, topLevelOffsetCount); + indexOutOfBounds = ensureCapacity(indexOutOfBounds, topLevelOffsetCount); + Arrays.fill(indexOutOfBounds, 0, topLevelOffsetCount, false); + + int elementPosition = 0; + if (parent == null) { // top-level filter + for (int i = 0; i < positionCount; i++) { + topLevelOffsets[i] = elementPosition; + + for (int j = 0; j < lengths[i]; j++) { + if (j < subscriptFilters.length) { + elementFilters[elementPosition] = subscriptFilters[j]; + } + else { + elementFilters[elementPosition] = 0; + } + elementPosition++; + } + + if (nulls == null || !nulls[i]) { + for (int j = lengths[i]; j < subscriptFilters.length; j++) { + if (subscriptFilters[j] > 0) { + // this entry doesn't have enough elements for all filters, + // hence, raise "Array index out of bound" error + indexOutOfBounds[i] = true; + break; + } + } + } + } + } + else { + int[] parentTopLevelOffsets = updateParentTopLevelOffsets(); + long[] parentElementFilters = parent.getElementFilters(); + + int offsetIndex = 0; + int nextOffset = parentTopLevelOffsets[offsetIndex + 1]; + for (int i = 0; i < positionCount; i++) { + while (i == nextOffset) { // there might be duplicate offsets (if some offsets failed) + offsetIndex++; + topLevelOffsets[offsetIndex] = elementPosition; + nextOffset = parentTopLevelOffsets[offsetIndex + 1]; + } + + long parentElementFilter = parentElementFilters[i]; + for (int j = 0; j < lengths[i]; j++) { + if (subscriptFilters != null && j < subscriptFilters.length) { + elementFilters[elementPosition] = parentElementFilter & subscriptFilters[j]; + } + else { + elementFilters[elementPosition] = 0; + } + elementPosition++; + } + + if (subscriptFilters != null && (nulls == null || !nulls[i])) { + for (int j = lengths[i]; j < subscriptFilters.length; j++) { + if ((parentElementFilter & subscriptFilters[j]) > 0) { + // this entry doesn't have enough elements for all filters, + // hence, raise "Array index out of bound" error + indexOutOfBounds[offsetIndex] = true; + break; + } + } + } + } + while (offsetIndex < topLevelOffsetCount - 1) { + offsetIndex++; + topLevelOffsets[offsetIndex] = elementPosition; + } + } + topLevelOffsets[topLevelOffsetCount - 1] = elementPosition; + + if (positionalFilter != null) { + setupPositionalFilter(); + } + else if (nullsFilter != null) { + setupNullsFilter(); + } + } + + // update parentTopLevelOffsets to take into account failed positions + private int[] updateParentTopLevelOffsets() + { + int[] parentTopLevelOffsets = parent.getTopLevelOffsets(); + + NullsFilter nullsFilter = parent.getNullsFilter(); + if (nullsFilter != null) { + boolean[] failed = nullsFilter.getFailed(); + int skipped = 0; + for (int i = 0; i < topLevelOffsetCount - 1; i++) { + parentTopLevelOffsets[i] -= skipped; + if (failed[i]) { + skipped += parentTopLevelOffsets[i + 1] - parentTopLevelOffsets[i] - skipped; + } + } + parentTopLevelOffsets[topLevelOffsetCount - 1] -= skipped; + } + + return parentTopLevelOffsets; + } + + private void setupPositionalFilter() + { + int count = topLevelOffsets[topLevelOffsetCount - 1]; + if (elementTupleDomainFilters == null || elementTupleDomainFilters.length < count) { + elementTupleDomainFilters = new TupleDomainFilter[count]; + } + for (int i = 0; i < count; i++) { + long filter = elementFilters[i]; + if (filter > 0) { + elementTupleDomainFilters[i] = tupleDomainFilters[numberOfTrailingZeros(filter)]; + } + else { + elementTupleDomainFilters[i] = null; + } + } + positionalFilter.setFilters(elementTupleDomainFilters, topLevelOffsets); + } + + private void setupNullsFilter() + { + int count = topLevelOffsets[topLevelOffsetCount - 1]; + nullsAllowed = ensureCapacity(nullsAllowed, count); + nonNullsAllowed = ensureCapacity(nonNullsAllowed, count); + + for (int i = 0; i < count; i++) { + long filter = elementFilters[i] & nullFilters; + if (filter > 0) { + TupleDomainFilter tupleDomainFilter = tupleDomainFilters[numberOfTrailingZeros(filter)]; + nullsAllowed[i] = tupleDomainFilter == IS_NULL; + nonNullsAllowed[i] = tupleDomainFilter == IS_NOT_NULL; + } + else { + nullsAllowed[i] = true; + nonNullsAllowed[i] = true; + } + } + nullsFilter.setup(nullsAllowed, nonNullsAllowed, topLevelOffsets); + } + + @Override + public boolean[] getTopLevelFailed() + { + if (child == null) { + return positionalFilter.getFailed(); + } + + boolean[] failed = child.getTopLevelFailed(); + if (nullsFilter != null) { + for (int i = 0; i < failed.length; i++) { + failed[i] |= nullsFilter.getFailed()[i]; + } + } + + return failed; + } + + @Override + public boolean[] getTopLevelIndexOutOfBounds() + { + if (child == null) { + return indexOutOfBounds; + } + + boolean[] indexOutOfBounds = child.getTopLevelIndexOutOfBounds(); + for (int i = 0; i < indexOutOfBounds.length; i++) { + indexOutOfBounds[i] |= this.indexOutOfBounds[i]; + } + + return indexOutOfBounds; + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + sizeOf(elementFilters) + sizeOf(tupleDomainFilters) + sizeOf(subscriptFilters) + + sizeOf(elementTupleDomainFilters) + sizeOf(topLevelOffsets) + + sizeOf(nonNullsAllowed) + sizeOf(nullsAllowed) + sizeOf(indexOutOfBounds) + + (child != null ? child.getRetainedSizeInBytes() : 0); + } + + private static long[] extractFilters(Map filters, int level, Predicate predicate) + { + int maxSubscript = -1; + int[] subscripts = new int[filters.size()]; + int index = 0; + for (Subfield subfield : filters.keySet()) { + if (predicate.test(subfield)) { + subscripts[index] = toSubscript(subfield, level); + maxSubscript = Math.max(maxSubscript, subscripts[index]); + } + else { + subscripts[index] = -1; + } + index++; + } + + if (maxSubscript == -1) { + return null; + } + + long[] filterCodes = new long[maxSubscript + 1]; + for (int i = 0; i < subscripts.length; i++) { + if (subscripts[i] != -1) { + filterCodes[subscripts[i]] |= 1 << i; + } + } + + return filterCodes; + } + + private static int toSubscript(Subfield subfield, int level) + { + checkArgument(subfield.getPath().size() > level); + checkArgument(subfield.getPath().get(level) instanceof Subfield.LongSubscript); + + return toIntExact(((Subfield.LongSubscript) subfield.getPath().get(level)).getIndex()) - 1; + } + + private static long combineFilters(long[] filters) + { + if (filters == null) { + return 0; + } + + int combined = 0; + for (long filter : filters) { + combined |= filter; + } + return combined; + } + + private static int[] ensureCapacity(int[] buffer, int capacity) + { + if (buffer == null || buffer.length < capacity) { + return new int[capacity]; + } + + return buffer; + } + + private static long[] ensureCapacity(long[] buffer, int capacity) + { + if (buffer == null || buffer.length < capacity) { + return new long[capacity]; + } + + return buffer; + } + + private static boolean[] ensureCapacity(boolean[] buffer, int capacity) + { + if (buffer == null || buffer.length < capacity) { + return new boolean[capacity]; + } + + return buffer; + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListSelectiveStreamReader.java new file mode 100644 index 0000000000000..dda88bb0e0890 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ListSelectiveStreamReader.java @@ -0,0 +1,740 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.AggregatedMemoryContext; +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.TupleDomainFilter.NullsFilter; +import com.facebook.presto.orc.metadata.ColumnEncoding; +import com.facebook.presto.orc.stream.BooleanInputStream; +import com.facebook.presto.orc.stream.InputStreamSource; +import com.facebook.presto.orc.stream.InputStreamSources; +import com.facebook.presto.orc.stream.LongInputStream; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.block.ArrayBlock; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import org.joda.time.DateTimeZone; +import org.openjdk.jol.info.ClassLayout; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NOT_NULL; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NULL; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.LENGTH; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.createNestedStreamReader; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; +import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.spi.block.ClosingBlockLease.newLease; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; + +public class ListSelectiveStreamReader + implements SelectiveStreamReader +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(ListSelectiveStreamReader.class).instanceSize(); + private static final int ELEMENT_LENGTH_UNBOUNDED = -1; + + private final StreamDescriptor streamDescriptor; + private final int level; + private final ListFilter listFilter; + private final NullsFilter nullsFilter; + private final boolean nullsAllowed; + private final boolean nonNullsAllowed; + private final boolean outputRequired; + @Nullable + private final ArrayType outputType; + private final int maxElementLength; + // elementStreamReader is null if output is not required and filter is a simple IS [NOT] NULL + @Nullable + private final SelectiveStreamReader elementStreamReader; + private final LocalMemoryContext systemMemoryContext; + + private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); + @Nullable + private BooleanInputStream presentStream; + + private InputStreamSource lengthStreamSource = missingStreamSource(LongInputStream.class); + @Nullable + private LongInputStream lengthStream; + + private boolean rowGroupOpen; + private int readOffset; + @Nullable + private int[] offsets; + @Nullable + private boolean[] nulls; + @Nullable + private int[] outputPositions; + private int outputPositionCount; + private boolean[] indexOutOfBounds; + private boolean allNulls; + + private int elementReadOffset; // offset within elementStream relative to row group start + private int[] elementOffsets; // offsets within elementStream relative to elementReadOffset + private int[] elementLengths; // aligned with elementOffsets + private int[] elementPositions; // positions in elementStream corresponding to positions passed to read(); relative to elementReadOffset + private int elementOutputPositionCount; + private int[] elementOutputPositions; + + private boolean valuesInUse; + + public ListSelectiveStreamReader( + StreamDescriptor streamDescriptor, + Map filters, + List subfields, + ListFilter listFilter, + int subfieldLevel, // 0 - top level + Optional outputType, + DateTimeZone hiveStorageTimeZone, + AggregatedMemoryContext systemMemoryContext) + { + requireNonNull(filters, "filters is null"); + requireNonNull(subfields, "subfields is null"); + requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + + // TODO Implement subfield pruning + + if (listFilter != null) { + checkArgument(subfieldLevel > 0, "SubfieldFilter is not expected at the top level"); + checkArgument(filters.isEmpty(), "Range filters are not expected at mid level"); + } + + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + this.outputRequired = requireNonNull(outputType, "outputType is null").isPresent(); + this.outputType = (ArrayType) outputType.orElse(null); + this.level = subfieldLevel; + + if (subfields.stream() + .map(Subfield::getPath) + .map(path -> path.get(0)) + .anyMatch(Subfield.AllSubscripts.class::isInstance)) { + maxElementLength = ELEMENT_LENGTH_UNBOUNDED; + } + else { + maxElementLength = subfields.stream() + .map(Subfield::getPath) + .map(path -> path.get(0)) + .map(Subfield.LongSubscript.class::cast) + .map(Subfield.LongSubscript::getIndex) + .mapToInt(Long::intValue) + .max() + .orElse(ELEMENT_LENGTH_UNBOUNDED); + } + + if (listFilter != null) { + nullsAllowed = true; + nonNullsAllowed = true; + this.listFilter = listFilter; + nullsFilter = listFilter.getParent().getNullsFilter(); + } + else if (!filters.isEmpty()) { + Optional topLevelFilter = getTopLevelFilter(filters); + if (topLevelFilter.isPresent()) { + nullsAllowed = topLevelFilter.get() == IS_NULL; + nonNullsAllowed = !nullsAllowed; + } + else { + nullsAllowed = filters.values().stream().allMatch(TupleDomainFilter::testNull); + nonNullsAllowed = true; + } + + if (filters.keySet().stream().anyMatch(path -> !path.getPath().isEmpty())) { + this.listFilter = new ListFilter(streamDescriptor, filters); + } + else { + this.listFilter = null; + } + + nullsFilter = null; + } + else { + nullsAllowed = true; + nonNullsAllowed = true; + this.listFilter = null; + nullsFilter = null; + } + + StreamDescriptor elementStreamDescriptor = streamDescriptor.getNestedStreams().get(0); + Optional elementOutputType = outputType.map(type -> type.getTypeParameters().get(0)); + + List elementSubfields = ImmutableList.of(); + if (subfields.stream().map(Subfield::getPath).allMatch(path -> path.size() > 1)) { + elementSubfields = subfields.stream() + .map(subfield -> subfield.tail(subfield.getRootName())) + .distinct() + .collect(toImmutableList()); + } + + this.elementStreamReader = createNestedStreamReader(elementStreamDescriptor, level + 1, Optional.ofNullable(this.listFilter), elementOutputType, elementSubfields, hiveStorageTimeZone, systemMemoryContext); + this.systemMemoryContext = systemMemoryContext.newLocalMemoryContext(ListSelectiveStreamReader.class.getSimpleName()); + } + + private static Optional getTopLevelFilter(Map filters) + { + Map topLevelFilters = Maps.filterEntries(filters, entry -> entry.getKey().getPath().isEmpty()); + if (topLevelFilters.isEmpty()) { + return Optional.empty(); + } + + checkArgument(topLevelFilters.size() == 1, "ARRAY column may have at most one top-level range filter"); + TupleDomainFilter filter = Iterables.getOnlyElement(topLevelFilters.values()); + checkArgument(filter == IS_NULL || filter == IS_NOT_NULL, "Top-level range filter on ARRAY column must be IS NULL or IS NOT NULL"); + return Optional.of(filter); + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + checkArgument(positionCount > 0 || listFilter != null, "positionCount must be greater than zero"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (!rowGroupOpen) { + openRowGroup(); + } + + allNulls = false; + + if (outputRequired) { + offsets = ensureCapacity(offsets, positionCount + 1); + } + + if (nullsAllowed && presentStream != null && (outputRequired || listFilter != null)) { + nulls = ensureCapacity(nulls, positionCount); + } + + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); + + systemMemoryContext.setBytes(getRetainedSizeInBytes()); + + if (readOffset < offset) { + elementReadOffset += skip(offset - readOffset); + } + + int streamPosition; + if (lengthStream == null && presentStream != null) { + streamPosition = readAllNulls(positions, positionCount); + } + else { + streamPosition = readNotAllNulls(positions, positionCount); + } + + readOffset = offset + streamPosition; + return outputPositionCount; + } + + private void openRowGroup() + throws IOException + { + presentStream = presentStreamSource.openStream(); + lengthStream = lengthStreamSource.openStream(); + + rowGroupOpen = true; + } + + private int skip(int items) + throws IOException + { + if (lengthStream == null) { + presentStream.skip(items); + return 0; + } + + if (presentStream != null) { + int lengthsToSkip = presentStream.countBitsSet(items); + return toIntExact(lengthStream.sum(lengthsToSkip)); + } + + return toIntExact(lengthStream.sum(items)); + } + + private int readAllNulls(int[] positions, int positionCount) + throws IOException + { + presentStream.skip(positions[positionCount - 1]); + if (nullsAllowed) { + if (nullsFilter != null) { + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + if (nullsFilter.testNull()) { + outputPositions[outputPositionCount] = positions[i]; + outputPositionCount++; + } + else { + outputPositionCount -= nullsFilter.getPrecedingPositionsToFail(); + i += nullsFilter.getSucceedingPositionsToFail(); + } + } + } + else { + outputPositionCount = positionCount; + } + } + else { + outputPositionCount = 0; + } + + if (listFilter != null) { + listFilter.populateElementFilters(0, null, null, 0); + } + + allNulls = true; + return positions[positionCount - 1] + 1; + } + + private int readNotAllNulls(int[] positions, int positionCount) + throws IOException + { + // populate elementOffsets, elementLengths, and nulls + elementOffsets = ensureCapacity(elementOffsets, positionCount + 1); + elementLengths = ensureCapacity(elementLengths, positionCount); + + systemMemoryContext.setBytes(getRetainedSizeInBytes()); + + int streamPosition = 0; + int skippedElements = 0; + int elementPositionCount = 0; + + outputPositionCount = 0; + + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skippedElements += skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + if (nullsAllowed && (nullsFilter == null || nullsFilter.testNull())) { + if (outputRequired || listFilter != null) { + nulls[outputPositionCount] = true; + } + + elementOffsets[outputPositionCount] = elementPositionCount + skippedElements; + elementLengths[outputPositionCount] = 0; + + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + else { + int length = toIntExact(lengthStream.next()); + + if (nonNullsAllowed && (nullsFilter == null || nullsFilter.testNonNull())) { + elementOffsets[outputPositionCount] = elementPositionCount + skippedElements; + elementLengths[outputPositionCount] = length; + + elementPositionCount += length; + + if ((outputRequired || listFilter != null) && nullsAllowed && presentStream != null) { + nulls[outputPositionCount] = false; + } + + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + else { + skippedElements += length; + } + } + + streamPosition++; + + if (nullsFilter != null) { + int precedingPositionsToFail = nullsFilter.getPrecedingPositionsToFail(); + + for (int j = 0; j < precedingPositionsToFail; j++) { + int length = elementLengths[outputPositionCount - 1 - j]; + skippedElements += length; + elementPositionCount -= length; + } + outputPositionCount -= precedingPositionsToFail; + + int succeedingPositionsToFail = nullsFilter.getSucceedingPositionsToFail(); + if (succeedingPositionsToFail > 0) { + int positionsToSkip = 0; + for (int j = 0; j < succeedingPositionsToFail; j++) { + i++; + int nextPosition = positions[i]; + positionsToSkip += 1 + nextPosition - streamPosition; + streamPosition = nextPosition + 1; + } + skippedElements += skip(positionsToSkip); + } + } + } + elementOffsets[outputPositionCount] = elementPositionCount + skippedElements; + + elementPositionCount = populateElementPositions(elementPositionCount); + + if (listFilter != null) { + listFilter.populateElementFilters(outputPositionCount, nulls, elementLengths, elementPositionCount); + } + + if (elementStreamReader != null && elementPositionCount > 0) { + elementStreamReader.read(elementReadOffset, elementPositions, elementPositionCount); + } + else if (listFilter != null && listFilter.getChild() != null) { + elementStreamReader.read(elementReadOffset, elementPositions, elementPositionCount); + } + elementReadOffset += elementOffsets[outputPositionCount]; + + if (listFilter == null || level > 0) { + populateOutputPositionsNoFilter(elementPositionCount); + } + else { + populateOutputPositionsWithFilter(elementPositionCount); + } + + return streamPosition; + } + + private int populateElementPositions(int elementPositionCount) + { + elementPositions = ensureCapacity(elementPositions, elementPositionCount); + + int index = 0; + for (int i = 0; i < outputPositionCount; i++) { + int length = elementLengths[i]; + if (maxElementLength != ELEMENT_LENGTH_UNBOUNDED && length > maxElementLength) { + length = maxElementLength; + elementLengths[i] = length; + } + + for (int j = 0; j < length; j++) { + elementPositions[index] = elementOffsets[i] + j; + index++; + } + } + return index; + } + + private void populateOutputPositionsNoFilter(int elementPositionCount) + { + if (outputRequired) { + elementOutputPositionCount = elementPositionCount; + elementOutputPositions = ensureCapacity(elementOutputPositions, elementPositionCount); + System.arraycopy(elementPositions, 0, elementOutputPositions, 0, elementPositionCount); + + int offset = 0; + for (int i = 0; i < outputPositionCount; i++) { + offsets[i] = offset; + offset += elementLengths[i]; + } + offsets[outputPositionCount] = offset; + } + } + + private void populateOutputPositionsWithFilter(int elementPositionCount) + { + elementOutputPositionCount = 0; + elementOutputPositions = ensureCapacity(elementOutputPositions, elementPositionCount); + + indexOutOfBounds = listFilter.getTopLevelIndexOutOfBounds(); + + int outputPosition = 0; + int elementOffset = 0; + boolean[] positionsFailed = listFilter.getTopLevelFailed(); + for (int i = 0; i < outputPositionCount; i++) { + if (!positionsFailed[i]) { + indexOutOfBounds[outputPosition] = indexOutOfBounds[i]; + outputPositions[outputPosition] = outputPositions[i]; + if (outputRequired) { + if (nullsAllowed && presentStream != null) { + nulls[outputPosition] = nulls[i]; + } + offsets[outputPosition] = elementOutputPositionCount; + for (int j = 0; j < elementLengths[i]; j++) { + elementOutputPositions[elementOutputPositionCount] = elementPositions[elementOffset + j]; + elementOutputPositionCount++; + } + } + outputPosition++; + } + elementOffset += elementLengths[i]; + } + if (outputRequired) { + this.offsets[outputPosition] = elementOutputPositionCount; + } + + outputPositionCount = outputPosition; + } + + @Override + public int[] getReadPositions() + { + return outputPositions; + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values: " + outputPositionCount + ", " + positionCount); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), positionCount); + } + + boolean mayHaveNulls = nullsAllowed && presentStream != null; + + if (positionCount == outputPositionCount) { + Block block = ArrayBlock.fromElementBlock(positionCount, Optional.ofNullable(mayHaveNulls ? nulls : null), offsets, makeElementBlock()); + nulls = null; + offsets = null; + return block; + } + + int[] offsetsCopy = new int[positionCount + 1]; + boolean[] nullsCopy = null; + if (mayHaveNulls) { + nullsCopy = new boolean[positionCount]; + } + + elementOutputPositionCount = 0; + + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + int skippedElements = 0; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + skippedElements += offsets[i + 1] - offsets[i]; + continue; + } + + assert outputPositions[i] == nextPosition; + + offsetsCopy[positionIndex] = this.offsets[i] - skippedElements; + if (nullsCopy != null) { + nullsCopy[positionIndex] = this.nulls[i]; + } + for (int j = 0; j < offsets[i + 1] - offsets[i]; j++) { + elementOutputPositions[elementOutputPositionCount] = elementOutputPositions[elementOutputPositionCount + skippedElements]; + elementOutputPositionCount++; + } + + positionIndex++; + if (positionIndex >= positionCount) { + offsetsCopy[positionCount] = this.offsets[i + 1] - skippedElements; + break; + } + + nextPosition = positions[positionIndex]; + } + return ArrayBlock.fromElementBlock(positionCount, Optional.ofNullable(nullsCopy), offsetsCopy, makeElementBlock()); + } + + private Block makeElementBlock() + { + if (elementOutputPositionCount == 0) { + return outputType.getElementType().createBlockBuilder(null, 0).build(); + } + return elementStreamReader.getBlock(elementOutputPositions, elementOutputPositionCount); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return newLease(new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), positionCount)); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount != outputPositionCount) { + compactValues(positions, positionCount, includeNulls); + } + + BlockLease elementBlockLease; + if (elementOutputPositionCount == 0) { + elementBlockLease = newLease(outputType.getElementType().createBlockBuilder(null, 0).build()); + } + else { + elementBlockLease = elementStreamReader.getBlockView(elementOutputPositions, elementOutputPositionCount); + } + + valuesInUse = true; + Block block = ArrayBlock.fromElementBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), offsets, elementBlockLease.get()); + return newLease(block, () -> closeBlockLease(elementBlockLease)); + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + if (indexOutOfBounds == null) { + return; + } + + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + if (indexOutOfBounds[i]) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Array subscript out of bounds"); + } + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + + nextPosition = positions[positionIndex]; + } + } + + private void closeBlockLease(BlockLease elementBlockLease) + { + elementBlockLease.close(); + valuesInUse = false; + } + + private void compactValues(int[] positions, int positionCount, boolean compactNulls) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + int skippedElements = 0; + elementOutputPositionCount = 0; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + skippedElements += offsets[i + 1] - offsets[i]; + continue; + } + + assert outputPositions[i] == nextPosition; + + for (int j = 0; j < offsets[i + 1] - offsets[i]; j++) { + elementOutputPositions[elementOutputPositionCount] = elementOutputPositions[elementOutputPositionCount + skippedElements]; + elementOutputPositionCount++; + } + + offsets[positionIndex] = offsets[i] - skippedElements; + if (compactNulls) { + nulls[positionIndex] = nulls[i]; + } + outputPositions[positionIndex] = nextPosition; + if (indexOutOfBounds != null) { + indexOutOfBounds[positionIndex] = indexOutOfBounds[i]; + } + + positionIndex++; + if (positionIndex >= positionCount) { + offsets[positionCount] = offsets[i + 1] - skippedElements; + break; + } + nextPosition = positions[positionIndex]; + } + + outputPositionCount = positionCount; + } + + @Override + public void close() + { + if (elementStreamReader != null) { + elementStreamReader.close(); + } + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + throws IOException + { + presentStreamSource = missingStreamSource(BooleanInputStream.class); + lengthStreamSource = missingStreamSource(LongInputStream.class); + + readOffset = 0; + elementReadOffset = 0; + + presentStream = null; + lengthStream = null; + + rowGroupOpen = false; + + if (elementStreamReader != null) { + elementStreamReader.startStripe(dictionaryStreamSources, encoding); + } + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + throws IOException + { + presentStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, PRESENT, BooleanInputStream.class); + lengthStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, LENGTH, LongInputStream.class); + + readOffset = 0; + elementReadOffset = 0; + + presentStream = null; + lengthStream = null; + + rowGroupOpen = false; + + if (elementStreamReader != null) { + elementStreamReader.startRowGroup(dataStreamSources); + } + } + + @Override + public String toString() + { + return toStringHelper(this) + .addValue(streamDescriptor) + .toString(); + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + sizeOf(offsets) + sizeOf(nulls) + sizeOf(outputPositions) + sizeOf(indexOutOfBounds) + + sizeOf(elementOffsets) + sizeOf(elementLengths) + sizeOf(elementPositions) + + sizeOf(elementOutputPositions) + + (listFilter != null ? listFilter.getRetainedSizeInBytes() : 0) + + (elementStreamReader != null ? elementStreamReader.getRetainedSizeInBytes() : 0); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongBatchStreamReader.java index 1cad6c0c75619..e67ffedd1dc85 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongBatchStreamReader.java @@ -13,15 +13,19 @@ */ package com.facebook.presto.orc.reader; +import com.facebook.presto.memory.context.AggregatedMemoryContext; +import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.metadata.ColumnEncoding; import com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind; import com.facebook.presto.orc.stream.InputStreamSources; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.Type; +import com.google.common.io.Closer; import org.openjdk.jol.info.ClassLayout; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.List; import static com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind.DICTIONARY; @@ -41,11 +45,12 @@ public class LongBatchStreamReader private final LongDictionaryBatchStreamReader dictionaryReader; private BatchStreamReader currentReader; - public LongBatchStreamReader(StreamDescriptor streamDescriptor) + public LongBatchStreamReader(Type type, StreamDescriptor streamDescriptor, AggregatedMemoryContext systemMemoryContext) + throws OrcCorruptionException { this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); - directReader = new LongDirectBatchStreamReader(streamDescriptor); - dictionaryReader = new LongDictionaryBatchStreamReader(streamDescriptor); + directReader = new LongDirectBatchStreamReader(type, streamDescriptor, systemMemoryContext.newLocalMemoryContext(LongBatchStreamReader.class.getSimpleName())); + dictionaryReader = new LongDictionaryBatchStreamReader(type, streamDescriptor, systemMemoryContext.newLocalMemoryContext(LongBatchStreamReader.class.getSimpleName())); } @Override @@ -55,10 +60,10 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { - return currentReader.readBlock(type); + return currentReader.readBlock(); } @Override @@ -96,6 +101,18 @@ public String toString() .toString(); } + @Override + public void close() + { + try (Closer closer = Closer.create()) { + closer.register(directReader::close); + closer.register(dictionaryReader::close); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDecimalSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDecimalSelectiveStreamReader.java new file mode 100644 index 0000000000000..753304e293ba8 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDecimalSelectiveStreamReader.java @@ -0,0 +1,202 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.Int128ArrayBlock; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.UnscaledDecimal128Arithmetic; +import io.airlift.slice.Slice; +import io.airlift.slice.UnsafeSlice; + +import java.io.IOException; +import java.util.Optional; + +import static com.facebook.presto.spi.type.UnscaledDecimal128Arithmetic.rescale; + +public class LongDecimalSelectiveStreamReader + extends AbstractDecimalSelectiveStreamReader +{ + public LongDecimalSelectiveStreamReader( + StreamDescriptor streamDescriptor, + Optional filter, + Optional outputType, + LocalMemoryContext systemMemoryContext) + { + super(streamDescriptor, filter, outputType, systemMemoryContext, 2); + } + + @Override + protected int readNoFilter(int[] positions, int positionCount) + throws IOException + { + Slice decimal = UnscaledDecimal128Arithmetic.unscaledDecimal(); + Slice rescaledDecimal = UnscaledDecimal128Arithmetic.unscaledDecimal(); + + int streamPosition = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + nulls[i] = true; + } + else { + int scale = (int) scaleStream.next(); + dataStream.nextLongDecimal(decimal); + rescale(decimal, this.scale - scale, rescaledDecimal); + values[2 * i] = UnsafeSlice.getLongUnchecked(rescaledDecimal, 0); + values[2 * i + 1] = UnsafeSlice.getLongUnchecked(rescaledDecimal, Long.BYTES); + if (presentStream != null) { + nulls[i] = false; + } + } + streamPosition++; + } + outputPositionCount = positionCount; + return streamPosition; + } + + @Override + protected int readWithFilter(int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + outputPositionCount = 0; + Slice decimal = UnscaledDecimal128Arithmetic.unscaledDecimal(); + Slice rescaledDecimal = UnscaledDecimal128Arithmetic.unscaledDecimal(); + + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + if ((nonDeterministicFilter && filter.testNull()) || nullsAllowed) { + if (outputRequired) { + nulls[outputPositionCount] = true; + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + else { + int scale = (int) scaleStream.next(); + dataStream.nextLongDecimal(decimal); + rescale(decimal, this.scale - scale, rescaledDecimal); + long low = UnsafeSlice.getLongUnchecked(rescaledDecimal, 0); + long high = UnsafeSlice.getLongUnchecked(rescaledDecimal, Long.BYTES); + if (filter.testDecimal(low, high)) { + if (outputRequired) { + values[2 * outputPositionCount] = low; + values[2 * outputPositionCount + 1] = high; + if (nullsAllowed && presentStream != null) { + nulls[outputPositionCount] = false; + } + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + streamPosition++; + + if (filter != null) { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + int succeedingPositionsToFail = filter.getSucceedingPositionsToFail(); + if (succeedingPositionsToFail > 0) { + int positionsToSkip = 0; + for (int j = 0; j < succeedingPositionsToFail; j++) { + i++; + int nextPosition = positions[i]; + positionsToSkip += 1 + nextPosition - streamPosition; + streamPosition = nextPosition + 1; + } + skip(positionsToSkip); + } + } + } + return streamPosition; + } + + @Override + protected void copyValues(int[] positions, int positionsCount, long[] valuesCopy, boolean[] nullsCopy) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + valuesCopy[2 * positionIndex] = this.values[2 * i]; + valuesCopy[2 * positionIndex + 1] = this.values[2 * i + 1]; + + if (nullsCopy != null) { + nullsCopy[positionIndex] = this.nulls[i]; + } + positionIndex++; + + if (positionIndex >= positionsCount) { + break; + } + + nextPosition = positions[positionIndex]; + } + } + + @Override + protected void compactValues(int[] positions, int positionCount, boolean compactNulls) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + values[2 * positionIndex] = values[2 * i]; + values[2 * positionIndex + 1] = values[2 * i + 1]; + if (compactNulls) { + nulls[positionIndex] = nulls[i]; + } + outputPositions[positionIndex] = nextPosition; + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + nextPosition = positions[positionIndex]; + } + + outputPositionCount = positionCount; + } + + @Override + protected Block makeBlock(int positionCount, boolean includeNulls, boolean[] nulls, long[] values) + { + return new Int128ArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDictionaryBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDictionaryBatchStreamReader.java index 19415893ccd19..f14c13f8ac63b 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDictionaryBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDictionaryBatchStreamReader.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc.reader; +import com.facebook.presto.memory.context.LocalMemoryContext; import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.metadata.ColumnEncoding; @@ -22,6 +23,10 @@ import com.facebook.presto.orc.stream.LongInputStream; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.DateType; +import com.facebook.presto.spi.type.IntegerType; +import com.facebook.presto.spi.type.SmallintType; import com.facebook.presto.spi.type.Type; import org.openjdk.jol.info.ClassLayout; @@ -34,6 +39,7 @@ import static com.facebook.presto.orc.metadata.Stream.StreamKind.DICTIONARY_DATA; import static com.facebook.presto.orc.metadata.Stream.StreamKind.IN_DICTIONARY; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.ReaderUtils.verifyStreamType; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; import static com.google.common.base.MoreObjects.toStringHelper; import static io.airlift.slice.SizeOf.sizeOf; @@ -44,6 +50,7 @@ public class LongDictionaryBatchStreamReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(LongDictionaryBatchStreamReader.class).instanceSize(); + private final Type type; private final StreamDescriptor streamDescriptor; private int readOffset; @@ -68,9 +75,16 @@ public class LongDictionaryBatchStreamReader private boolean dictionaryOpen; private boolean rowGroupOpen; - public LongDictionaryBatchStreamReader(StreamDescriptor streamDescriptor) + private final LocalMemoryContext systemMemoryContext; + + public LongDictionaryBatchStreamReader(Type type, StreamDescriptor streamDescriptor, LocalMemoryContext systemMemoryContext) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, t -> t instanceof BigintType || t instanceof IntegerType || t instanceof SmallintType || t instanceof DateType); + this.type = type; this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); } @Override @@ -81,7 +95,7 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -172,13 +186,14 @@ private void openRowGroup() if (!dictionaryOpen && dictionarySize > 0) { if (dictionary.length < dictionarySize) { dictionary = new long[dictionarySize]; + systemMemoryContext.setBytes(sizeOf(dictionary)); } LongInputStream dictionaryStream = dictionaryDataStreamSource.openStream(); if (dictionaryStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Dictionary is not empty but data stream is not present"); } - dictionaryStream.nextLongVector(dictionarySize, dictionary); + dictionaryStream.next(dictionary, dictionarySize); } dictionaryOpen = true; @@ -237,6 +252,13 @@ public String toString() .toString(); } + @Override + public void close() + { + systemMemoryContext.close(); + dictionary = null; + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDictionarySelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDictionarySelectiveStreamReader.java index b33a338a44e53..4b773b2f0831b 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDictionarySelectiveStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDictionarySelectiveStreamReader.java @@ -37,6 +37,7 @@ import static com.facebook.presto.orc.metadata.Stream.StreamKind.DICTIONARY_DATA; import static com.facebook.presto.orc.metadata.Stream.StreamKind.IN_DICTIONARY; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; @@ -52,6 +53,7 @@ public class LongDictionarySelectiveStreamReader private final StreamDescriptor streamDescriptor; @Nullable private final TupleDomainFilter filter; + private final boolean nonDeterministicFilter; private final boolean nullsAllowed; private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); @@ -83,29 +85,29 @@ public LongDictionarySelectiveStreamReader( LocalMemoryContext systemMemoryContext) { super(outputType); + requireNonNull(filter, "filter is null"); + checkArgument(filter.isPresent() || outputRequired, "filter must be present if output is not required"); this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); - this.filter = requireNonNull(filter, "filter is null").orElse(null); + this.filter = filter.orElse(null); this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); - nullsAllowed = this.filter == null || this.filter.testNull(); + nonDeterministicFilter = this.filter != null && !this.filter.isDeterministic(); + nullsAllowed = this.filter == null || nonDeterministicFilter || this.filter.testNull(); } @Override public int read(int offset, int[] positions, int positionCount) throws IOException { + checkArgument(positionCount > 0, "positionCount must be greater than zero"); + if (!rowGroupOpen) { openRowGroup(); } prepareNextRead(positionCount, presentStream != null && nullsAllowed); - if (filter != null) { - ensureOutputPositionsCapacity(positionCount); - } - else { - outputPositions = positions; - } + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); // account memory used by values, nulls and outputPositions systemMemoryContext.setBytes(getRetainedSizeInBytes()); @@ -125,9 +127,10 @@ public int read(int offset, int[] positions, int positionCount) } if (presentStream != null && !presentStream.nextBit()) { - if (nullsAllowed) { + if ((nonDeterministicFilter && filter.testNull()) || nullsAllowed) { if (outputRequired) { nulls[outputPositionCount] = true; + values[outputPositionCount] = 0; } if (filter != null) { outputPositions[outputPositionCount] = position; @@ -143,7 +146,7 @@ public int read(int offset, int[] positions, int positionCount) if (filter == null || filter.testLong(value)) { if (outputRequired) { values[outputPositionCount] = value; - if (nulls != null) { + if (presentStream != null && nullsAllowed) { nulls[outputPositionCount] = false; } } @@ -153,7 +156,24 @@ public int read(int offset, int[] positions, int positionCount) outputPositionCount++; } } + streamPosition++; + + if (filter != null) { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + + int succeedingPositionsToFail = filter.getSucceedingPositionsToFail(); + if (succeedingPositionsToFail > 0) { + int positionsToSkip = 0; + for (int j = 0; j < succeedingPositionsToFail; j++) { + i++; + int nextPosition = positions[i]; + positionsToSkip += 1 + nextPosition - streamPosition; + streamPosition = nextPosition + 1; + } + skip(positionsToSkip); + } + } } readOffset = offset + streamPosition; @@ -169,7 +189,9 @@ private void skip(int items) if (inDictionaryStream != null) { inDictionaryStream.skip(dataToSkip); } - dataStream.skip(dataToSkip); + if (dataStream != null) { + dataStream.skip(dataToSkip); + } } else { if (inDictionaryStream != null) { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDirectBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDirectBatchStreamReader.java index 197db41d1ddd5..c707a640c502b 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDirectBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDirectBatchStreamReader.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc.reader; +import com.facebook.presto.memory.context.LocalMemoryContext; import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.metadata.ColumnEncoding; @@ -21,8 +22,14 @@ import com.facebook.presto.orc.stream.InputStreamSources; import com.facebook.presto.orc.stream.LongInputStream; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.block.IntArrayBlock; +import com.facebook.presto.spi.block.LongArrayBlock; import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.block.ShortArrayBlock; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.DateType; +import com.facebook.presto.spi.type.IntegerType; +import com.facebook.presto.spi.type.SmallintType; import com.facebook.presto.spi.type.Type; import org.openjdk.jol.info.ClassLayout; @@ -30,11 +37,19 @@ import java.io.IOException; import java.util.List; +import java.util.Optional; import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.ReaderUtils.minNonNullValueSize; +import static com.facebook.presto.orc.reader.ReaderUtils.unpackIntNulls; +import static com.facebook.presto.orc.reader.ReaderUtils.unpackLongNulls; +import static com.facebook.presto.orc.reader.ReaderUtils.unpackShortNulls; +import static com.facebook.presto.orc.reader.ReaderUtils.verifyStreamType; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Verify.verify; +import static io.airlift.slice.SizeOf.sizeOf; import static java.util.Objects.requireNonNull; public class LongDirectBatchStreamReader @@ -42,6 +57,7 @@ public class LongDirectBatchStreamReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(LongDirectBatchStreamReader.class).instanceSize(); + private final Type type; private final StreamDescriptor streamDescriptor; private int readOffset; @@ -57,9 +73,21 @@ public class LongDirectBatchStreamReader private boolean rowGroupOpen; - public LongDirectBatchStreamReader(StreamDescriptor streamDescriptor) + // only one of the three arrays will be used + private short[] shortNonNullValueTemp = new short[0]; + private int[] intNonNullValueTemp = new int[0]; + private long[] longNonNullValueTemp = new long[0]; + + private LocalMemoryContext systemMemoryContext; + + public LongDirectBatchStreamReader(Type type, StreamDescriptor streamDescriptor, LocalMemoryContext systemMemoryContext) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, t -> t instanceof BigintType || t instanceof IntegerType || t instanceof SmallintType || t instanceof DateType); + this.type = type; this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); } @Override @@ -70,7 +98,7 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -85,44 +113,128 @@ public Block readBlock(Type type) } if (readOffset > 0) { if (dataStream == null) { - throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is missing"); } dataStream.skip(readOffset); } } - if (dataStream == null && presentStream != null) { + Block block; + if (dataStream == null) { + if (presentStream == null) { + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is null but present stream is missing"); + } presentStream.skip(nextBatchSize); - Block nullValueBlock = new RunLengthEncodedBlock( - type.createBlockBuilder(null, 1).appendNull().build(), - nextBatchSize); - readOffset = 0; - nextBatchSize = 0; - return nullValueBlock; + block = RunLengthEncodedBlock.create(type, null, nextBatchSize); } - - BlockBuilder builder = type.createBlockBuilder(null, nextBatchSize); - if (presentStream == null) { - if (dataStream == null) { - throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); - } - dataStream.nextLongVector(type, nextBatchSize, builder); + else if (presentStream == null) { + block = readNonNullBlock(); } else { - for (int i = 0; i < nextBatchSize; i++) { - if (presentStream.nextBit()) { - type.writeLong(builder, dataStream.next()); - } - else { - builder.appendNull(); - } + boolean[] isNull = new boolean[nextBatchSize]; + int nullCount = presentStream.getUnsetBits(nextBatchSize, isNull); + if (nullCount == 0) { + block = readNonNullBlock(); + } + else if (nullCount != nextBatchSize) { + block = readNullBlock(isNull, nextBatchSize - nullCount); + } + else { + block = RunLengthEncodedBlock.create(type, null, nextBatchSize); } } - readOffset = 0; nextBatchSize = 0; - return builder.build(); + return block; + } + + private Block readNonNullBlock() + throws IOException + { + verify(dataStream != null); + if (type instanceof BigintType) { + long[] values = new long[nextBatchSize]; + dataStream.next(values, nextBatchSize); + return new LongArrayBlock(nextBatchSize, Optional.empty(), values); + } + if (type instanceof IntegerType || type instanceof DateType) { + int[] values = new int[nextBatchSize]; + dataStream.next(values, nextBatchSize); + return new IntArrayBlock(nextBatchSize, Optional.empty(), values); + } + if (type instanceof SmallintType) { + short[] values = new short[nextBatchSize]; + dataStream.next(values, nextBatchSize); + return new ShortArrayBlock(nextBatchSize, Optional.empty(), values); + } + throw new VerifyError("Unsupported type " + type); + } + + private Block readNullBlock(boolean[] isNull, int nonNullCount) + throws IOException + { + if (type instanceof BigintType) { + return longReadNullBlock(isNull, nonNullCount); + } + if (type instanceof IntegerType || type instanceof DateType) { + return intReadNullBlock(isNull, nonNullCount); + } + if (type instanceof SmallintType) { + return shortReadNullBlock(isNull, nonNullCount); + } + throw new VerifyError("Unsupported type " + type); + } + + private Block longReadNullBlock(boolean[] isNull, int nonNullCount) + throws IOException + { + verify(dataStream != null); + int minNonNullValueSize = minNonNullValueSize(nonNullCount); + if (longNonNullValueTemp.length < minNonNullValueSize) { + longNonNullValueTemp = new long[minNonNullValueSize]; + systemMemoryContext.setBytes(sizeOf(longNonNullValueTemp)); + } + + dataStream.next(longNonNullValueTemp, nonNullCount); + + long[] result = unpackLongNulls(longNonNullValueTemp, isNull); + + return new LongArrayBlock(nextBatchSize, Optional.of(isNull), result); + } + + private Block intReadNullBlock(boolean[] isNull, int nonNullCount) + throws IOException + { + verify(dataStream != null); + int minNonNullValueSize = minNonNullValueSize(nonNullCount); + if (intNonNullValueTemp.length < minNonNullValueSize) { + intNonNullValueTemp = new int[minNonNullValueSize]; + systemMemoryContext.setBytes(sizeOf(intNonNullValueTemp)); + } + + dataStream.next(intNonNullValueTemp, nonNullCount); + + int[] result = unpackIntNulls(intNonNullValueTemp, isNull); + + return new IntArrayBlock(nextBatchSize, Optional.of(isNull), result); + } + + private Block shortReadNullBlock(boolean[] isNull, int nonNullCount) + throws IOException + { + verify(dataStream != null); + int minNonNullValueSize = minNonNullValueSize(nonNullCount); + if (shortNonNullValueTemp.length < minNonNullValueSize) { + shortNonNullValueTemp = new short[minNonNullValueSize]; + systemMemoryContext.setBytes(sizeOf(shortNonNullValueTemp)); + } + + dataStream.next(shortNonNullValueTemp, nonNullCount); + + short[] result = unpackShortNulls(shortNonNullValueTemp, isNull); + + return new ShortArrayBlock(nextBatchSize, Optional.of(isNull), result); } private void openRowGroup() @@ -172,6 +284,15 @@ public String toString() .toString(); } + @Override + public void close() + { + systemMemoryContext.close(); + shortNonNullValueTemp = null; + intNonNullValueTemp = null; + longNonNullValueTemp = null; + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDirectSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDirectSelectiveStreamReader.java index 35cdb16b3839f..d240e0c8f80c2 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDirectSelectiveStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongDirectSelectiveStreamReader.java @@ -35,6 +35,7 @@ import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; import static com.facebook.presto.spi.block.ClosingBlockLease.newLease; import static com.google.common.base.Preconditions.checkArgument; @@ -50,6 +51,7 @@ public class LongDirectSelectiveStreamReader private final StreamDescriptor streamDescriptor; @Nullable private final TupleDomainFilter filter; + private final boolean nonDeterministicFilter; private final boolean nullsAllowed; private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); @@ -74,17 +76,22 @@ public LongDirectSelectiveStreamReader( LocalMemoryContext systemMemoryContext) { super(outputType); + requireNonNull(filter, "filter is null"); + checkArgument(filter.isPresent() || outputRequired, "filter must be present if output is not required"); this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); - this.filter = requireNonNull(filter, "filter is null").orElse(null); + this.filter = filter.orElse(null); this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); - nullsAllowed = this.filter == null || this.filter.testNull(); + nonDeterministicFilter = this.filter != null && !this.filter.isDeterministic(); + nullsAllowed = this.filter == null || nonDeterministicFilter || this.filter.testNull(); } @Override public int read(int offset, int[] positions, int positionCount) throws IOException { + checkArgument(positionCount > 0, "positionCount must be greater than zero"); + if (!rowGroupOpen) { openRowGroup(); } @@ -93,26 +100,21 @@ public int read(int offset, int[] positions, int positionCount) allNulls = false; - if (filter != null) { - ensureOutputPositionsCapacity(positionCount); - } - else { - outputPositions = positions; - } + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); // account memory used by values, nulls and outputPositions systemMemoryContext.setBytes(getRetainedSizeInBytes()); - if (readOffset < offset) { - skip(offset - readOffset); - } - outputPositionCount = 0; int streamPosition = 0; if (dataStream == null && presentStream != null) { streamPosition = readAllNulls(positions, positionCount); } else { + if (readOffset < offset) { + skip(offset - readOffset); + } + for (int i = 0; i < positionCount; i++) { int position = positions[i]; if (position > streamPosition) { @@ -121,9 +123,10 @@ public int read(int offset, int[] positions, int positionCount) } if (presentStream != null && !presentStream.nextBit()) { - if (nullsAllowed) { + if ((nonDeterministicFilter && filter.testNull()) || nullsAllowed) { if (outputRequired) { nulls[outputPositionCount] = true; + values[outputPositionCount] = 0; } if (filter != null) { outputPositions[outputPositionCount] = position; @@ -146,7 +149,24 @@ public int read(int offset, int[] positions, int positionCount) outputPositionCount++; } } + streamPosition++; + + if (filter != null) { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + + int succeedingPositionsToFail = filter.getSucceedingPositionsToFail(); + if (succeedingPositionsToFail > 0) { + int positionsToSkip = 0; + for (int j = 0; j < succeedingPositionsToFail; j++) { + i++; + int nextPosition = positions[i]; + positionsToSkip += 1 + nextPosition - streamPosition; + streamPosition = nextPosition + 1; + } + skip(positionsToSkip); + } + } } } @@ -160,14 +180,26 @@ private int readAllNulls(int[] positions, int positionCount) { presentStream.skip(positions[positionCount - 1]); - if (nullsAllowed) { + if (nonDeterministicFilter) { + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + if (filter.testNull()) { + outputPositionCount++; + } + else { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + i += filter.getSucceedingPositionsToFail(); + } + } + } + else if (nullsAllowed) { outputPositionCount = positionCount; - allNulls = true; } else { outputPositionCount = 0; } + allNulls = true; return positions[positionCount - 1] + 1; } @@ -200,7 +232,7 @@ public Block getBlock(int[] positions, int positionCount) checkState(positionCount <= outputPositionCount, "Not enough values"); if (allNulls) { - return new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), outputPositionCount); + return new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), positionCount); } return buildOutputBlock(positions, positionCount, nullsAllowed && presentStream != null); @@ -214,7 +246,7 @@ public BlockLease getBlockView(int[] positions, int positionCount) checkState(positionCount <= outputPositionCount, "Not enough values"); if (allNulls) { - return newLease(new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), outputPositionCount)); + return newLease(new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), positionCount)); } return buildOutputBlockView(positions, positionCount, nullsAllowed && presentStream != null); diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongSelectiveStreamReader.java index 80926ae53d35c..a8a84b99e2a3e 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongSelectiveStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/LongSelectiveStreamReader.java @@ -133,4 +133,10 @@ public BlockLease getBlockView(int[] positions, int positionCount) { return currentReader.getBlockView(positions, positionCount); } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + currentReader.throwAnyError(positions, positionCount); + } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapBatchStreamReader.java index 0a30a3a1aedd8..48e6bfbc6d2f5 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapBatchStreamReader.java @@ -13,16 +13,20 @@ */ package com.facebook.presto.orc.reader; +import com.facebook.presto.memory.context.AggregatedMemoryContext; +import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.metadata.ColumnEncoding; import com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind; import com.facebook.presto.orc.stream.InputStreamSources; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.Type; +import com.google.common.io.Closer; import org.joda.time.DateTimeZone; import org.openjdk.jol.info.ClassLayout; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.List; import static com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT; @@ -42,11 +46,12 @@ public class MapBatchStreamReader private final MapFlatBatchStreamReader flatReader; private BatchStreamReader currentReader; - public MapBatchStreamReader(StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone) + public MapBatchStreamReader(Type type, StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone, AggregatedMemoryContext systemMemoryContext) + throws OrcCorruptionException { this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); - directReader = new MapDirectBatchStreamReader(streamDescriptor, hiveStorageTimeZone); - flatReader = new MapFlatBatchStreamReader(streamDescriptor, hiveStorageTimeZone); + this.directReader = new MapDirectBatchStreamReader(type, streamDescriptor, hiveStorageTimeZone, systemMemoryContext); + this.flatReader = new MapFlatBatchStreamReader(type, streamDescriptor, hiveStorageTimeZone, systemMemoryContext); } @Override @@ -56,10 +61,10 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { - return currentReader.readBlock(type); + return currentReader.readBlock(); } @Override @@ -97,6 +102,18 @@ public String toString() .toString(); } + @Override + public void close() + { + try (Closer closer = Closer.create()) { + closer.register(directReader::close); + closer.register(flatReader::close); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapDirectBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapDirectBatchStreamReader.java index 2b6af9bfe19da..96419173d9e9c 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapDirectBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapDirectBatchStreamReader.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc.reader; +import com.facebook.presto.memory.context.AggregatedMemoryContext; import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.metadata.ColumnEncoding; @@ -23,6 +24,7 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.Type; +import com.google.common.io.Closer; import it.unimi.dsi.fastutil.ints.IntArrayList; import org.joda.time.DateTimeZone; import org.openjdk.jol.info.ClassLayout; @@ -30,12 +32,16 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.List; import java.util.Optional; import static com.facebook.presto.orc.metadata.Stream.StreamKind.LENGTH; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; import static com.facebook.presto.orc.reader.BatchStreamReaders.createStreamReader; +import static com.facebook.presto.orc.reader.ReaderUtils.convertLengthVectorToOffsetVector; +import static com.facebook.presto.orc.reader.ReaderUtils.unpackLengthNulls; +import static com.facebook.presto.orc.reader.ReaderUtils.verifyStreamType; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; import static com.google.common.base.MoreObjects.toStringHelper; import static java.lang.Math.toIntExact; @@ -46,6 +52,7 @@ public class MapDirectBatchStreamReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(MapDirectBatchStreamReader.class).instanceSize(); + private final MapType type; private final StreamDescriptor streamDescriptor; private final BatchStreamReader keyStreamReader; @@ -64,11 +71,15 @@ public class MapDirectBatchStreamReader private boolean rowGroupOpen; - public MapDirectBatchStreamReader(StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone) + public MapDirectBatchStreamReader(Type type, StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone, AggregatedMemoryContext systemMemoryContext) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, MapType.class::isInstance); + this.type = (MapType) type; this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); - this.keyStreamReader = createStreamReader(streamDescriptor.getNestedStreams().get(0), hiveStorageTimeZone); - this.valueStreamReader = createStreamReader(streamDescriptor.getNestedStreams().get(1), hiveStorageTimeZone); + this.keyStreamReader = createStreamReader(this.type.getKeyType(), streamDescriptor.getNestedStreams().get(0), hiveStorageTimeZone, systemMemoryContext); + this.valueStreamReader = createStreamReader(this.type.getValueType(), streamDescriptor.getNestedStreams().get(1), hiveStorageTimeZone, systemMemoryContext); } @Override @@ -79,7 +90,7 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -111,7 +122,7 @@ public Block readBlock(Type type) if (lengthStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); } - lengthStream.nextIntVector(nextBatchSize, offsetVector, 0); + lengthStream.next(offsetVector, nextBatchSize); } else { nullVector = new boolean[nextBatchSize]; @@ -120,7 +131,8 @@ public Block readBlock(Type type) if (lengthStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); } - lengthStream.nextIntVector(nextBatchSize, offsetVector, 0, nullVector); + lengthStream.next(offsetVector, nextBatchSize - nullValues); + unpackLengthNulls(offsetVector, nullVector, nextBatchSize - nullValues); } } @@ -139,8 +151,8 @@ public Block readBlock(Type type) if (entryCount > 0) { keyStreamReader.prepareNextRead(entryCount); valueStreamReader.prepareNextRead(entryCount); - keys = keyStreamReader.readBlock(keyType); - values = valueStreamReader.readBlock(valueType); + keys = keyStreamReader.readBlock(); + values = valueStreamReader.readBlock(); } else { keys = keyType.createBlockBuilder(null, 0).build(); @@ -149,19 +161,14 @@ public Block readBlock(Type type) Block[] keyValueBlock = createKeyValueBlock(nextBatchSize, keys, values, offsetVector); - // Convert the length values in the offsetVector to offset values in-place - int currentLength = offsetVector[0]; - offsetVector[0] = 0; - for (int i = 1; i < offsetVector.length; i++) { - int lastLength = offsetVector[i]; - offsetVector[i] = offsetVector[i - 1] + currentLength; - currentLength = lastLength; - } + convertLengthVectorToOffsetVector(offsetVector); + + Block block = mapType.createBlockFromKeyValue(nextBatchSize, Optional.ofNullable(nullVector), offsetVector, keyValueBlock[0], keyValueBlock[1]); readOffset = 0; nextBatchSize = 0; - return mapType.createBlockFromKeyValue(Optional.ofNullable(nullVector), offsetVector, keyValueBlock[0], keyValueBlock[1]); + return block; } private static Block[] createKeyValueBlock(int positionCount, Block keys, Block values, int[] lengths) @@ -261,6 +268,18 @@ public String toString() .toString(); } + @Override + public void close() + { + try (Closer closer = Closer.create()) { + closer.register(keyStreamReader::close); + closer.register(valueStreamReader::close); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapDirectSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapDirectSelectiveStreamReader.java new file mode 100644 index 0000000000000..d26e710c19fc4 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapDirectSelectiveStreamReader.java @@ -0,0 +1,713 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.AggregatedMemoryContext; +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.TupleDomainFilter.BigintRange; +import com.facebook.presto.orc.TupleDomainFilter.BigintValues; +import com.facebook.presto.orc.TupleDomainFilter.BytesRange; +import com.facebook.presto.orc.TupleDomainFilter.BytesValues; +import com.facebook.presto.orc.metadata.ColumnEncoding; +import com.facebook.presto.orc.metadata.OrcType; +import com.facebook.presto.orc.stream.BooleanInputStream; +import com.facebook.presto.orc.stream.InputStreamSource; +import com.facebook.presto.orc.stream.InputStreamSources; +import com.facebook.presto.orc.stream.LongInputStream; +import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.block.ClosingBlockLease; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.MapType; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import org.joda.time.DateTimeZone; +import org.openjdk.jol.info.ClassLayout; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NOT_NULL; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NULL; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.LENGTH; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; +import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; + +public class MapDirectSelectiveStreamReader + implements SelectiveStreamReader +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(MapDirectSelectiveStreamReader.class).instanceSize(); + + private final StreamDescriptor streamDescriptor; + private final boolean nullsAllowed; + private final boolean nonNullsAllowed; + private final boolean outputRequired; + @Nullable + private final MapType outputType; + + private final SelectiveStreamReader keyReader; + private final SelectiveStreamReader valueReader; + + private final LocalMemoryContext systemMemoryContext; + + private int readOffset; + private int nestedReadOffset; + + private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); + @Nullable + private BooleanInputStream presentStream; + + private InputStreamSource lengthStreamSource = missingStreamSource(LongInputStream.class); + @Nullable + private LongInputStream lengthStream; + + private boolean rowGroupOpen; + @Nullable + private int[] offsets; + private boolean[] nulls; + private int[] outputPositions; + private int outputPositionCount; + private boolean allNulls; + private int[] nestedLengths; + private int[] nestedOffsets; + private int[] nestedPositions; + private int[] nestedOutputPositions; + private int nestedOutputPositionCount; + + private boolean valuesInUse; + + public MapDirectSelectiveStreamReader( + StreamDescriptor streamDescriptor, + Map filters, + List requiredSubfields, + Optional outputType, + DateTimeZone hiveStorageTimeZone, + AggregatedMemoryContext systemMemoryContext) + { + checkArgument(filters.keySet().stream().map(Subfield::getPath).allMatch(List::isEmpty), "filters on nested columns are not supported yet"); + + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null").newLocalMemoryContext(MapDirectSelectiveStreamReader.class.getSimpleName()); + this.outputRequired = requireNonNull(outputType, "outputType is null").isPresent(); + this.outputType = outputType.map(MapType.class::cast).orElse(null); + + TupleDomainFilter filter = getTopLevelFilter(filters).orElse(null); + this.nullsAllowed = filter == null || filter.testNull(); + this.nonNullsAllowed = filter == null || filter.testNonNull(); + + List nestedStreams = streamDescriptor.getNestedStreams(); + + Optional keyOutputType = outputType.map(MapType.class::cast).map(MapType::getKeyType); + Optional valueOutputType = outputType.map(MapType.class::cast).map(MapType::getValueType); + + if (outputRequired) { + Map keyFilter = ImmutableMap.of(new Subfield("c"), makeKeyFilter(nestedStreams.get(0).getOrcTypeKind(), requiredSubfields)); + + List elementRequiredSubfields = ImmutableList.of(); + if (requiredSubfields.stream().map(Subfield::getPath).allMatch(path -> path.size() > 1)) { + elementRequiredSubfields = requiredSubfields.stream() + .map(subfield -> subfield.tail(subfield.getRootName())) + .distinct() + .collect(toImmutableList()); + } + + this.keyReader = SelectiveStreamReaders.createStreamReader(nestedStreams.get(0), keyFilter, keyOutputType, ImmutableList.of(), hiveStorageTimeZone, systemMemoryContext.newAggregatedMemoryContext()); + this.valueReader = SelectiveStreamReaders.createStreamReader(nestedStreams.get(1), ImmutableMap.of(), valueOutputType, elementRequiredSubfields, hiveStorageTimeZone, systemMemoryContext.newAggregatedMemoryContext()); + } + else { + this.keyReader = null; + this.valueReader = null; + } + } + + private static TupleDomainFilter makeKeyFilter(OrcType.OrcTypeKind orcType, List requiredSubfields) + { + // Map entries with a null key are skipped in the Hive ORC reader, so skip them here also + + if (requiredSubfields.isEmpty()) { + return IS_NOT_NULL; + } + + if (requiredSubfields.stream() + .map(Subfield::getPath) + .map(path -> path.get(0)) + .anyMatch(Subfield.AllSubscripts.class::isInstance)) { + return IS_NOT_NULL; + } + + switch (orcType) { + case BYTE: + case SHORT: + case INT: + case LONG: { + long[] requiredIndices = requiredSubfields.stream() + .map(Subfield::getPath) + .map(path -> path.get(0)) + .map(Subfield.LongSubscript.class::cast) + .mapToLong(Subfield.LongSubscript::getIndex) + .toArray(); + + if (requiredIndices.length == 0) { + return IS_NOT_NULL; + } + + if (requiredIndices.length == 1) { + return BigintRange.of(requiredIndices[0], requiredIndices[0], false); + } + + return BigintValues.of(requiredIndices, false); + } + case STRING: + case CHAR: + case VARCHAR: { + byte[][] requiredIndices = requiredSubfields.stream() + .map(Subfield::getPath) + .map(path -> path.get(0)) + .map(Subfield.StringSubscript.class::cast) + .map(Subfield.StringSubscript::getIndex) + .map(String::getBytes) + .toArray(byte[][]::new); + + if (requiredIndices.length == 0) { + return IS_NOT_NULL; + } + + if (requiredIndices.length == 1) { + return BytesRange.of(requiredIndices[0], false, requiredIndices[0], false, false); + } + + return BytesValues.of(requiredIndices, false); + } + default: + return IS_NOT_NULL; + } + } + + private static Optional getTopLevelFilter(Map filters) + { + Map topLevelFilters = Maps.filterEntries(filters, entry -> entry.getKey().getPath().isEmpty()); + if (topLevelFilters.isEmpty()) { + return Optional.empty(); + } + + checkArgument(topLevelFilters.size() == 1, "MAP column may have at most one top-level range filter"); + TupleDomainFilter filter = Iterables.getOnlyElement(topLevelFilters.values()); + checkArgument(filter == IS_NULL || filter == IS_NOT_NULL, "Top-level range filter on MAP column must be IS NULL or IS NOT NULL"); + return Optional.of(filter); + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (!rowGroupOpen) { + openRowGroup(); + } + + allNulls = false; + + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); + + offsets = ensureCapacity(offsets, positionCount + 1); + offsets[0] = 0; + + nestedLengths = ensureCapacity(nestedLengths, positionCount); + nestedOffsets = ensureCapacity(nestedOffsets, positionCount + 1); + + systemMemoryContext.setBytes(getRetainedSizeInBytes()); + + if (lengthStream == null) { + readAllNulls(positions, positionCount); + } + else if (presentStream == null) { + readNoNulls(offset, positions, positionCount); + } + else { + readWithNulls(offset, positions, positionCount); + } + + return outputPositionCount; + } + + private int readAllNulls(int[] positions, int positionCount) + { + if (nullsAllowed) { + outputPositionCount = positionCount; + } + else { + outputPositionCount = 0; + } + + allNulls = true; + return positions[positionCount - 1] + 1; + } + + private void readNoNulls(int offset, int[] positions, int positionCount) + throws IOException + { + if (!nonNullsAllowed) { + outputPositionCount = 0; + return; + } + + if (readOffset < offset) { + nestedReadOffset += lengthStream.sum(offset - readOffset); + } + + int streamPosition = 0; + int nestedOffset = 0; + + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + nestedOffset += lengthStream.sum(position - streamPosition); + streamPosition = position; + } + + streamPosition++; + + int length = toIntExact(lengthStream.next()); + offsets[i + 1] = offsets[i] + length; + nestedLengths[i] = length; + nestedOffsets[i] = nestedOffset; + nestedOffset += length; + } + + outputPositionCount = positionCount; + readOffset = offset + streamPosition; + + if (outputRequired) { + nestedOffsets[positionCount] = nestedOffset; + int nestedPositionCount = populateNestedPositions(positionCount, nestedOffset); + readKeyValueStreams(nestedPositionCount); + } + nestedReadOffset += nestedOffset; + } + + private void readWithNulls(int offset, int[] positions, int positionCount) + throws IOException + { + if (readOffset < offset) { + int dataToSkip = presentStream.countBitsSet(offset - readOffset); + nestedReadOffset += lengthStream.sum(dataToSkip); + } + + if (outputRequired) { + nulls = ensureCapacity(nulls, positionCount); + } + outputPositionCount = 0; + + int streamPosition = 0; + int nonNullPositionCount = 0; + int nestedOffset = 0; + + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + int dataToSkip = presentStream.countBitsSet(position - streamPosition); + nestedOffset += lengthStream.sum(dataToSkip); + streamPosition = position; + } + + streamPosition++; + + if (presentStream.nextBit()) { + // not null + int length = toIntExact(lengthStream.next()); + if (nonNullsAllowed) { + if (outputRequired) { + nulls[outputPositionCount] = false; + offsets[outputPositionCount + 1] = offsets[outputPositionCount] + length; + + nestedLengths[nonNullPositionCount] = length; + nestedOffsets[nonNullPositionCount] = nestedOffset; + nonNullPositionCount++; + } + + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + nestedOffset += length; + } + else { + // null + if (nullsAllowed) { + if (outputRequired) { + nulls[outputPositionCount] = true; + offsets[outputPositionCount + 1] = offsets[outputPositionCount]; + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + } + + if (nonNullPositionCount == 0) { + allNulls = true; + } + else if (outputRequired) { + nestedOffsets[nonNullPositionCount] = nestedOffset; + int nestedPositionCount = populateNestedPositions(nonNullPositionCount, nestedOffset); + readKeyValueStreams(nestedPositionCount); + } + + readOffset = offset + streamPosition; + nestedReadOffset += nestedOffset; + } + + private int populateNestedPositions(int positionCount, int nestedOffset) + { + nestedPositions = ensureCapacity(nestedPositions, nestedOffset); + int nestedPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + for (int j = 0; j < nestedLengths[i]; j++) { + nestedPositions[nestedPositionCount++] = nestedOffsets[i] + j; + } + } + return nestedPositionCount; + } + + private void readKeyValueStreams(int positionCount) + throws IOException + { + if (positionCount == 0) { + nestedOutputPositionCount = 0; + return; + } + + int readCount = keyReader.read(nestedReadOffset, nestedPositions, positionCount); + int[] readPositions = keyReader.getReadPositions(); + + if (readCount == 0) { + nestedOutputPositionCount = 0; + for (int i = 0; i <= outputPositionCount; i++) { + offsets[i] = 0; + } + return; + } + + if (readCount < positionCount) { + int positionIndex = 0; + int nextPosition = readPositions[positionIndex]; + int offset = 0; + int previousOffset = 0; + for (int i = 0; i < outputPositionCount; i++) { + int length = 0; + for (int j = previousOffset; j < offsets[i + 1]; j++) { + if (nestedPositions[j] == nextPosition) { + length++; + positionIndex++; + if (positionIndex >= readCount) { + break; + } + nextPosition = readPositions[positionIndex]; + } + } + offset += length; + previousOffset = offsets[i + 1]; + offsets[i + 1] = offset; + + if (positionIndex >= readCount) { + for (int j = i + 1; j < outputPositionCount; j++) { + offsets[j + 1] = offset; + } + break; + } + } + } + + int valueReadCount = valueReader.read(nestedReadOffset, readPositions, readCount); + assert valueReadCount == readCount; + + nestedOutputPositions = ensureCapacity(nestedOutputPositions, readCount); + System.arraycopy(readPositions, 0, nestedOutputPositions, 0, readCount); + nestedOutputPositionCount = readCount; + } + + private void openRowGroup() + throws IOException + { + presentStream = presentStreamSource.openStream(); + lengthStream = lengthStreamSource.openStream(); + + rowGroupOpen = true; + } + + @Override + public int[] getReadPositions() + { + return outputPositions; + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return createNullBlock(outputType, positionCount); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (outputPositionCount == positionCount) { + Block keyBlock; + Block valueBlock; + if (nestedOutputPositionCount == 0) { + keyBlock = createEmptyBlock(outputType.getKeyType()); + valueBlock = createEmptyBlock(outputType.getValueType()); + } + else { + keyBlock = keyReader.getBlock(nestedOutputPositions, nestedOutputPositionCount); + valueBlock = valueReader.getBlock(nestedOutputPositions, nestedOutputPositionCount); + } + + Block block = outputType.createBlockFromKeyValue(positionCount, Optional.ofNullable(includeNulls ? nulls : null), offsets, keyBlock, valueBlock); + nulls = null; + offsets = null; + return block; + } + + int[] offsetsCopy = new int[positionCount + 1]; + boolean[] nullsCopy = null; + if (includeNulls) { + nullsCopy = new boolean[positionCount]; + } + + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + int nestedSkipped = 0; + nestedOutputPositionCount = 0; + + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + nestedSkipped += offsets[i + 1] - offsets[i]; + continue; + } + + assert outputPositions[i] == nextPosition; + + offsetsCopy[positionIndex + 1] = offsets[i + 1] - nestedSkipped; + for (int j = offsetsCopy[positionIndex]; j < offsetsCopy[positionIndex + 1]; j++) { + nestedOutputPositions[nestedOutputPositionCount] = nestedOutputPositions[nestedOutputPositionCount + nestedSkipped]; + nestedOutputPositionCount++; + } + + if (nullsCopy != null) { + nullsCopy[positionIndex] = this.nulls[i]; + } + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + + nextPosition = positions[positionIndex]; + } + + Block keyBlock; + Block valueBlock; + if (nestedOutputPositionCount == 0) { + keyBlock = createEmptyBlock(outputType.getKeyType()); + valueBlock = createEmptyBlock(outputType.getValueType()); + } + else { + keyBlock = keyReader.getBlock(nestedOutputPositions, nestedOutputPositionCount); + valueBlock = valueReader.getBlock(nestedOutputPositions, nestedOutputPositionCount); + } + + return outputType.createBlockFromKeyValue(positionCount, Optional.ofNullable(includeNulls ? nullsCopy : null), offsetsCopy, keyBlock, valueBlock); + } + + private static RunLengthEncodedBlock createNullBlock(Type type, int positionCount) + { + return new RunLengthEncodedBlock(type.createBlockBuilder(null, 1).appendNull().build(), positionCount); + } + + private static Block createEmptyBlock(Type type) + { + return type.createBlockBuilder(null, 0).build(); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return newLease(createNullBlock(outputType, positionCount)); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount != outputPositionCount) { + compactValues(positions, positionCount, includeNulls); + } + + if (nestedOutputPositionCount == 0) { + return newLease(outputType.createBlockFromKeyValue(positionCount, Optional.ofNullable(includeNulls ? nulls : null), offsets, createEmptyBlock(outputType.getKeyType()), createEmptyBlock(outputType.getValueType()))); + } + + BlockLease keyBlockLease = keyReader.getBlockView(nestedOutputPositions, nestedOutputPositionCount); + BlockLease valueBlockLease = valueReader.getBlockView(nestedOutputPositions, nestedOutputPositionCount); + return newLease(outputType.createBlockFromKeyValue(positionCount, Optional.ofNullable(includeNulls ? nulls : null), offsets, keyBlockLease.get(), valueBlockLease.get()), keyBlockLease, valueBlockLease); + } + + private void compactValues(int[] positions, int positionCount, boolean compactNulls) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + int nestedSkipped = 0; + nestedOutputPositionCount = 0; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + nestedSkipped += offsets[i + 1] - offsets[i]; + continue; + } + + assert outputPositions[i] == nextPosition; + + offsets[positionIndex + 1] = offsets[i + 1] - nestedSkipped; + for (int j = offsets[positionIndex]; j < offsets[positionIndex + 1]; j++) { + nestedOutputPositions[nestedOutputPositionCount] = nestedOutputPositions[nestedOutputPositionCount + nestedSkipped]; + nestedOutputPositionCount++; + } + + if (compactNulls) { + nulls[positionIndex] = nulls[i]; + } + outputPositions[positionIndex] = nextPosition; + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + nextPosition = positions[positionIndex]; + } + + outputPositionCount = positionCount; + } + + private BlockLease newLease(Block block, BlockLease...fieldBlockLeases) + { + valuesInUse = true; + return ClosingBlockLease.newLease(block, () -> { + for (BlockLease lease : fieldBlockLeases) { + lease.close(); + } + valuesInUse = false; + }); + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } + + @Override + public String toString() + { + return toStringHelper(this) + .addValue(streamDescriptor) + .toString(); + } + + @Override + public void close() + { + systemMemoryContext.close(); + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + throws IOException + { + presentStreamSource = missingStreamSource(BooleanInputStream.class); + lengthStreamSource = missingStreamSource(LongInputStream.class); + + readOffset = 0; + nestedReadOffset = 0; + + presentStream = null; + lengthStream = null; + + rowGroupOpen = false; + + if (outputRequired) { + keyReader.startStripe(dictionaryStreamSources, encoding); + valueReader.startStripe(dictionaryStreamSources, encoding); + } + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + throws IOException + { + presentStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, PRESENT, BooleanInputStream.class); + lengthStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, LENGTH, LongInputStream.class); + + readOffset = 0; + nestedReadOffset = 0; + + presentStream = null; + lengthStream = null; + + rowGroupOpen = false; + + if (outputRequired) { + keyReader.startRowGroup(dataStreamSources); + valueReader.startRowGroup(dataStreamSources); + } + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + + sizeOf(outputPositions) + + sizeOf(offsets) + + sizeOf(nulls) + + sizeOf(nestedLengths) + + sizeOf(nestedOffsets) + + sizeOf(nestedPositions) + + sizeOf(nestedOutputPositions) + + (keyReader != null ? keyReader.getRetainedSizeInBytes() : 0) + + (valueReader != null ? valueReader.getRetainedSizeInBytes() : 0); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapFlatBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapFlatBatchStreamReader.java index 56587aac43f33..82e74ca80e876 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapFlatBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapFlatBatchStreamReader.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.orc.reader; +import com.facebook.presto.memory.context.AggregatedMemoryContext; +import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.metadata.ColumnEncoding; import com.facebook.presto.orc.metadata.DwrfSequenceEncoding; @@ -30,6 +32,7 @@ import com.facebook.presto.spi.type.SmallintType; import com.facebook.presto.spi.type.TinyintType; import com.facebook.presto.spi.type.Type; +import com.google.common.io.Closer; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import org.joda.time.DateTimeZone; @@ -38,6 +41,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -47,6 +51,7 @@ import static com.facebook.presto.orc.metadata.Stream.StreamKind.IN_MAP; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.ReaderUtils.verifyStreamType; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -68,6 +73,7 @@ public class MapFlatBatchStreamReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(MapFlatBatchStreamReader.class).instanceSize(); + private final MapType type; private final StreamDescriptor streamDescriptor; private final DateTimeZone hiveStorageTimeZone; @@ -91,11 +97,18 @@ public class MapFlatBatchStreamReader private boolean rowGroupOpen; - public MapFlatBatchStreamReader(StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone) + private AggregatedMemoryContext systemMemoryContext; + + public MapFlatBatchStreamReader(Type type, StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone, AggregatedMemoryContext systemMemoryContext) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, MapType.class::isInstance); + this.type = (MapType) type; this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); this.hiveStorageTimeZone = requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null"); - this.keyOrcType = streamDescriptor.getNestedStreams().get(0).getStreamType(); + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.keyOrcType = streamDescriptor.getNestedStreams().get(0).getOrcTypeKind(); this.baseValueStreamDescriptor = streamDescriptor.getNestedStreams().get(1); } @@ -107,7 +120,7 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -172,7 +185,7 @@ public Block readBlock(Type type) if (mapsContainingKey > 0) { BatchStreamReader streamReader = valueStreamReaders.get(keyIndex); streamReader.prepareNextRead(mapsContainingKey); - valueBlocks[keyIndex] = streamReader.readBlock(valueType); + valueBlocks[keyIndex] = streamReader.readBlock(); } else { valueBlocks[keyIndex] = valueType.createBlockBuilder(null, 0).build(); @@ -200,10 +213,12 @@ public Block readBlock(Type type) mapOffsets[mapIndex + 1] = mapOffsets[mapIndex] + mapLength; } + Block block = mapType.createBlockFromKeyValue(nextBatchSize, Optional.ofNullable(nullVector), mapOffsets, new DictionaryBlock(keyBlockTemplate, keyIds), valueBlockBuilder); + readOffset = 0; nextBatchSize = 0; - return mapType.createBlockFromKeyValue(Optional.ofNullable(nullVector), mapOffsets, new DictionaryBlock(keyBlockTemplate, keyIds), valueBlockBuilder); + return block; } private void openRowGroup() @@ -239,7 +254,7 @@ public void startStripe(InputStreamSources dictionaryStreamSources, List requiredLongKeys; + @Nullable + private final Set requiredStringKeys; + + private int[] keyIndices; // indices of the required keys in the encodings array passed to startStripe + private int keyCount; // number of valid entries in keyIndices array + + private final List> inMapStreamSources = new ArrayList<>(); + private final List inMapStreams = new ArrayList<>(); + private final List valueStreamReaders = new ArrayList<>(); + private final List valueStreamDescriptors = new ArrayList<>(); + + private Block keyBlock; + private int readOffset; + private int[] nestedReadOffsets; + + private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); + @Nullable + private BooleanInputStream presentStream; + + private boolean rowGroupOpen; + private int[] offsets; + private boolean[] nulls; + private int[] outputPositions; + private int outputPositionCount; + private boolean outputPositionsReadOnly; + private boolean allNulls; + private boolean valuesInUse; + + private int[] nestedLengths; + private int[][] nestedPositions; + private int[] nestedPositionCounts; + private int[][] nestedOutputPositions; + private boolean[][] inMap; + + private final AggregatedMemoryContext systemMemoryContext; + private final LocalMemoryContext localMemoryContext; + + public MapFlatSelectiveStreamReader( + StreamDescriptor streamDescriptor, + Map filters, + List requiredSubfields, + Optional outputType, + DateTimeZone hiveStorageTimeZone, + AggregatedMemoryContext systemMemoryContext) + { + checkArgument(filters.keySet().stream().map(Subfield::getPath).allMatch(List::isEmpty), "filters on nested columns are not supported yet"); + + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + checkArgument(streamDescriptor.getNestedStreams().size() == 2, "there must be exactly 2 nested stream descriptor"); + this.keyOrcTypeKind = streamDescriptor.getNestedStreams().get(0).getOrcTypeKind(); + this.baseValueStreamDescriptor = streamDescriptor.getNestedStreams().get(1); + this.hiveStorageTimeZone = requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null"); + + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.localMemoryContext = systemMemoryContext.newLocalMemoryContext(MapFlatSelectiveStreamReader.class.getSimpleName()); + this.outputRequired = requireNonNull(outputType, "outputType is null").isPresent(); + this.outputType = outputType.map(MapType.class::cast).orElse(null); + + TupleDomainFilter filter = getTopLevelFilter(filters).orElse(null); + this.nullsAllowed = filter == null || filter.testNull(); + this.nonNullsAllowed = filter == null || filter.testNonNull(); + + requireNonNull(requiredSubfields, "requiredSubfields is null"); + if (requiredSubfields.stream() + .map(Subfield::getPath) + .map(path -> path.get(0)) + .anyMatch(Subfield.AllSubscripts.class::isInstance)) { + requiredLongKeys = null; + requiredStringKeys = null; + } + else { + switch (keyOrcTypeKind) { + case BYTE: + case SHORT: + case INT: + case LONG: + requiredLongKeys = requiredSubfields.stream() + .map(Subfield::getPath) + .map(path -> path.get(0)) + .map(Subfield.LongSubscript.class::cast) + .map(Subfield.LongSubscript::getIndex) + .collect(toImmutableSet()); + requiredStringKeys = null; + return; + case STRING: + case BINARY: + requiredStringKeys = requiredSubfields.stream() + .map(Subfield::getPath) + .map(path -> path.get(0)) + .map(Subfield.StringSubscript.class::cast) + .map(Subfield.StringSubscript::getIndex) + .collect(toImmutableSet()); + requiredLongKeys = null; + return; + default: + requiredStringKeys = null; + requiredLongKeys = null; + } + } + } + + private static Optional getTopLevelFilter(Map filters) + { + Map topLevelFilters = Maps.filterEntries(filters, entry -> entry.getKey().getPath().isEmpty()); + if (topLevelFilters.isEmpty()) { + return Optional.empty(); + } + + checkArgument(topLevelFilters.size() == 1, "MAP column may have at most one top-level range filter"); + TupleDomainFilter filter = Iterables.getOnlyElement(topLevelFilters.values()); + checkArgument(filter == IS_NULL || filter == IS_NOT_NULL, "Top-level range filter on MAP column must be IS NULL or IS NOT NULL"); + return Optional.of(filter); + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (!rowGroupOpen) { + openRowGroup(); + } + + allNulls = false; + + if (outputRequired && nullsAllowed && presentStream != null) { + nulls = ensureCapacity(nulls, positionCount); + } + + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); + + if (presentStream != null && keyCount == 0) { + readAllNulls(positions, positionCount); + } + else { + readNotAllNulls(offset, positions, positionCount); + } + + localMemoryContext.setBytes(getRetainedSizeInBytes()); + + return outputPositionCount; + } + + private void readAllNulls(int[] positions, int positionCount) + throws IOException + { + presentStream.skip(positions[positionCount - 1]); + + allNulls = true; + + if (!nullsAllowed) { + outputPositionCount = 0; + } + else { + outputPositionCount = positionCount; + outputPositions = positions; + outputPositionsReadOnly = true; + } + } + + private void readNotAllNulls(int offset, int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + + int[] nonNullPositions = new int[positionCount]; + int[] nullPositions = new int[positionCount]; + int nonNullPositionCount = 0; + int nullPositionCount = 0; + int nonNullSkipped = 0; + + if (presentStream == null) { + if (readOffset < offset) { + for (int i = 0; i < keyCount; i++) { + nestedReadOffsets[i] += inMapStreams.get(i).countBitsSet(offset - readOffset); + } + } + + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + nonNullSkipped += position - streamPosition; + streamPosition = position; + } + + nonNullPositions[i] = i + nonNullSkipped; + streamPosition++; + } + nonNullPositionCount = positionCount; + } + else { + if (readOffset < offset) { + int skipped = presentStream.countBitsSet(offset - readOffset); + if (skipped > 0) { + for (int i = 0; i < keyCount; i++) { + nestedReadOffsets[i] += inMapStreams.get(i).countBitsSet(skipped); + } + } + } + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + nonNullSkipped += presentStream.countBitsSet(position - streamPosition); + streamPosition = position; + } + + streamPosition++; + if (presentStream.nextBit()) { + // not null + if (nullsAllowed) { + nulls[i] = false; + } + nonNullPositions[nonNullPositionCount] = nonNullPositionCount + nonNullSkipped; + nonNullPositionCount++; + } + else { + if (nullsAllowed) { + nulls[i] = true; + nullPositions[nullPositionCount] = positions[i]; + nullPositionCount++; + } + } + } + } + + readOffset = offset + streamPosition; + + if (!nonNullsAllowed) { + checkState(nullPositionCount == (positionCount - nonNullPositionCount), "nullPositionCount should be equal to postitionCount - nonNullPositionCount"); + outputPositionCount = nullPositionCount; + allNulls = true; + System.arraycopy(nullPositions, 0, outputPositions, 0, nullPositionCount); + } + else { + nestedLengths = ensureCapacity(nestedLengths, nonNullPositionCount); + Arrays.fill(nestedLengths, 0); + + for (int keyIndex = 0; keyIndex < keyCount; keyIndex++) { + BooleanInputStream inMapStream = inMapStreams.get(keyIndex); + + nestedPositions[keyIndex] = ensureCapacity(nestedPositions[keyIndex], nonNullPositionCount); + inMap[keyIndex] = ensureCapacity(inMap[keyIndex], nonNullPositionCount); + + int nestedStreamPosition = 0; + + int nestedSkipped = 0; + int nestedPositionCount = 0; + for (int i = 0; i < nonNullPositionCount; i++) { + int position = nonNullPositions[i]; + if (position > nestedStreamPosition) { + nestedSkipped += inMapStream.countBitsSet(position - nestedStreamPosition); + nestedStreamPosition = position; + } + + nestedStreamPosition++; + if (inMapStream.nextBit()) { + nestedPositions[keyIndex][nestedPositionCount] = nestedPositionCount + nestedSkipped; + nestedPositionCount++; + + nestedLengths[i]++; + inMap[keyIndex][i] = true; + } + else { + inMap[keyIndex][i] = false; + } + } + + if (nonNullSkipped > nestedStreamPosition - nonNullPositionCount) { + inMapStream.skip(nonNullSkipped - (nestedStreamPosition - nonNullPositionCount)); + } + + nestedPositionCounts[keyIndex] = nestedPositionCount; + + if (nestedPositionCount > 0) { + int readCount = valueStreamReaders.get(keyIndex).read(nestedReadOffsets[keyIndex], nestedPositions[keyIndex], nestedPositionCount); + verify(readCount == nestedPositionCount); + } + nestedReadOffsets[keyIndex] += nestedSkipped + nestedPositionCount; + } + + if (nullsAllowed) { + outputPositionCount = positionCount; + } + else { + System.arraycopy(nonNullPositions, 0, outputPositions, 0, nonNullPositionCount); + outputPositionCount = nonNullPositionCount; + } + } + + if (outputRequired) { + nestedOutputPositions = ensureCapacity(nestedOutputPositions, keyCount); + for (int i = 0; i < keyCount; i++) { + int nestedPositionCount = nestedPositionCounts[i]; + if (nestedPositionCount > 0) { + nestedOutputPositions[i] = ensureCapacity(nestedOutputPositions[i], nestedPositionCount); + System.arraycopy(nestedPositions[i], 0, nestedOutputPositions[i], 0, nestedPositionCount); + } + } + } + } + + private void openRowGroup() + throws IOException + { + presentStream = presentStreamSource.openStream(); + + for (int i = 0; i < keyCount; i++) { + BooleanInputStream inMapStream = requireNonNull(inMapStreamSources.get(i).openStream(), "missing inMapStream at position " + i); + inMapStreams.add(inMapStream); + } + + rowGroupOpen = true; + } + + @Override + public int[] getReadPositions() + { + return outputPositions; + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return createNullBlock(outputType, positionCount); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (outputPositionCount != positionCount) { + compactValues(positions, positionCount, includeNulls); + } + + Block block = assembleMapBlock(includeNulls); + nulls = null; + offsets = null; + return block; + } + + private Block assembleMapBlock(boolean includeNulls) + { + offsets = ensureCapacity(offsets, outputPositionCount + 1); + offsets[0] = 0; + + int offset = 0; + int inMapIndex = 0; + for (int i = 0; i < outputPositionCount; i++) { + if (!includeNulls || !nulls[i]) { + offset += nestedLengths[inMapIndex]; + inMapIndex++; + } + offsets[i + 1] = offset; + } + + BlockLease[] valueBlockLeases = new BlockLease[keyCount]; + Block[] valueBlocks = new Block[keyCount]; + for (int i = 0; i < keyCount; i++) { + if (nestedPositionCounts[i] > 0) { + valueBlockLeases[i] = valueStreamReaders.get(i).getBlockView(nestedOutputPositions[i], nestedPositionCounts[i]); + valueBlocks[i] = valueBlockLeases[i].get(); + } + else { + valueBlocks[i] = outputType.getKeyType().createBlockBuilder(null, 0).build(); + } + } + + int[] keyIds = new int[offset]; + int count = 0; + + Type valueType = outputType.getValueType(); + BlockBuilder valueBlockBuilder = valueType.createBlockBuilder(null, offset); + + int[] valueBlockPositions = new int[keyCount]; + + inMapIndex = 0; + for (int i = 0; i < outputPositionCount; i++) { + if (includeNulls && nulls[i]) { + continue; + } + for (int keyIndex = 0; keyIndex < keyCount; keyIndex++) { + if (inMap[keyIndex][inMapIndex]) { + valueType.appendTo(valueBlocks[keyIndex], valueBlockPositions[keyIndex], valueBlockBuilder); + valueBlockPositions[keyIndex]++; + keyIds[count++] = keyIndex; + } + } + inMapIndex++; + } + + for (int i = 0; i < keyCount; i++) { + if (valueBlockLeases[i] != null) { + valueBlockLeases[i].close(); + } + } + + return outputType.createBlockFromKeyValue(outputPositionCount, Optional.ofNullable(includeNulls ? nulls : null), offsets, new DictionaryBlock(keyBlock, keyIds), valueBlockBuilder); + } + + private static RunLengthEncodedBlock createNullBlock(Type type, int positionCount) + { + return new RunLengthEncodedBlock(type.createBlockBuilder(null, 1).appendNull().build(), positionCount); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return newLease(createNullBlock(outputType, positionCount)); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount != outputPositionCount) { + compactValues(positions, positionCount, includeNulls); + } + + return newLease(assembleMapBlock(includeNulls)); + } + + private BlockLease newLease(Block block) + { + valuesInUse = true; + return ClosingBlockLease.newLease(block, () -> valuesInUse = false); + } + + private void compactValues(int[] positions, int positionCount, boolean includeNulls) + { + if (outputPositionsReadOnly) { + outputPositions = Arrays.copyOf(outputPositions, outputPositionCount); + outputPositionsReadOnly = false; + } + + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + + int skipped = 0; + int inMapSkipped = 0; + int inMapIndex = 0; + int[] nestedSkipped = new int[keyCount]; + int[] nestedIndex = new int[keyCount]; + + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + // skip this position + if (!includeNulls || !nulls[i]) { + // not null + for (int keyIndex = 0; keyIndex < keyCount; keyIndex++) { + if (inMap[keyIndex][inMapIndex]) { + nestedSkipped[keyIndex]++; + nestedIndex[keyIndex]++; + } + } + inMapSkipped++; + inMapIndex++; + } + skipped++; + continue; + } + + outputPositions[i - skipped] = outputPositions[i]; + if (includeNulls) { + nulls[i - skipped] = nulls[i]; + } + if (!includeNulls || !nulls[i]) { + // not null + nestedLengths[inMapIndex - inMapSkipped] = nestedLengths[inMapIndex]; + for (int keyIndex = 0; keyIndex < keyCount; keyIndex++) { + inMap[keyIndex][inMapIndex - inMapSkipped] = inMap[keyIndex][inMapIndex]; + if (inMap[keyIndex][inMapIndex]) { + nestedOutputPositions[keyIndex][nestedIndex[keyIndex] - nestedSkipped[keyIndex]] = nestedOutputPositions[keyIndex][nestedIndex[keyIndex]]; + nestedIndex[keyIndex]++; + } + } + inMapIndex++; + } + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + + nextPosition = positions[positionIndex]; + } + + outputPositionCount = positionCount; + for (int keyIndex = 0; keyIndex < keyCount; keyIndex++) { + nestedPositionCounts[keyIndex] = nestedIndex[keyIndex] - nestedSkipped[keyIndex]; + } + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } + + @Override + public void close() + { + localMemoryContext.close(); + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encodings) + throws IOException + { + presentStreamSource = missingStreamSource(BooleanInputStream.class); + + inMapStreamSources.clear(); + valueStreamDescriptors.clear(); + valueStreamReaders.clear(); + + ColumnEncoding encoding = encodings.get(baseValueStreamDescriptor.getStreamId()); + // encoding.getAdditionalSequenceEncodings() may not be present when every map is empty or null + SortedMap additionalSequenceEncodings = encoding.getAdditionalSequenceEncodings().orElse(Collections.emptySortedMap()); + + keyIndices = ensureCapacity(keyIndices, additionalSequenceEncodings.size()); + keyCount = 0; + + // The ColumnEncoding with sequence ID 0 doesn't have any data associated with it + int keyIndex = 0; + for (Map.Entry entry : additionalSequenceEncodings.entrySet()) { + if (!isRequiredKey(entry.getValue())) { + keyIndex++; + continue; + } + + keyIndices[keyCount] = keyIndex; + keyCount++; + keyIndex++; + + int sequence = entry.getKey(); + + inMapStreamSources.add(missingStreamSource(BooleanInputStream.class)); + + StreamDescriptor valueStreamDescriptor = copyStreamDescriptorWithSequence(baseValueStreamDescriptor, sequence); + valueStreamDescriptors.add(valueStreamDescriptor); + + SelectiveStreamReader valueStreamReader = SelectiveStreamReaders.createStreamReader(valueStreamDescriptor, ImmutableBiMap.of(), Optional.ofNullable(outputType).map(MapType::getValueType), ImmutableList.of(), hiveStorageTimeZone, systemMemoryContext.newAggregatedMemoryContext()); + valueStreamReader.startStripe(dictionaryStreamSources, encodings); + valueStreamReaders.add(valueStreamReader); + } + + keyBlock = getKeysBlock(ImmutableList.copyOf(additionalSequenceEncodings.values())); + readOffset = 0; + + presentStream = null; + + rowGroupOpen = false; + } + + private boolean isRequiredKey(DwrfSequenceEncoding value) + { + if (requiredLongKeys != null) { + return requiredLongKeys.isEmpty() || requiredLongKeys.contains(value.getKey().getIntKey()); + } + + return requiredStringKeys.isEmpty() || requiredStringKeys.contains(value.getKey().getBytesKey().toStringUtf8()); + } + + /** + * Creates StreamDescriptor which is a copy of this one with the value of sequence changed to + * the value passed in. Recursively calls itself on the nested streams. + */ + private static StreamDescriptor copyStreamDescriptorWithSequence(StreamDescriptor streamDescriptor, int sequence) + { + List streamDescriptors = streamDescriptor.getNestedStreams().stream() + .map(stream -> copyStreamDescriptorWithSequence(stream, sequence)) + .collect(toImmutableList()); + + return new StreamDescriptor( + streamDescriptor.getStreamName(), + streamDescriptor.getStreamId(), + streamDescriptor.getFieldName(), + streamDescriptor.getOrcType(), + streamDescriptor.getOrcDataSource(), + streamDescriptors, + sequence); + } + + private Block getKeysBlock(List sequenceEncodings) + { + switch (keyOrcTypeKind) { + case BYTE: + case SHORT: + case INT: + case LONG: + return getIntegerKeysBlock(sequenceEncodings); + case STRING: + case BINARY: + return getSliceKeysBlock(sequenceEncodings); + default: + throw new IllegalArgumentException("Unsupported flat map key type: " + keyOrcTypeKind); + } + } + + private Block getIntegerKeysBlock(List sequenceEncodings) + { + Type keyType; + + switch (keyOrcTypeKind) { + case BYTE: + keyType = TinyintType.TINYINT; + break; + case SHORT: + keyType = SmallintType.SMALLINT; + break; + case INT: + keyType = IntegerType.INTEGER; + break; + case LONG: + keyType = BigintType.BIGINT; + break; + default: + throw new IllegalArgumentException("Unsupported flat map key type: " + keyOrcTypeKind); + } + + BlockBuilder blockBuilder = keyType.createBlockBuilder(null, sequenceEncodings.size()); + + for (int i = 0; i < keyCount; i++) { + keyType.writeLong(blockBuilder, sequenceEncodings.get(keyIndices[i]).getKey().getIntKey()); + } + + return blockBuilder.build(); + } + + private Block getSliceKeysBlock(List sequenceEncodings) + { + int bytes = 0; + + for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) { + bytes += sequenceEncoding.getKey().getBytesKey().size(); + } + + VariableWidthBlockBuilder builder = new VariableWidthBlockBuilder(null, sequenceEncodings.size(), bytes); + + for (int i = 0; i < keyCount; i++) { + Slice key = Slices.wrappedBuffer(sequenceEncodings.get(keyIndices[i]).getKey().getBytesKey().toByteArray()); + builder.writeBytes(key, 0, key.length()); + builder.closeEntry(); + } + + return builder.build(); + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + throws IOException + { + presentStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, PRESENT, BooleanInputStream.class); + + for (int i = 0; i < keyCount; i++) { + InputStreamSource inMapStreamSource = dataStreamSources.getInputStreamSource(valueStreamDescriptors.get(i), IN_MAP, BooleanInputStream.class); + inMapStreamSources.set(i, inMapStreamSource); + } + + readOffset = 0; + nestedReadOffsets = ensureCapacity(nestedReadOffsets, keyCount); + Arrays.fill(nestedReadOffsets, 0); + + nestedPositions = ensureCapacity(nestedPositions, keyCount); + nestedPositionCounts = ensureCapacity(nestedPositionCounts, keyCount); + inMap = ensureCapacity(inMap, keyCount); + + presentStream = null; + inMapStreams.clear(); + + rowGroupOpen = false; + + for (SelectiveStreamReader valueStreamReader : valueStreamReaders) { + valueStreamReader.startRowGroup(dataStreamSources); + } + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + + sizeOf(keyIndices) + + sizeOf(nestedReadOffsets) + + sizeOf(offsets) + + sizeOf(nulls) + + sizeOf(outputPositions) + + sizeOf(nestedLengths) + + (nestedPositions != null ? Arrays.stream(nestedPositions).mapToLong(SizeOf::sizeOf).sum() : 0) + + sizeOf(nestedPositionCounts) + + (nestedOutputPositions != null ? Arrays.stream(nestedOutputPositions).mapToLong(SizeOf::sizeOf).sum() : 0) + + (inMap != null ? Arrays.stream(inMap).mapToLong(SizeOf::sizeOf).sum() : 0) + + (valueStreamReaders != null ? valueStreamReaders.stream().mapToLong(StreamReader::getRetainedSizeInBytes).sum() : 0); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapSelectiveStreamReader.java new file mode 100644 index 0000000000000..ea7c44784739b --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/MapSelectiveStreamReader.java @@ -0,0 +1,143 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.AggregatedMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.metadata.ColumnEncoding; +import com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind; +import com.facebook.presto.orc.stream.InputStreamSources; +import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.type.Type; +import org.joda.time.DateTimeZone; +import org.openjdk.jol.info.ClassLayout; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT; +import static com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT_V2; +import static com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind.DWRF_DIRECT; +import static com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind.DWRF_MAP_FLAT; +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class MapSelectiveStreamReader + implements SelectiveStreamReader +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(MapSelectiveStreamReader.class).instanceSize(); + + private final StreamDescriptor streamDescriptor; + private final MapDirectSelectiveStreamReader directReader; + private final MapFlatSelectiveStreamReader flatReader; + private SelectiveStreamReader currentReader; + + public MapSelectiveStreamReader( + StreamDescriptor streamDescriptor, + Map filters, + List requiredSubfields, + Optional outputType, + DateTimeZone hiveStorageTimeZone, + AggregatedMemoryContext systemMemoryContext) + { + this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); + directReader = new MapDirectSelectiveStreamReader(streamDescriptor, filters, requiredSubfields, outputType, hiveStorageTimeZone, systemMemoryContext); + flatReader = new MapFlatSelectiveStreamReader(streamDescriptor, filters, requiredSubfields, outputType, hiveStorageTimeZone, systemMemoryContext); + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + throws IOException + { + ColumnEncodingKind kind = encoding.get(streamDescriptor.getStreamId()) + .getColumnEncoding(streamDescriptor.getSequence()) + .getColumnEncodingKind(); + if (kind == DIRECT || kind == DIRECT_V2 || kind == DWRF_DIRECT) { + currentReader = directReader; + } + else if (kind == DWRF_MAP_FLAT) { + currentReader = flatReader; + } + else { + throw new IllegalArgumentException("Unsupported encoding " + kind); + } + + currentReader.startStripe(dictionaryStreamSources, encoding); + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + throws IOException + { + currentReader.startRowGroup(dataStreamSources); + } + + @Override + public String toString() + { + return toStringHelper(this) + .addValue(streamDescriptor) + .toString(); + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + directReader.getRetainedSizeInBytes() + flatReader.getRetainedSizeInBytes(); + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + return currentReader.read(offset, positions, positionCount); + } + + @Override + public int[] getReadPositions() + { + return currentReader.getReadPositions(); + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + return currentReader.getBlock(positions, positionCount); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + return currentReader.getBlockView(positions, positionCount); + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + currentReader.throwAnyError(positions, positionCount); + } + + @Override + public void close() + { + if (currentReader != null) { + currentReader.close(); + } + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/ReaderUtils.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ReaderUtils.java new file mode 100644 index 0000000000000..720023399de5c --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ReaderUtils.java @@ -0,0 +1,128 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.orc.OrcCorruptionException; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.spi.type.Type; + +import java.util.function.Predicate; + +import static java.lang.Math.max; + +final class ReaderUtils +{ + private ReaderUtils() {} + + public static void verifyStreamType(StreamDescriptor streamDescriptor, Type actual, Predicate validTypes) + throws OrcCorruptionException + { + if (validTypes.test(actual)) { + return; + } + + throw new OrcCorruptionException( + streamDescriptor.getOrcDataSourceId(), + "Can not read SQL type %s from ORC stream %s of type %s", + actual, + streamDescriptor.getStreamName(), + streamDescriptor.getOrcTypeKind()); + } + + public static int minNonNullValueSize(int nonNullCount) + { + return max(nonNullCount + 1, 1025); + } + + public static byte[] unpackByteNulls(byte[] values, boolean[] isNull) + { + byte[] result = new byte[isNull.length]; + + int position = 0; + for (int i = 0; i < isNull.length; i++) { + if (!isNull[i]) { + result[i] = values[position]; + position++; + } + } + return result; + } + + public static short[] unpackShortNulls(short[] values, boolean[] isNull) + { + short[] result = new short[isNull.length]; + + int position = 0; + for (int i = 0; i < isNull.length; i++) { + if (!isNull[i]) { + result[i] = values[position]; + position++; + } + } + return result; + } + + public static int[] unpackIntNulls(int[] values, boolean[] isNull) + { + int[] result = new int[isNull.length]; + + int position = 0; + for (int i = 0; i < isNull.length; i++) { + if (!isNull[i]) { + result[i] = values[position]; + position++; + } + } + return result; + } + + public static long[] unpackLongNulls(long[] values, boolean[] isNull) + { + long[] result = new long[isNull.length]; + + int position = 0; + for (int i = 0; i < isNull.length; i++) { + if (!isNull[i]) { + result[i] = values[position]; + position++; + } + } + return result; + } + + public static void unpackLengthNulls(int[] values, boolean[] isNull, int nonNullCount) + { + int nullSuppressedPosition = nonNullCount - 1; + for (int outputPosition = isNull.length - 1; outputPosition >= 0; outputPosition--) { + if (isNull[outputPosition]) { + values[outputPosition] = 0; + } + else { + values[outputPosition] = values[nullSuppressedPosition]; + nullSuppressedPosition--; + } + } + } + + public static void convertLengthVectorToOffsetVector(int[] vector) + { + int currentLength = vector[0]; + vector[0] = 0; + for (int i = 1; i < vector.length; i++) { + int nextLength = vector[i]; + vector[i] = vector[i - 1] + currentLength; + currentLength = nextLength; + } + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SelectiveStreamReader.java index 113d5c306faea..6c4da035ddce3 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SelectiveStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SelectiveStreamReader.java @@ -64,5 +64,10 @@ int read(int offset, int[] positions, int positionCount) */ BlockLease getBlockView(int[] positions, int positionCount); - void close(); + /** + * Throws any error that occurred while reading specified positions. + * + * Used by list and map readers to raise "subscript out of bounds" error. + */ + void throwAnyError(int[] positions, int positionCount); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SelectiveStreamReaders.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SelectiveStreamReaders.java index 3a9d7e123d10d..515384abbf546 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SelectiveStreamReaders.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SelectiveStreamReaders.java @@ -16,14 +16,21 @@ import com.facebook.presto.memory.context.AggregatedMemoryContext; import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.metadata.OrcType.OrcTypeKind; import com.facebook.presto.spi.Subfield; import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import org.joda.time.DateTimeZone; import java.util.List; +import java.util.Map; import java.util.Optional; +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.spi.type.Decimals.MAX_SHORT_PRECISION; import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; public final class SelectiveStreamReaders { @@ -31,24 +38,92 @@ private SelectiveStreamReaders() {} public static SelectiveStreamReader createStreamReader( StreamDescriptor streamDescriptor, - Optional filter, + Map filters, Optional outputType, List requiredSubfields, DateTimeZone hiveStorageTimeZone, AggregatedMemoryContext systemMemoryContext) { - switch (streamDescriptor.getStreamType()) { - case BOOLEAN: + OrcTypeKind type = streamDescriptor.getOrcTypeKind(); + switch (type) { + case BOOLEAN: { checkArgument(requiredSubfields.isEmpty(), "Boolean stream reader doesn't support subfields"); - return new BooleanSelectiveStreamReader(streamDescriptor, filter, outputType.isPresent(), systemMemoryContext.newLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName())); + return new BooleanSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType.isPresent(), systemMemoryContext.newLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName())); + } + case BYTE: { + checkArgument(requiredSubfields.isEmpty(), "Byte stream reader doesn't support subfields"); + return new ByteSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType.isPresent(), systemMemoryContext.newLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName())); + } + case SHORT: + case INT: + case LONG: + case DATE: { + checkArgument(requiredSubfields.isEmpty(), "Primitive type stream reader doesn't support subfields"); + return new LongSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext); + } + case FLOAT: { + checkArgument(requiredSubfields.isEmpty(), "Float type stream reader doesn't support subfields"); + return new FloatSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType.isPresent(), systemMemoryContext.newLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName())); + } + case DOUBLE: + checkArgument(requiredSubfields.isEmpty(), "Double stream reader doesn't support subfields"); + return new DoubleSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType.isPresent(), systemMemoryContext.newLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName())); + case BINARY: + case STRING: + case VARCHAR: + case CHAR: + checkArgument(requiredSubfields.isEmpty(), "Primitive stream reader doesn't support subfields"); + return new SliceSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext); + case TIMESTAMP: { + checkArgument(requiredSubfields.isEmpty(), "Timestamp stream reader doesn't support subfields"); + return new TimestampSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), hiveStorageTimeZone, outputType.isPresent(), systemMemoryContext.newLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName())); + } + case LIST: + return new ListSelectiveStreamReader(streamDescriptor, filters, requiredSubfields, null, 0, outputType, hiveStorageTimeZone, systemMemoryContext); + case STRUCT: + return new StructSelectiveStreamReader(streamDescriptor, filters, requiredSubfields, outputType, hiveStorageTimeZone, systemMemoryContext); + case MAP: + return new MapSelectiveStreamReader(streamDescriptor, filters, requiredSubfields, outputType, hiveStorageTimeZone, systemMemoryContext); + case DECIMAL: { + if (streamDescriptor.getOrcType().getPrecision().get() <= MAX_SHORT_PRECISION) { + return new ShortDecimalSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext.newLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName())); + } + else { + return new LongDecimalSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext.newLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName())); + } + } + case UNION: + default: + throw new IllegalArgumentException("Unsupported type: " + type); + } + } + + private static Optional getOptionalOnlyFilter(OrcTypeKind type, Map filters) + { + if (filters.isEmpty()) { + return Optional.empty(); + } + + checkArgument(filters.size() == 1, format("Stream reader for %s doesn't support multiple range filters", type)); + return Optional.of(Iterables.getOnlyElement(filters.values())); + } + + public static SelectiveStreamReader createNestedStreamReader( + StreamDescriptor streamDescriptor, + int level, + Optional parentFilter, + Optional outputType, + List requiredSubfields, + DateTimeZone hiveStorageTimeZone, + AggregatedMemoryContext systemMemoryContext) + { + switch (streamDescriptor.getOrcTypeKind()) { + case BOOLEAN: case BYTE: - throw new IllegalArgumentException("Unsupported type: " + streamDescriptor.getStreamType()); case SHORT: case INT: case LONG: case DATE: - checkArgument(requiredSubfields.isEmpty(), "Primitive type stream reader doesn't support subfields"); - return new LongSelectiveStreamReader(streamDescriptor, filter, outputType, systemMemoryContext); case FLOAT: case DOUBLE: case BINARY: @@ -56,13 +131,38 @@ public static SelectiveStreamReader createStreamReader( case VARCHAR: case CHAR: case TIMESTAMP: + case DECIMAL: + Map elementFilters = ImmutableMap.of(); + if (parentFilter.isPresent()) { + TupleDomainFilter.PositionalFilter positionalFilter = parentFilter.get().getPositionalFilter(); + if (positionalFilter != null) { + elementFilters = ImmutableMap.of(new Subfield("c"), positionalFilter); + } + } + if (!outputType.isPresent() && elementFilters.isEmpty()) { + // No need to read the elements when output is not required and the filter is a simple IS [NOT] NULL + return null; + } + return createStreamReader(streamDescriptor, elementFilters, outputType, requiredSubfields, hiveStorageTimeZone, systemMemoryContext.newAggregatedMemoryContext()); case LIST: + Optional childFilter = parentFilter.map(HierarchicalFilter::getChild).map(ListFilter.class::cast); + return new ListSelectiveStreamReader(streamDescriptor, ImmutableMap.of(), requiredSubfields, childFilter.orElse(null), level, outputType, hiveStorageTimeZone, systemMemoryContext.newAggregatedMemoryContext()); case STRUCT: + checkArgument(!parentFilter.isPresent(), "Filters on nested structs are not supported yet"); + return new StructSelectiveStreamReader(streamDescriptor, ImmutableMap.of(), requiredSubfields, outputType, hiveStorageTimeZone, systemMemoryContext.newAggregatedMemoryContext()); case MAP: - case DECIMAL: + checkArgument(!parentFilter.isPresent(), "Filters on nested maps are not supported yet"); + return new MapSelectiveStreamReader(streamDescriptor, ImmutableMap.of(), requiredSubfields, outputType, hiveStorageTimeZone, systemMemoryContext.newAggregatedMemoryContext()); case UNION: default: - throw new IllegalArgumentException("Unsupported type: " + streamDescriptor.getStreamType()); + throw new IllegalArgumentException("Unsupported type: " + streamDescriptor.getOrcTypeKind()); } } + + public static int[] initializeOutputPositions(int[] outputPositions, int[] positions, int positionCount) + { + outputPositions = ensureCapacity(outputPositions, positionCount); + System.arraycopy(positions, 0, outputPositions, 0, positionCount); + return outputPositions; + } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/ShortDecimalSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ShortDecimalSelectiveStreamReader.java new file mode 100644 index 0000000000000..76b15fff3be0a --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/ShortDecimalSelectiveStreamReader.java @@ -0,0 +1,177 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.LongArrayBlock; +import com.facebook.presto.spi.type.Decimals; +import com.facebook.presto.spi.type.Type; + +import java.io.IOException; +import java.util.Optional; + +public class ShortDecimalSelectiveStreamReader + extends AbstractDecimalSelectiveStreamReader +{ + public ShortDecimalSelectiveStreamReader( + StreamDescriptor streamDescriptor, + Optional filter, + Optional outputType, + LocalMemoryContext systemMemoryContext) + { + super(streamDescriptor, filter, outputType, systemMemoryContext, 1); + } + + @Override + protected int readNoFilter(int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + if (presentStream != null && !presentStream.nextBit()) { + nulls[i] = true; + } + else { + values[i] = Decimals.rescale(dataStream.nextLong(), (int) scaleStream.next(), this.scale); + if (presentStream != null) { + nulls[i] = false; + } + } + streamPosition++; + } + outputPositionCount = positionCount; + return streamPosition; + } + + @Override + protected int readWithFilter(int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + if ((nonDeterministicFilter && filter.testNull()) || nullsAllowed) { + if (outputRequired) { + nulls[outputPositionCount] = true; + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + else { + long rescale = Decimals.rescale(dataStream.nextLong(), (int) scaleStream.next(), this.scale); + if (filter.testLong(rescale)) { + if (outputRequired) { + values[outputPositionCount] = rescale; + if (nullsAllowed && presentStream != null) { + nulls[outputPositionCount] = false; + } + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + streamPosition++; + + if (filter != null) { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + int succeedingPositionsToFail = filter.getSucceedingPositionsToFail(); + if (succeedingPositionsToFail > 0) { + int positionsToSkip = 0; + for (int j = 0; j < succeedingPositionsToFail; j++) { + i++; + int nextPosition = positions[i]; + positionsToSkip += 1 + nextPosition - streamPosition; + streamPosition = nextPosition + 1; + } + skip(positionsToSkip); + } + } + } + return streamPosition; + } + + @Override + protected void copyValues(int[] positions, int positionCount, long[] valuesCopy, boolean[] nullsCopy) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + assert outputPositions[i] == nextPosition; + + valuesCopy[positionIndex] = this.values[i]; + if (nullsCopy != null) { + nullsCopy[positionIndex] = this.nulls[i]; + } + positionIndex++; + + if (positionIndex >= positionCount) { + break; + } + nextPosition = positions[positionIndex]; + } + } + + @Override + protected void compactValues(int[] positions, int positionCount, boolean compactNulls) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + values[positionIndex] = values[i]; + if (compactNulls) { + nulls[positionIndex] = nulls[i]; + } + outputPositions[positionIndex] = nextPosition; + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + nextPosition = positions[positionIndex]; + } + + outputPositionCount = positionCount; + } + + @Override + protected Block makeBlock(int positionCount, boolean includeNulls, boolean[] nulls, long[] values) + { + return new LongArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceBatchStreamReader.java index 3c63faabd7aa3..7786141dadb4e 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceBatchStreamReader.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.orc.reader; +import com.facebook.presto.memory.context.AggregatedMemoryContext; +import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.metadata.ColumnEncoding; import com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind; @@ -20,11 +22,14 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.VarbinaryType; import com.facebook.presto.spi.type.VarcharType; +import com.google.common.io.Closer; import io.airlift.slice.Slice; import org.openjdk.jol.info.ClassLayout; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.List; import static com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind.DICTIONARY; @@ -32,6 +37,7 @@ import static com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT; import static com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT_V2; import static com.facebook.presto.orc.metadata.ColumnEncoding.ColumnEncodingKind.DWRF_DIRECT; +import static com.facebook.presto.orc.reader.ReaderUtils.verifyStreamType; import static com.facebook.presto.spi.type.Chars.byteCountWithoutTrailingSpace; import static com.facebook.presto.spi.type.Chars.isCharType; import static com.facebook.presto.spi.type.VarbinaryType.isVarbinaryType; @@ -50,18 +56,21 @@ public class SliceBatchStreamReader private final SliceDictionaryBatchStreamReader dictionaryReader; private BatchStreamReader currentReader; - public SliceBatchStreamReader(StreamDescriptor streamDescriptor) + public SliceBatchStreamReader(Type type, StreamDescriptor streamDescriptor, AggregatedMemoryContext systemMemoryContext) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, t -> t instanceof VarcharType || t instanceof CharType || t instanceof VarbinaryType); this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); - directReader = new SliceDirectBatchStreamReader(streamDescriptor); - dictionaryReader = new SliceDictionaryBatchStreamReader(streamDescriptor); + this.directReader = new SliceDirectBatchStreamReader(streamDescriptor, getMaxCodePointCount(type), isCharType(type)); + this.dictionaryReader = new SliceDictionaryBatchStreamReader(streamDescriptor, getMaxCodePointCount(type), isCharType(type), systemMemoryContext.newLocalMemoryContext(SliceBatchStreamReader.class.getSimpleName())); } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { - return currentReader.readBlock(type); + return currentReader.readBlock(); } @Override @@ -132,6 +141,18 @@ public static int computeTruncatedLength(Slice slice, int offset, int length, in return length; } + @Override + public void close() + { + try (Closer closer = Closer.create()) { + closer.register(directReader::close); + closer.register(dictionaryReader::close); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDictionaryBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDictionaryBatchStreamReader.java index 47747d12cfc54..a05c2cb2fafb6 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDictionaryBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDictionaryBatchStreamReader.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc.reader; +import com.facebook.presto.memory.context.LocalMemoryContext; import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.StreamDescriptor; import com.facebook.presto.orc.metadata.ColumnEncoding; @@ -25,7 +26,6 @@ import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.DictionaryBlock; import com.facebook.presto.spi.block.VariableWidthBlock; -import com.facebook.presto.spi.type.Type; import io.airlift.slice.Slice; import org.openjdk.jol.info.ClassLayout; @@ -44,11 +44,10 @@ import static com.facebook.presto.orc.metadata.Stream.StreamKind.ROW_GROUP_DICTIONARY; import static com.facebook.presto.orc.metadata.Stream.StreamKind.ROW_GROUP_DICTIONARY_LENGTH; import static com.facebook.presto.orc.reader.SliceBatchStreamReader.computeTruncatedLength; -import static com.facebook.presto.orc.reader.SliceBatchStreamReader.getMaxCodePointCount; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; -import static com.facebook.presto.spi.type.Chars.isCharType; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Verify.verify; +import static io.airlift.slice.SizeOf.sizeOf; import static io.airlift.slice.Slices.wrappedBuffer; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; @@ -63,6 +62,8 @@ public class SliceDictionaryBatchStreamReader private static final int[] EMPTY_DICTIONARY_OFFSETS = new int[2]; private final StreamDescriptor streamDescriptor; + private final int maxCodePointCount; + private final boolean isCharType; private int readOffset; private int nextBatchSize; @@ -98,9 +99,14 @@ public class SliceDictionaryBatchStreamReader private boolean rowGroupOpen; - public SliceDictionaryBatchStreamReader(StreamDescriptor streamDescriptor) + private final LocalMemoryContext systemMemoryContext; + + public SliceDictionaryBatchStreamReader(StreamDescriptor streamDescriptor, int maxCodePointCount, boolean isCharType, LocalMemoryContext systemMemoryContext) { + this.maxCodePointCount = maxCodePointCount; + this.isCharType = isCharType; this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); } @Override @@ -111,11 +117,11 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { - openRowGroup(type); + openRowGroup(); } if (readOffset > 0) { @@ -142,7 +148,7 @@ public Block readBlock(Type type) throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); } if (inDictionaryStream == null) { - dataStream.nextIntVector(nextBatchSize, idsVector, 0); + dataStream.next(idsVector, nextBatchSize); } else { for (int i = 0; i < nextBatchSize; i++) { @@ -202,7 +208,7 @@ private void setDictionaryBlockData(byte[] dictionaryData, int[] dictionaryOffse } } - private void openRowGroup(Type type) + private void openRowGroup() throws IOException { // read the dictionary @@ -211,6 +217,7 @@ private void openRowGroup(Type type) // resize the dictionary lengths array if necessary if (stripeDictionaryLength.length < stripeDictionarySize) { stripeDictionaryLength = new int[stripeDictionarySize]; + systemMemoryContext.setBytes(sizeOf(stripeDictionaryLength)); } // read the lengths @@ -218,7 +225,7 @@ private void openRowGroup(Type type) if (lengthStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Dictionary is not empty but dictionary length stream is not present"); } - lengthStream.nextIntVector(stripeDictionarySize, stripeDictionaryLength, 0); + lengthStream.next(stripeDictionaryLength, stripeDictionarySize); long dataLength = 0; for (int i = 0; i < stripeDictionarySize; i++) { @@ -227,12 +234,15 @@ private void openRowGroup(Type type) // we must always create a new dictionary array because the previous dictionary may still be referenced stripeDictionaryData = new byte[toIntExact(dataLength)]; + systemMemoryContext.setBytes(sizeOf(stripeDictionaryData)); + // add one extra entry for null stripeDictionaryOffsetVector = new int[stripeDictionarySize + 2]; + systemMemoryContext.setBytes(sizeOf(stripeDictionaryOffsetVector)); // read dictionary values ByteArrayInputStream dictionaryDataStream = stripeDictionaryDataStreamSource.openStream(); - readDictionary(dictionaryDataStream, stripeDictionarySize, stripeDictionaryLength, 0, stripeDictionaryData, stripeDictionaryOffsetVector, type); + readDictionary(dictionaryDataStream, stripeDictionarySize, stripeDictionaryLength, 0, stripeDictionaryData, stripeDictionaryOffsetVector, maxCodePointCount, isCharType); } else { stripeDictionaryData = EMPTY_DICTIONARY_DATA; @@ -252,7 +262,7 @@ private void openRowGroup(Type type) } // read the lengths - dictionaryLengthStream.nextIntVector(rowGroupDictionarySize, rowGroupDictionaryLength, 0); + dictionaryLengthStream.next(rowGroupDictionaryLength, rowGroupDictionarySize); long dataLength = 0; for (int i = 0; i < rowGroupDictionarySize; i++) { dataLength += rowGroupDictionaryLength[i]; @@ -265,7 +275,7 @@ private void openRowGroup(Type type) // read dictionary values ByteArrayInputStream dictionaryDataStream = rowGroupDictionaryDataStreamSource.openStream(); - readDictionary(dictionaryDataStream, rowGroupDictionarySize, rowGroupDictionaryLength, stripeDictionarySize, rowGroupDictionaryData, rowGroupDictionaryOffsetVector, type); + readDictionary(dictionaryDataStream, rowGroupDictionarySize, rowGroupDictionaryLength, stripeDictionarySize, rowGroupDictionaryData, rowGroupDictionaryOffsetVector, maxCodePointCount, isCharType); setDictionaryBlockData(rowGroupDictionaryData, rowGroupDictionaryOffsetVector, stripeDictionarySize + rowGroupDictionarySize + 1); } else { @@ -288,7 +298,8 @@ private static void readDictionary( int offsetVectorOffset, byte[] data, int[] offsetVector, - Type type) + int maxCodePointCount, + boolean isCharType) throws IOException { Slice slice = wrappedBuffer(data); @@ -306,8 +317,6 @@ private static void readDictionary( int length = dictionaryLengthVector[i]; int truncatedLength; - int maxCodePointCount = getMaxCodePointCount(type); - boolean isCharType = isCharType(type); if (length > 0) { // read data without truncation dictionaryDataStream.next(data, offset, offset + length); @@ -379,9 +388,18 @@ public String toString() .toString(); } + @Override + public void close() + { + systemMemoryContext.close(); + stripeDictionaryLength = null; + stripeDictionaryData = null; + stripeDictionaryOffsetVector = null; + } + @Override public long getRetainedSizeInBytes() { - return INSTANCE_SIZE; + return INSTANCE_SIZE + sizeOf(stripeDictionaryLength); } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDictionarySelectiveReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDictionarySelectiveReader.java new file mode 100644 index 0000000000000..b06ac8550ae10 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDictionarySelectiveReader.java @@ -0,0 +1,642 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.orc.OrcCorruptionException; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.metadata.ColumnEncoding; +import com.facebook.presto.orc.metadata.OrcType; +import com.facebook.presto.orc.stream.BooleanInputStream; +import com.facebook.presto.orc.stream.ByteArrayInputStream; +import com.facebook.presto.orc.stream.InputStreamSource; +import com.facebook.presto.orc.stream.InputStreamSources; +import com.facebook.presto.orc.stream.LongInputStream; +import com.facebook.presto.orc.stream.RowGroupDictionaryLengthInputStream; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.block.ClosingBlockLease; +import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.block.VariableWidthBlock; +import com.facebook.presto.spi.type.Chars; +import com.facebook.presto.spi.type.Type; +import io.airlift.slice.Slice; +import org.openjdk.jol.info.ClassLayout; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.orc.metadata.OrcType.OrcTypeKind.CHAR; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.DICTIONARY_DATA; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.IN_DICTIONARY; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.LENGTH; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.ROW_GROUP_DICTIONARY; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.ROW_GROUP_DICTIONARY_LENGTH; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; +import static com.facebook.presto.orc.reader.SliceSelectiveStreamReader.computeTruncatedLength; +import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Verify.verify; +import static io.airlift.slice.SizeOf.sizeOf; +import static io.airlift.slice.Slices.wrappedBuffer; +import static java.lang.Math.toIntExact; +import static java.util.Arrays.fill; +import static java.util.Objects.requireNonNull; + +public class SliceDictionarySelectiveReader + implements SelectiveStreamReader +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceDictionarySelectiveReader.class).instanceSize(); + + // filter evaluation states, using byte constants instead of enum as its memory efficient + private static final byte FILTER_NOT_EVALUATED = 0; + private static final byte FILTER_PASSED = 1; + private static final byte FILTER_FAILED = 2; + + private static final byte[] EMPTY_DICTIONARY_DATA = new byte[0]; + // add one extra entry for null after stripe/rowGroup dictionary + private static final int[] EMPTY_DICTIONARY_OFFSETS = new int[2]; + + private final TupleDomainFilter filter; + private final boolean nonDeterministicFilter; + private final boolean nullsAllowed; + private final Type outputType; + private final boolean outputRequired; + private final StreamDescriptor streamDescriptor; + private final int maxCodePointCount; + private final boolean isCharType; + + private byte[] stripeDictionaryData = EMPTY_DICTIONARY_DATA; + private int[] stripeDictionaryOffsetVector = EMPTY_DICTIONARY_OFFSETS; + private byte[] currentDictionaryData = EMPTY_DICTIONARY_DATA; + private int[] stripeDictionaryLength = new int[0]; + private int[] rowGroupDictionaryLength = new int[0]; + private byte[] evaluationStatus; + + private int readOffset; + + private VariableWidthBlock dictionary = new VariableWidthBlock(1, wrappedBuffer(EMPTY_DICTIONARY_DATA), EMPTY_DICTIONARY_OFFSETS, Optional.of(new boolean[] {true})); + + private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); + private BooleanInputStream presentStream; + + private BooleanInputStream inDictionaryStream; + + private InputStreamSource stripeDictionaryDataStreamSource = missingStreamSource(ByteArrayInputStream.class); + private InputStreamSource stripeDictionaryLengthStreamSource = missingStreamSource(LongInputStream.class); + private boolean stripeDictionaryOpen; + private int stripeDictionarySize; + + private InputStreamSource rowGroupDictionaryDataStreamSource = missingStreamSource(ByteArrayInputStream.class); + private InputStreamSource inDictionaryStreamSource = missingStreamSource(BooleanInputStream.class); + private InputStreamSource rowGroupDictionaryLengthStreamSource = missingStreamSource(RowGroupDictionaryLengthInputStream.class); + + private InputStreamSource dataStreamSource = missingStreamSource(LongInputStream.class); + private LongInputStream dataStream; + + private boolean rowGroupOpen; + private LocalMemoryContext systemMemoryContext; + + private int[] values; + private boolean allNulls; + private int[] outputPositions; + private int outputPositionCount; + private boolean valuesInUse; + + public SliceDictionarySelectiveReader(StreamDescriptor streamDescriptor, Optional filter, Optional outputType, LocalMemoryContext systemMemoryContext) + { + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + this.filter = requireNonNull(filter, "filter is null").orElse(null); + this.systemMemoryContext = systemMemoryContext; + this.nonDeterministicFilter = this.filter != null && !this.filter.isDeterministic(); + this.nullsAllowed = this.filter == null || nonDeterministicFilter || this.filter.testNull(); + this.outputType = requireNonNull(outputType, "outputType is null").orElse(null); + OrcType orcType = streamDescriptor.getOrcType(); + this.maxCodePointCount = orcType == null ? 0 : orcType.getLength().orElse(-1); + this.isCharType = orcType.getOrcTypeKind() == CHAR; + this.outputRequired = outputType.isPresent(); + checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false"); + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (!rowGroupOpen) { + openRowGroup(); + } + + allNulls = false; + + if (outputRequired) { + values = ensureCapacity(values, positionCount); + } + + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); + + systemMemoryContext.setBytes(getRetainedSizeInBytes()); + + if (readOffset < offset) { + skip(offset - readOffset); + } + + outputPositionCount = 0; + int streamPosition; + + if (dataStream == null && presentStream != null) { + streamPosition = readAllNulls(positions, positionCount); + } + else if (filter == null) { + streamPosition = readNoFilter(positions, positionCount); + } + else { + streamPosition = readWithFilter(positions, positionCount); + } + + readOffset = offset + streamPosition; + + return outputPositionCount; + } + + private int readNoFilter(int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + values[i] = dictionary.getPositionCount() - 1; + } + else { + boolean isInRowDictionary = inDictionaryStream != null && !inDictionaryStream.nextBit(); + int index = toIntExact(dataStream.next()); + values[i] = isInRowDictionary ? stripeDictionarySize + index : index; + } + streamPosition++; + } + outputPositionCount = positionCount; + return streamPosition; + } + + private int readWithFilter(int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + if ((nonDeterministicFilter && filter.testNull()) || nullsAllowed) { + if (outputRequired) { + values[outputPositionCount] = dictionary.getPositionCount() - 1; + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + else { + boolean inRowDictionary = inDictionaryStream != null && !inDictionaryStream.nextBit(); + int rawIndex = toIntExact(dataStream.next()); + int index = inRowDictionary ? stripeDictionarySize + rawIndex : rawIndex; + int length; + if (isCharType) { + length = maxCodePointCount; + } + else { + length = inRowDictionary ? rowGroupDictionaryLength[rawIndex] : stripeDictionaryLength[rawIndex]; + } + if (nonDeterministicFilter) { + evaluateFilter(position, index, length); + } + else { + switch (evaluationStatus[index]) { + case FILTER_FAILED: { + break; + } + case FILTER_PASSED: { + if (outputRequired) { + values[outputPositionCount] = index; + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + break; + } + case FILTER_NOT_EVALUATED: { + evaluationStatus[index] = evaluateFilter(position, index, length); + break; + } + default: { + throw new IllegalStateException("invalid evaluation state"); + } + } + } + } + streamPosition++; + + if (filter != null) { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + int succeedingPositionsToFail = filter.getSucceedingPositionsToFail(); + if (succeedingPositionsToFail > 0) { + int positionsToSkip = 0; + for (int j = 0; j < succeedingPositionsToFail; j++) { + i++; + int nextPosition = positions[i]; + positionsToSkip += 1 + nextPosition - streamPosition; + streamPosition = nextPosition + 1; + } + skip(positionsToSkip); + } + } + } + return streamPosition; + } + + private byte evaluateFilter(int position, int index, int length) + { + if (filter.testLength(length)) { + int currentLength = dictionary.getSliceLength(index); + Slice data = dictionary.getSlice(index, 0, currentLength); + if (isCharType && length != currentLength) { + data = Chars.padSpaces(data, maxCodePointCount); + } + if (filter.testBytes(data.getBytes(), 0, length)) { + if (outputRequired) { + values[outputPositionCount] = index; + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + return FILTER_PASSED; + } + } + return FILTER_FAILED; + } + + private int readAllNulls(int[] positions, int positionCount) + throws IOException + { + presentStream.skip(positions[positionCount - 1]); + + if (nonDeterministicFilter) { + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + if (filter.testNull()) { + outputPositionCount++; + } + else { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + i += filter.getSucceedingPositionsToFail(); + } + } + } + else if (nullsAllowed) { + outputPositionCount = positionCount; + } + else { + outputPositionCount = 0; + } + + allNulls = true; + return positions[positionCount - 1] + 1; + } + + private void skip(int items) + throws IOException + { + if (presentStream != null) { + int dataToSkip = presentStream.countBitsSet(items); + if (inDictionaryStream != null) { + inDictionaryStream.skip(dataToSkip); + } + if (dataStream != null) { + dataStream.skip(dataToSkip); + } + } + else { + if (inDictionaryStream != null) { + inDictionaryStream.skip(items); + } + dataStream.skip(items); + } + } + + @Override + public int[] getReadPositions() + { + return outputPositions; + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), positionCount); + } + + if (positionCount == outputPositionCount) { + DictionaryBlock block = new DictionaryBlock(positionCount, dictionary, values); + values = null; + return block; + } + + int[] valuesCopy = new int[positionCount]; + + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + assert outputPositions[i] == nextPosition; + valuesCopy[positionIndex] = this.values[i]; + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + nextPosition = positions[positionIndex]; + } + + return new DictionaryBlock(positionCount, dictionary, valuesCopy); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return newLease(new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), positionCount)); + } + if (positionCount < outputPositionCount) { + compactValues(positions, positionCount); + } + return newLease(new DictionaryBlock(positionCount, dictionary, values)); + } + + private void compactValues(int[] positions, int positionCount) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + values[positionIndex] = values[i]; + outputPositions[positionIndex] = nextPosition; + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + nextPosition = positions[positionIndex]; + } + + outputPositionCount = positionCount; + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } + + private void openRowGroup() + throws IOException + { + // read the dictionary + if (!stripeDictionaryOpen) { + if (stripeDictionarySize > 0) { + // resize the dictionary lengths array if necessary + if (stripeDictionaryLength.length < stripeDictionarySize) { + stripeDictionaryLength = new int[stripeDictionarySize]; + } + + // read the lengths + LongInputStream lengthStream = stripeDictionaryLengthStreamSource.openStream(); + if (lengthStream == null) { + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Dictionary is not empty but dictionary length stream is not present"); + } + lengthStream.nextIntVector(stripeDictionarySize, stripeDictionaryLength, 0); + + long dataLength = 0; + for (int i = 0; i < stripeDictionarySize; i++) { + dataLength += stripeDictionaryLength[i]; + } + + // we must always create a new dictionary array because the previous dictionary may still be referenced + stripeDictionaryData = new byte[toIntExact(dataLength)]; + // add one extra entry for null + stripeDictionaryOffsetVector = new int[stripeDictionarySize + 2]; + + // read dictionary values + ByteArrayInputStream dictionaryDataStream = stripeDictionaryDataStreamSource.openStream(); + readDictionary(dictionaryDataStream, stripeDictionarySize, stripeDictionaryLength, 0, stripeDictionaryData, stripeDictionaryOffsetVector, maxCodePointCount, isCharType); + } + else { + stripeDictionaryData = EMPTY_DICTIONARY_DATA; + stripeDictionaryOffsetVector = EMPTY_DICTIONARY_OFFSETS; + } + } + stripeDictionaryOpen = true; + + // read row group dictionary + RowGroupDictionaryLengthInputStream dictionaryLengthStream = rowGroupDictionaryLengthStreamSource.openStream(); + if (dictionaryLengthStream != null) { + int rowGroupDictionarySize = dictionaryLengthStream.getEntryCount(); + + // resize the dictionary lengths array if necessary + if (rowGroupDictionaryLength.length < rowGroupDictionarySize) { + rowGroupDictionaryLength = new int[rowGroupDictionarySize]; + } + + // read the lengths + dictionaryLengthStream.nextIntVector(rowGroupDictionarySize, rowGroupDictionaryLength, 0); + long dataLength = 0; + for (int i = 0; i < rowGroupDictionarySize; i++) { + dataLength += rowGroupDictionaryLength[i]; + } + + // We must always create a new dictionary array because the previous dictionary may still be referenced + // The first elements of the dictionary are from the stripe dictionary, then the row group dictionary elements, and then a null + byte[] rowGroupDictionaryData = java.util.Arrays.copyOf(stripeDictionaryData, stripeDictionaryOffsetVector[stripeDictionarySize] + toIntExact(dataLength)); + int[] rowGroupDictionaryOffsetVector = Arrays.copyOf(stripeDictionaryOffsetVector, stripeDictionarySize + rowGroupDictionarySize + 2); + + // read dictionary values + ByteArrayInputStream dictionaryDataStream = rowGroupDictionaryDataStreamSource.openStream(); + readDictionary(dictionaryDataStream, rowGroupDictionarySize, rowGroupDictionaryLength, stripeDictionarySize, rowGroupDictionaryData, rowGroupDictionaryOffsetVector, maxCodePointCount, isCharType); + setDictionaryBlockData(rowGroupDictionaryData, rowGroupDictionaryOffsetVector, stripeDictionarySize + rowGroupDictionarySize + 1); + } + else { + // there is no row group dictionary so use the stripe dictionary + setDictionaryBlockData(stripeDictionaryData, stripeDictionaryOffsetVector, stripeDictionarySize + 1); + } + + presentStream = presentStreamSource.openStream(); + inDictionaryStream = inDictionaryStreamSource.openStream(); + dataStream = dataStreamSource.openStream(); + + rowGroupOpen = true; + } + + // Reads dictionary into data and offsetVector + private static void readDictionary( + @Nullable ByteArrayInputStream dictionaryDataStream, + int dictionarySize, + int[] dictionaryLengthVector, + int offsetVectorOffset, + byte[] data, + int[] offsetVector, + int maxCodePointCount, + boolean isCharType) + throws IOException + { + Slice slice = wrappedBuffer(data); + + // initialize the offset if necessary; + // otherwise, use the previous offset + if (offsetVectorOffset == 0) { + offsetVector[0] = 0; + } + + // truncate string and update offsets + for (int i = 0; i < dictionarySize; i++) { + int offsetIndex = offsetVectorOffset + i; + int offset = offsetVector[offsetIndex]; + int length = dictionaryLengthVector[i]; + + int truncatedLength; + if (length > 0) { + // read data without truncation + dictionaryDataStream.next(data, offset, offset + length); + // adjust offsets with truncated length + truncatedLength = computeTruncatedLength(slice, offset, length, maxCodePointCount, isCharType); + verify(truncatedLength >= 0); + } + else { + truncatedLength = 0; + } + offsetVector[offsetIndex + 1] = offsetVector[offsetIndex] + truncatedLength; + } + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + { + stripeDictionaryDataStreamSource = dictionaryStreamSources.getInputStreamSource(streamDescriptor, DICTIONARY_DATA, ByteArrayInputStream.class); + stripeDictionaryLengthStreamSource = dictionaryStreamSources.getInputStreamSource(streamDescriptor, LENGTH, LongInputStream.class); + stripeDictionarySize = encoding.get(streamDescriptor.getStreamId()) + .getColumnEncoding(streamDescriptor.getSequence()) + .getDictionarySize(); + stripeDictionaryOpen = false; + + presentStreamSource = missingStreamSource(BooleanInputStream.class); + dataStreamSource = missingStreamSource(LongInputStream.class); + + inDictionaryStreamSource = missingStreamSource(BooleanInputStream.class); + rowGroupDictionaryLengthStreamSource = missingStreamSource(RowGroupDictionaryLengthInputStream.class); + rowGroupDictionaryDataStreamSource = missingStreamSource(ByteArrayInputStream.class); + + readOffset = 0; + + presentStream = null; + inDictionaryStream = null; + dataStream = null; + + rowGroupOpen = false; + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + { + presentStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, PRESENT, BooleanInputStream.class); + dataStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, DATA, LongInputStream.class); + + // the "in dictionary" stream signals if the value is in the stripe or row group dictionary + inDictionaryStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, IN_DICTIONARY, BooleanInputStream.class); + rowGroupDictionaryLengthStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, ROW_GROUP_DICTIONARY_LENGTH, RowGroupDictionaryLengthInputStream.class); + rowGroupDictionaryDataStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, ROW_GROUP_DICTIONARY, ByteArrayInputStream.class); + + readOffset = 0; + + presentStream = null; + inDictionaryStream = null; + dataStream = null; + + rowGroupOpen = false; + } + + @Override + public void close() + { + systemMemoryContext.close(); + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + sizeOf(currentDictionaryData) + sizeOf(values) + sizeOf(outputPositions); + } + + private void setDictionaryBlockData(byte[] dictionaryData, int[] dictionaryOffsets, int positionCount) + { + verify(positionCount > 0); + // only update the block if the array changed to prevent creation of new Block objects, since + // the engine currently uses identity equality to test if dictionaries are the same + if (currentDictionaryData != dictionaryData) { + boolean[] isNullVector = new boolean[positionCount]; + isNullVector[positionCount - 1] = true; + dictionaryOffsets[positionCount] = dictionaryOffsets[positionCount - 1]; + dictionary = new VariableWidthBlock(positionCount, wrappedBuffer(dictionaryData), dictionaryOffsets, Optional.of(isNullVector)); + currentDictionaryData = dictionaryData; + evaluationStatus = ensureCapacity(evaluationStatus, positionCount - 1); + fill(evaluationStatus, 0, evaluationStatus.length, FILTER_NOT_EVALUATED); + } + } + + private BlockLease newLease(Block block) + { + valuesInUse = true; + return ClosingBlockLease.newLease(block, () -> valuesInUse = false); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDirectBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDirectBatchStreamReader.java index f4ba7a937ed77..b0f89a0c14e4c 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDirectBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDirectBatchStreamReader.java @@ -23,8 +23,8 @@ import com.facebook.presto.orc.stream.LongInputStream; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; import com.facebook.presto.spi.block.VariableWidthBlock; -import com.facebook.presto.spi.type.Type; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import io.airlift.units.DataSize; @@ -39,11 +39,11 @@ import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; import static com.facebook.presto.orc.metadata.Stream.StreamKind.LENGTH; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.ReaderUtils.convertLengthVectorToOffsetVector; +import static com.facebook.presto.orc.reader.ReaderUtils.unpackLengthNulls; import static com.facebook.presto.orc.reader.SliceBatchStreamReader.computeTruncatedLength; -import static com.facebook.presto.orc.reader.SliceBatchStreamReader.getMaxCodePointCount; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static com.facebook.presto.spi.type.Chars.isCharType; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; @@ -60,6 +60,8 @@ public class SliceDirectBatchStreamReader private static final int ONE_GIGABYTE = toIntExact(new DataSize(1, GIGABYTE).toBytes()); private final StreamDescriptor streamDescriptor; + private final int maxCodePointCount; + private final boolean isCharType; private int readOffset; private int nextBatchSize; @@ -78,8 +80,10 @@ public class SliceDirectBatchStreamReader private boolean rowGroupOpen; - public SliceDirectBatchStreamReader(StreamDescriptor streamDescriptor) + public SliceDirectBatchStreamReader(StreamDescriptor streamDescriptor, int maxCodePointCount, boolean isCharType) { + this.maxCodePointCount = maxCodePointCount; + this.isCharType = isCharType; this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); } @@ -91,7 +95,7 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -118,6 +122,17 @@ public Block readBlock(Type type) } } + if (lengthStream == null) { + if (presentStream == null) { + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is null but present stream is missing"); + } + presentStream.skip(nextBatchSize); + Block nullValueBlock = readAllNullsBlock(); + readOffset = 0; + nextBatchSize = 0; + return nullValueBlock; + } + // create new isNullVector and offsetVector for VariableWidthBlock boolean[] isNullVector = null; @@ -126,35 +141,33 @@ public Block readBlock(Type type) int[] offsetVector = new int[nextBatchSize + 1]; if (presentStream == null) { - if (lengthStream == null) { - throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but length stream is not present"); - } - lengthStream.nextIntVector(nextBatchSize, offsetVector, 0); + lengthStream.next(offsetVector, nextBatchSize); } else { isNullVector = new boolean[nextBatchSize]; - int nullValues = presentStream.getUnsetBits(nextBatchSize, isNullVector); - if (nullValues != nextBatchSize) { - if (lengthStream == null) { - throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but length stream is not present"); - } + int nullCount = presentStream.getUnsetBits(nextBatchSize, isNullVector); + if (nullCount == nextBatchSize) { + // all nulls + Block nullValueBlock = readAllNullsBlock(); + readOffset = 0; + nextBatchSize = 0; + return nullValueBlock; + } - if (nullValues == 0) { - isNullVector = null; - lengthStream.nextIntVector(nextBatchSize, offsetVector, 0); - } - else { - lengthStream.nextIntVector(nextBatchSize, offsetVector, 0, isNullVector); - } + if (nullCount == 0) { + isNullVector = null; + lengthStream.next(offsetVector, nextBatchSize); + } + else { + lengthStream.next(offsetVector, nextBatchSize - nullCount); + unpackLengthNulls(offsetVector, isNullVector, nextBatchSize - nullCount); } } // Calculate the total length for all entries. Note that the values in the offsetVector are still length values now. long totalLength = 0; for (int i = 0; i < nextBatchSize; i++) { - if (isNullVector == null || !isNullVector[i]) { - totalLength += offsetVector[i]; - } + totalLength += offsetVector[i]; } int currentBatchSize = nextBatchSize; @@ -165,48 +178,58 @@ public Block readBlock(Type type) } if (totalLength > ONE_GIGABYTE) { throw new PrestoException(GENERIC_INTERNAL_ERROR, - format("Values in column \"%s\" are too large to process for Presto. %s column values are larger than 1GB [%s]", streamDescriptor.getFieldName(), nextBatchSize, streamDescriptor.getOrcDataSourceId())); + format("Values in column \"%s\" are too large to process for Presto. %s column values are larger than 1GB [%s]", streamDescriptor.getFieldName(), currentBatchSize, streamDescriptor.getOrcDataSourceId())); } if (dataStream == null) { - throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is missing"); } // allocate enough space to read byte[] data = new byte[toIntExact(totalLength)]; Slice slice = Slices.wrappedBuffer(data); - // We do the following operations together in the for loop: - // * truncate strings - // * convert original length values in offsetVector into truncated offset values - int currentLength = offsetVector[0]; - offsetVector[0] = 0; - int maxCodePointCount = getMaxCodePointCount(type); - boolean isCharType = isCharType(type); - for (int i = 1; i <= currentBatchSize; i++) { - int nextLength = offsetVector[i]; - if (isNullVector != null && isNullVector[i - 1]) { - checkState(currentLength == 0, "Corruption in slice direct stream: length is non-zero for null entry"); - offsetVector[i] = offsetVector[i - 1]; - currentLength = nextLength; - continue; - } - int offset = offsetVector[i - 1]; + if (maxCodePointCount < 0) { + // unbounded, simply read all data in on shot + dataStream.next(data, 0, data.length); + convertLengthVectorToOffsetVector(offsetVector); + } + else { + // We do the following operations together in the for loop: + // * truncate strings + // * convert original length values in offsetVector into truncated offset values + int currentLength = offsetVector[0]; + offsetVector[0] = 0; + for (int i = 1; i <= currentBatchSize; i++) { + int nextLength = offsetVector[i]; + if (isNullVector != null && isNullVector[i - 1]) { + checkState(currentLength == 0, "Corruption in slice direct stream: length is non-zero for null entry"); + offsetVector[i] = offsetVector[i - 1]; + currentLength = nextLength; + continue; + } + int offset = offsetVector[i - 1]; - // read data without truncation - dataStream.next(data, offset, offset + currentLength); + // read data without truncation + dataStream.next(data, offset, offset + currentLength); - // adjust offsetVector with truncated length - int truncatedLength = computeTruncatedLength(slice, offset, currentLength, maxCodePointCount, isCharType); - verify(truncatedLength >= 0); - offsetVector[i] = offset + truncatedLength; + // adjust offsetVector with truncated length + int truncatedLength = computeTruncatedLength(slice, offset, currentLength, maxCodePointCount, isCharType); + verify(truncatedLength >= 0); + offsetVector[i] = offset + truncatedLength; - currentLength = nextLength; + currentLength = nextLength; + } } // this can lead to over-retention but unlikely to happen given truncation rarely happens return new VariableWidthBlock(currentBatchSize, slice, offsetVector, Optional.ofNullable(isNullVector)); } + private RunLengthEncodedBlock readAllNullsBlock() + { + return new RunLengthEncodedBlock(new VariableWidthBlock(1, EMPTY_SLICE, new int[2], Optional.of(new boolean[] {true})), nextBatchSize); + } + private void openRowGroup() throws IOException { @@ -259,6 +282,11 @@ public String toString() .toString(); } + @Override + public void close() + { + } + @Override public long getRetainedSizeInBytes() { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDirectSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDirectSelectiveStreamReader.java new file mode 100644 index 0000000000000..1ebd3dd0ce728 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceDirectSelectiveStreamReader.java @@ -0,0 +1,531 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.metadata.ColumnEncoding; +import com.facebook.presto.orc.metadata.OrcType; +import com.facebook.presto.orc.stream.BooleanInputStream; +import com.facebook.presto.orc.stream.ByteArrayInputStream; +import com.facebook.presto.orc.stream.InputStreamSource; +import com.facebook.presto.orc.stream.InputStreamSources; +import com.facebook.presto.orc.stream.LongInputStream; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.block.ClosingBlockLease; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.block.VariableWidthBlock; +import com.facebook.presto.spi.type.Type; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import io.airlift.units.DataSize; +import org.openjdk.jol.info.ClassLayout; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.LENGTH; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; +import static com.facebook.presto.orc.reader.SliceSelectiveStreamReader.computeTruncatedLength; +import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static io.airlift.slice.SizeOf.sizeOf; +import static io.airlift.units.DataSize.Unit.GIGABYTE; +import static java.lang.Math.toIntExact; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class SliceDirectSelectiveStreamReader + implements SelectiveStreamReader +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceDirectSelectiveStreamReader.class).instanceSize(); + private static final int ONE_GIGABYTE = toIntExact(new DataSize(1, GIGABYTE).toBytes()); + + private final TupleDomainFilter filter; + private final boolean nonDeterministicFilter; + private final boolean nullsAllowed; + + private final StreamDescriptor streamDescriptor; + private final boolean outputRequired; + private final Type outputType; + private final boolean isCharType; + private final int maxCodePointCount; + + private int readOffset; + + private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); + private BooleanInputStream presentStream; + private InputStreamSource dataStreamSource = missingStreamSource(ByteArrayInputStream.class); + private ByteArrayInputStream dataStream; + private InputStreamSource lengthStreamSource = missingStreamSource(LongInputStream.class); + private LongInputStream lengthStream; + + private boolean rowGroupOpen; + private LocalMemoryContext systemMemoryContext; + private boolean[] nulls; + + private int[] outputPositions; + private int outputPositionCount; + + private boolean allNulls; // true if all requested positions are null + private boolean[] isNullVector; // isNull flags for all positions up to the last positions requested in read() + private int[] lengthVector; // lengths for all positions up to the last positions requested in read() + private int lengthIndex; // index into lengthVector array + private int[] offsets; // offsets of requested positions only; specifies position boundaries for the data array + private byte[] data; // data for requested positions only + private Slice dataAsSlice; // data array wrapped in Slice + private boolean valuesInUse; + + public SliceDirectSelectiveStreamReader(StreamDescriptor streamDescriptor, Optional filter, Optional outputType, LocalMemoryContext newLocalMemoryContext) + { + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + this.filter = requireNonNull(filter, "filter is null").orElse(null); + this.systemMemoryContext = newLocalMemoryContext; + this.nonDeterministicFilter = this.filter != null && !this.filter.isDeterministic(); + this.nullsAllowed = this.filter == null || nonDeterministicFilter || this.filter.testNull(); + this.outputType = requireNonNull(outputType, "outputType is null").orElse(null); + this.outputRequired = outputType.isPresent(); + this.isCharType = streamDescriptor.getOrcType().getOrcTypeKind() == OrcType.OrcTypeKind.CHAR; + this.maxCodePointCount = streamDescriptor.getOrcType().getLength().orElse(-1); + checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false"); + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (!rowGroupOpen) { + openRowGroup(); + } + + allNulls = false; + + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); + + systemMemoryContext.setBytes(getRetainedSizeInBytes()); + + if (readOffset < offset) { + skip(offset - readOffset); + } + + prepareForNextRead(positionCount, positions); + + int streamPosition; + + if (lengthStream == null) { + streamPosition = readAllNulls(positions, positionCount); + } + else if (filter == null) { + streamPosition = readNoFilter(positions, positionCount); + } + else { + streamPosition = readWithFilter(positions, positionCount); + } + + readOffset = offset + streamPosition; + return outputPositionCount; + } + + private int readNoFilter(int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skipData(streamPosition, position - streamPosition); + streamPosition = position; + } + + int offset = offsets[i]; + if (presentStream != null && isNullVector[position]) { + if (offsets != null) { + offsets[i + 1] = offset; + } + nulls[i] = true; + } + else { + int length = lengthVector[lengthIndex]; + int truncatedLength = 0; + if (length > 0) { + dataStream.next(data, offset, offset + length); + truncatedLength = computeTruncatedLength(dataAsSlice, offset, length, maxCodePointCount, isCharType); + } + offsets[i + 1] = offset + truncatedLength; + lengthIndex++; + if (presentStream != null) { + nulls[i] = false; + } + } + streamPosition++; + } + outputPositionCount = positionCount; + return streamPosition; + } + + private int readWithFilter(int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + int dataToSkip = 0; + + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skipData(streamPosition, position - streamPosition); + streamPosition = position; + } + + int offset = outputRequired ? offsets[outputPositionCount] : 0; + if (presentStream != null && isNullVector[position]) { + if ((nonDeterministicFilter && filter.testNull()) || nullsAllowed) { + if (outputRequired) { + offsets[outputPositionCount + 1] = offset; + nulls[outputPositionCount] = true; + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + else { + int length = lengthVector[lengthIndex]; + int dataOffset = outputRequired ? offset : 0; + if (filter.testLength(length)) { + if (dataStream != null) { + dataStream.skip(dataToSkip); + dataToSkip = 0; + dataStream.next(data, dataOffset, dataOffset + length); + if (filter.testBytes(data, dataOffset, length)) { + if (outputRequired) { + int truncatedLength = computeTruncatedLength(dataAsSlice, dataOffset, length, maxCodePointCount, isCharType); + offsets[outputPositionCount + 1] = offset + truncatedLength; + if (nullsAllowed && isNullVector != null) { + nulls[outputPositionCount] = false; + } + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + else { + assert length == 0; + if (filter.testBytes("".getBytes(), 0, 0)) { + if (outputRequired) { + offsets[outputPositionCount + 1] = offset; + if (nullsAllowed && isNullVector != null) { + nulls[outputPositionCount] = false; + } + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + } + else { + dataToSkip += length; + } + lengthIndex++; + } + + streamPosition++; + + if (filter != null) { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + int succeedingPositionsToFail = filter.getSucceedingPositionsToFail(); + if (succeedingPositionsToFail > 0) { + int positionsToSkip = 0; + for (int j = 0; j < succeedingPositionsToFail; j++) { + i++; + int nextPosition = positions[i]; + positionsToSkip += 1 + nextPosition - streamPosition; + streamPosition = nextPosition + 1; + } + skipData(streamPosition, positionsToSkip); + } + } + } + if (dataToSkip > 0) { + dataStream.skip(dataToSkip); + } + return streamPosition; + } + + private int readAllNulls(int[] positions, int positionCount) + { + if (nonDeterministicFilter) { + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + if (filter.testNull()) { + outputPositionCount++; + } + else { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + i += filter.getSucceedingPositionsToFail(); + } + } + } + else if (nullsAllowed) { + outputPositionCount = positionCount; + } + else { + outputPositionCount = 0; + } + + allNulls = true; + return positions[positionCount - 1] + 1; + } + + private void skip(int items) + throws IOException + { + // in case of an empty varbinary both the presentStream and dataStream are null and only lengthStream is present. + if (dataStream == null && presentStream != null) { + presentStream.skip(items); + } + else if (presentStream != null) { + int lengthToSkip = presentStream.countBitsSet(items); + dataStream.skip(lengthStream.sum(lengthToSkip)); + } + else { + long sum = lengthStream.sum(items); + if (dataStream != null) { + dataStream.skip(sum); + } + } + } + + private void skipData(int start, int items) + throws IOException + { + int dataToSkip = 0; + for (int i = 0; i < items; i++) { + if (presentStream == null || !isNullVector[start + i]) { + dataToSkip += lengthVector[lengthIndex]; + lengthIndex++; + } + } + // in case of an empty varbinary both the presentStream and dataStream are null and only lengthStream is present. + if (dataStream != null) { + dataStream.skip(dataToSkip); + } + } + + @Override + public int[] getReadPositions() + { + return outputPositions; + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), positionCount); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + + if (positionCount != outputPositionCount) { + compactValues(positions, positionCount, includeNulls); + } + + Block block = new VariableWidthBlock(positionCount, dataAsSlice, offsets, Optional.ofNullable(includeNulls ? nulls : null)); + dataAsSlice = null; + data = null; + offsets = null; + nulls = null; + return block; + } + + private void compactValues(int[] positions, int positionCount, boolean includeNulls) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + int length = offsets[i + 1] - offsets[i]; + if (length > 0) { + System.arraycopy(data, offsets[i], data, offsets[positionIndex], length); + } + offsets[positionIndex + 1] = offsets[positionIndex] + length; + outputPositions[positionIndex] = nextPosition; + + if (includeNulls) { + nulls[positionIndex] = nulls[i]; + } + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + nextPosition = positions[positionIndex]; + } + + outputPositionCount = positionCount; + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return newLease(new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), positionCount)); + } + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount != outputPositionCount) { + compactValues(positions, positionCount, includeNulls); + } + return newLease(new VariableWidthBlock(positionCount, dataAsSlice, offsets, Optional.ofNullable(includeNulls ? nulls : null))); + } + + private BlockLease newLease(Block block) + { + valuesInUse = true; + return ClosingBlockLease.newLease(block, () -> valuesInUse = false); + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } + + @Override + public void close() + { + systemMemoryContext.close(); + } + + private void openRowGroup() + throws IOException + { + presentStream = presentStreamSource.openStream(); + lengthStream = lengthStreamSource.openStream(); + dataStream = dataStreamSource.openStream(); + + rowGroupOpen = true; + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + { + presentStreamSource = missingStreamSource(BooleanInputStream.class); + lengthStreamSource = missingStreamSource(LongInputStream.class); + dataStreamSource = missingStreamSource(ByteArrayInputStream.class); + + readOffset = 0; + + presentStream = null; + lengthStream = null; + dataStream = null; + + rowGroupOpen = false; + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + { + presentStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, PRESENT, BooleanInputStream.class); + lengthStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, LENGTH, LongInputStream.class); + dataStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, DATA, ByteArrayInputStream.class); + + readOffset = 0; + + presentStream = null; + lengthStream = null; + dataStream = null; + + rowGroupOpen = false; + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + sizeOf(offsets) + sizeOf(outputPositions) + sizeOf(data) + sizeOf(nulls) + sizeOf(lengthVector) + sizeOf(isNullVector); + } + + private void prepareForNextRead(int positionCount, int[] positions) + throws IOException + { + lengthIndex = 0; + outputPositionCount = 0; + + int totalLength = 0; + int maxLength = 0; + + int totalPositions = positions[positionCount - 1] + 1; + int nullCount = 0; + if (presentStream != null) { + isNullVector = ensureCapacity(isNullVector, totalPositions); + nullCount = presentStream.getUnsetBits(totalPositions, isNullVector); + } + + if (lengthStream != null) { + int nonNullCount = totalPositions - nullCount; + lengthVector = ensureCapacity(lengthVector, nonNullCount); + lengthStream.nextIntVector(nonNullCount, lengthVector, 0); + + //TODO calculate totalLength for only requested positions + for (int i = 0; i < nonNullCount; i++) { + totalLength += lengthVector[i]; + maxLength = Math.max(maxLength, lengthVector[i]); + } + + if (totalLength > ONE_GIGABYTE) { + throw new PrestoException( + GENERIC_INTERNAL_ERROR, + format("Values in column \"%s\" are too large to process for Presto. %s column values are larger than 1GB [%s]", + streamDescriptor.getFieldName(), positionCount, + streamDescriptor.getOrcDataSourceId())); + } + } + + if (outputRequired) { + if (presentStream != null && nullsAllowed) { + nulls = ensureCapacity(nulls, positionCount); + } + data = ensureCapacity(data, totalLength); + offsets = ensureCapacity(offsets, totalPositions + 1); + } + else { + data = ensureCapacity(data, maxLength); + } + + dataAsSlice = Slices.wrappedBuffer(data); + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceSelectiveStreamReader.java new file mode 100644 index 0000000000000..265f0c7a9e19a --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/SliceSelectiveStreamReader.java @@ -0,0 +1,154 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.AggregatedMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.metadata.ColumnEncoding; +import com.facebook.presto.orc.stream.InputStreamSources; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.type.Chars; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.Varchars; +import com.google.common.io.Closer; +import io.airlift.slice.Slice; +import org.openjdk.jol.info.ClassLayout; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class SliceSelectiveStreamReader + implements SelectiveStreamReader +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceSelectiveStreamReader.class).instanceSize(); + + private final StreamDescriptor streamDescriptor; + private final SliceDirectSelectiveStreamReader directReader; + private final SliceDictionarySelectiveReader dictionaryReader; + private SelectiveStreamReader currentReader; + + public SliceSelectiveStreamReader(StreamDescriptor streamDescriptor, Optional filter, Optional outputType, AggregatedMemoryContext systemMemoryContext) + { + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + this.directReader = new SliceDirectSelectiveStreamReader(streamDescriptor, filter, outputType, systemMemoryContext.newLocalMemoryContext(SliceDirectSelectiveStreamReader.class.getSimpleName())); + this.dictionaryReader = new SliceDictionarySelectiveReader(streamDescriptor, filter, outputType, systemMemoryContext.newLocalMemoryContext(SliceDictionarySelectiveReader.class.getSimpleName())); + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + throws IOException + { + ColumnEncoding.ColumnEncodingKind kind = encoding.get(streamDescriptor.getStreamId()) + .getColumnEncoding(streamDescriptor.getSequence()) + .getColumnEncodingKind(); + switch (kind) { + case DIRECT: + case DIRECT_V2: + case DWRF_DIRECT: + currentReader = directReader; + break; + case DICTIONARY: + case DICTIONARY_V2: + currentReader = dictionaryReader; + break; + default: + throw new IllegalArgumentException("Unsupported encoding " + kind); + } + + currentReader.startStripe(dictionaryStreamSources, encoding); + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + throws IOException + { + currentReader.startRowGroup(dataStreamSources); + } + + @Override + public String toString() + { + return toStringHelper(this) + .addValue(streamDescriptor) + .toString(); + } + + @Override + public void close() + { + try (Closer closer = Closer.create()) { + closer.register(directReader::close); + closer.register(dictionaryReader::close); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + directReader.getRetainedSizeInBytes() + dictionaryReader.getRetainedSizeInBytes(); + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + return currentReader.read(offset, positions, positionCount); + } + + @Override + public int[] getReadPositions() + { + return currentReader.getReadPositions(); + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + return currentReader.getBlock(positions, positionCount); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + return currentReader.getBlockView(positions, positionCount); + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + currentReader.throwAnyError(positions, positionCount); + } + + public static int computeTruncatedLength(Slice slice, int offset, int length, int maxCodePointCount, boolean isCharType) + { + if (isCharType) { + // truncate the characters and then remove the trailing white spaces + return Chars.byteCountWithoutTrailingSpace(slice, offset, length, maxCodePointCount); + } + if (maxCodePointCount >= 0 && length > maxCodePointCount) { + return Varchars.byteCount(slice, offset, length, maxCodePointCount); + } + return length; + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/StreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/StreamReader.java index 0746cae305fa8..c20a535bc4272 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/StreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/StreamReader.java @@ -27,5 +27,7 @@ void startStripe(InputStreamSources dictionaryStreamSources, List structFields; + private final RowType type; + private final List fieldNames; private int readOffset; private int nextBatchSize; @@ -61,11 +71,31 @@ public class StructBatchStreamReader private boolean rowGroupOpen; - StructBatchStreamReader(StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone) + StructBatchStreamReader(Type type, StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone, AggregatedMemoryContext systemMemoryContext) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, RowType.class::isInstance); + this.type = (RowType) type; this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); - this.structFields = streamDescriptor.getNestedStreams().stream() - .collect(toImmutableMap(stream -> stream.getFieldName().toLowerCase(Locale.ENGLISH), stream -> createStreamReader(stream, hiveStorageTimeZone))); + + Map nestedStreams = Maps.uniqueIndex( + streamDescriptor.getNestedStreams(), stream -> stream.getFieldName().toLowerCase(Locale.ENGLISH)); + ImmutableList.Builder fieldNames = ImmutableList.builder(); + ImmutableMap.Builder structFields = ImmutableMap.builder(); + for (Field field : this.type.getFields()) { + String fieldName = field.getName() + .orElseThrow(() -> new IllegalArgumentException("ROW type does not have field names declared: " + type)) + .toLowerCase(Locale.ENGLISH); + fieldNames.add(fieldName); + + StreamDescriptor fieldStream = nestedStreams.get(fieldName); + if (fieldStream != null) { + structFields.put(fieldName, createStreamReader(field.getType(), fieldStream, hiveStorageTimeZone, systemMemoryContext)); + } + } + this.fieldNames = fieldNames.build(); + this.structFields = structFields.build(); } @Override @@ -76,7 +106,7 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -98,13 +128,13 @@ public Block readBlock(Type type) Block[] blocks; if (presentStream == null) { - blocks = getBlocksForType(type, nextBatchSize); + blocks = getBlocksForType(nextBatchSize); } else { nullVector = new boolean[nextBatchSize]; int nullValues = presentStream.getUnsetBits(nextBatchSize, nullVector); if (nullValues != nextBatchSize) { - blocks = getBlocksForType(type, nextBatchSize - nullValues); + blocks = getBlocksForType(nextBatchSize - nullValues); } else { List typeParameters = type.getTypeParameters(); @@ -181,40 +211,36 @@ public String toString() .toString(); } - private Block[] getBlocksForType(Type type, int positionCount) + private Block[] getBlocksForType(int positionCount) throws IOException { - RowType rowType = (RowType) type; + Block[] blocks = new Block[fieldNames.size()]; - Block[] blocks = new Block[rowType.getFields().size()]; - - for (int i = 0; i < rowType.getFields().size(); i++) { - Optional fieldName = rowType.getFields().get(i).getName(); - Type fieldType = rowType.getFields().get(i).getType(); - - if (!fieldName.isPresent()) { - throw new IllegalArgumentException("Missing struct field name in type " + rowType); - } - - String lowerCaseFieldName = fieldName.get().toLowerCase(Locale.ENGLISH); - BatchStreamReader streamReader = structFields.get(lowerCaseFieldName); + for (int i = 0; i < fieldNames.size(); i++) { + String fieldName = fieldNames.get(i); + BatchStreamReader streamReader = structFields.get(fieldName); if (streamReader != null) { streamReader.prepareNextRead(positionCount); - blocks[i] = streamReader.readBlock(fieldType); + blocks[i] = streamReader.readBlock(); } else { - blocks[i] = getNullBlock(fieldType, positionCount); + blocks[i] = RunLengthEncodedBlock.create(type.getFields().get(i).getType(), null, positionCount); } } return blocks; } - private static Block getNullBlock(Type type, int positionCount) + @Override + public void close() { - Block nullValueBlock = type.createBlockBuilder(null, 1) - .appendNull() - .build(); - return new RunLengthEncodedBlock(nullValueBlock, positionCount); + try (Closer closer = Closer.create()) { + for (BatchStreamReader batchStreamReader : structFields.values()) { + closer.register(batchStreamReader::close); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } } @Override diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/StructSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/StructSelectiveStreamReader.java new file mode 100644 index 0000000000000..b8e9229a3fac0 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/StructSelectiveStreamReader.java @@ -0,0 +1,752 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.AggregatedMemoryContext; +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.metadata.ColumnEncoding; +import com.facebook.presto.orc.stream.BooleanInputStream; +import com.facebook.presto.orc.stream.InputStreamSource; +import com.facebook.presto.orc.stream.InputStreamSources; +import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.block.ClosingBlockLease; +import com.facebook.presto.spi.block.RowBlock; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import org.joda.time.DateTimeZone; +import org.openjdk.jol.info.ClassLayout; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NOT_NULL; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NULL; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; +import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.util.Objects.requireNonNull; + +public class StructSelectiveStreamReader + implements SelectiveStreamReader +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(StructSelectiveStreamReader.class).instanceSize(); + + private final StreamDescriptor streamDescriptor; + private final boolean nullsAllowed; + private final boolean nonNullsAllowed; + private final boolean outputRequired; + @Nullable + private final Type outputType; + + private final Map nestedReaders; + private final SelectiveStreamReader[] orderedNestedReaders; + private final boolean missingFieldFilterIsFalse; + + private final LocalMemoryContext systemMemoryContext; + + private int readOffset; + private int nestedReadOffset; + + private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); + @Nullable + private BooleanInputStream presentStream; + + private boolean rowGroupOpen; + private boolean[] nulls; + private int[] outputPositions; + private int outputPositionCount; + private boolean allNulls; + private int[] nestedPositions; + private int[] nestedOutputPositions; + private int nestedOutputPositionCount; + + private boolean valuesInUse; + + public StructSelectiveStreamReader( + StreamDescriptor streamDescriptor, + Map filters, + List requiredSubfields, + Optional outputType, + DateTimeZone hiveStorageTimeZone, + AggregatedMemoryContext systemMemoryContext) + { + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null").newLocalMemoryContext(StructSelectiveStreamReader.class.getSimpleName()); + this.outputRequired = requireNonNull(outputType, "outputType is null").isPresent(); + this.outputType = outputType.orElse(null); + + if (filters.isEmpty()) { + nullsAllowed = true; + nonNullsAllowed = true; + } + else { + Optional topLevelFilter = getTopLevelFilter(filters); + if (topLevelFilter.isPresent()) { + nullsAllowed = topLevelFilter.get() == IS_NULL; + nonNullsAllowed = !nullsAllowed; + } + else { + nullsAllowed = filters.values().stream().allMatch(TupleDomainFilter::testNull); + nonNullsAllowed = true; + } + } + + Optional> nestedTypes = outputType.map(type -> type.getTypeParameters()); + List nestedStreams = streamDescriptor.getNestedStreams(); + + Optional>> requiredFields = getRequiredFields(requiredSubfields); + + // TODO streamDescriptor may be missing some fields (due to schema evolution, e.g. add field?) + // TODO fields in streamDescriptor may be out of order (due to schema evolution, e.g. remove field?) + + Set fieldsWithFilters = filters.keySet().stream() + .map(Subfield::getPath) + .filter(path -> path.size() > 0) + .map(path -> path.get(0)) + .filter(Subfield.NestedField.class::isInstance) + .map(Subfield.NestedField.class::cast) + .map(Subfield.NestedField::getName) + .collect(toImmutableSet()); + + if (!checkMissingFieldFilters(nestedStreams, filters)) { + this.missingFieldFilterIsFalse = true; + this.nestedReaders = ImmutableMap.of(); + this.orderedNestedReaders = new SelectiveStreamReader[0]; + } + else if (outputRequired || !fieldsWithFilters.isEmpty()) { + ImmutableMap.Builder nestedReaders = ImmutableMap.builder(); + for (int i = 0; i < nestedStreams.size(); i++) { + StreamDescriptor nestedStream = nestedStreams.get(i); + String fieldName = nestedStream.getFieldName().toLowerCase(Locale.ENGLISH); + Optional fieldOutputType = nestedTypes.isPresent() ? Optional.of(nestedTypes.get().get(i)) : Optional.empty(); + boolean requiredField = requiredFields.map(names -> names.containsKey(fieldName)).orElse(true); + + if (requiredField || fieldsWithFilters.contains(fieldName)) { + Map nestedFilters = filters.entrySet().stream() + .filter(entry -> entry.getKey().getPath().size() > 0) + .filter(entry -> ((Subfield.NestedField) entry.getKey().getPath().get(0)).getName().equalsIgnoreCase(fieldName)) + .collect(toImmutableMap(entry -> entry.getKey().tail(fieldName), Map.Entry::getValue)); + List nestedRequiredSubfields = requiredFields.map(names -> names.get(fieldName)).orElse(ImmutableList.of()); + SelectiveStreamReader nestedReader = SelectiveStreamReaders.createStreamReader( + nestedStream, + nestedFilters, + fieldOutputType, + nestedRequiredSubfields, + hiveStorageTimeZone, + systemMemoryContext.newAggregatedMemoryContext()); + nestedReaders.put(fieldName, nestedReader); + } + else { + nestedReaders.put(fieldName, new PruningStreamReader(nestedStream, fieldOutputType)); + } + } + + this.missingFieldFilterIsFalse = false; + this.nestedReaders = nestedReaders.build(); + this.orderedNestedReaders = orderNestedReaders(this.nestedReaders, fieldsWithFilters); + } + else { + // No need to read the elements when output is not required and the filter is a simple IS [NOT] NULL + this.missingFieldFilterIsFalse = false; + this.nestedReaders = ImmutableMap.of(); + this.orderedNestedReaders = new SelectiveStreamReader[0]; + } + } + + private boolean checkMissingFieldFilters(List nestedStreams, Map filters) + { + if (filters.isEmpty()) { + return true; + } + + Set presentFieldNames = nestedStreams.stream() + .map(StreamDescriptor::getFieldName) + .map(name -> name.toLowerCase(Locale.ENGLISH)) + .collect(toImmutableSet()); + + for (Map.Entry entry : filters.entrySet()) { + Subfield subfield = entry.getKey(); + if (subfield.getPath().isEmpty()) { + continue; + } + + String fieldName = ((Subfield.NestedField) subfield.getPath().get(0)).getName(); + if (presentFieldNames.contains(fieldName)) { + continue; + } + + // Check out the filter. If filter allows nulls, then all rows pass, otherwise, no row passes. + TupleDomainFilter filter = entry.getValue(); + checkArgument(filter.isDeterministic(), "Non-deterministic range filters are not supported yet"); + + if (!filter.testNull()) { + return false; + } + } + + return true; + } + + private static SelectiveStreamReader[] orderNestedReaders(Map nestedReaders, Set fieldsWithFilters) + { + SelectiveStreamReader[] order = new SelectiveStreamReader[nestedReaders.size()]; + + int index = 0; + for (Map.Entry entry : nestedReaders.entrySet()) { + if (fieldsWithFilters.contains(entry.getKey())) { + order[index++] = entry.getValue(); + } + } + for (Map.Entry entry : nestedReaders.entrySet()) { + if (!fieldsWithFilters.contains(entry.getKey())) { + order[index++] = entry.getValue(); + } + } + + return order; + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + checkArgument(positionCount > 0, "positionCount must be greater than zero"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (missingFieldFilterIsFalse) { + outputPositionCount = 0; + return 0; + } + + if (!rowGroupOpen) { + openRowGroup(); + } + + allNulls = false; + + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); + + systemMemoryContext.setBytes(getRetainedSizeInBytes()); + + if (presentStream == null) { + // no nulls + if (nonNullsAllowed) { + if (nestedReaders.isEmpty()) { + outputPositionCount = positionCount; + } + else { + readNestedStreams(offset, positions, positionCount); + if (nestedOutputPositionCount > 0) { + outputPositions = initializeOutputPositions(outputPositions, nestedOutputPositions, positionCount); + } + outputPositionCount = nestedOutputPositionCount; + } + readOffset = offset + positions[positionCount - 1]; + } + else { + outputPositionCount = 0; + } + } + else { + // some or all nulls + if (readOffset < offset) { + nestedReadOffset += presentStream.countBitsSet(offset - readOffset); + } + + nulls = ensureCapacity(nulls, positionCount); + nestedPositions = ensureCapacity(nestedPositions, positionCount); + outputPositionCount = 0; + + int streamPosition = 0; + int nestedPositionCount = 0; + int nullCount = 0; + + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + int nonNullCount = presentStream.countBitsSet(position - streamPosition); + nullCount += position - streamPosition - nonNullCount; + streamPosition = position; + } + + streamPosition++; + + if (presentStream.nextBit()) { + // not null + if (nonNullsAllowed) { + nulls[outputPositionCount] = false; + if (!nullsAllowed) { + outputPositions[outputPositionCount] = position; + } + outputPositionCount++; + nestedPositions[nestedPositionCount++] = position - nullCount; + } + } + else { + // null + if (nullsAllowed) { + nulls[outputPositionCount] = true; + if (!nonNullsAllowed) { + outputPositions[outputPositionCount] = position; + } + outputPositionCount++; + } + nullCount++; + } + } + + if (!nestedReaders.isEmpty()) { + if (nestedPositionCount == 0) { + allNulls = true; + } + else { + readNestedStreams(nestedReadOffset, nestedPositions, nestedPositionCount); + pruneOutputPositions(nestedPositionCount); + } + nestedReadOffset += streamPosition - nullCount; + } + + readOffset = offset + streamPosition; + } + + return outputPositionCount; + } + + private void pruneOutputPositions(int nestedPositionCount) + { + if (nestedOutputPositionCount == 0) { + allNulls = true; + } + + if (nestedOutputPositionCount < nestedPositionCount) { + int nestedIndex = 0; + int skipped = 0; + int nestedOutputIndex = 0; + for (int i = 0; i < outputPositionCount; i++) { + outputPositions[i - skipped] = outputPositions[i]; + if (nullsAllowed) { + nulls[i - skipped] = nulls[i]; + + if (nulls[i]) { + continue; + } + } + + if (nestedOutputIndex >= nestedOutputPositionCount) { + skipped++; + } + else if (nestedPositions[nestedIndex] < nestedOutputPositions[nestedOutputIndex]) { + skipped++; + } + else { + nestedOutputIndex++; + } + + nestedIndex++; + } + } + outputPositionCount -= nestedPositionCount - nestedOutputPositionCount; + } + + private void readNestedStreams(int offset, int[] positions, int positionCount) + throws IOException + { + int[] readPositions = positions; + int readPositionCount = positionCount; + for (SelectiveStreamReader reader : orderedNestedReaders) { + readPositionCount = reader.read(offset, readPositions, readPositionCount); + if (readPositionCount == 0) { + break; + } + + readPositions = reader.getReadPositions(); + } + + if (readPositionCount > 0) { + nestedOutputPositions = ensureCapacity(nestedOutputPositions, positionCount); + System.arraycopy(readPositions, 0, nestedOutputPositions, 0, readPositionCount); + } + nestedOutputPositionCount = readPositionCount; + } + + private void openRowGroup() + throws IOException + { + presentStream = presentStreamSource.openStream(); + + rowGroupOpen = true; + } + + @Override + public int[] getReadPositions() + { + return outputPositions; + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return createNullBlock(outputType, positionCount); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (outputPositionCount == positionCount) { + Block block = RowBlock.fromFieldBlocks(positionCount, Optional.ofNullable(includeNulls ? nulls : null), getFieldBlocks()); + nulls = null; + return block; + } + + boolean[] nullsCopy = null; + if (includeNulls) { + nullsCopy = new boolean[positionCount]; + } + + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + int nestedIndex = 0; + nestedOutputPositionCount = 0; + + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + if (!includeNulls || !nulls[i]) { + nestedIndex++; + } + continue; + } + + assert outputPositions[i] == nextPosition; + + if (!includeNulls || !nulls[i]) { + nestedOutputPositions[nestedOutputPositionCount++] = nestedOutputPositions[nestedIndex]; + nestedIndex++; + } + if (nullsCopy != null) { + nullsCopy[positionIndex] = this.nulls[i]; + } + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + + nextPosition = positions[positionIndex]; + } + + if (nestedOutputPositionCount == 0) { + return createNullBlock(outputType, positionCount); + } + + return RowBlock.fromFieldBlocks(positionCount, Optional.ofNullable(includeNulls ? nullsCopy : null), getFieldBlocks()); + } + + private Block[] getFieldBlocks() + { + Block[] blocks = new Block[nestedReaders.size()]; + int i = 0; + for (SelectiveStreamReader reader : nestedReaders.values()) { + blocks[i++] = reader.getBlock(nestedOutputPositions, nestedOutputPositionCount); + } + return blocks; + } + + private static RunLengthEncodedBlock createNullBlock(Type type, int positionCount) + { + return new RunLengthEncodedBlock(type.createBlockBuilder(null, 1).appendNull().build(), positionCount); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return newLease(createNullBlock(outputType, positionCount)); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount != outputPositionCount) { + compactValues(positions, positionCount, includeNulls); + + if (nestedOutputPositionCount == 0) { + allNulls = true; + return newLease(createNullBlock(outputType, positionCount)); + } + } + + BlockLease[] fieldBlockLeases = new BlockLease[nestedReaders.size()]; + Block[] fieldBlocks = new Block[nestedReaders.size()]; + int i = 0; + for (SelectiveStreamReader reader : nestedReaders.values()) { + fieldBlockLeases[i] = reader.getBlockView(nestedOutputPositions, nestedOutputPositionCount); + fieldBlocks[i] = fieldBlockLeases[i].get(); + i++; + } + + return newLease(RowBlock.fromFieldBlocks(positionCount, Optional.ofNullable(includeNulls ? nulls : null), fieldBlocks), fieldBlockLeases); + } + + private void compactValues(int[] positions, int positionCount, boolean compactNulls) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + int nestedIndex = 0; + nestedOutputPositionCount = 0; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + if (!compactNulls || !nulls[i]) { + nestedIndex++; + } + continue; + } + + assert outputPositions[i] == nextPosition; + + if (!compactNulls || !nulls[i]) { + nestedOutputPositions[nestedOutputPositionCount++] = nestedOutputPositions[nestedIndex]; + nestedIndex++; + } + if (compactNulls) { + nulls[positionIndex] = nulls[i]; + } + outputPositions[positionIndex] = nextPosition; + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + nextPosition = positions[positionIndex]; + } + + outputPositionCount = positionCount; + } + + private BlockLease newLease(Block block, BlockLease...fieldBlockLeases) + { + valuesInUse = true; + return ClosingBlockLease.newLease(block, () -> { + for (BlockLease lease : fieldBlockLeases) { + lease.close(); + } + valuesInUse = false; + }); + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } + + @Override + public String toString() + { + return toStringHelper(this) + .addValue(streamDescriptor) + .toString(); + } + + @Override + public void close() + { + systemMemoryContext.close(); + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + throws IOException + { + presentStreamSource = missingStreamSource(BooleanInputStream.class); + + readOffset = 0; + nestedReadOffset = 0; + + presentStream = null; + + rowGroupOpen = false; + + for (SelectiveStreamReader reader : nestedReaders.values()) { + reader.startStripe(dictionaryStreamSources, encoding); + } + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + throws IOException + { + presentStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, PRESENT, BooleanInputStream.class); + + readOffset = 0; + nestedReadOffset = 0; + + presentStream = null; + + rowGroupOpen = false; + + for (SelectiveStreamReader reader : nestedReaders.values()) { + reader.startRowGroup(dataStreamSources); + } + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + sizeOf(outputPositions) + sizeOf(nestedPositions) + sizeOf(nestedOutputPositions) + sizeOf(nulls) + + nestedReaders.values().stream() + .mapToLong(SelectiveStreamReader::getRetainedSizeInBytes) + .sum(); + } + + private static Optional getTopLevelFilter(Map filters) + { + Map topLevelFilters = Maps.filterEntries(filters, entry -> entry.getKey().getPath().isEmpty()); + if (topLevelFilters.isEmpty()) { + return Optional.empty(); + } + + checkArgument(topLevelFilters.size() == 1, "ROW column may have at most one top-level range filter"); + TupleDomainFilter filter = Iterables.getOnlyElement(topLevelFilters.values()); + checkArgument(filter == IS_NULL || filter == IS_NOT_NULL, "Top-level range filter on ROW column must be IS NULL or IS NOT NULL"); + return Optional.of(filter); + } + + private static Optional>> getRequiredFields(List requiredSubfields) + { + if (requiredSubfields.isEmpty()) { + return Optional.empty(); + } + + Map> fields = new HashMap<>(); + for (Subfield subfield : requiredSubfields) { + List path = subfield.getPath(); + String name = ((Subfield.NestedField) path.get(0)).getName().toLowerCase(Locale.ENGLISH); + fields.computeIfAbsent(name, k -> new ArrayList<>()); + if (path.size() > 1) { + fields.get(name).add(new Subfield("c", path.subList(1, path.size()))); + } + } + + return Optional.of(ImmutableMap.copyOf(fields)); + } + + private static final class PruningStreamReader + implements SelectiveStreamReader + { + private static final int INSTANCE_SIZE = ClassLayout.parseClass(PruningStreamReader.class).instanceSize(); + + private final StreamDescriptor streamDescriptor; + @Nullable + private final Type outputType; + private int[] outputPositions; + private int outputPositionCount; + + private PruningStreamReader(StreamDescriptor streamDescriptor, Optional outputType) + { + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + this.outputType = requireNonNull(outputType, "outputType is null").orElse(null); + } + + @Override + public int read(int offset, int[] positions, int positionCount) + { + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); + outputPositionCount = positionCount; + return outputPositionCount; + } + + @Override + public int[] getReadPositions() + { + return outputPositions; + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + checkState(outputType != null, "This stream reader doesn't produce output"); + return createNullBlock(outputType, positionCount); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + checkState(outputType != null, "This stream reader doesn't produce output"); + return ClosingBlockLease.newLease(createNullBlock(outputType, positionCount)); + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } + + @Override + public String toString() + { + return toStringHelper(this) + .addValue(streamDescriptor) + .toString(); + } + + @Override + public void close() + { + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + { + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + { + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + sizeOf(outputPositions); + } + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/TimestampBatchStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/TimestampBatchStreamReader.java index 10809eca1c270..6085a2951b177 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/reader/TimestampBatchStreamReader.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/TimestampBatchStreamReader.java @@ -21,8 +21,9 @@ import com.facebook.presto.orc.stream.InputStreamSources; import com.facebook.presto.orc.stream.LongInputStream; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.block.LongArrayBlock; import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import com.facebook.presto.spi.type.TimestampType; import com.facebook.presto.spi.type.Type; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -32,13 +33,16 @@ import java.io.IOException; import java.util.List; +import java.util.Optional; import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; import static com.facebook.presto.orc.metadata.Stream.StreamKind.SECONDARY; +import static com.facebook.presto.orc.reader.ApacheHiveTimestampDecoder.decodeTimestamp; +import static com.facebook.presto.orc.reader.ReaderUtils.verifyStreamType; import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Verify.verify; import static java.util.Objects.requireNonNull; public class TimestampBatchStreamReader @@ -68,8 +72,11 @@ public class TimestampBatchStreamReader private boolean rowGroupOpen; - public TimestampBatchStreamReader(StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone) + public TimestampBatchStreamReader(Type type, StreamDescriptor streamDescriptor, DateTimeZone hiveStorageTimeZone) + throws OrcCorruptionException { + requireNonNull(type, "type is null"); + verifyStreamType(streamDescriptor, type, TimestampType.class::isInstance); this.streamDescriptor = requireNonNull(streamDescriptor, "stream is null"); this.baseTimestampInSeconds = new DateTime(2015, 1, 1, 0, 0, requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null")).getMillis() / MILLIS_PER_SECOND; } @@ -82,7 +89,7 @@ public void prepareNextRead(int batchSize) } @Override - public Block readBlock(Type type) + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -97,10 +104,10 @@ public Block readBlock(Type type) } if (readOffset > 0) { if (secondsStream == null) { - throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but seconds stream is not present"); + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but seconds stream is missing"); } if (nanosStream == null) { - throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but nanos stream is not present"); + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but nanos stream is missing"); } secondsStream.skip(readOffset); @@ -108,46 +115,70 @@ public Block readBlock(Type type) } } - if (secondsStream == null && nanosStream == null && presentStream != null) { + Block block; + if (secondsStream == null && nanosStream == null) { + if (presentStream == null) { + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is null but present stream is missing"); + } presentStream.skip(nextBatchSize); - Block nullValueBlock = new RunLengthEncodedBlock( - type.createBlockBuilder(null, 1).appendNull().build(), - nextBatchSize); - readOffset = 0; - nextBatchSize = 0; - return nullValueBlock; + block = RunLengthEncodedBlock.create(TIMESTAMP, null, nextBatchSize); } - - BlockBuilder builder = type.createBlockBuilder(null, nextBatchSize); - - if (presentStream == null) { - if (secondsStream == null) { - throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but seconds stream is not present"); - } - if (nanosStream == null) { - throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but nanos stream is not present"); - } - - for (int i = 0; i < nextBatchSize; i++) { - type.writeLong(builder, decodeTimestamp(secondsStream.next(), nanosStream.next(), baseTimestampInSeconds)); - } + else if (presentStream == null) { + block = readNonNullBlock(); } else { - verify(secondsStream != null, "Value is not null but seconds stream is not present"); - verify(nanosStream != null, "Value is not null but nanos stream is not present"); - for (int i = 0; i < nextBatchSize; i++) { - if (presentStream.nextBit()) { - type.writeLong(builder, decodeTimestamp(secondsStream.next(), nanosStream.next(), baseTimestampInSeconds)); - } - else { - builder.appendNull(); - } + boolean[] isNull = new boolean[nextBatchSize]; + int nullCount = presentStream.getUnsetBits(nextBatchSize, isNull); + if (nullCount == 0) { + block = readNonNullBlock(); + } + else if (nullCount != nextBatchSize) { + block = readNullBlock(isNull); + } + else { + block = RunLengthEncodedBlock.create(TIMESTAMP, null, nextBatchSize); } } readOffset = 0; nextBatchSize = 0; - return builder.build(); + return block; + } + + private Block readNonNullBlock() + throws IOException + { + if (secondsStream == null) { + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but seconds stream is missing"); + } + if (nanosStream == null) { + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but nanos stream is missing"); + } + + long[] values = new long[nextBatchSize]; + for (int i = 0; i < nextBatchSize; i++) { + values[i] = decodeTimestamp(secondsStream.next(), nanosStream.next(), baseTimestampInSeconds); + } + return new LongArrayBlock(nextBatchSize, Optional.empty(), values); + } + + private Block readNullBlock(boolean[] isNull) + throws IOException + { + if (secondsStream == null) { + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but seconds stream is missing"); + } + if (nanosStream == null) { + throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but nanos stream is missing"); + } + + long[] values = new long[isNull.length]; + for (int i = 0; i < isNull.length; i++) { + if (!isNull[i]) { + values[i] = decodeTimestamp(secondsStream.next(), nanosStream.next(), baseTimestampInSeconds); + } + } + return new LongArrayBlock(isNull.length, Optional.of(isNull), values); } private void openRowGroup() @@ -202,38 +233,9 @@ public String toString() .toString(); } - // This comes from the Apache Hive ORC code - public static long decodeTimestamp(long seconds, long serializedNanos, long baseTimestampInSeconds) - { - long millis = (seconds + baseTimestampInSeconds) * MILLIS_PER_SECOND; - long nanos = parseNanos(serializedNanos); - if (nanos > 999999999 || nanos < 0) { - throw new IllegalArgumentException("nanos field of an encoded timestamp in ORC must be between 0 and 999999999 inclusive, got " + nanos); - } - - // the rounding error exists because java always rounds up when dividing integers - // -42001/1000 = -42; and -42001 % 1000 = -1 (+ 1000) - // to get the correct value we need - // (-42 - 1)*1000 + 999 = -42001 - // (42)*1000 + 1 = 42001 - if (millis < 0 && nanos != 0) { - millis -= 1000; - } - // Truncate nanos to millis and add to mills - return millis + (nanos / 1_000_000); - } - - // This comes from the Apache Hive ORC code - private static int parseNanos(long serialized) + @Override + public void close() { - int zeros = ((int) serialized) & 0b111; - int result = (int) (serialized >>> 3); - if (zeros != 0) { - for (int i = 0; i <= zeros; ++i) { - result *= 10; - } - } - return result; } @Override diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/reader/TimestampSelectiveStreamReader.java b/presto-orc/src/main/java/com/facebook/presto/orc/reader/TimestampSelectiveStreamReader.java new file mode 100644 index 0000000000000..f83616ac730b6 --- /dev/null +++ b/presto-orc/src/main/java/com/facebook/presto/orc/reader/TimestampSelectiveStreamReader.java @@ -0,0 +1,452 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc.reader; + +import com.facebook.presto.memory.context.LocalMemoryContext; +import com.facebook.presto.orc.StreamDescriptor; +import com.facebook.presto.orc.TupleDomainFilter; +import com.facebook.presto.orc.metadata.ColumnEncoding; +import com.facebook.presto.orc.stream.BooleanInputStream; +import com.facebook.presto.orc.stream.InputStreamSource; +import com.facebook.presto.orc.stream.InputStreamSources; +import com.facebook.presto.orc.stream.LongInputStream; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockLease; +import com.facebook.presto.spi.block.ClosingBlockLease; +import com.facebook.presto.spi.block.LongArrayBlock; +import com.facebook.presto.spi.block.RunLengthEncodedBlock; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.openjdk.jol.info.ClassLayout; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.array.Arrays.ensureCapacity; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.PRESENT; +import static com.facebook.presto.orc.metadata.Stream.StreamKind.SECONDARY; +import static com.facebook.presto.orc.reader.ApacheHiveTimestampDecoder.decodeTimestamp; +import static com.facebook.presto.orc.reader.SelectiveStreamReaders.initializeOutputPositions; +import static com.facebook.presto.orc.stream.MissingInputStreamSource.missingStreamSource; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static io.airlift.slice.SizeOf.sizeOf; +import static java.util.Objects.requireNonNull; + +public class TimestampSelectiveStreamReader + implements SelectiveStreamReader +{ + private static final int INSTANCE_SIZE = ClassLayout.parseClass(TimestampSelectiveStreamReader.class).instanceSize(); + private static final Block NULL_BLOCK = TIMESTAMP.createBlockBuilder(null, 1).appendNull().build(); + + private final StreamDescriptor streamDescriptor; + private final TupleDomainFilter filter; + private final boolean nullsAllowed; + private final boolean outputRequired; + private final LocalMemoryContext systemMemoryContext; + private final long baseTimestampInSeconds; + private final boolean nonDeterministicFilter; + + private InputStreamSource presentStreamSource = missingStreamSource(BooleanInputStream.class); + private InputStreamSource secondsStreamSource = missingStreamSource(LongInputStream.class); + private InputStreamSource nanosStreamSource = missingStreamSource(LongInputStream.class); + + @Nullable + private BooleanInputStream presentStream; + private LongInputStream secondsStream; + private LongInputStream nanosStream; + private boolean rowGroupOpen; + + private int readOffset; + @Nullable + private long[] values; + @Nullable + private boolean[] nulls; + @Nullable + private int[] outputPositions; + private int outputPositionCount; + private boolean allNulls; + private boolean valuesInUse; + + public TimestampSelectiveStreamReader( + StreamDescriptor streamDescriptor, + Optional filter, + DateTimeZone hiveStorageTimeZone, + boolean outputRequired, + LocalMemoryContext systemMemoryContext) + { + requireNonNull(filter, "filter is null"); + checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false"); + this.streamDescriptor = requireNonNull(streamDescriptor, "streamDescriptor is null"); + this.filter = filter.orElse(null); + this.outputRequired = outputRequired; + this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); + this.nonDeterministicFilter = this.filter != null && !this.filter.isDeterministic(); + this.nullsAllowed = this.filter == null || nonDeterministicFilter || this.filter.testNull(); + this.baseTimestampInSeconds = new DateTime(2015, 1, 1, 0, 0, requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null")).getMillis() / 1000; + } + + @Override + public void startStripe(InputStreamSources dictionaryStreamSources, List encoding) + { + presentStreamSource = missingStreamSource(BooleanInputStream.class); + secondsStreamSource = missingStreamSource(LongInputStream.class); + nanosStreamSource = missingStreamSource(LongInputStream.class); + readOffset = 0; + presentStream = null; + secondsStream = null; + nanosStream = null; + rowGroupOpen = false; + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) + { + presentStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, PRESENT, BooleanInputStream.class); + secondsStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, DATA, LongInputStream.class); + nanosStreamSource = dataStreamSources.getInputStreamSource(streamDescriptor, SECONDARY, LongInputStream.class); + readOffset = 0; + presentStream = null; + secondsStream = null; + nanosStream = null; + rowGroupOpen = false; + } + + @Override + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + sizeOf(values) + sizeOf(nulls) + sizeOf(outputPositions); + } + + private void openRowGroup() + throws IOException + { + presentStream = presentStreamSource.openStream(); + secondsStream = secondsStreamSource.openStream(); + nanosStream = nanosStreamSource.openStream(); + rowGroupOpen = true; + } + + @Override + public int read(int offset, int[] positions, int positionCount) + throws IOException + { + checkArgument(positionCount > 0, "positionCount must be greater than zero"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (!rowGroupOpen) { + openRowGroup(); + } + + allNulls = false; + + if (outputRequired) { + ensureValuesCapacity(positionCount, nullsAllowed && presentStream != null); + } + + outputPositions = initializeOutputPositions(outputPositions, positions, positionCount); + + // account memory used by values, nulls and outputPositions + systemMemoryContext.setBytes(getRetainedSizeInBytes()); + + if (readOffset < offset) { + skip(offset - readOffset); + } + + int streamPosition = 0; + if (secondsStream == null && nanosStream == null && presentStream != null) { + streamPosition = readAllNulls(positions, positionCount); + } + else if (filter == null) { + streamPosition = readNoFilter(positions, positionCount); + } + else { + streamPosition = readWithFilter(positions, positionCount); + } + + readOffset = offset + streamPosition; + return outputPositionCount; + } + + private int readWithFilter(int[] positions, int positionCount) + throws IOException + { + int streamPosition = 0; + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + if ((nonDeterministicFilter && filter.testNull()) || nullsAllowed) { + if (outputRequired) { + nulls[outputPositionCount] = true; + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + else { + long value = decodeTimestamp(secondsStream.next(), nanosStream.next(), baseTimestampInSeconds); + if (filter.testLong(value)) { + if (outputRequired) { + values[outputPositionCount] = value; + if (nullsAllowed && presentStream != null) { + nulls[outputPositionCount] = false; + } + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; + } + } + streamPosition++; + + if (filter != null) { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + + int succeedingPositionsToFail = filter.getSucceedingPositionsToFail(); + if (succeedingPositionsToFail > 0) { + int positionsToSkip = 0; + for (int j = 0; j < succeedingPositionsToFail; j++) { + i++; + int nextPosition = positions[i]; + positionsToSkip += 1 + nextPosition - streamPosition; + streamPosition = nextPosition + 1; + } + skip(positionsToSkip); + } + } + } + return streamPosition; + } + + private int readAllNulls(int[] positions, int positionCount) + throws IOException + { + presentStream.skip(positions[positionCount - 1]); + + if (nonDeterministicFilter) { + outputPositionCount = 0; + for (int i = 0; i < positionCount; i++) { + if (filter.testNull()) { + outputPositionCount++; + } + else { + outputPositionCount -= filter.getPrecedingPositionsToFail(); + i += filter.getSucceedingPositionsToFail(); + } + } + } + else if (nullsAllowed) { + outputPositionCount = positionCount; + } + else { + outputPositionCount = 0; + } + + allNulls = true; + return positions[positionCount - 1] + 1; + } + + private int readNoFilter(int[] positions, int positionCount) + throws IOException + { + // filter == null implies outputRequired == true + int streamPosition = 0; + for (int i = 0; i < positionCount; i++) { + int position = positions[i]; + if (position > streamPosition) { + skip(position - streamPosition); + streamPosition = position; + } + + if (presentStream != null && !presentStream.nextBit()) { + nulls[i] = true; + } + else { + values[i] = decodeTimestamp(secondsStream.next(), nanosStream.next(), baseTimestampInSeconds); + if (presentStream != null) { + nulls[i] = false; + } + } + streamPosition++; + } + outputPositionCount = positionCount; + return streamPosition; + } + + private void skip(int items) + throws IOException + { + if (secondsStream == null && nanosStream == null) { + presentStream.skip(items); + } + else if (presentStream != null) { + int dataToSkip = presentStream.countBitsSet(items); + secondsStream.skip(dataToSkip); + nanosStream.skip(dataToSkip); + } + else { + secondsStream.skip(items); + nanosStream.skip(items); + } + } + + private void ensureValuesCapacity(int capacity, boolean recordNulls) + { + values = ensureCapacity(values, capacity); + + if (recordNulls) { + nulls = ensureCapacity(nulls, capacity); + } + } + + @Override + public int[] getReadPositions() + { + return outputPositions; + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return new RunLengthEncodedBlock(NULL_BLOCK, positionCount); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount == outputPositionCount) { + Block block = new LongArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values); + nulls = null; + values = null; + return block; + } + + long[] valuesCopy = new long[positionCount]; + boolean[] nullsCopy = null; + if (includeNulls) { + nullsCopy = new boolean[positionCount]; + } + + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + valuesCopy[positionIndex] = this.values[i]; + if (nullsCopy != null) { + nullsCopy[positionIndex] = this.nulls[i]; + } + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + + nextPosition = positions[positionIndex]; + } + + return new LongArrayBlock(positionCount, Optional.ofNullable(nullsCopy), valuesCopy); + } + + @Override + public BlockLease getBlockView(int[] positions, int positionCount) + { + checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); + checkState(outputRequired, "This stream reader doesn't produce output"); + checkState(positionCount <= outputPositionCount, "Not enough values"); + checkState(!valuesInUse, "BlockLease hasn't been closed yet"); + + if (allNulls) { + return newLease(new RunLengthEncodedBlock(NULL_BLOCK, positionCount)); + } + + boolean includeNulls = nullsAllowed && presentStream != null; + if (positionCount != outputPositionCount) { + compactValues(positions, positionCount, includeNulls); + } + + return newLease(new LongArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values)); + } + + private BlockLease newLease(Block block) + { + valuesInUse = true; + return ClosingBlockLease.newLease(block, () -> valuesInUse = false); + } + + private void compactValues(int[] positions, int positionCount, boolean compactNulls) + { + int positionIndex = 0; + int nextPosition = positions[positionIndex]; + for (int i = 0; i < outputPositionCount; i++) { + if (outputPositions[i] < nextPosition) { + continue; + } + + assert outputPositions[i] == nextPosition; + + values[positionIndex] = values[i]; + if (compactNulls) { + nulls[positionIndex] = nulls[i]; + } + outputPositions[positionIndex] = nextPosition; + + positionIndex++; + if (positionIndex >= positionCount) { + break; + } + nextPosition = positions[positionIndex]; + } + + outputPositionCount = positionCount; + } + + @Override + public String toString() + { + return toStringHelper(this) + .addValue(streamDescriptor) + .toString(); + } + + @Override + public void close() + { + systemMemoryContext.close(); + } + + @Override + public void throwAnyError(int[] positions, int positionCount) + { + } +} diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/stream/BooleanInputStream.java b/presto-orc/src/main/java/com/facebook/presto/orc/stream/BooleanInputStream.java index 4665f1bbe6d40..b5ae0ebcdf9ce 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/stream/BooleanInputStream.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/stream/BooleanInputStream.java @@ -14,8 +14,6 @@ package com.facebook.presto.orc.stream; import com.facebook.presto.orc.checkpoint.BooleanStreamCheckpoint; -import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.spi.type.Type; import java.io.IOException; @@ -126,68 +124,270 @@ public int countBitsSet(int items) } /** - * Sets the vector element to true if the bit is set. + * Gets a vector of bytes set to 1 if the bit is set. */ - public int getSetBits(int batchSize, boolean[] vector) + public byte[] getSetBits(int batchSize) throws IOException { - int count = 0; - for (int i = 0; i < batchSize; i++) { - vector[i] = nextBit(); - count += vector[i] ? 1 : 0; - } - return count; + byte[] vector = new byte[batchSize]; + getSetBits(batchSize, vector); + return vector; } /** - * Sets the vector element to true if the bit is set, skipping the null values. + * Sets the vector element to 1 if the bit is set. */ - public int getSetBits(int batchSize, boolean[] vector, boolean[] isNull) + @SuppressWarnings({"PointlessBitwiseExpression", "PointlessArithmeticExpression", "UnusedAssignment"}) + public void getSetBits(int batchSize, byte[] vector) throws IOException { - int count = 0; - for (int i = 0; i < batchSize; i++) { - if (!isNull[i]) { - vector[i] = nextBit(); - count += vector[i] ? 1 : 0; + int offset = 0; + // handle the head + int count = Math.min(batchSize, bitsInData); + if (count != 0) { + int value = data >>> (8 - count); + switch (count) { + case 7: + vector[offset++] = (byte) ((value & 64) >>> 6); + case 6: + vector[offset++] = (byte) ((value & 32) >>> 5); + case 5: + vector[offset++] = (byte) ((value & 16) >>> 4); + case 4: + vector[offset++] = (byte) ((value & 8) >>> 3); + case 3: + vector[offset++] = (byte) ((value & 4) >>> 2); + case 2: + vector[offset++] = (byte) ((value & 2) >>> 1); + case 1: + vector[offset++] = (byte) ((value & 1) >>> 0); + } + data <<= count; + bitsInData -= count; + + if (count == batchSize) { + return; } } - return count; + + // the middle part + while (offset < batchSize - 7) { + byte value = byteStream.next(); + vector[offset + 0] = (byte) ((value & 128) >>> 7); + vector[offset + 1] = (byte) ((value & 64) >>> 6); + vector[offset + 2] = (byte) ((value & 32) >>> 5); + vector[offset + 3] = (byte) ((value & 16) >>> 4); + vector[offset + 4] = (byte) ((value & 8) >>> 3); + vector[offset + 5] = (byte) ((value & 4) >>> 2); + vector[offset + 6] = (byte) ((value & 2) >>> 1); + vector[offset + 7] = (byte) ((value & 1)); + offset += 8; + } + + // the tail + int remaining = batchSize - offset; + if (remaining > 0) { + byte data = byteStream.next(); + int value = data >>> (8 - remaining); + switch (remaining) { + case 7: + vector[offset++] = (byte) ((value & 64) >>> 6); + case 6: + vector[offset++] = (byte) ((value & 32) >>> 5); + case 5: + vector[offset++] = (byte) ((value & 16) >>> 4); + case 4: + vector[offset++] = (byte) ((value & 8) >>> 3); + case 3: + vector[offset++] = (byte) ((value & 4) >>> 2); + case 2: + vector[offset++] = (byte) ((value & 2) >>> 1); + case 1: + vector[offset++] = (byte) ((value & 1) >>> 0); + } + this.data = (byte) (data << remaining); + bitsInData = 8 - remaining; + } } /** * Sets the vector element to true if the bit is set. */ - public void getSetBits(Type type, int batchSize, BlockBuilder builder) + @SuppressWarnings({"PointlessBitwiseExpression", "PointlessArithmeticExpression", "UnusedAssignment"}) + public int getSetBits(int batchSize, boolean[] vector) throws IOException { - for (int i = 0; i < batchSize; i++) { - type.writeBoolean(builder, nextBit()); + int offset = 0; + int countBitsSet = 0; + // handle the head + int count = Math.min(batchSize, bitsInData); + if (count != 0) { + int value = (data & 0xFF) >>> (8 - count); + countBitsSet += Integer.bitCount(value); + switch (count) { + case 7: + vector[offset++] = ((value & 64) >>> 6) == 1; + case 6: + vector[offset++] = ((value & 32) >>> 5) == 1; + case 5: + vector[offset++] = ((value & 16) >>> 4) == 1; + case 4: + vector[offset++] = ((value & 8) >>> 3) == 1; + case 3: + vector[offset++] = ((value & 4) >>> 2) == 1; + case 2: + vector[offset++] = ((value & 2) >>> 1) == 1; + case 1: + vector[offset++] = ((value & 1) >>> 0) == 1; + } + data <<= count; + bitsInData -= count; + + if (count == batchSize) { + return countBitsSet; + } } + + // the middle part + while (offset < batchSize - 7) { + int value = byteStream.next() & 0xFF; + countBitsSet += Integer.bitCount(value); + vector[offset + 0] = ((value & 128) >>> 7) == 1; + vector[offset + 1] = ((value & 64) >>> 6) == 1; + vector[offset + 2] = ((value & 32) >>> 5) == 1; + vector[offset + 3] = ((value & 16) >>> 4) == 1; + vector[offset + 4] = ((value & 8) >>> 3) == 1; + vector[offset + 5] = ((value & 4) >>> 2) == 1; + vector[offset + 6] = ((value & 2) >>> 1) == 1; + vector[offset + 7] = ((value & 1)) == 1; + offset += 8; + } + + // the tail + int remaining = batchSize - offset; + if (remaining > 0) { + byte data = byteStream.next(); + int value = (data & 0xFF) >> (8 - remaining); + countBitsSet += Integer.bitCount(value); + switch (remaining) { + case 7: + vector[offset++] = ((value & 64) >>> 6) == 1; + case 6: + vector[offset++] = ((value & 32) >>> 5) == 1; + case 5: + vector[offset++] = ((value & 16) >>> 4) == 1; + case 4: + vector[offset++] = ((value & 8) >>> 3) == 1; + case 3: + vector[offset++] = ((value & 4) >>> 2) == 1; + case 2: + vector[offset++] = ((value & 2) >>> 1) == 1; + case 1: + vector[offset++] = ((value & 1) >>> 0) == 1; + } + this.data = (byte) (data << remaining); + bitsInData = 8 - remaining; + } + return countBitsSet; } /** - * Sets the vector element to true if the bit is not set. + * Sets the vector element to true if the bit is set, skipping the null values. */ - public int getUnsetBits(int batchSize, boolean[] vector) + public int getSetBits(int batchSize, boolean[] vector, boolean[] isNull) throws IOException { - return getUnsetBits(batchSize, vector, 0); + int count = 0; + for (int i = 0; i < batchSize; i++) { + if (!isNull[i]) { + vector[i] = nextBit(); + count += vector[i] ? 1 : 0; + } + } + return count; } /** - * Sets the vector element to true for the batchSize number of elements starting at offset - * if the bit is not set. + * Sets the vector element to true if the bit is not set. */ - public int getUnsetBits(int batchSize, boolean[] vector, int offset) + @SuppressWarnings({"PointlessArithmeticExpression", "UnusedAssignment"}) + public int getUnsetBits(int batchSize, boolean[] vector) throws IOException { - int count = 0; - for (int i = offset; i < batchSize + offset; i++) { - vector[i] = !nextBit(); - count += vector[i] ? 1 : 0; + int unsetCount = 0; + int offset = 0; + + // handle the head + int count = Math.min(batchSize, bitsInData); + if (count != 0) { + int value = (data & 0xFF) >>> (8 - count); + unsetCount += (count - Integer.bitCount(value)); + switch (count) { + case 7: + vector[offset++] = (value & 64) == 0; + case 6: + vector[offset++] = (value & 32) == 0; + case 5: + vector[offset++] = (value & 16) == 0; + case 4: + vector[offset++] = (value & 8) == 0; + case 3: + vector[offset++] = (value & 4) == 0; + case 2: + vector[offset++] = (value & 2) == 0; + case 1: + vector[offset++] = (value & 1) == 0; + } + data <<= count; + bitsInData -= count; + + if (count == batchSize) { + return unsetCount; + } } - return count; + + // the middle part + while (offset < batchSize - 7) { + byte value = byteStream.next(); + unsetCount += (8 - Integer.bitCount(value & 0xFF)); + vector[offset + 0] = (value & 128) == 0; + vector[offset + 1] = (value & 64) == 0; + vector[offset + 2] = (value & 32) == 0; + vector[offset + 3] = (value & 16) == 0; + vector[offset + 4] = (value & 8) == 0; + vector[offset + 5] = (value & 4) == 0; + vector[offset + 6] = (value & 2) == 0; + vector[offset + 7] = (value & 1) == 0; + offset += 8; + } + + // the tail + int remaining = batchSize - offset; + if (remaining > 0) { + byte data = byteStream.next(); + int value = (data & 0xFF) >> (8 - remaining); + unsetCount += (remaining - Integer.bitCount(value)); + switch (remaining) { + case 7: + vector[offset++] = (value & 64) == 0; + case 6: + vector[offset++] = (value & 32) == 0; + case 5: + vector[offset++] = (value & 16) == 0; + case 4: + vector[offset++] = (value & 8) == 0; + case 3: + vector[offset++] = (value & 4) == 0; + case 2: + vector[offset++] = (value & 2) == 0; + case 1: + vector[offset++] = (value & 1) == 0; + } + this.data = (byte) (data << remaining); + bitsInData = 8 - remaining; + } + + return unsetCount; } /** diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/stream/ByteInputStream.java b/presto-orc/src/main/java/com/facebook/presto/orc/stream/ByteInputStream.java index 21edc123ebedb..331c804b658f2 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/stream/ByteInputStream.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/stream/ByteInputStream.java @@ -15,12 +15,12 @@ import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.checkpoint.ByteStreamCheckpoint; -import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.spi.type.Type; import java.io.IOException; import java.util.Arrays; +import static java.lang.Math.min; + public class ByteInputStream implements ValueInputStream { @@ -104,7 +104,7 @@ public void skip(long items) if (offset == length) { readNextBlock(); } - long consume = Math.min(items, length - offset); + long consume = min(items, length - offset); offset += consume; items -= consume; } @@ -119,11 +119,31 @@ public byte next() return buffer[offset++]; } - public void nextVector(Type type, long items, BlockBuilder builder) + public byte[] next(int items) + throws IOException + { + byte[] values = new byte[items]; + next(values, items); + return values; + } + + public void next(byte[] values, int items) throws IOException { - for (int i = 0; i < items; i++) { - type.writeLong(builder, next()); + int outputOffset = 0; + while (outputOffset < items) { + if (offset == length) { + readNextBlock(); + } + if (length == 0) { + throw new OrcCorruptionException(input.getOrcDataSourceId(), "Unexpected end of stream"); + } + + int chunkSize = min(items - outputOffset, length - offset); + System.arraycopy(buffer, offset, values, outputOffset, chunkSize); + + outputOffset += chunkSize; + offset += chunkSize; } } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/stream/DoubleInputStream.java b/presto-orc/src/main/java/com/facebook/presto/orc/stream/DoubleInputStream.java index fcc14b79f4020..e6c5f9293666d 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/stream/DoubleInputStream.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/stream/DoubleInputStream.java @@ -16,8 +16,6 @@ import com.facebook.presto.orc.checkpoint.DoubleStreamCheckpoint; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.type.Type; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; import java.io.IOException; @@ -27,8 +25,6 @@ public class DoubleInputStream implements ValueInputStream { private final OrcInputStream input; - private final byte[] buffer = new byte[SIZE_OF_DOUBLE]; - private final Slice slice = Slices.wrappedBuffer(buffer); public DoubleInputStream(OrcInputStream input) { @@ -59,8 +55,7 @@ public void skip(long items) public double next() throws IOException { - input.readFully(buffer, 0, SIZE_OF_DOUBLE); - return slice.getDouble(0); + return input.readDouble(); } public void nextVector(Type type, int items, BlockBuilder builder) diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/stream/FloatInputStream.java b/presto-orc/src/main/java/com/facebook/presto/orc/stream/FloatInputStream.java index 34525389d2eb2..a8f7812606f65 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/stream/FloatInputStream.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/stream/FloatInputStream.java @@ -16,8 +16,6 @@ import com.facebook.presto.orc.checkpoint.FloatStreamCheckpoint; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.type.Type; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; import java.io.IOException; @@ -28,8 +26,6 @@ public class FloatInputStream implements ValueInputStream { private final OrcInputStream input; - private final byte[] buffer = new byte[SIZE_OF_FLOAT]; - private final Slice slice = Slices.wrappedBuffer(buffer); public FloatInputStream(OrcInputStream input) { @@ -60,8 +56,7 @@ public void skip(long items) public float next() throws IOException { - input.readFully(buffer, 0, SIZE_OF_FLOAT); - return slice.getFloat(0); + return input.readFloat(); } public void nextVector(Type type, int items, BlockBuilder builder) diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStream.java b/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStream.java index 6a6c4ac0c14e7..22d3de2bc3393 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStream.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStream.java @@ -14,8 +14,6 @@ package com.facebook.presto.orc.stream; import com.facebook.presto.orc.checkpoint.LongStreamCheckpoint; -import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.spi.type.Type; import java.io.IOException; @@ -28,6 +26,15 @@ public interface LongInputStream long next() throws IOException; + void next(long[] values, int items) + throws IOException; + + void next(int[] values, int items) + throws IOException; + + void next(short[] values, int items) + throws IOException; + default void nextIntVector(int items, int[] vector, int offset) throws IOException { @@ -38,19 +45,6 @@ default void nextIntVector(int items, int[] vector, int offset) } } - default void nextIntVector(int items, int[] vector, int vectorOffset, boolean[] isNull) - throws IOException - { - checkPositionIndex(items + vectorOffset, vector.length); - checkPositionIndex(items, isNull.length); - - for (int i = 0; i < items; i++) { - if (!isNull[i]) { - vector[i + vectorOffset] = toIntExact(next()); - } - } - } - default void nextLongVector(int items, long[] vector) throws IOException { @@ -61,14 +55,6 @@ default void nextLongVector(int items, long[] vector) } } - default void nextLongVector(Type type, int items, BlockBuilder builder) - throws IOException - { - for (int i = 0; i < items; i++) { - type.writeLong(builder, next()); - } - } - default long sum(int items) throws IOException { diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStreamDwrf.java b/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStreamDwrf.java index c6764cb65a544..5c909126e7090 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStreamDwrf.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStreamDwrf.java @@ -19,8 +19,6 @@ import java.io.IOException; -import static com.facebook.presto.orc.stream.LongDecode.readDwrfLong; - public class LongInputStreamDwrf implements LongInputStream { @@ -55,9 +53,11 @@ public void seekToCheckpoint(LongStreamCheckpoint checkpoint) public void skip(long items) throws IOException { - // there is no fast way to skip values - for (long i = 0; i < items; i++) { - next(); + if (usesVInt) { + input.skipVarints(items); + } + else { + input.skipDwrfLong(orcTypeKind, items); } } @@ -65,6 +65,36 @@ public void skip(long items) public long next() throws IOException { - return readDwrfLong(input, orcTypeKind, signed, usesVInt); + if (usesVInt) { + return input.readVarint(signed); + } + return input.readDwrfLong(orcTypeKind); + } + + @Override + public void next(long[] values, int items) + throws IOException + { + for (int i = 0; i < items; i++) { + values[i] = next(); + } + } + + @Override + public void next(int[] values, int items) + throws IOException + { + for (int i = 0; i < items; i++) { + values[i] = (int) next(); + } + } + + @Override + public void next(short[] values, int items) + throws IOException + { + for (int i = 0; i < items; i++) { + values[i] = (short) next(); + } } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStreamV1.java b/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStreamV1.java index 77598aac7db89..1941db66c7439 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStreamV1.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStreamV1.java @@ -19,41 +19,37 @@ import java.io.IOException; +import static java.lang.Math.min; + public class LongInputStreamV1 implements LongInputStream { private static final int MIN_REPEAT_SIZE = 3; - private static final int MAX_LITERAL_SIZE = 128; private final OrcInputStream input; private final boolean signed; - private final long[] literals = new long[MAX_LITERAL_SIZE]; - private int numLiterals; + private long repeatBase; + private int numValuesInRun; private int delta; private int used; private boolean repeat; - private long lastReadInputCheckpoint; public LongInputStreamV1(OrcInputStream input, boolean signed) { this.input = input; this.signed = signed; - lastReadInputCheckpoint = input.getCheckpoint(); } - // This comes from the Apache Hive ORC code - private void readValues() + private void readHeader() throws IOException { - lastReadInputCheckpoint = input.getCheckpoint(); - int control = input.read(); if (control == -1) { throw new OrcCorruptionException(input.getOrcDataSourceId(), "Read past end of RLE integer"); } if (control < 0x80) { - numLiterals = control + MIN_REPEAT_SIZE; + numValuesInRun = control + MIN_REPEAT_SIZE; used = 0; repeat = true; delta = input.read(); @@ -62,17 +58,13 @@ private void readValues() } // convert from 0 to 255 to -128 to 127 by converting to a signed byte - // noinspection SillyAssignment delta = (byte) delta; - literals[0] = LongDecode.readVInt(signed, input); + repeatBase = input.readVarint(signed); } else { - numLiterals = 0x100 - control; + numValuesInRun = 0x100 - control; used = 0; repeat = false; - for (int i = 0; i < numLiterals; ++i) { - literals[i] = LongDecode.readVInt(signed, input); - } } } @@ -82,18 +74,126 @@ public long next() throws IOException { long result; - if (used == numLiterals) { - readValues(); + if (used == numValuesInRun) { + readHeader(); } if (repeat) { - result = literals[0] + (used++) * delta; + result = repeatBase + used * delta; } else { - result = literals[used++]; + result = input.readVarint(signed); } + used++; return result; } + @Override + public void next(long[] values, int items) + throws IOException + { + int offset = 0; + while (items > 0) { + if (used == numValuesInRun) { + numValuesInRun = 0; + used = 0; + readHeader(); + } + + int chunkSize = min(numValuesInRun - used, items); + if (repeat) { + for (int i = 0; i < chunkSize; i++) { + values[offset + i] = repeatBase + ((used + i) * delta); + } + } + else { + for (int i = 0; i < chunkSize; ++i) { + values[offset + i] = input.readVarint(signed); + } + } + used += chunkSize; + offset += chunkSize; + items -= chunkSize; + } + } + + @Override + public void next(int[] values, int items) + throws IOException + { + int offset = 0; + while (items > 0) { + if (used == numValuesInRun) { + numValuesInRun = 0; + used = 0; + readHeader(); + } + + int chunkSize = min(numValuesInRun - used, items); + if (repeat) { + for (int i = 0; i < chunkSize; i++) { + long literal = repeatBase + ((used + i) * delta); + int value = (int) literal; + if (literal != value) { + throw new OrcCorruptionException(input.getOrcDataSourceId(), "Decoded value out of range for a 32bit number"); + } + values[offset + i] = value; + } + } + else { + for (int i = 0; i < chunkSize; i++) { + long literal = input.readVarint(signed); + int value = (int) literal; + if (literal != value) { + throw new OrcCorruptionException(input.getOrcDataSourceId(), "Decoded value out of range for a 32bit number"); + } + values[offset + i] = value; + } + } + used += chunkSize; + offset += chunkSize; + items -= chunkSize; + } + } + + @Override + public void next(short[] values, int items) + throws IOException + { + int offset = 0; + while (items > 0) { + if (used == numValuesInRun) { + numValuesInRun = 0; + used = 0; + readHeader(); + } + + int chunkSize = min(numValuesInRun - used, items); + if (repeat) { + for (int i = 0; i < chunkSize; i++) { + long literal = repeatBase + ((used + i) * delta); + short value = (short) literal; + if (literal != value) { + throw new OrcCorruptionException(input.getOrcDataSourceId(), "Decoded value out of range for a 16bit number"); + } + values[offset + i] = value; + } + } + else { + for (int i = 0; i < chunkSize; i++) { + long literal = input.readVarint(signed); + short value = (short) literal; + if (literal != value) { + throw new OrcCorruptionException(input.getOrcDataSourceId(), "Decoded value out of range for a 16bit number"); + } + values[offset + i] = value; + } + } + used += chunkSize; + offset += chunkSize; + items -= chunkSize; + } + } + @Override public Class getCheckpointType() { @@ -105,18 +205,11 @@ public void seekToCheckpoint(LongStreamCheckpoint checkpoint) throws IOException { LongStreamV1Checkpoint v1Checkpoint = (LongStreamV1Checkpoint) checkpoint; - - // if the checkpoint is within the current buffer, just adjust the pointer - if (lastReadInputCheckpoint == v1Checkpoint.getInputStreamCheckpoint() && v1Checkpoint.getOffset() <= numLiterals) { - used = v1Checkpoint.getOffset(); - } - else { - // otherwise, discard the buffer and start over - input.seekToCheckpoint(v1Checkpoint.getInputStreamCheckpoint()); - numLiterals = 0; - used = 0; - skip(v1Checkpoint.getOffset()); - } + // Discard the buffer and start over + input.seekToCheckpoint(v1Checkpoint.getInputStreamCheckpoint()); + numValuesInRun = 0; + used = 0; + skip(v1Checkpoint.getOffset()); } @Override @@ -124,10 +217,13 @@ public void skip(long items) throws IOException { while (items > 0) { - if (used == numLiterals) { - readValues(); + if (used == numValuesInRun) { + readHeader(); + } + long consume = Math.min(items, numValuesInRun - used); + if (!repeat) { + input.skipVarints(consume); } - long consume = Math.min(items, numLiterals - used); used += consume; items -= consume; } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStreamV2.java b/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStreamV2.java index 63d8f3480e664..f7ef658ec247d 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStreamV2.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/stream/LongInputStreamV2.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.io.InputStream; +import static java.lang.Math.min; + /** * @see {@link org.apache.hadoop.hive.ql.io.orc.RunLengthIntegerWriterV2} for description of various lightweight compression techniques. */ @@ -332,6 +334,80 @@ public long next() return literals[used++]; } + @Override + public void next(long[] values, int items) + throws IOException + { + int offset = 0; + while (items > 0) { + if (used == numLiterals) { + numLiterals = 0; + used = 0; + readValues(); + } + + int chunkSize = min(numLiterals - used, items); + System.arraycopy(literals, used, values, offset, chunkSize); + used += chunkSize; + offset += chunkSize; + items -= chunkSize; + } + } + + @Override + public void next(int[] values, int items) + throws IOException + { + int offset = 0; + while (items > 0) { + if (used == numLiterals) { + numLiterals = 0; + used = 0; + readValues(); + } + + int chunkSize = min(numLiterals - used, items); + for (int i = 0; i < chunkSize; i++) { + long literal = literals[used + i]; + int value = (int) literal; + if (literal != value) { + throw new OrcCorruptionException(input.getOrcDataSourceId(), "Decoded value out of range for a 32bit number"); + } + values[offset + i] = value; + } + used += chunkSize; + offset += chunkSize; + items -= chunkSize; + } + } + + @Override + public void next(short[] values, int items) + throws IOException + { + int offset = 0; + while (items > 0) { + if (used == numLiterals) { + numLiterals = 0; + used = 0; + readValues(); + } + + int chunkSize = min(numLiterals - used, items); + for (int i = 0; i < chunkSize; i++) { + long literal = literals[used + i]; + short value = (short) literal; + if (literal != value) { + throw new OrcCorruptionException(input.getOrcDataSourceId(), "Decoded value out of range for a 16bit number"); + } + values[offset + i] = value; + } + used += chunkSize; + offset += chunkSize; + items -= chunkSize; + } + } + @Override public Class getCheckpointType() { @@ -367,7 +443,7 @@ public void skip(long items) used = 0; readValues(); } - long consume = Math.min(items, numLiterals - used); + long consume = min(items, numLiterals - used); used += consume; items -= consume; } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/stream/OrcInputStream.java b/presto-orc/src/main/java/com/facebook/presto/orc/stream/OrcInputStream.java index d786c332c4159..407b094a8403c 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/stream/OrcInputStream.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/stream/OrcInputStream.java @@ -18,9 +18,10 @@ import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.OrcDataSourceId; import com.facebook.presto.orc.OrcDecompressor; +import com.facebook.presto.orc.metadata.OrcType.OrcTypeKind; +import io.airlift.slice.ByteArrays; import io.airlift.slice.FixedLengthSliceInput; import io.airlift.slice.Slice; -import io.airlift.slice.Slices; import java.io.IOException; import java.io.InputStream; @@ -30,8 +31,14 @@ import static com.facebook.presto.orc.checkpoint.InputStreamCheckpoint.createInputStreamCheckpoint; import static com.facebook.presto.orc.checkpoint.InputStreamCheckpoint.decodeCompressedBlockOffset; import static com.facebook.presto.orc.checkpoint.InputStreamCheckpoint.decodeDecompressedOffset; +import static com.facebook.presto.orc.stream.LongDecode.zigzagDecode; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; +import static io.airlift.slice.SizeOf.SIZE_OF_DOUBLE; +import static io.airlift.slice.SizeOf.SIZE_OF_FLOAT; +import static io.airlift.slice.SizeOf.SIZE_OF_INT; +import static io.airlift.slice.SizeOf.SIZE_OF_LONG; +import static io.airlift.slice.SizeOf.SIZE_OF_SHORT; import static io.airlift.slice.Slices.EMPTY_SLICE; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; @@ -40,14 +47,24 @@ public final class OrcInputStream extends InputStream { + private static final long VARINT_MASK = 0x8080_8080_8080_8080L; + private static final int MAX_VARINT_LENGTH = 10; + private final OrcDataSourceId orcDataSourceId; private final FixedLengthSliceInput compressedSliceInput; private final Optional decompressor; private int currentCompressedBlockOffset; - private FixedLengthSliceInput current; private byte[] buffer; + private byte[] decompressionResultBuffer; + private int position; + private int length; + private int uncompressedOffset; + + // Temporary memory for reading a float or double at buffer boundary. + private byte[] temporaryBuffer = new byte[SIZE_OF_DOUBLE]; + private final LocalMemoryContext bufferMemoryUsage; public OrcInputStream( @@ -71,12 +88,16 @@ public OrcInputStream( systemMemoryContext.newLocalMemoryContext(OrcInputStream.class.getSimpleName()).setBytes(sliceInputRetainedSizeInBytes); if (!decompressor.isPresent()) { - this.current = sliceInput; + int sliceInputPosition = toIntExact(sliceInput.position()); + int sliceInputRemaining = toIntExact(sliceInput.remaining()); + this.buffer = new byte[sliceInputRemaining]; + this.length = buffer.length; + sliceInput.readFully(buffer, sliceInputPosition, sliceInputRemaining); this.compressedSliceInput = EMPTY_SLICE.getInput(); } else { this.compressedSliceInput = sliceInput; - this.current = EMPTY_SLICE.getInput(); + this.buffer = new byte[0]; } } @@ -89,10 +110,10 @@ public void close() @Override public int available() { - if (current == null) { + if (buffer == null) { return 0; } - return current.available(); + return length - position; } @Override @@ -105,13 +126,11 @@ public boolean markSupported() public int read() throws IOException { - if (current == null) { + if (buffer == null) { return -1; } - - int result = current.read(); - if (result != -1) { - return result; + if (available() > 0) { + return 0xff & buffer[position++]; } advance(); @@ -122,18 +141,20 @@ public int read() public int read(byte[] b, int off, int length) throws IOException { - if (current == null) { + if (buffer == null) { return -1; } - if (current.remaining() == 0) { + if (available() == 0) { advance(); - if (current == null) { + if (buffer == null) { return -1; } } - - return current.read(b, off, length); + length = Math.min(length, available()); + System.arraycopy(buffer, position, b, off, length); + position += length; + return length; } public void skipFully(long length) @@ -168,11 +189,12 @@ public OrcDataSourceId getOrcDataSourceId() public long getCheckpoint() { // if the decompressed buffer is empty, return a checkpoint starting at the next block - if (current == null || (current.position() == 0 && current.remaining() == 0)) { + if (buffer == null || (position == 0 && available() == 0)) { return createInputStreamCheckpoint(toIntExact(compressedSliceInput.position()), 0); } // otherwise return a checkpoint at the last compressed block read and the current position in the buffer - return createInputStreamCheckpoint(currentCompressedBlockOffset, toIntExact(current.position())); + // If we have uncompressed data uncompressedOffset is not included in the offset. + return createInputStreamCheckpoint(currentCompressedBlockOffset, toIntExact(position - uncompressedOffset)); } public boolean seekToCheckpoint(long checkpoint) @@ -186,20 +208,27 @@ public boolean seekToCheckpoint(long checkpoint) throw new OrcCorruptionException(orcDataSourceId, "Reset stream has a compressed block offset but stream is not compressed"); } compressedSliceInput.setPosition(compressedBlockOffset); - current = EMPTY_SLICE.getInput(); + buffer = new byte[0]; + position = 0; + length = 0; + uncompressedOffset = 0; discardedBuffer = true; } else { discardedBuffer = false; } - if (decompressedOffset != current.position()) { - current.setPosition(0); - if (current.remaining() < decompressedOffset) { - decompressedOffset -= current.remaining(); + if (decompressedOffset != position - uncompressedOffset) { + position = uncompressedOffset; + if (available() < decompressedOffset) { + decompressedOffset -= available(); advance(); } - current.setPosition(decompressedOffset); + position += decompressedOffset; + } + else if (length == 0) { + advance(); + position += decompressedOffset; } return discardedBuffer; } @@ -208,18 +237,202 @@ public boolean seekToCheckpoint(long checkpoint) public long skip(long n) throws IOException { - if (current == null || n <= 0) { + if (buffer == null || n <= 0) { return -1; } - long result = current.skip(n); + long result = Math.min(available(), n); + position += toIntExact(result); if (result != 0) { return result; } if (read() == -1) { return 0; } - return 1 + current.skip(n - 1); + result = Math.min(available(), n - 1); + position += toIntExact(result); + return 1 + result; + } + + public long readDwrfLong(OrcTypeKind type) + throws IOException + { + switch (type) { + case SHORT: + return read() | (read() << 8); + case INT: + return read() | (read() << 8) | (read() << 16) | (read() << 24); + case LONG: + return ((long) read()) | + (((long) read()) << 8) | + (((long) read()) << 16) | + (((long) read()) << 24) | + (((long) read()) << 32) | + (((long) read()) << 40) | + (((long) read()) << 48) | + (((long) read()) << 56); + default: + throw new IllegalStateException(); + } + } + + public void skipDwrfLong(OrcTypeKind type, long items) + throws IOException + { + if (items == 0) { + return; + } + long bytes = items; + switch (type) { + case SHORT: + bytes *= SIZE_OF_SHORT; + break; + case INT: + bytes *= SIZE_OF_INT; + break; + case LONG: + bytes *= SIZE_OF_LONG; + break; + default: + throw new IllegalStateException(); + } + skip(bytes); + } + + public long readVarint(boolean signed) + throws IOException + { + long result = 0; + int shift = 0; + int available = available(); + if (available >= 2 * Long.BYTES) { + long word = ByteArrays.getLong(buffer, position); + int count = 1; + boolean atEnd = false; + result = word & 0x7f; + if ((word & 0x80) != 0) { + long control = word >>> 8; + long mask = 0x7f << 7; + while (true) { + word = word >>> 1; + result |= word & mask; + count++; + if ((control & 0x80) == 0) { + atEnd = true; + break; + } + if (mask == 0x7fL << (7 * 7)) { + break; + } + mask = mask << 7; + control = control >>> 8; + } + if (!atEnd) { + word = ByteArrays.getLong(buffer, position + 8); + result |= (word & 0x7f) << 56; + if ((word & 0x80) == 0) { + count++; + } + else { + result |= 1L << 63; + count += 2; + } + } + } + position += count; + } + else { + do { + if (available == 0) { + advance(); + available = available(); + if (available == 0) { + throw new OrcCorruptionException(orcDataSourceId, "End of stream in RLE Integer"); + } + } + available--; + result |= (long) (buffer[position] & 0x7f) << shift; + shift += 7; + } + while ((buffer[position++] & 0x80) != 0); + } + if (signed) { + return zigzagDecode(result); + } + else { + return result; + } + } + + public void skipVarints(long items) + throws IOException + { + if (items == 0) { + return; + } + + while (items > 0) { + items -= skipVarintsInBuffer(items); + } + } + + private long skipVarintsInBuffer(long items) + throws IOException + { + if (available() == 0) { + advance(); + if (available() == 0) { + throw new OrcCorruptionException(orcDataSourceId, "Unexpected end of stream"); + } + } + long skipped = 0; + // If items to skip is > SIZE_OF_LONG it is safe to skip entire longs + while (items - skipped > SIZE_OF_LONG && available() > MAX_VARINT_LENGTH) { + long value = ByteArrays.getLong(buffer, position); + position += SIZE_OF_LONG; + long mask = (value & VARINT_MASK) ^ VARINT_MASK; + skipped += Long.bitCount(mask); + } + while (skipped < items && available() > 0) { + if ((buffer[position++] & 0x80) == 0) { + skipped++; + } + } + return skipped; + } + + public double readDouble() + throws IOException + { + int readPosition = ensureContiguousBytesAndAdvance(SIZE_OF_DOUBLE); + if (readPosition < 0) { + return ByteArrays.getDouble(temporaryBuffer, 0); + } + return ByteArrays.getDouble(buffer, readPosition); + } + + public float readFloat() + throws IOException + { + int readPosition = ensureContiguousBytesAndAdvance(SIZE_OF_FLOAT); + if (readPosition < 0) { + return ByteArrays.getFloat(temporaryBuffer, 0); + } + return ByteArrays.getFloat(buffer, readPosition); + } + + private int ensureContiguousBytesAndAdvance(int bytes) + throws IOException + { + // If there are numBytes in the buffer, return the offset of the start and advance by numBytes. If not, copy numBytes + // into temporaryBuffer, advance by numBytes and return -1. + if (available() >= bytes) { + int startPosition = position; + position += bytes; + return startPosition; + } + readFully(temporaryBuffer, 0, bytes); + return -1; } // This comes from the Apache Hive ORC code @@ -227,7 +440,10 @@ private void advance() throws IOException { if (compressedSliceInput == null || compressedSliceInput.remaining() == 0) { - current = null; + buffer = null; + position = 0; + length = 0; + uncompressedOffset = 0; return; } @@ -243,13 +459,15 @@ private void advance() if (chunkLength < 0 || chunkLength > compressedSliceInput.remaining()) { throw new OrcCorruptionException(orcDataSourceId, "The chunkLength (%s) must not be negative or greater than remaining size (%s)", chunkLength, compressedSliceInput.remaining()); } - Slice chunk = compressedSliceInput.readSlice(chunkLength); if (isUncompressed) { - current = chunk.getInput(); + buffer = (byte[]) chunk.getBase(); + position = toIntExact(chunk.getAddress() - ARRAY_BYTE_BASE_OFFSET); + length = toIntExact(position + chunk.length()); } else { + buffer = decompressionResultBuffer; OrcDecompressor.OutputBuffer output = new OrcDecompressor.OutputBuffer() { @Override @@ -258,6 +476,8 @@ public byte[] initialize(int size) if (buffer == null || size > buffer.length) { buffer = new byte[size]; bufferMemoryUsage.setBytes(buffer.length); + position = 0; + length = size; } return buffer; } @@ -272,10 +492,11 @@ public byte[] grow(int size) return buffer; } }; - - int uncompressedSize = decompressor.get().decompress((byte[]) chunk.getBase(), (int) (chunk.getAddress() - ARRAY_BYTE_BASE_OFFSET), chunk.length(), output); - current = Slices.wrappedBuffer(buffer, 0, uncompressedSize).getInput(); + length = decompressor.get().decompress((byte[]) chunk.getBase(), (int) (chunk.getAddress() - ARRAY_BYTE_BASE_OFFSET), chunk.length(), output); + decompressionResultBuffer = buffer; + position = 0; } + uncompressedOffset = position; } @Override @@ -284,7 +505,7 @@ public String toString() return toStringHelper(this) .add("source", orcDataSourceId) .add("compressedOffset", compressedSliceInput.position()) - .add("uncompressedOffset", current == null ? null : current.position()) + .add("uncompressedOffset", buffer == null ? null : position) .add("decompressor", decompressor.map(Object::toString).orElse("none")) .toString(); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/writer/BooleanColumnWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/writer/BooleanColumnWriter.java index 03408610663d1..62b099886895c 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/writer/BooleanColumnWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/writer/BooleanColumnWriter.java @@ -57,6 +57,7 @@ public class BooleanColumnWriter private final PresentOutputStream presentStream; private final List rowGroupColumnStatistics = new ArrayList<>(); + private long columnStatisticsRetainedSizeInBytes; private BooleanStatisticsBuilder statisticsBuilder = new BooleanStatisticsBuilder(); @@ -112,6 +113,7 @@ public Map finishRowGroup() checkState(!closed); ColumnStatistics statistics = statisticsBuilder.buildColumnStatistics(); rowGroupColumnStatistics.add(statistics); + columnStatisticsRetainedSizeInBytes += statistics.getRetainedSizeInBytes(); statisticsBuilder = new BooleanStatisticsBuilder(); return ImmutableMap.of(column, statistics); } @@ -186,11 +188,7 @@ public long getBufferedBytes() @Override public long getRetainedBytes() { - long retainedBytes = INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes(); - for (ColumnStatistics statistics : rowGroupColumnStatistics) { - retainedBytes += statistics.getRetainedSizeInBytes(); - } - return retainedBytes; + return INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes() + columnStatisticsRetainedSizeInBytes; } @Override @@ -200,6 +198,7 @@ public void reset() dataStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); + columnStatisticsRetainedSizeInBytes = 0; statisticsBuilder = new BooleanStatisticsBuilder(); } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/writer/ByteColumnWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/writer/ByteColumnWriter.java index 7f009cff7b33f..4253727345e34 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/writer/ByteColumnWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/writer/ByteColumnWriter.java @@ -57,6 +57,7 @@ public class ByteColumnWriter private final PresentOutputStream presentStream; private final List rowGroupColumnStatistics = new ArrayList<>(); + private long columnStatisticsRetainedSizeInBytes; private int nonNullValueCount; @@ -111,6 +112,7 @@ public Map finishRowGroup() checkState(!closed); ColumnStatistics statistics = new ColumnStatistics((long) nonNullValueCount, 0, null, null, null, null, null, null, null, null); rowGroupColumnStatistics.add(statistics); + columnStatisticsRetainedSizeInBytes += statistics.getRetainedSizeInBytes(); nonNullValueCount = 0; return ImmutableMap.of(column, statistics); } @@ -185,11 +187,7 @@ public long getBufferedBytes() @Override public long getRetainedBytes() { - long retainedBytes = INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes(); - for (ColumnStatistics statistics : rowGroupColumnStatistics) { - retainedBytes += statistics.getRetainedSizeInBytes(); - } - return retainedBytes; + return INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes() + columnStatisticsRetainedSizeInBytes; } @Override @@ -199,6 +197,7 @@ public void reset() dataStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); + columnStatisticsRetainedSizeInBytes = 0; nonNullValueCount = 0; } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/writer/DecimalColumnWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/writer/DecimalColumnWriter.java index 3bf812a5972d1..4ade69612b35d 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/writer/DecimalColumnWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/writer/DecimalColumnWriter.java @@ -68,6 +68,7 @@ public class DecimalColumnWriter private final PresentOutputStream presentStream; private final List rowGroupColumnStatistics = new ArrayList<>(); + private long columnStatisticsRetainedSizeInBytes; private ShortDecimalStatisticsBuilder shortDecimalStatisticsBuilder; private LongDecimalStatisticsBuilder longDecimalStatisticsBuilder; @@ -159,6 +160,7 @@ public Map finishRowGroup() longDecimalStatisticsBuilder = new LongDecimalStatisticsBuilder(); } rowGroupColumnStatistics.add(statistics); + columnStatisticsRetainedSizeInBytes += statistics.getRetainedSizeInBytes(); return ImmutableMap.of(column, statistics); } @@ -239,11 +241,7 @@ public long getBufferedBytes() @Override public long getRetainedBytes() { - long retainedBytes = INSTANCE_SIZE + dataStream.getRetainedBytes() + scaleStream.getRetainedBytes() + presentStream.getRetainedBytes(); - for (ColumnStatistics statistics : rowGroupColumnStatistics) { - retainedBytes += statistics.getRetainedSizeInBytes(); - } - return retainedBytes; + return INSTANCE_SIZE + dataStream.getRetainedBytes() + scaleStream.getRetainedBytes() + presentStream.getRetainedBytes() + columnStatisticsRetainedSizeInBytes; } @Override @@ -254,6 +252,7 @@ public void reset() scaleStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); + columnStatisticsRetainedSizeInBytes = 0; shortDecimalStatisticsBuilder = new ShortDecimalStatisticsBuilder(this.type.getScale()); longDecimalStatisticsBuilder = new LongDecimalStatisticsBuilder(); } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/writer/DoubleColumnWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/writer/DoubleColumnWriter.java index 627bc10cb8d00..0c1d02cccba04 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/writer/DoubleColumnWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/writer/DoubleColumnWriter.java @@ -58,6 +58,7 @@ public class DoubleColumnWriter private final PresentOutputStream presentStream; private final List rowGroupColumnStatistics = new ArrayList<>(); + private long columnStatisticsRetainedSizeInBytes; private DoubleStatisticsBuilder statisticsBuilder = new DoubleStatisticsBuilder(); @@ -113,6 +114,7 @@ public Map finishRowGroup() checkState(!closed); ColumnStatistics statistics = statisticsBuilder.buildColumnStatistics(); rowGroupColumnStatistics.add(statistics); + columnStatisticsRetainedSizeInBytes += statistics.getRetainedSizeInBytes(); statisticsBuilder = new DoubleStatisticsBuilder(); return ImmutableMap.of(column, statistics); } @@ -187,11 +189,7 @@ public long getBufferedBytes() @Override public long getRetainedBytes() { - long retainedBytes = INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes(); - for (ColumnStatistics statistics : rowGroupColumnStatistics) { - retainedBytes += statistics.getRetainedSizeInBytes(); - } - return retainedBytes; + return INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes() + columnStatisticsRetainedSizeInBytes; } @Override @@ -201,6 +199,7 @@ public void reset() dataStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); + columnStatisticsRetainedSizeInBytes = 0; statisticsBuilder = new DoubleStatisticsBuilder(); } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/writer/FloatColumnWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/writer/FloatColumnWriter.java index fa2d48d897c9a..752b2b1a1b176 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/writer/FloatColumnWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/writer/FloatColumnWriter.java @@ -59,6 +59,7 @@ public class FloatColumnWriter private final PresentOutputStream presentStream; private final List rowGroupColumnStatistics = new ArrayList<>(); + private long columnStatisticsRetainedSizeInBytes; private DoubleStatisticsBuilder statisticsBuilder = new DoubleStatisticsBuilder(); @@ -115,6 +116,7 @@ public Map finishRowGroup() checkState(!closed); ColumnStatistics statistics = statisticsBuilder.buildColumnStatistics(); rowGroupColumnStatistics.add(statistics); + columnStatisticsRetainedSizeInBytes += statistics.getRetainedSizeInBytes(); statisticsBuilder = new DoubleStatisticsBuilder(); return ImmutableMap.of(column, statistics); } @@ -189,11 +191,7 @@ public long getBufferedBytes() @Override public long getRetainedBytes() { - long retainedBytes = INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes(); - for (ColumnStatistics statistics : rowGroupColumnStatistics) { - retainedBytes += statistics.getRetainedSizeInBytes(); - } - return retainedBytes; + return INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes() + columnStatisticsRetainedSizeInBytes; } @Override @@ -203,6 +201,7 @@ public void reset() dataStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); + columnStatisticsRetainedSizeInBytes = 0; statisticsBuilder = new DoubleStatisticsBuilder(); } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/writer/ListColumnWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/writer/ListColumnWriter.java index bfd2a0ddb5fff..275bf8c2eec4f 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/writer/ListColumnWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/writer/ListColumnWriter.java @@ -61,6 +61,7 @@ public class ListColumnWriter private final ColumnWriter elementWriter; private final List rowGroupColumnStatistics = new ArrayList<>(); + private long columnStatisticsRetainedSizeInBytes; private int nonNullValueCount; @@ -140,6 +141,7 @@ public Map finishRowGroup() ColumnStatistics statistics = new ColumnStatistics((long) nonNullValueCount, 0, null, null, null, null, null, null, null, null); rowGroupColumnStatistics.add(statistics); + columnStatisticsRetainedSizeInBytes += statistics.getRetainedSizeInBytes(); nonNullValueCount = 0; ImmutableMap.Builder columnStatistics = ImmutableMap.builder(); @@ -227,11 +229,7 @@ public long getBufferedBytes() @Override public long getRetainedBytes() { - long retainedBytes = INSTANCE_SIZE + lengthStream.getRetainedBytes() + presentStream.getRetainedBytes() + elementWriter.getRetainedBytes(); - for (ColumnStatistics statistics : rowGroupColumnStatistics) { - retainedBytes += statistics.getRetainedSizeInBytes(); - } - return retainedBytes; + return INSTANCE_SIZE + lengthStream.getRetainedBytes() + presentStream.getRetainedBytes() + elementWriter.getRetainedBytes() + columnStatisticsRetainedSizeInBytes; } @Override @@ -242,6 +240,7 @@ public void reset() presentStream.reset(); elementWriter.reset(); rowGroupColumnStatistics.clear(); + columnStatisticsRetainedSizeInBytes = 0; nonNullValueCount = 0; } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/writer/LongColumnWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/writer/LongColumnWriter.java index b0413bbacfb72..3622b95b52807 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/writer/LongColumnWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/writer/LongColumnWriter.java @@ -64,6 +64,7 @@ public class LongColumnWriter private final PresentOutputStream presentStream; private final List rowGroupColumnStatistics = new ArrayList<>(); + private long columnStatisticsRetainedSizeInBytes; private final Supplier statisticsBuilderSupplier; private LongValueStatisticsBuilder statisticsBuilder; @@ -129,6 +130,7 @@ public Map finishRowGroup() checkState(!closed); ColumnStatistics statistics = statisticsBuilder.buildColumnStatistics(); rowGroupColumnStatistics.add(statistics); + columnStatisticsRetainedSizeInBytes += statistics.getRetainedSizeInBytes(); statisticsBuilder = statisticsBuilderSupplier.get(); return ImmutableMap.of(column, statistics); } @@ -203,11 +205,7 @@ public long getBufferedBytes() @Override public long getRetainedBytes() { - long retainedBytes = INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes(); - for (ColumnStatistics statistics : rowGroupColumnStatistics) { - retainedBytes += statistics.getRetainedSizeInBytes(); - } - return retainedBytes; + return INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes() + columnStatisticsRetainedSizeInBytes; } @Override @@ -217,6 +215,7 @@ public void reset() dataStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); + columnStatisticsRetainedSizeInBytes = 0; statisticsBuilder = statisticsBuilderSupplier.get(); } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/writer/MapColumnWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/writer/MapColumnWriter.java index 563b954ff16c9..87d17b24519dc 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/writer/MapColumnWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/writer/MapColumnWriter.java @@ -62,6 +62,7 @@ public class MapColumnWriter private final ColumnWriter valueWriter; private final List rowGroupColumnStatistics = new ArrayList<>(); + private long columnStatisticsRetainedSizeInBytes; private int nonNullValueCount; @@ -147,6 +148,7 @@ public Map finishRowGroup() ColumnStatistics statistics = new ColumnStatistics((long) nonNullValueCount, 0, null, null, null, null, null, null, null, null); rowGroupColumnStatistics.add(statistics); + columnStatisticsRetainedSizeInBytes += statistics.getRetainedSizeInBytes(); nonNullValueCount = 0; ImmutableMap.Builder columnStatistics = ImmutableMap.builder(); @@ -239,11 +241,7 @@ public long getBufferedBytes() @Override public long getRetainedBytes() { - long retainedBytes = INSTANCE_SIZE + lengthStream.getRetainedBytes() + presentStream.getRetainedBytes() + keyWriter.getRetainedBytes() + valueWriter.getRetainedBytes(); - for (ColumnStatistics statistics : rowGroupColumnStatistics) { - retainedBytes += statistics.getRetainedSizeInBytes(); - } - return retainedBytes; + return INSTANCE_SIZE + lengthStream.getRetainedBytes() + presentStream.getRetainedBytes() + keyWriter.getRetainedBytes() + valueWriter.getRetainedBytes() + columnStatisticsRetainedSizeInBytes; } @Override @@ -255,6 +253,7 @@ public void reset() keyWriter.reset(); valueWriter.reset(); rowGroupColumnStatistics.clear(); + columnStatisticsRetainedSizeInBytes = 0; nonNullValueCount = 0; } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/writer/SliceDictionaryColumnWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/writer/SliceDictionaryColumnWriter.java index 570acc8ae7ed4..b52a1f6d57395 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/writer/SliceDictionaryColumnWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/writer/SliceDictionaryColumnWriter.java @@ -85,6 +85,7 @@ public class SliceDictionaryColumnWriter private final DictionaryBuilder dictionary = new DictionaryBuilder(10000); private final List rowGroups = new ArrayList<>(); + private long columnStatisticsRetainedSizeInBytes; private IntBigArray values; private int rowGroupValueCount; @@ -320,7 +321,9 @@ public Map finishRowGroup() } ColumnStatistics statistics = statisticsBuilder.buildColumnStatistics(); - rowGroups.add(new DictionaryRowGroup(values, rowGroupValueCount, statistics)); + DictionaryRowGroup rowGroup = new DictionaryRowGroup(values, rowGroupValueCount, statistics); + rowGroups.add(rowGroup); + columnStatisticsRetainedSizeInBytes += rowGroup.getColumnStatistics().getRetainedSizeInBytes(); rowGroupValueCount = 0; statisticsBuilder = newStringStatisticsBuilder(); values = new IntBigArray(); @@ -523,19 +526,15 @@ public long getBufferedBytes() @Override public long getRetainedBytes() { - long retainedBytes = INSTANCE_SIZE + + return INSTANCE_SIZE + values.sizeOf() + dataStream.getRetainedBytes() + presentStream.getRetainedBytes() + dictionaryDataStream.getRetainedBytes() + dictionaryLengthStream.getRetainedBytes() + dictionary.getRetainedSizeInBytes() + - (directColumnWriter == null ? 0 : directColumnWriter.getRetainedBytes()); - - for (DictionaryRowGroup rowGroup : rowGroups) { - retainedBytes += rowGroup.getColumnStatistics().getRetainedSizeInBytes(); - } - return retainedBytes; + (directColumnWriter == null ? 0 : directColumnWriter.getRetainedBytes()) + + columnStatisticsRetainedSizeInBytes; } @Override @@ -548,6 +547,7 @@ public void reset() dictionaryDataStream.reset(); dictionaryLengthStream.reset(); rowGroups.clear(); + columnStatisticsRetainedSizeInBytes = 0; rowGroupValueCount = 0; statisticsBuilder = newStringStatisticsBuilder(); columnEncoding = null; diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/writer/SliceDirectColumnWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/writer/SliceDirectColumnWriter.java index 5d4f50ccbe7c2..bd5897f719033 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/writer/SliceDirectColumnWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/writer/SliceDirectColumnWriter.java @@ -65,6 +65,7 @@ public class SliceDirectColumnWriter private final PresentOutputStream presentStream; private final List rowGroupColumnStatistics = new ArrayList<>(); + private long columnStatisticsRetainedSizeInBytes; private final Supplier statisticsBuilderSupplier; private SliceColumnStatisticsBuilder statisticsBuilder; @@ -129,6 +130,7 @@ public Map finishRowGroup() ColumnStatistics statistics = statisticsBuilder.buildColumnStatistics(); rowGroupColumnStatistics.add(statistics); + columnStatisticsRetainedSizeInBytes += statistics.getRetainedSizeInBytes(); statisticsBuilder = statisticsBuilderSupplier.get(); return ImmutableMap.of(column, statistics); @@ -211,11 +213,7 @@ public long getBufferedBytes() @Override public long getRetainedBytes() { - long retainedBytes = INSTANCE_SIZE + lengthStream.getRetainedBytes() + dataStream.getRetainedBytes() + presentStream.getRetainedBytes(); - for (ColumnStatistics statistics : rowGroupColumnStatistics) { - retainedBytes += statistics.getRetainedSizeInBytes(); - } - return retainedBytes; + return INSTANCE_SIZE + lengthStream.getRetainedBytes() + dataStream.getRetainedBytes() + presentStream.getRetainedBytes() + columnStatisticsRetainedSizeInBytes; } @Override @@ -227,6 +225,7 @@ public void reset() dataStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); + columnStatisticsRetainedSizeInBytes = 0; statisticsBuilder = statisticsBuilderSupplier.get(); } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/writer/StructColumnWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/writer/StructColumnWriter.java index d39f0811155e1..df660402cc34d 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/writer/StructColumnWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/writer/StructColumnWriter.java @@ -55,6 +55,7 @@ public class StructColumnWriter private final List structFields; private final List rowGroupColumnStatistics = new ArrayList<>(); + private long columnStatisticsRetainedSizeInBytes; private int nonNullValueCount; @@ -137,6 +138,7 @@ public Map finishRowGroup() checkState(!closed); ColumnStatistics statistics = new ColumnStatistics((long) nonNullValueCount, 0, null, null, null, null, null, null, null, null); rowGroupColumnStatistics.add(statistics); + columnStatisticsRetainedSizeInBytes += statistics.getRetainedSizeInBytes(); nonNullValueCount = 0; ImmutableMap.Builder columnStatistics = ImmutableMap.builder(); @@ -234,9 +236,7 @@ public long getRetainedBytes() for (ColumnWriter structField : structFields) { retainedBytes += structField.getRetainedBytes(); } - for (ColumnStatistics statistics : rowGroupColumnStatistics) { - retainedBytes += statistics.getRetainedSizeInBytes(); - } + retainedBytes += columnStatisticsRetainedSizeInBytes; return retainedBytes; } @@ -247,6 +247,7 @@ public void reset() presentStream.reset(); structFields.forEach(ColumnWriter::reset); rowGroupColumnStatistics.clear(); + columnStatisticsRetainedSizeInBytes = 0; nonNullValueCount = 0; } } diff --git a/presto-orc/src/main/java/com/facebook/presto/orc/writer/TimestampColumnWriter.java b/presto-orc/src/main/java/com/facebook/presto/orc/writer/TimestampColumnWriter.java index 83271f1b39ebe..c34f8d2c1e004 100644 --- a/presto-orc/src/main/java/com/facebook/presto/orc/writer/TimestampColumnWriter.java +++ b/presto-orc/src/main/java/com/facebook/presto/orc/writer/TimestampColumnWriter.java @@ -69,6 +69,8 @@ public class TimestampColumnWriter private final PresentOutputStream presentStream; private final List rowGroupColumnStatistics = new ArrayList<>(); + private long columnStatisticsRetainedSizeInBytes; + private final long baseTimestampInSeconds; private int nonNullValueCount; @@ -163,6 +165,7 @@ public Map finishRowGroup() checkState(!closed); ColumnStatistics statistics = new ColumnStatistics((long) nonNullValueCount, 0, null, null, null, null, null, null, null, null); rowGroupColumnStatistics.add(statistics); + columnStatisticsRetainedSizeInBytes += statistics.getRetainedSizeInBytes(); nonNullValueCount = 0; return ImmutableMap.of(column, statistics); } @@ -243,11 +246,7 @@ public long getBufferedBytes() @Override public long getRetainedBytes() { - long retainedBytes = secondsStream.getRetainedBytes() + nanosStream.getRetainedBytes() + presentStream.getRetainedBytes(); - for (ColumnStatistics statistics : rowGroupColumnStatistics) { - retainedBytes += statistics.getRetainedSizeInBytes(); - } - return retainedBytes; + return INSTANCE_SIZE + secondsStream.getRetainedBytes() + nanosStream.getRetainedBytes() + presentStream.getRetainedBytes() + columnStatisticsRetainedSizeInBytes; } @Override @@ -258,6 +257,7 @@ public void reset() nanosStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); + columnStatisticsRetainedSizeInBytes = 0; nonNullValueCount = 0; } } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/AbstractTestOrcReader.java b/presto-orc/src/test/java/com/facebook/presto/orc/AbstractTestOrcReader.java index e61520da51ddf..5c7ad0224fbda 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/AbstractTestOrcReader.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/AbstractTestOrcReader.java @@ -13,22 +13,42 @@ */ package com.facebook.presto.orc; +import com.facebook.presto.orc.StripeReader.StripeId; +import com.facebook.presto.orc.StripeReader.StripeStreamId; +import com.facebook.presto.orc.cache.CachingOrcFileTailSource; +import com.facebook.presto.orc.cache.OrcFileTailSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; +import com.facebook.presto.orc.metadata.CompressionKind; +import com.facebook.presto.orc.metadata.OrcFileTail; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.SqlDate; import com.facebook.presto.spi.type.SqlDecimal; import com.facebook.presto.spi.type.SqlVarbinary; import com.google.common.base.Strings; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ContiguousSet; import com.google.common.collect.DiscreteDomain; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.Range; +import io.airlift.slice.Slice; +import io.airlift.units.DataSize; +import io.airlift.units.Duration; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; +import org.apache.hadoop.hive.ql.io.orc.OrcSerde; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.Serializer; +import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.io.Writable; import org.joda.time.DateTimeZone; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.io.IOException; import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; @@ -36,7 +56,13 @@ import java.util.List; import java.util.Random; +import static com.facebook.presto.orc.OrcEncoding.ORC; +import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE; +import static com.facebook.presto.orc.OrcTester.Format.ORC_12; import static com.facebook.presto.orc.OrcTester.HIVE_STORAGE_TIME_ZONE; +import static com.facebook.presto.orc.OrcTester.createCustomOrcRecordReader; +import static com.facebook.presto.orc.OrcTester.createOrcRecordWriter; +import static com.facebook.presto.orc.OrcTester.createSettableStructObjectInspector; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.CharType.createCharType; @@ -55,8 +81,11 @@ import static com.google.common.collect.Iterables.cycle; import static com.google.common.collect.Iterables.limit; import static com.google.common.collect.Lists.newArrayList; +import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.nCopies; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.stream.Collectors.toList; import static org.testng.Assert.assertEquals; @@ -156,6 +185,78 @@ public void testLongStrideDictionary() testRoundTripNumeric(concat(ImmutableList.of(1), nCopies(9999, 123), ImmutableList.of(2), nCopies(9999, 123))); } + @Test + public void testCaching() + throws Exception + { + Cache orcFileTailCache = CacheBuilder.newBuilder() + .maximumWeight(new DataSize(1, MEGABYTE).toBytes()) + .weigher((id, tail) -> ((OrcFileTail) tail).getFooterSize() + ((OrcFileTail) tail).getMetadataSize()) + .expireAfterAccess(new Duration(10, MINUTES).toMillis(), MILLISECONDS) + .recordStats() + .build(); + OrcFileTailSource orcFileTailSource = new CachingOrcFileTailSource(new StorageOrcFileTailSource(), orcFileTailCache); + + Cache stripeFootercache = CacheBuilder.newBuilder() + .maximumWeight(new DataSize(1, MEGABYTE).toBytes()) + .weigher((id, footer) -> ((Slice) footer).length()) + .expireAfterAccess(new Duration(10, MINUTES).toMillis(), MILLISECONDS) + .recordStats() + .build(); + Cache stripeStreamCache = CacheBuilder.newBuilder() + .maximumWeight(new DataSize(1, MEGABYTE).toBytes()) + .weigher((id, stream) -> ((Slice) stream).length()) + .expireAfterAccess(new Duration(10, MINUTES).toMillis(), MILLISECONDS) + .recordStats() + .build(); + StripeMetadataSource stripeMetadataSource = new CachingStripeMetadataSource(new StorageStripeMetadataSource(), stripeFootercache, stripeStreamCache); + + try (TempFile tempFile = createTempFile()) { + OrcBatchRecordReader storageReader = createCustomOrcRecordReader(tempFile, ORC, OrcPredicate.TRUE, ImmutableList.of(BIGINT), INITIAL_BATCH_SIZE, orcFileTailSource, stripeMetadataSource); + assertEquals(orcFileTailCache.stats().missCount(), 1); + assertEquals(orcFileTailCache.stats().hitCount(), 0); + + OrcBatchRecordReader cacheReader = createCustomOrcRecordReader(tempFile, ORC, OrcPredicate.TRUE, ImmutableList.of(BIGINT), INITIAL_BATCH_SIZE, orcFileTailSource, stripeMetadataSource); + assertEquals(orcFileTailCache.stats().missCount(), 1); + assertEquals(orcFileTailCache.stats().hitCount(), 1); + + assertEquals(storageReader.getRetainedSizeInBytes(), cacheReader.getRetainedSizeInBytes()); + assertEquals(storageReader.getFileRowCount(), cacheReader.getFileRowCount()); + assertEquals(storageReader.getSplitLength(), cacheReader.getSplitLength()); + + storageReader.nextBatch(); + assertEquals(stripeFootercache.stats().missCount(), 1); + assertEquals(stripeFootercache.stats().hitCount(), 0); + assertEquals(stripeStreamCache.stats().missCount(), 2); + assertEquals(stripeStreamCache.stats().hitCount(), 0); + cacheReader.nextBatch(); + assertEquals(stripeFootercache.stats().missCount(), 1); + assertEquals(stripeFootercache.stats().hitCount(), 1); + assertEquals(stripeStreamCache.stats().missCount(), 2); + assertEquals(stripeStreamCache.stats().hitCount(), 2); + assertEquals(storageReader.readBlock(0).getInt(0), cacheReader.readBlock(0).getInt(0)); + } + } + + private static TempFile createTempFile() + throws IOException, SerDeException + { + TempFile file = new TempFile(); + RecordWriter writer = createOrcRecordWriter(file.getFile(), ORC_12, CompressionKind.NONE, BIGINT); + + @SuppressWarnings("deprecation") Serializer serde = new OrcSerde(); + SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", BIGINT); + Object row = objectInspector.create(); + StructField field = objectInspector.getAllStructFieldRefs().get(0); + + objectInspector.setStructFieldData(row, field, 1L); + Writable record = serde.serialize(row, objectInspector); + writer.write(record); + + writer.close(false); + return file; + } + private void testRoundTripNumeric(Iterable values) throws Exception { diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkBatchStreamReaders.java b/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkBatchStreamReaders.java index 03a9143a6998f..e61da8527d4c0 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkBatchStreamReaders.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkBatchStreamReaders.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.SqlDecimal; @@ -21,6 +22,7 @@ import com.facebook.presto.spi.type.TypeSignature; import com.facebook.presto.type.TypeRegistry; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.units.DataSize; import org.openjdk.jmh.annotations.Benchmark; @@ -39,15 +41,16 @@ import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; import org.openjdk.jmh.runner.options.VerboseMode; +import org.openjdk.jmh.runner.options.WarmupMode; import java.io.File; import java.io.IOException; import java.math.BigInteger; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Random; import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.OrcEncoding.ORC; @@ -55,715 +58,176 @@ import static com.facebook.presto.orc.OrcTester.Format.ORC_12; import static com.facebook.presto.orc.OrcTester.writeOrcColumnHive; import static com.facebook.presto.orc.metadata.CompressionKind.NONE; -import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DecimalType.createDecimalType; -import static com.facebook.presto.spi.type.DoubleType.DOUBLE; -import static com.facebook.presto.spi.type.RealType.REAL; import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY; -import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; -import static com.facebook.presto.spi.type.TinyintType.TINYINT; -import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.google.common.io.Files.createTempDir; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.util.UUID.randomUUID; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.stream.Collectors.toList; import static org.joda.time.DateTimeZone.UTC; @SuppressWarnings("MethodMayBeStatic") @State(Scope.Thread) -@OutputTimeUnit(TimeUnit.SECONDS) +@OutputTimeUnit(TimeUnit.MILLISECONDS) @Fork(3) @Warmup(iterations = 20, time = 500, timeUnit = MILLISECONDS) @Measurement(iterations = 20, time = 500, timeUnit = MILLISECONDS) @BenchmarkMode(Mode.AverageTime) public class BenchmarkBatchStreamReaders { - public static final DecimalType SHORT_DECIMAL_TYPE = createDecimalType(10, 5); - public static final DecimalType LONG_DECIMAL_TYPE = createDecimalType(30, 10); - public static final int ROWS = 10_000_000; - public static final List NULL_VALUES = Collections.nCopies(ROWS, null); + private static final DecimalType SHORT_DECIMAL_TYPE = createDecimalType(10, 5); + private static final DecimalType LONG_DECIMAL_TYPE = createDecimalType(30, 10); + private static final int ROWS = 10_000_000; + private static final int MAX_STRING = 10; + private static final List NULL_VALUES = Collections.nCopies(ROWS, null); @Benchmark - public Object readBooleanNoNull(BooleanNoNullBenchmarkData data) + public Object readBlocks(BenchmarkData data) throws Throwable { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readBooleanWithNull(BooleanWithNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readAllNull(AllNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readByteNoNull(TinyIntNoNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readByteWithNull(TinyIntWithNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readShortDecimalNoNull(ShortDecimalNoNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readShortDecimalWithNull(ShortDecimalWithNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readLongDecimalNoNull(LongDecimalNoNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readLongDecimalWithNull(LongDecimalWithNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readDoubleNoNull(DoubleNoNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readDoubleWithNull(DoubleWithNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readFloatNoNull(FloatNoNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readFloatWithNull(FloatWithNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readLongDirectNoNull(BigintNoNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readLongDirectWithNull(BigintWithNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readSliceDictionaryNoNull(VarcharNoNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readSliceDictionaryWithNull(VarcharWithNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readTimestampNoNull(TimestampNoNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - @Benchmark - public Object readTimestampWithNull(TimestampWithNullBenchmarkData data) - throws Throwable - { - return readAllBlocks(data.createRecordReader(), data.getType()); - } - - private Object readAllBlocks(OrcBatchRecordReader recordReader, Type type) - throws IOException - { - List blocks = new ArrayList<>(); + OrcBatchRecordReader recordReader = data.createRecordReader(); + ImmutableList.Builder blocks = new ImmutableList.Builder<>(); while (recordReader.nextBatch() > 0) { - Block block = recordReader.readBlock(type, 0); + Block block = recordReader.readBlock(0); blocks.add(block); } - return blocks; + return blocks.build(); } - private abstract static class BenchmarkData + @State(Scope.Thread) + public static class BenchmarkData { - protected final Random random = new Random(0); + private final Random random = new Random(0); + private Type type; private File temporaryDirectory; private File orcFile; - public void setup(Type type) - throws Exception - { - this.type = type; - temporaryDirectory = createTempDir(); - orcFile = new File(temporaryDirectory, randomUUID().toString()); - writeOrcColumnHive(orcFile, ORC_12, NONE, type, createValues()); - } - - @TearDown - public void tearDown() - throws IOException - { - deleteRecursively(temporaryDirectory.toPath(), ALLOW_INSECURE); - } - - public Type getType() - { - return type; - } - - protected abstract List createValues(); - - OrcBatchRecordReader createRecordReader() - throws IOException - { - OrcDataSource dataSource = new FileOrcDataSource(orcFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); - OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); - return orcReader.createBatchRecordReader( - ImmutableMap.of(0, type), - OrcPredicate.TRUE, - UTC, // arbitrary - newSimpleAggregatedMemoryContext(), - INITIAL_BATCH_SIZE); - } - } - - @State(Scope.Thread) - public static class AllNullBenchmarkData - extends BenchmarkData - { @SuppressWarnings("unused") @Param({ "boolean", - "tinyint", + "smallint", "integer", "bigint", "decimal(10,5)", "decimal(30,10)", - "timestamp", - "real", "double", - - "varchar", - "varbinary", + "varchar_direct", + "varchar_dictionary" }) private String typeSignature; - @Setup - public void setup() - throws Exception - { - Type type = new TypeRegistry().getType(TypeSignature.parseTypeSignature(typeSignature)); - setup(type); - } - - @Override - protected final List createValues() - { - return NULL_VALUES; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class BooleanNoNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(BOOLEAN); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - values.add(random.nextBoolean()); - } - return values; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class BooleanWithNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(BOOLEAN); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - values.add(random.nextBoolean() ? random.nextBoolean() : null); - } - return values; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class TinyIntNoNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(TINYINT); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - values.add(Long.valueOf(random.nextLong()).byteValue()); - } - return values; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class TinyIntWithNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(TINYINT); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - if (random.nextBoolean()) { - values.add(Long.valueOf(random.nextLong()).byteValue()); - } - else { - values.add(null); - } - } - return values; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class ShortDecimalNoNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(SHORT_DECIMAL_TYPE); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - values.add(new SqlDecimal(BigInteger.valueOf(random.nextLong() % 10_000_000_000L), SHORT_DECIMAL_TYPE.getPrecision(), SHORT_DECIMAL_TYPE.getScale())); - } - return values; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class ShortDecimalWithNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(SHORT_DECIMAL_TYPE); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - if (random.nextBoolean()) { - values.add(new SqlDecimal(BigInteger.valueOf(random.nextLong() % 10_000_000_000L), SHORT_DECIMAL_TYPE.getPrecision(), SHORT_DECIMAL_TYPE.getScale())); - } - else { - values.add(null); - } - } - return values; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class LongDecimalNoNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(LONG_DECIMAL_TYPE); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - values.add(new SqlDecimal(BigInteger.valueOf(random.nextLong() % 10_000_000_000L), LONG_DECIMAL_TYPE.getPrecision(), LONG_DECIMAL_TYPE.getScale())); - } - return values; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class LongDecimalWithNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(LONG_DECIMAL_TYPE); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - if (random.nextBoolean()) { - values.add(new SqlDecimal(BigInteger.valueOf(random.nextLong() % 10_000_000_000L), LONG_DECIMAL_TYPE.getPrecision(), LONG_DECIMAL_TYPE.getScale())); - } - else { - values.add(null); - } - } - return values; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class DoubleNoNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(DOUBLE); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - values.add(random.nextDouble()); - } - return values; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class DoubleWithNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(DOUBLE); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - if (random.nextBoolean()) { - values.add(random.nextDouble()); - } - else { - values.add(null); - } - } - return values; - } - } + @SuppressWarnings("unused") + @Param({ + "PARTIAL", + "NONE", + "ALL" + }) + private Nulls withNulls; - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class FloatNoNullBenchmarkData - extends BenchmarkData - { @Setup public void setup() throws Exception { - setup(REAL); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - values.add(random.nextFloat()); + if (typeSignature.startsWith("varchar")) { + type = new TypeRegistry().getType(TypeSignature.parseTypeSignature("varchar")); } - return values; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class FloatWithNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(REAL); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - if (random.nextBoolean()) { - values.add(random.nextFloat()); - } - else { - values.add(null); - } + else { + type = new TypeRegistry().getType(TypeSignature.parseTypeSignature(typeSignature)); } - return values; - } - } - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class BigintNoNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(BIGINT); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - values.add(random.nextLong()); - } - return values; + temporaryDirectory = createTempDir(); + orcFile = new File(temporaryDirectory, randomUUID().toString()); + writeOrcColumnHive(orcFile, ORC_12, NONE, type, createValues()); } - } - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class BigintWithNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception + @TearDown + public void tearDown() + throws IOException { - setup(BIGINT); + deleteRecursively(temporaryDirectory.toPath(), ALLOW_INSECURE); } - @Override protected List createValues() { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - if (random.nextBoolean()) { - values.add(Long.valueOf(random.nextLong())); - } - else { - values.add(null); - } + switch (withNulls) { + case ALL: + return NULL_VALUES; + case PARTIAL: + return IntStream.range(0, ROWS).mapToObj(i -> i % 2 == 0 ? createValue() : null).collect(toList()); + default: + return IntStream.range(0, ROWS).mapToObj(i -> createValue()).collect(toList()); } - return values; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class VarcharNoNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(VARCHAR); } - @Override - protected List createValues() + private Object createValue() { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - values.add(Strings.repeat("0", 4)); + switch (typeSignature) { + case "boolean": + return random.nextBoolean(); + case "tinyint": + return Long.valueOf(random.nextLong()).byteValue(); + case "smallint": + return (short) random.nextInt(); + case "integer": + return random.nextInt(); + case "bigint": + return random.nextLong(); + case "decimal(10,5)": + return new SqlDecimal(BigInteger.valueOf(random.nextLong() % 10_000_000_000L), SHORT_DECIMAL_TYPE.getPrecision(), SHORT_DECIMAL_TYPE.getScale()); + case "decimal(30,10)": + return new SqlDecimal(BigInteger.valueOf(random.nextLong() % 10_000_000_000L), LONG_DECIMAL_TYPE.getPrecision(), LONG_DECIMAL_TYPE.getScale()); + case "timestamp": + return new SqlTimestamp((random.nextLong()), UTC_KEY); + case "real": + return random.nextFloat(); + case "double": + return random.nextDouble(); + case "varchar_dictionary": + return Strings.repeat("0", MAX_STRING); + case "varchar_direct": + return randomAsciiString(random); } - return values; - } - } - - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class VarcharWithNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception - { - setup(VARCHAR); - } - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - if (random.nextBoolean()) { - values.add(Strings.repeat("0", 4)); - } - else { - values.add(null); - } - } - return values; + throw new UnsupportedOperationException("Unsupported type: " + typeSignature); } - } - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class TimestampNoNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception + private OrcBatchRecordReader createRecordReader() + throws IOException { - setup(TIMESTAMP); + OrcDataSource dataSource = new FileOrcDataSource(orcFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); + OrcReader orcReader = new OrcReader( + dataSource, + ORC, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + OrcReaderTestingUtils.createDefaultTestConfig()); + return orcReader.createBatchRecordReader( + ImmutableMap.of(0, type), + OrcPredicate.TRUE, + UTC, // arbitrary + newSimpleAggregatedMemoryContext(), + INITIAL_BATCH_SIZE); } - @Override - protected List createValues() + private static String randomAsciiString(Random random) { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - values.add(new SqlTimestamp((random.nextLong()), UTC_KEY)); + char[] value = new char[random.nextInt(MAX_STRING)]; + for (int i = 0; i < value.length; i++) { + value[i] = (char) random.nextInt(Byte.MAX_VALUE); } - return values; + return new String(value); } - } - @SuppressWarnings("FieldMayBeFinal") - @State(Scope.Thread) - public static class TimestampWithNullBenchmarkData - extends BenchmarkData - { - @Setup - public void setup() - throws Exception + public enum Nulls { - setup(TIMESTAMP); - } - - @Override - protected List createValues() - { - List values = new ArrayList<>(); - for (int i = 0; i < ROWS; ++i) { - if (random.nextBoolean()) { - values.add(new SqlTimestamp(random.nextLong(), UTC_KEY)); - } - else { - values.add(null); - } - } - return values; + PARTIAL, NONE, ALL; } } @@ -773,6 +237,7 @@ public static void main(String[] args) Options options = new OptionsBuilder() .verbosity(VerboseMode.NORMAL) .include(".*" + BenchmarkBatchStreamReaders.class.getSimpleName() + ".*") + .warmupMode(WarmupMode.BULK) .build(); new Runner(options).run(); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkBatchStreamReadersWithZstd.java b/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkBatchStreamReadersWithZstd.java new file mode 100644 index 0000000000000..9a43b83e60330 --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkBatchStreamReadersWithZstd.java @@ -0,0 +1,256 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.type.DecimalType; +import com.facebook.presto.spi.type.SqlDecimal; +import com.facebook.presto.spi.type.SqlTimestamp; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeSignature; +import com.facebook.presto.type.TypeRegistry; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.units.DataSize; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.VerboseMode; + +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; + +import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE; +import static com.facebook.presto.orc.OrcTester.Format.DWRF; +import static com.facebook.presto.orc.OrcTester.writeOrcColumnPresto; +import static com.facebook.presto.orc.metadata.CompressionKind.ZSTD; +import static com.facebook.presto.spi.type.DecimalType.createDecimalType; +import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY; +import static com.google.common.io.Files.createTempDir; +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static io.airlift.units.DataSize.Unit.MEGABYTE; +import static java.util.UUID.randomUUID; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.stream.Collectors.toList; +import static org.joda.time.DateTimeZone.UTC; + +@SuppressWarnings("MethodMayBeStatic") +@State(Scope.Thread) +@OutputTimeUnit(TimeUnit.SECONDS) +@Fork(3) +@Warmup(iterations = 20, time = 500, timeUnit = MILLISECONDS) +@Measurement(iterations = 20, time = 500, timeUnit = MILLISECONDS) +@BenchmarkMode(Mode.AverageTime) +public class BenchmarkBatchStreamReadersWithZstd +{ + private static final DecimalType SHORT_DECIMAL_TYPE = createDecimalType(10, 5); + private static final DecimalType LONG_DECIMAL_TYPE = createDecimalType(30, 10); + private static final int ROWS = 10_000_000; + private static final int MAX_STRING = 10; + private static final List NULL_VALUES = Collections.nCopies(ROWS, null); + + @Benchmark + public Object readBlocksWithoutJni(BenchmarkData data) + throws Throwable + { + OrcBatchRecordReader recordReader = data.createRecordReader(false); + ImmutableList.Builder blocks = new ImmutableList.Builder<>(); + while (recordReader.nextBatch() > 0) { + Block block = recordReader.readBlock(0); + blocks.add(block); + } + return blocks.build(); + } + + @Benchmark + public Object readBlocksWithJni(BenchmarkData data) + throws Throwable + { + OrcBatchRecordReader recordReader = data.createRecordReader(true); + ImmutableList.Builder blocks = new ImmutableList.Builder<>(); + while (recordReader.nextBatch() > 0) { + Block block = recordReader.readBlock(0); + blocks.add(block); + } + return blocks.build(); + } + + @State(Scope.Thread) + public static class BenchmarkData + { + private final Random random = new Random(0); + + private Type type; + private File temporaryDirectory; + private File orcFile; + private final OrcTester.Format format = DWRF; + + @SuppressWarnings("unused") + @Param({ + "boolean", + "tinyint", + "smallint", + "integer", + "bigint", + "decimal(10,5)", + "decimal(30,10)", + "timestamp", + "real", + "double", + "varchar_direct", + "varchar_dictionary", + }) + private String typeSignature; + + @SuppressWarnings("unused") + @Param({ + "PARTIAL", + "NONE", + "ALL" + }) + private Nulls withNulls; + + @Setup + public void setup() + throws Exception + { + if (typeSignature.startsWith("varchar")) { + type = new TypeRegistry().getType(TypeSignature.parseTypeSignature("varchar")); + } + else { + type = new TypeRegistry().getType(TypeSignature.parseTypeSignature(typeSignature)); + } + + temporaryDirectory = createTempDir(); + orcFile = new File(temporaryDirectory, randomUUID().toString()); + writeOrcColumnPresto(orcFile, format, ZSTD, type, createValues()); + } + + @TearDown + public void tearDown() + throws IOException + { + deleteRecursively(temporaryDirectory.toPath(), ALLOW_INSECURE); + } + + protected List createValues() + { + switch (withNulls) { + case ALL: + return NULL_VALUES; + case PARTIAL: + return IntStream.range(0, ROWS).mapToObj(i -> i % 2 == 0 ? createValue() : null).collect(toList()); + default: + return IntStream.range(0, ROWS).mapToObj(i -> createValue()).collect(toList()); + } + } + + private Object createValue() + { + switch (typeSignature) { + case "boolean": + return random.nextBoolean(); + case "tinyint": + return Long.valueOf(random.nextLong()).byteValue(); + case "smallint": + return (short) random.nextInt(); + case "integer": + return random.nextInt(); + case "bigint": + return random.nextLong(); + case "decimal(10,5)": + return new SqlDecimal(BigInteger.valueOf(random.nextLong() % 10_000_000_000L), SHORT_DECIMAL_TYPE.getPrecision(), SHORT_DECIMAL_TYPE.getScale()); + case "decimal(30,10)": + return new SqlDecimal(BigInteger.valueOf(random.nextLong() % 10_000_000_000L), LONG_DECIMAL_TYPE.getPrecision(), LONG_DECIMAL_TYPE.getScale()); + case "timestamp": + return new SqlTimestamp((random.nextLong()), UTC_KEY); + case "real": + return random.nextFloat(); + case "double": + return random.nextDouble(); + case "varchar_dictionary": + return Strings.repeat("0", MAX_STRING); + case "varchar_direct": + return randomAsciiString(random); + } + + throw new UnsupportedOperationException("Unsupported type: " + typeSignature); + } + + private OrcBatchRecordReader createRecordReader(boolean zstdJniDecompressionEnabled) + throws IOException + { + OrcDataSource dataSource = new FileOrcDataSource(orcFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); + OrcReader orcReader = new OrcReader( + dataSource, + format.getOrcEncoding(), + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + OrcReaderTestingUtils.createTestingReaderOptions(zstdJniDecompressionEnabled)); + return orcReader.createBatchRecordReader( + ImmutableMap.of(0, type), + OrcPredicate.TRUE, + UTC, // arbitrary + newSimpleAggregatedMemoryContext(), + INITIAL_BATCH_SIZE); + } + + private static String randomAsciiString(Random random) + { + char[] value = new char[random.nextInt(MAX_STRING)]; + for (int i = 0; i < value.length; i++) { + value[i] = (char) random.nextInt(Byte.MAX_VALUE); + } + return new String(value); + } + + public enum Nulls + { + PARTIAL, NONE, ALL; + } + } + + public static void main(String[] args) + throws Throwable + { + Options options = new OptionsBuilder() + .verbosity(VerboseMode.NORMAL) + .include(".*" + BenchmarkBatchStreamReadersWithZstd.class.getSimpleName() + ".*") + .build(); + + new Runner(options).run(); + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkSelectiveStreamReaders.java b/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkSelectiveStreamReaders.java index a94e77276c4bc..af94c121c2801 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkSelectiveStreamReaders.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkSelectiveStreamReaders.java @@ -15,14 +15,27 @@ import com.facebook.presto.orc.TupleDomainFilter.BigintRange; import com.facebook.presto.orc.TupleDomainFilter.BooleanValue; +import com.facebook.presto.orc.TupleDomainFilter.BytesRange; +import com.facebook.presto.orc.TupleDomainFilter.DoubleRange; +import com.facebook.presto.orc.TupleDomainFilter.FloatRange; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.Subfield; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.type.DecimalType; +import com.facebook.presto.spi.type.Decimals; import com.facebook.presto.spi.type.SqlDate; +import com.facebook.presto.spi.type.SqlDecimal; +import com.facebook.presto.spi.type.SqlTimestamp; +import com.facebook.presto.spi.type.TimeZoneKey; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeSignature; +import com.facebook.presto.spi.type.VarcharType; import com.facebook.presto.type.TypeRegistry; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.primitives.Longs; import io.airlift.units.DataSize; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -43,6 +56,7 @@ import java.io.File; import java.io.IOException; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -56,12 +70,18 @@ import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE; import static com.facebook.presto.orc.OrcTester.Format.ORC_12; import static com.facebook.presto.orc.OrcTester.writeOrcColumnHive; +import static com.facebook.presto.orc.TupleDomainFilter.LongDecimalRange; import static com.facebook.presto.orc.metadata.CompressionKind.NONE; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DateType.DATE; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RealType.REAL; import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.google.common.io.Files.createTempDir; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; @@ -82,6 +102,9 @@ public class BenchmarkSelectiveStreamReaders { public static final int ROWS = 10_000_000; public static final List NULL_VALUES = Collections.nCopies(ROWS, null); + private static final DecimalType SHORT_DECIMAL_TYPE = DecimalType.createDecimalType(10, 5); + private static final DecimalType LONG_DECIMAL_TYPE = DecimalType.createDecimalType(30, 10); + private static final int MAX_STRING_LENGTH = 10; @Benchmark public Object readAllNull(AllNullBenchmarkData data) @@ -127,11 +150,18 @@ private abstract static class AbstractBenchmarkData private Type type; private File temporaryDirectory; private File orcFile; + private String typeSignature; public void setup(String typeSignature) throws Exception { - type = new TypeRegistry().getType(TypeSignature.parseTypeSignature(typeSignature)); + if (typeSignature.startsWith("varchar")) { + type = new TypeRegistry().getType(TypeSignature.parseTypeSignature("varchar")); + } + else { + type = new TypeRegistry().getType(TypeSignature.parseTypeSignature(typeSignature)); + } + this.typeSignature = typeSignature; temporaryDirectory = createTempDir(); orcFile = new File(temporaryDirectory, randomUUID().toString()); writeOrcColumnHive(orcFile, ORC_12, NONE, type, createValues()); @@ -149,22 +179,33 @@ public Type getType() return type; } + public String getTypeSignature() + { + return typeSignature; + } + protected abstract List createValues(); public OrcSelectiveRecordReader createRecordReader(Optional filter) throws IOException { OrcDataSource dataSource = new FileOrcDataSource(orcFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); - OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); + OrcReader orcReader = new OrcReader( + dataSource, + ORC, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + OrcReaderTestingUtils.createDefaultTestConfig()); return orcReader.createSelectiveRecordReader( ImmutableMap.of(0, type), ImmutableList.of(0), - filter.map(f -> ImmutableMap.of(0, f)).orElse(ImmutableMap.of()), + filter.isPresent() ? ImmutableMap.of(0, ImmutableMap.of(new Subfield("c"), filter.get())) : ImmutableMap.of(), ImmutableList.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(), + ImmutableMap.of(), OrcPredicate.TRUE, 0, dataSource.getSize(), @@ -182,10 +223,22 @@ public static class AllNullBenchmarkData @SuppressWarnings("unused") @Param({ "boolean", + "integer", "bigint", "smallint", - "date" + "tinyint", + + "date", + "timestamp", + + "real", + "double", + "decimal(10,5)", + "decimal(30,10)", + + "varchar_direct", + "varchar_dictionary" }) private String typeSignature; @@ -214,10 +267,22 @@ public static class BenchmarkData "integer", "bigint", "smallint", - "date" + "tinyint", + + "date", + "timestamp", + + "real", + "double", + "decimal(10,5)", + "decimal(30,10)", + + "varchar_direct", + "varchar_dictionary" }) private String typeSignature; + @SuppressWarnings("unused") @Param({"true", "false"}) private boolean withNulls; @@ -243,10 +308,29 @@ private static Optional getFilter(Type type) return Optional.of(BooleanValue.of(true, true)); } - if (type == BIGINT || type == INTEGER || type == SMALLINT || type == DATE) { + if (type == TINYINT || type == BIGINT || type == INTEGER || type == SMALLINT || type == DATE || type == TIMESTAMP) { return Optional.of(BigintRange.of(0, Long.MAX_VALUE, true)); } + if (type == REAL) { + return Optional.of(FloatRange.of(0, true, true, Integer.MAX_VALUE, true, true, true)); + } + + if (type == DOUBLE) { + return Optional.of(DoubleRange.of(.5, false, false, 2, false, false, false)); + } + + if (type instanceof DecimalType) { + if (((DecimalType) type).isShort()) { + return Optional.of(BigintRange.of(0, Long.MAX_VALUE, true)); + } + return Optional.of(LongDecimalRange.of(0, 0, false, true, Long.MAX_VALUE, Long.MAX_VALUE, false, true, true)); + } + + if (type instanceof VarcharType) { + return Optional.of(BytesRange.of("0".getBytes(), false, Longs.toByteArray(Long.MAX_VALUE), false, true)); + } + throw new UnsupportedOperationException("Unsupported type: " + type); } @@ -277,14 +361,55 @@ private final Object createValue() return (short) random.nextInt(); } + if (getType() == TINYINT) { + return (byte) random.nextInt(); + } + if (getType() == DATE) { return new SqlDate(random.nextInt()); } + if (getType() == TIMESTAMP) { + return new SqlTimestamp(random.nextLong(), TimeZoneKey.UTC_KEY); + } + + if (getType() == REAL) { + return random.nextFloat(); + } + + if (getType() == DOUBLE) { + return random.nextDouble(); + } + + if (getType() instanceof DecimalType) { + if (Decimals.isShortDecimal(getType())) { + return new SqlDecimal(BigInteger.valueOf(random.nextLong() % 10_000_000_000L), SHORT_DECIMAL_TYPE.getPrecision(), SHORT_DECIMAL_TYPE.getScale()); + } + else { + return new SqlDecimal(BigInteger.valueOf(random.nextLong() % 10_000_000_000L), LONG_DECIMAL_TYPE.getPrecision(), LONG_DECIMAL_TYPE.getScale()); + } + } + + if (getType() == VARCHAR) { + if (typeSignature.equals("varchar_dictionary")) { + return Strings.repeat("0", MAX_STRING_LENGTH); + } + return randomAsciiString(random, MAX_STRING_LENGTH); + } + throw new UnsupportedOperationException("Unsupported type: " + getType()); } } + private static String randomAsciiString(Random random, int maxLength) + { + char[] value = new char[random.nextInt(maxLength)]; + for (int i = 0; i < value.length; i++) { + value[i] = (char) random.nextInt(Byte.MAX_VALUE); + } + return new String(value); + } + public static void main(String[] args) throws Throwable { diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkZstdJniDecompression.java b/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkZstdJniDecompression.java new file mode 100644 index 0000000000000..28c9f56f90482 --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/BenchmarkZstdJniDecompression.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.facebook.presto.orc.zstd.ZstdJniCompressor; +import com.google.common.collect.ImmutableList; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.List; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.testng.Assert.assertEquals; + +@SuppressWarnings("MethodMayBeStatic") +@State(Scope.Thread) +@OutputTimeUnit(MILLISECONDS) +@Fork(3) +@Warmup(iterations = 20, time = 500, timeUnit = MILLISECONDS) +@Measurement(iterations = 20, time = 500, timeUnit = MILLISECONDS) +@BenchmarkMode(Mode.AverageTime) + +public class BenchmarkZstdJniDecompression +{ + private static final ZstdJniCompressor compressor = new ZstdJniCompressor(); + private static final List list = generateWorkload(); + private static final int sourceLength = 256 * 1024; + private static byte[] decompressedBytes = new byte[sourceLength]; + + @Benchmark + public void decompressJni() + throws OrcCorruptionException + { + decompressList(createOrcDecompressor(true)); + } + + @Benchmark + public void decompressJava() + throws OrcCorruptionException + { + decompressList(createOrcDecompressor(false)); + } + + private void decompressList(OrcDecompressor decompressor) + throws OrcCorruptionException + { + for (Unit unit : list) { + int outputSize = decompressor.decompress(unit.compressedBytes, 0, unit.compressedLength, new OrcDecompressor.OutputBuffer() + { + @Override + public byte[] initialize(int size) + { + return decompressedBytes; + } + + @Override + public byte[] grow(int size) + { + throw new RuntimeException(); + } + }); + assertEquals(outputSize, unit.sourceLength); + } + } + + private static List generateWorkload() + { + ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (int i = 0; i < 10; i++) { + byte[] sourceBytes = getAlphaNumericString(sourceLength).getBytes(); + byte[] compressedBytes = new byte[sourceLength * 32]; + int size = compressor.compress(sourceBytes, 0, sourceBytes.length, compressedBytes, 0, compressedBytes.length); + builder.add(new Unit(sourceBytes, sourceLength, compressedBytes, size)); + } + return builder.build(); + } + + private OrcDecompressor createOrcDecompressor(boolean zstdJniDecompressionEnabled) + { + return new OrcZstdDecompressor(new OrcDataSourceId("orc"), sourceLength, zstdJniDecompressionEnabled); + } + + private static String getAlphaNumericString(int length) + { + String alphaNumericString = "USINDIA"; + + StringBuilder stringBuilder = new StringBuilder(length); + + for (int index = 0; index < length; index++) { + int arrayIndex = (int) (alphaNumericString.length() * Math.random()); + + stringBuilder.append(alphaNumericString.charAt(arrayIndex)); + } + return stringBuilder.toString(); + } + + static class Unit + { + final byte[] sourceBytes; + final int sourceLength; + final byte[] compressedBytes; + final int compressedLength; + + public Unit(byte[] sourceBytes, int sourceLength, byte[] compressedBytes, int compressedLength) + { + this.sourceBytes = sourceBytes; + this.sourceLength = sourceLength; + this.compressedBytes = compressedBytes; + this.compressedLength = compressedLength; + } + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/OrcReaderTestingUtils.java b/presto-orc/src/test/java/com/facebook/presto/orc/OrcReaderTestingUtils.java new file mode 100644 index 0000000000000..2b91af7b2bf0c --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/OrcReaderTestingUtils.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import io.airlift.units.DataSize; + +import static io.airlift.units.DataSize.Unit.MEGABYTE; + +public class OrcReaderTestingUtils +{ + private OrcReaderTestingUtils() {} + + public static OrcReaderOptions createDefaultTestConfig() + { + return createTestingReaderOptions(false); + } + + public static OrcReaderOptions createTestingReaderOptions(boolean zstdJniDecompressionEnabled) + { + return new OrcReaderOptions( + new DataSize(1, MEGABYTE), + new DataSize(1, MEGABYTE), + new DataSize(1, MEGABYTE), + zstdJniDecompressionEnabled); + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/OrcTester.java b/presto-orc/src/test/java/com/facebook/presto/orc/OrcTester.java index c9962442081c8..34ded96c18cbb 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/OrcTester.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/OrcTester.java @@ -17,15 +17,25 @@ import com.facebook.hive.orc.lazy.OrcLazyObject; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.orc.TrackingTupleDomainFilter.TestBigintRange; +import com.facebook.presto.orc.TrackingTupleDomainFilter.TestDoubleRange; +import com.facebook.presto.orc.TupleDomainFilter.BigintRange; +import com.facebook.presto.orc.TupleDomainFilter.DoubleRange; +import com.facebook.presto.orc.cache.OrcFileTailSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.Subfield; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.Decimals; +import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.NamedTypeSignature; import com.facebook.presto.spi.type.RowFieldName; +import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.SqlDate; import com.facebook.presto.spi.type.SqlDecimal; import com.facebook.presto.spi.type.SqlTimestamp; @@ -45,6 +55,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import io.airlift.units.DataSize; @@ -90,6 +101,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.lang.reflect.Field; +import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Date; import java.sql.Timestamp; @@ -118,6 +130,8 @@ import static com.facebook.presto.orc.OrcTester.Format.ORC_12; import static com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode.BOTH; import static com.facebook.presto.orc.TestingOrcPredicate.createOrcPredicate; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NOT_NULL; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NULL; import static com.facebook.presto.orc.metadata.CompressionKind.LZ4; import static com.facebook.presto.orc.metadata.CompressionKind.NONE; import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; @@ -135,10 +149,14 @@ import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; import static com.facebook.presto.spi.type.TinyintType.TINYINT; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.spi.type.Varchars.truncateToLength; import static com.facebook.presto.testing.DateTimeTestingUtils.sqlTimestampOf; import static com.facebook.presto.testing.TestingConnectorSession.SESSION; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Lists.newArrayList; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.units.DataSize.Unit.MEGABYTE; @@ -167,7 +185,8 @@ import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaTimestampObjectInspector; import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getCharTypeInfo; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -287,11 +306,14 @@ public static OrcTester fullOrcTester() public static OrcTester quickSelectiveOrcTester() { OrcTester orcTester = new OrcTester(); + orcTester.listTestsEnabled = true; + orcTester.structTestsEnabled = true; orcTester.nullTestsEnabled = true; orcTester.skipBatchTestsEnabled = true; orcTester.formats = ImmutableSet.of(ORC_12, ORC_11, DWRF); orcTester.compressions = ImmutableSet.of(ZLIB); orcTester.useSelectiveOrcReader = true; + return orcTester; } @@ -301,11 +323,19 @@ public void testRoundTrip(Type type, List readValues) testRoundTrip(type, readValues, ImmutableList.of()); } - public void testRoundTrip(Type type, List readValues, List> filters) + public void testRoundTrip(Type type, List readValues, TupleDomainFilter... filters) + throws Exception + { + testRoundTrip(type, readValues, Arrays.stream(filters).map(filter -> ImmutableMap.of(new Subfield("c"), filter)).collect(toImmutableList())); + } + + public void testRoundTrip(Type type, List readValues, List> filters) throws Exception { + List>> columnFilters = filters.stream().map(filter -> ImmutableMap.of(0, filter)).collect(toImmutableList()); + // just the values - testRoundTripTypes(ImmutableList.of(type), ImmutableList.of(readValues), filters); + testRoundTripTypes(ImmutableList.of(type), ImmutableList.of(readValues), columnFilters); // all nulls if (nullTestsEnabled) { @@ -314,7 +344,7 @@ public void testRoundTrip(Type type, List readValues, List null) .collect(toList()), - filters); + columnFilters); } // values wrapped in struct @@ -460,30 +490,59 @@ private void testRoundTripType(Type type, List readValues) testRoundTripTypes(ImmutableList.of(type), ImmutableList.of(readValues), ImmutableList.of()); } - public void testRoundTripTypes(List types, List> readValues, List> filters) + public void testRoundTripTypes(List types, List> readValues, List>> filters) + throws Exception + { + testRoundTripTypes(types, readValues, filters, ImmutableList.of()); + } + + public void testRoundTripTypes(List types, List> readValues, List>> filters, List> expectedFilterOrder) throws Exception { assertEquals(types.size(), readValues.size()); + if (!expectedFilterOrder.isEmpty()) { + assertEquals(filters.size(), expectedFilterOrder.size()); + } // forward order - assertRoundTrip(types, readValues, filters); + assertRoundTrip(types, readValues, filters, expectedFilterOrder); // reverse order if (reverseTestsEnabled) { - assertRoundTrip(types, Lists.transform(readValues, OrcTester::reverse), filters); + assertRoundTrip(types, Lists.transform(readValues, OrcTester::reverse), filters, expectedFilterOrder); } if (nullTestsEnabled) { // forward order with nulls - assertRoundTrip(types, insertNulls(types, readValues), filters); + assertRoundTrip(types, insertNulls(types, readValues), filters, expectedFilterOrder); // reverse order with nulls if (reverseTestsEnabled) { - assertRoundTrip(types, insertNulls(types, Lists.transform(readValues, OrcTester::reverse)), filters); + assertRoundTrip(types, insertNulls(types, Lists.transform(readValues, OrcTester::reverse)), filters, expectedFilterOrder); } } } + public void testRoundTripTypesWithOrder(List types, List> readValues, List>> filters, List> expectedFilterOrder) + throws Exception + { + assertNotNull(expectedFilterOrder); + assertEquals(filters.size(), expectedFilterOrder.size()); + + // Forward order + testRoundTripTypes(types, readValues, filters, expectedFilterOrder); + + // Reverse order + int columnCount = types.size(); + List>> reverseFilters = filters.stream() + .map(filtersEntry -> filtersEntry.entrySet().stream().collect(toImmutableMap(entry -> columnCount - 1 - entry.getKey(), Entry::getValue))) + .collect(toImmutableList()); + List> reverseFilterOrder = expectedFilterOrder.stream() + .map(columns -> columns.stream().map(column -> columnCount - 1 - column).collect(toImmutableList())) + .collect(toImmutableList()); + testRoundTripTypes(Lists.reverse(types), Lists.reverse(readValues), reverseFilters, reverseFilterOrder); + } + private List> insertNulls(List types, List> values) { assertTrue(types.size() <= PRIME_NUMBERS.size()); @@ -499,10 +558,20 @@ public void assertRoundTrip(Type type, List readValues) assertRoundTrip(type, type, readValues, readValues, true, ImmutableList.of()); } - public void assertRoundTrip(Type type, List readValues, List> filters) + public void assertRoundTripWithSettings(Type type, List readValues, List settings) throws Exception { - assertRoundTrip(type, type, readValues, readValues, true, filters); + assertRoundTrip(type, type, readValues, readValues, true, settings); + } + + public void assertRoundTrip(Type type, List readValues, List>> filters) + throws Exception + { + List settings = filters.stream() + .map(entry -> OrcReaderSettings.builder().setColumnFilters(entry).build()) + .collect(toImmutableList()); + + assertRoundTrip(type, type, readValues, readValues, true, settings); } public void assertRoundTrip(Type type, List readValues, boolean verifyWithHiveReader) @@ -511,19 +580,25 @@ public void assertRoundTrip(Type type, List readValues, boolean verifyWithHiv assertRoundTrip(type, type, readValues, readValues, verifyWithHiveReader, ImmutableList.of()); } - public void assertRoundTrip(List types, List> readValues, List> filters) + public void assertRoundTrip(List types, List> readValues, List>> filters, List> expectedFilterOrder) throws Exception { - assertRoundTrip(types, types, readValues, readValues, true, filters); + List settings = IntStream.range(0, filters.size()) + .mapToObj(i -> OrcReaderSettings.builder() + .setColumnFilters(filters.get(i)) + .setExpectedFilterOrder(expectedFilterOrder.isEmpty() ? ImmutableList.of() : expectedFilterOrder.get(i)) + .build()) + .collect(toImmutableList()); + assertRoundTrip(types, types, readValues, readValues, true, settings); } - private void assertRoundTrip(Type writeType, Type readType, List writeValues, List readValues, boolean verifyWithHiveReader, List> filters) + private void assertRoundTrip(Type writeType, Type readType, List writeValues, List readValues, boolean verifyWithHiveReader, List settings) throws Exception { - assertRoundTrip(ImmutableList.of(writeType), ImmutableList.of(readType), ImmutableList.of(writeValues), ImmutableList.of(readValues), verifyWithHiveReader, filters); + assertRoundTrip(ImmutableList.of(writeType), ImmutableList.of(readType), ImmutableList.of(writeValues), ImmutableList.of(readValues), verifyWithHiveReader, settings); } - private void assertRoundTrip(List writeTypes, List readTypes, List> writeValues, List> readValues, boolean verifyWithHiveReader, List> filters) + private void assertRoundTrip(List writeTypes, List readTypes, List> writeValues, List> readValues, boolean verifyWithHiveReader, List settings) throws Exception { assertEquals(writeTypes.size(), readTypes.size()); @@ -544,7 +619,7 @@ private void assertRoundTrip(List writeTypes, List readTypes, List writeTypes, List readTypes, List writeTypes, List readTypes, List> columnFilters; + private final List expectedFilterOrder; + private final List filterFunctions; + private final Map filterFunctionInputMapping; + private final Map> requiredSubfields; + private final OrcFileTailSource orcFileTailSource; + + private OrcReaderSettings( + Map> columnFilters, + List expectedFilterOrder, + List filterFunctions, + Map filterFunctionInputMapping, + Map> requiredSubfields, + OrcFileTailSource orcFileTailSource) + { + this.columnFilters = requireNonNull(columnFilters, "columnFilters is null"); + this.expectedFilterOrder = requireNonNull(expectedFilterOrder, "expectedFilterOrder is null"); + this.filterFunctions = requireNonNull(filterFunctions, "filterFunctions is null"); + this.filterFunctionInputMapping = requireNonNull(filterFunctionInputMapping, "filterFunctionInputMapping is null"); + this.requiredSubfields = requireNonNull(requiredSubfields, "requiredSubfields is null"); + this.orcFileTailSource = requireNonNull(orcFileTailSource, "orcFileTailSource is null"); + } + + public Map> getColumnFilters() + { + return columnFilters; + } + + public List getExpectedFilterOrder() + { + return expectedFilterOrder; + } + + public List getFilterFunctions() + { + return filterFunctions; + } + + public Map getFilterFunctionInputMapping() + { + return filterFunctionInputMapping; + } + + public Map> getRequiredSubfields() + { + return requiredSubfields; + } + + public OrcFileTailSource getOrcFileTailSource() + { + return orcFileTailSource; + } + + public static Builder builder() + { + return new Builder(); + } + + public static class Builder + { + private Map> columnFilters = ImmutableMap.of(); + private List expectedFilterOrder = ImmutableList.of(); + private List filterFunctions = ImmutableList.of(); + private Map filterFunctionInputMapping = ImmutableMap.of(); + private Map> requiredSubfields = new HashMap<>(); + private OrcFileTailSource orcFileTailSource = new StorageOrcFileTailSource(); + + public Builder setColumnFilters(Map> columnFilters) + { + this.columnFilters = requireNonNull(columnFilters, "columnFilters is null"); + return this; + } + + public Builder setExpectedFilterOrder(List expectedFilterOrder) + { + this.expectedFilterOrder = requireNonNull(expectedFilterOrder, "expectedFilterOrder is null"); + return this; + } + + public Builder setFilterFunctions(List filterFunctions) + { + this.filterFunctions = requireNonNull(filterFunctions, "filterFunctions is null"); + return this; + } + + public Builder setFilterFunctionMapping(Map filterFunctionInputMapping) + { + this.filterFunctionInputMapping = requireNonNull(filterFunctionInputMapping, "filterFunctionInputMapping is null"); + return this; + } + + public Builder setRequiredSubfields(Map> requiredSubfields) + { + requireNonNull(requiredSubfields, "requiredSubfields is null"); + this.requiredSubfields.clear(); + this.requiredSubfields.putAll(requiredSubfields); + return this; + } + + public Builder addRequiredSubfields(int column, String... subfields) + { + this.requiredSubfields.put(column, Arrays.stream(subfields).map(subfield -> new Subfield(subfield)).collect(toImmutableList())); + return this; + } + + public Builder setOrcFileTailSource(OrcFileTailSource orcFileTailSource) + { + this.orcFileTailSource = requireNonNull(orcFileTailSource, "orcFileTailSource is null"); + return this; + } + + public OrcReaderSettings build() + { + return new OrcReaderSettings(columnFilters, expectedFilterOrder, filterFunctions, filterFunctionInputMapping, requiredSubfields, orcFileTailSource); + } + } + } + + public static void assertFileContentsPresto( List types, - TempFile tempFile, + File file, List> expectedValues, OrcEncoding orcEncoding, OrcPredicate orcPredicate, - Optional> filters) + Optional>> filters, + List filterFunctions, + Map filterFunctionInputMapping, + Map> requiredSubfields) throws IOException { - try (OrcSelectiveRecordReader recordReader = createCustomOrcSelectiveRecordReader(tempFile, orcEncoding, orcPredicate, types, MAX_BATCH_SIZE, filters.orElse(ImmutableMap.of()))) { + try (OrcSelectiveRecordReader recordReader = createCustomOrcSelectiveRecordReader(file, orcEncoding, orcPredicate, types, MAX_BATCH_SIZE, filters.orElse(ImmutableMap.of()), filterFunctions, filterFunctionInputMapping, requiredSubfields)) { assertEquals(recordReader.getReaderPosition(), 0); assertEquals(recordReader.getFilePosition(), 0); @@ -592,31 +790,52 @@ private static void assertFileContentsPresto( break; } - if (page.getPositionCount() == 0) { + int positionCount = page.getPositionCount(); + if (positionCount == 0) { continue; } + assertTrue(expectedValues.get(0).size() >= rowsProcessed + positionCount); + for (int i = 0; i < types.size(); i++) { Type type = types.get(i); Block block = page.getBlock(i); + assertEquals(block.getPositionCount(), positionCount); + checkNullValues(type, block); - List data = new ArrayList<>(block.getPositionCount()); - for (int position = 0; position < block.getPositionCount(); position++) { + List data = new ArrayList<>(positionCount); + for (int position = 0; position < positionCount; position++) { data.add(type.getObjectValue(SESSION, block, position)); } - for (int j = 0; j < block.getPositionCount(); j++) { - assertColumnValueEquals(type, data.get(j), expectedValues.get(i).get(rowsProcessed + j)); + for (int position = 0; position < positionCount; position++) { + assertColumnValueEquals(type, data.get(position), expectedValues.get(i).get(rowsProcessed + position)); } } - rowsProcessed += page.getPositionCount(); + rowsProcessed += positionCount; } assertEquals(rowsProcessed, expectedValues.get(0).size()); } } + private static Map> addOrderTracking(Map> filters, TupleDomainFilterOrderChecker orderChecker) + { + return Maps.transformEntries(filters, (column, columnFilters) -> Maps.transformValues(columnFilters, filter -> addOrderTracking(filter, orderChecker, column))); + } + + private static TupleDomainFilter addOrderTracking(TupleDomainFilter filter, TupleDomainFilterOrderChecker orderChecker, int column) + { + if (filter instanceof BigintRange) { + return TestBigintRange.of((BigintRange) filter, unused -> orderChecker.call(column)); + } + if (filter instanceof DoubleRange) { + return TestDoubleRange.of((DoubleRange) filter, unused -> orderChecker.call(column)); + } + throw new UnsupportedOperationException("Unsupported filter type: " + filter.getClass().getSimpleName()); + } + private static void assertFileContentsPresto( List types, TempFile tempFile, @@ -627,21 +846,36 @@ private static void assertFileContentsPresto( Format format, boolean isHiveWriter, boolean useSelectiveOrcReader, - List> filters) + List settings) throws IOException { OrcPredicate orcPredicate = createOrcPredicate(types, expectedValues, format, isHiveWriter); if (useSelectiveOrcReader) { - assertFileContentsPresto(types, tempFile, expectedValues, orcEncoding, orcPredicate, Optional.empty()); + assertFileContentsPresto(types, tempFile.getFile(), expectedValues, orcEncoding, orcPredicate, Optional.empty(), ImmutableList.of(), ImmutableMap.of(), ImmutableMap.of()); + for (OrcReaderSettings entry : settings) { + assertTrue(entry.getFilterFunctions().isEmpty(), "Filter functions are not supported yet"); + assertTrue(entry.getFilterFunctionInputMapping().isEmpty(), "Filter functions are not supported yet"); + + Map> columnFilters = entry.getColumnFilters(); + List> prunedAndFilteredRows = pruneValues(types, filterRows(types, expectedValues, columnFilters), entry.getRequiredSubfields()); + + Optional orderChecker = Optional.empty(); + List expectedFilterOrder = entry.getExpectedFilterOrder(); + if (!expectedFilterOrder.isEmpty()) { + orderChecker = Optional.of(new TupleDomainFilterOrderChecker(expectedFilterOrder)); + } + + Optional>> transformedFilters = Optional.of(orderChecker.map(checker -> addOrderTracking(columnFilters, checker)).orElse(columnFilters)); - for (Map columnFilters : filters) { - assertFileContentsPresto(types, tempFile, filterRows(types, expectedValues, columnFilters), orcEncoding, orcPredicate, Optional.of(columnFilters)); + assertFileContentsPresto(types, tempFile.getFile(), prunedAndFilteredRows, orcEncoding, orcPredicate, transformedFilters, entry.getFilterFunctions(), entry.getFilterFunctionInputMapping(), entry.getRequiredSubfields()); + + orderChecker.ifPresent(TupleDomainFilterOrderChecker::assertOrder); } return; } - try (OrcBatchRecordReader recordReader = createCustomOrcRecordReader(tempFile, orcEncoding, orcPredicate, types, MAX_BATCH_SIZE)) { + try (OrcBatchRecordReader recordReader = createCustomOrcRecordReader(tempFile, orcEncoding, orcPredicate, types, MAX_BATCH_SIZE, new StorageOrcFileTailSource(), new StorageStripeMetadataSource())) { assertEquals(recordReader.getReaderPosition(), 0); assertEquals(recordReader.getFilePosition(), 0); @@ -658,8 +892,9 @@ else if (skipFirstBatch && isFirst) { else { for (int i = 0; i < types.size(); i++) { Type type = types.get(i); - Block block = recordReader.readBlock(type, i); + Block block = recordReader.readBlock(i); assertEquals(block.getPositionCount(), batchSize); + checkNullValues(type, block); List data = new ArrayList<>(block.getPositionCount()); for (int position = 0; position < block.getPositionCount(); position++) { @@ -682,8 +917,12 @@ else if (skipFirstBatch && isFirst) { } } - private static List> filterRows(List types, List> values, Map columnFilters) + public static List> filterRows(List types, List> values, Map> columnFilters) { + if (columnFilters.isEmpty()) { + return values; + } + List passingRows = IntStream.range(0, values.get(0).size()) .filter(row -> testRow(types, values, row, columnFilters)) .boxed() @@ -693,56 +932,289 @@ private static List> filterRows(List types, List> values, .collect(toList()); } - private static boolean testRow(List types, List> values, int row, Map columnFilters) + private static boolean testRow(List types, List> values, int row, Map> columnFilters) { for (int column = 0; column < types.size(); column++) { - TupleDomainFilter filter = columnFilters.get(column); - if (filter == null) { + Map filters = columnFilters.get(column); + + if (filters == null) { continue; } + Type type = types.get(column); Object value = values.get(column).get(row); - if (value == null) { - if (!filter.testNull()) { + for (Map.Entry entry : filters.entrySet()) { + if (!testSubfieldValue(type, value, entry.getKey(), entry.getValue())) { return false; } } - else { - Type type = types.get(column); - if (type == BOOLEAN) { - if (!filter.testBoolean((Boolean) value)) { - return false; - } + } + + return true; + } + + private static boolean testSubfieldValue(Type type, Object value, Subfield subfield, TupleDomainFilter filter) + { + Type nestedType = type; + Object nestedValue = value; + for (Subfield.PathElement pathElement : subfield.getPath()) { + if (nestedType instanceof ArrayType) { + assertTrue(pathElement instanceof Subfield.LongSubscript); + if (nestedValue == null) { + return filter == IS_NULL; } - else if (type == BIGINT || type == INTEGER || type == SMALLINT) { - if (!filter.testLong(((Number) value).longValue())) { - return false; - } + int index = toIntExact(((Subfield.LongSubscript) pathElement).getIndex()) - 1; + nestedType = ((ArrayType) nestedType).getElementType(); + if (index >= ((List) nestedValue).size()) { + return true; } - else if (type == DATE) { - if (!filter.testLong(((SqlDate) value).getDays())) { - return false; - } + nestedValue = ((List) nestedValue).get(index); + } + else if (nestedType instanceof RowType) { + assertTrue(pathElement instanceof Subfield.NestedField); + if (nestedValue == null) { + return filter.testNull(); } - else { - fail("Unsupported type: " + type); + String fieldName = ((Subfield.NestedField) pathElement).getName(); + int index = -1; + List fields = ((RowType) nestedType).getFields(); + for (int i = 0; i < fields.size(); i++) { + if (fieldName.equalsIgnoreCase(fields.get(i).getName().get())) { + index = i; + nestedType = fields.get(i).getType(); + break; + } } + assertTrue(index >= 0, "Struct field not found: " + fieldName); + nestedValue = ((List) nestedValue).get(index); + } + else { + fail("Unsupported type: " + type); } } + return testValue(nestedType, nestedValue, filter); + } - return true; + private static boolean testValue(Type type, Object value, TupleDomainFilter filter) + { + if (value == null) { + return filter.testNull(); + } + + if (filter == IS_NULL) { + return false; + } + + if (filter == IS_NOT_NULL) { + return true; + } + + if (type == BOOLEAN) { + return filter.testBoolean((Boolean) value); + } + + if (type == TINYINT || type == BIGINT || type == INTEGER || type == SMALLINT) { + return filter.testLong(((Number) value).longValue()); + } + + if (type == REAL) { + return filter.testFloat(((Number) value).floatValue()); + } + + if (type == DOUBLE) { + return filter.testDouble((double) value); + } + + if (type == DATE) { + return filter.testLong(((SqlDate) value).getDays()); + } + + if (type == TIMESTAMP) { + return filter.testLong(((SqlTimestamp) value).getMillisUtc()); + } + + if (type instanceof DecimalType) { + DecimalType decimalType = (DecimalType) type; + BigDecimal bigDecimal = ((SqlDecimal) value).toBigDecimal(); + if (decimalType.isShort()) { + return filter.testLong(bigDecimal.unscaledValue().longValue()); + } + else { + Slice encodedDecimal = Decimals.encodeScaledValue(bigDecimal); + return filter.testDecimal(encodedDecimal.getLong(0), encodedDecimal.getLong(Long.BYTES)); + } + } + + if (type == VARCHAR) { + return filter.testBytes(((String) value).getBytes(), 0, ((String) value).length()); + } + if (type instanceof CharType) { + String charString = String.valueOf(value); + return filter.testBytes(charString.getBytes(), 0, charString.length()); + } + if (type == VARBINARY) { + byte[] binary = ((SqlVarbinary) value).getBytes(); + return filter.testBytes(binary, 0, binary.length); + } + fail("Unsupported type: " + type); + return false; + } + + private interface SubfieldPruner + { + Object prune(Object value); + } + + private static SubfieldPruner createSubfieldPruner(Type type, List requiredSubfields) + { + if (type instanceof ArrayType) { + return new ListSubfieldPruner(type, requiredSubfields); + } + + if (type instanceof MapType) { + return new MapSubfieldPruner(type, requiredSubfields); + } + + throw new UnsupportedOperationException("Unsupported type: " + type); + } + + private static class ListSubfieldPruner + implements SubfieldPruner + { + private final int maxIndex; + private final Optional nestedSubfieldPruner; + + public ListSubfieldPruner(Type type, List requiredSubfields) + { + checkArgument(type instanceof ArrayType, "type is not an array type: " + type); + + maxIndex = requiredSubfields.stream() + .map(Subfield::getPath) + .map(path -> path.get(0)) + .map(Subfield.LongSubscript.class::cast) + .map(Subfield.LongSubscript::getIndex) + .mapToInt(Long::intValue) + .max() + .orElse(-1); + + List elementSubfields = requiredSubfields.stream() + .filter(subfield -> subfield.getPath().size() > 1) + .map(subfield -> subfield.tail(subfield.getRootName())) + .distinct() + .collect(toImmutableList()); + + if (elementSubfields.isEmpty()) { + nestedSubfieldPruner = Optional.empty(); + } + else { + nestedSubfieldPruner = Optional.of(createSubfieldPruner(((ArrayType) type).getElementType(), elementSubfields)); + } + } + + @Override + public Object prune(Object value) + { + if (value == null) { + return null; + } + + List list = (List) value; + List prunedList; + if (maxIndex == -1) { + prunedList = list; + } + else { + prunedList = list.size() < maxIndex ? list : list.subList(0, maxIndex); + } + + return nestedSubfieldPruner.map(pruner -> prunedList.stream().map(pruner::prune).collect(toList())).orElse(prunedList); + } + } + + private static class MapSubfieldPruner + implements SubfieldPruner + { + private final Set keys; + private final Optional nestedSubfieldPruner; + + public MapSubfieldPruner(Type type, List requiredSubfields) + { + checkArgument(type instanceof MapType, "type is not a map type: " + type); + + keys = requiredSubfields.stream() + .map(Subfield::getPath) + .map(path -> path.get(0)) + .map(Subfield.LongSubscript.class::cast) + .map(Subfield.LongSubscript::getIndex) + .collect(toImmutableSet()); + + List elementSubfields = requiredSubfields.stream() + .filter(subfield -> subfield.getPath().size() > 1) + .map(subfield -> subfield.tail(subfield.getRootName())) + .distinct() + .collect(toImmutableList()); + + if (elementSubfields.isEmpty()) { + nestedSubfieldPruner = Optional.empty(); + } + else { + nestedSubfieldPruner = Optional.of(createSubfieldPruner(((MapType) type).getValueType(), elementSubfields)); + } + } + + @Override + public Object prune(Object value) + { + if (value == null) { + return null; + } + + Map map = (Map) value; + Map prunedMap; + if (keys.isEmpty()) { + prunedMap = map; + } + else { + prunedMap = Maps.filterKeys((Map) value, key -> keys.contains(Long.valueOf(((Number) key).longValue()))); + } + + return nestedSubfieldPruner.map(pruner -> Maps.transformValues(prunedMap, pruner::prune)).orElse(prunedMap); + } + } + + private static List> pruneValues(List types, List> values, Map> requiredSubfields) + { + if (requiredSubfields.isEmpty()) { + return values; + } + + ImmutableList.Builder> builder = ImmutableList.builder(); + for (int i = 0; i < types.size(); i++) { + List subfields = requiredSubfields.get(i); + + if (subfields.isEmpty()) { + builder.add(values.get(i)); + continue; + } + + SubfieldPruner subfieldPruner = createSubfieldPruner(types.get(i), subfields); + builder.add(values.get(i).stream().map(subfieldPruner::prune).collect(toList())); + } + + return builder.build(); } private static void assertColumnValueEquals(Type type, Object actual, Object expected) { if (actual == null) { - assertNull(expected); + assertEquals(actual, expected); return; } String baseType = type.getTypeSignature().getBase(); if (StandardTypes.ARRAY.equals(baseType)) { List actualArray = (List) actual; List expectedArray = (List) expected; + assertEquals(actualArray == null, expectedArray == null); assertEquals(actualArray.size(), expectedArray.size()); Type elementType = type.getTypeParameters().get(0); @@ -808,14 +1280,30 @@ else if (!Objects.equals(actual, expected)) { static OrcBatchRecordReader createCustomOrcRecordReader(TempFile tempFile, OrcEncoding orcEncoding, OrcPredicate predicate, Type type, int initialBatchSize) throws IOException { - return createCustomOrcRecordReader(tempFile, orcEncoding, predicate, ImmutableList.of(type), initialBatchSize); + return createCustomOrcRecordReader(tempFile, orcEncoding, predicate, ImmutableList.of(type), initialBatchSize, new StorageOrcFileTailSource(), new StorageStripeMetadataSource()); } - private static OrcBatchRecordReader createCustomOrcRecordReader(TempFile tempFile, OrcEncoding orcEncoding, OrcPredicate predicate, List types, int initialBatchSize) + static OrcBatchRecordReader createCustomOrcRecordReader( + TempFile tempFile, + OrcEncoding orcEncoding, + OrcPredicate predicate, + List types, + int initialBatchSize, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource) throws IOException { OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); - OrcReader orcReader = new OrcReader(orcDataSource, orcEncoding, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), MAX_BLOCK_SIZE); + OrcReader orcReader = new OrcReader( + orcDataSource, + orcEncoding, + orcFileTailSource, + stripeMetadataSource, + new OrcReaderOptions( + new DataSize(1, MEGABYTE), + new DataSize(1, MEGABYTE), + MAX_BLOCK_SIZE, + false)); assertEquals(orcReader.getColumnNames(), ImmutableList.of("test")); assertEquals(orcReader.getFooter().getRowsInRowGroup(), 10_000); @@ -827,6 +1315,12 @@ private static OrcBatchRecordReader createCustomOrcRecordReader(TempFile tempFil return orcReader.createBatchRecordReader(columnTypes, predicate, HIVE_STORAGE_TIME_ZONE, newSimpleAggregatedMemoryContext(), initialBatchSize); } + public static void writeOrcColumnPresto(File outputFile, Format format, CompressionKind compression, Type type, List values) + throws Exception + { + writeOrcColumnsPresto(outputFile, format, compression, ImmutableList.of(type), ImmutableList.of(values), new OrcWriterStats()); + } + private static void writeOrcColumnsPresto(File outputFile, Format format, CompressionKind compression, List types, List> values, OrcWriterStats stats) throws Exception { @@ -864,19 +1358,31 @@ private static void writeOrcColumnsPresto(File outputFile, Format format, Compre writer.validate(new FileOrcDataSource(outputFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true)); } - static OrcSelectiveRecordReader createCustomOrcSelectiveRecordReader( - TempFile tempFile, + private static OrcSelectiveRecordReader createCustomOrcSelectiveRecordReader( + File file, OrcEncoding orcEncoding, OrcPredicate predicate, List types, int initialBatchSize, - Map filters) + Map> filters, + List filterFunctions, + Map filterFunctionInputMapping, + Map> requiredSubfields) throws IOException { - OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); - OrcReader orcReader = new OrcReader(orcDataSource, orcEncoding, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), MAX_BLOCK_SIZE); - - assertEquals(orcReader.getColumnNames(), makeColumnNames(types.size())); + OrcDataSource orcDataSource = new FileOrcDataSource(file, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); + OrcReader orcReader = new OrcReader( + orcDataSource, + orcEncoding, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + new OrcReaderOptions( + new DataSize(1, MEGABYTE), + new DataSize(1, MEGABYTE), + MAX_BLOCK_SIZE, + false)); + + assertEquals(orcReader.getColumnNames().subList(0, types.size()), makeColumnNames(types.size())); assertEquals(orcReader.getFooter().getRowsInRowGroup(), 10_000); Map columnTypes = IntStream.range(0, types.size()) @@ -887,8 +1393,9 @@ static OrcSelectiveRecordReader createCustomOrcSelectiveRecordReader( columnTypes, IntStream.range(0, types.size()).boxed().collect(toList()), filters, - ImmutableList.of(), - ImmutableMap.of(), + filterFunctions, + filterFunctionInputMapping, + requiredSubfields, ImmutableMap.of(), ImmutableMap.of(), predicate, @@ -1394,6 +1901,29 @@ else if (type.getTypeSignature().getBase().equals(StandardTypes.ROW)) { throw new IllegalArgumentException("unsupported type: " + type); } + private static void checkNullValues(Type type, Block block) + { + if (!block.mayHaveNull()) { + return; + } + for (int position = 0; position < block.getPositionCount(); position++) { + if (block.isNull(position)) { + if (type.equals(TINYINT) || type.equals(SMALLINT) || type.equals(INTEGER) || type.equals(BIGINT) || type.equals(REAL) || type.equals(DATE) || type.equals(TIMESTAMP)) { + assertEquals(type.getLong(block, position), 0); + } + if (type.equals(BOOLEAN)) { + assertFalse(type.getBoolean(block, position)); + } + if (type.equals(DOUBLE)) { + assertEquals(type.getDouble(block, position), 0.0); + } + if (type instanceof VarcharType || type instanceof CharType || type.equals(VARBINARY)) { + assertEquals(type.getSlice(block, position).length(), 0); + } + } + } + } + private static void setDwrfLowMemoryFlag(RecordWriter recordWriter) { Object writer = getFieldValue(recordWriter, "writer"); @@ -1496,6 +2026,9 @@ private static Properties createTableProperties(List types) Properties orderTableProperties = new Properties(); orderTableProperties.setProperty("columns", String.join(", ", makeColumnNames(types.size()))); orderTableProperties.setProperty("columns.types", columnTypes); + orderTableProperties.setProperty("orc.bloom.filter.columns", String.join(", ", makeColumnNames(types.size()))); + orderTableProperties.setProperty("orc.bloom.filter.fpp", "0.50"); + orderTableProperties.setProperty("orc.bloom.filter.write.version", "original"); return orderTableProperties; } @@ -1577,17 +2110,17 @@ private static boolean hasType(Type testType, Set baseTypes) return baseTypes.contains(testBaseType); } - private static Type arrayType(Type elementType) + public static Type arrayType(Type elementType) { return TYPE_MANAGER.getParameterizedType(StandardTypes.ARRAY, ImmutableList.of(TypeSignatureParameter.of(elementType.getTypeSignature()))); } - private static Type mapType(Type keyType, Type valueType) + public static Type mapType(Type keyType, Type valueType) { return TYPE_MANAGER.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.of(keyType.getTypeSignature()), TypeSignatureParameter.of(valueType.getTypeSignature()))); } - private static Type rowType(Type... fieldTypes) + public static Type rowType(Type... fieldTypes) { ImmutableList.Builder typeSignatureParameters = ImmutableList.builder(); for (int i = 0; i < fieldTypes.length; i++) { diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestByteArrayUtils.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestByteArrayUtils.java new file mode 100644 index 0000000000000..6b2cea536c9ae --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestByteArrayUtils.java @@ -0,0 +1,29 @@ +package com.facebook.presto.orc; + +import org.testng.annotations.Test; + +import static com.facebook.presto.orc.ByteArrayUtils.compareRanges; +import static org.testng.Assert.assertEquals; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class TestByteArrayUtils +{ + @Test + public void testCompareRanges() + { + assertEquals(compareRanges("abcdefghij".getBytes(), 0, 10, "abcdefghij".getBytes(), 0, 10), 0); + assertEquals(compareRanges("123abcdefghij".getBytes(), 3, 10, "abcdefghij".getBytes(), 0, 10), 0); + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestCachingOrcDataSource.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestCachingOrcDataSource.java index 5dde0ff5cea87..21a6da0dd1171 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestCachingOrcDataSource.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestCachingOrcDataSource.java @@ -14,6 +14,7 @@ package com.facebook.presto.orc; import com.facebook.presto.orc.OrcTester.Format; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.orc.metadata.StripeInformation; import com.facebook.presto.spi.block.Block; @@ -39,6 +40,8 @@ import java.util.Random; import java.util.stream.Stream; +import static com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.AbstractOrcRecordReader.LinearProbeRangeFinder.createTinyStripesRangeFinder; import static com.facebook.presto.orc.OrcBatchRecordReader.wrapWithCacheIfTinyStripes; @@ -51,8 +54,6 @@ import static com.facebook.presto.orc.metadata.CompressionKind.ZLIB; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.testing.Assertions.assertGreaterThanOrEqual; -import static io.airlift.testing.Assertions.assertInstanceOf; import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; @@ -192,7 +193,12 @@ public void testIntegration() public void doIntegration(TestingOrcDataSource orcDataSource, DataSize maxMergeDistance, DataSize maxReadSize, DataSize tinyStripeThreshold) throws IOException { - OrcReader orcReader = new OrcReader(orcDataSource, ORC, maxMergeDistance, maxReadSize, tinyStripeThreshold, new DataSize(1, Unit.MEGABYTE)); + OrcReader orcReader = new OrcReader( + orcDataSource, + ORC, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + new OrcReaderOptions(maxMergeDistance, tinyStripeThreshold, new DataSize(1, Unit.MEGABYTE), false)); // 1 for reading file footer assertEquals(orcDataSource.getReadCount(), 1); List stripes = orcReader.getFooter().getStripes(); @@ -213,7 +219,7 @@ public void doIntegration(TestingOrcDataSource orcDataSource, DataSize maxMergeD if (batchSize <= 0) { break; } - Block block = orcRecordReader.readBlock(VARCHAR, 0); + Block block = orcRecordReader.readBlock(0); positionCount += block.getPositionCount(); } assertEquals(positionCount, POSITION_COUNT); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestDictionaryCompressionOptimizer.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestDictionaryCompressionOptimizer.java index 9ef2aca2ff38c..25dcac1834290 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestDictionaryCompressionOptimizer.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestDictionaryCompressionOptimizer.java @@ -24,11 +24,11 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static com.facebook.presto.orc.DictionaryCompressionOptimizer.DICTIONARY_MEMORY_MAX_RANGE; import static com.facebook.presto.orc.DictionaryCompressionOptimizer.estimateIndexBytesPerValue; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.testing.Assertions.assertGreaterThanOrEqual; -import static io.airlift.testing.Assertions.assertLessThan; import static java.lang.Math.min; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestFilterFunction.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestFilterFunction.java new file mode 100644 index 0000000000000..517400588ec0d --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestFilterFunction.java @@ -0,0 +1,141 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.DictionaryBlock; +import com.facebook.presto.spi.block.LongArrayBlock; +import com.facebook.presto.spi.relation.Predicate; +import com.facebook.presto.testing.TestingConnectorSession; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.Optional; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestFilterFunction +{ + private static final long UNLUCKY = 13; + + @Test + public void testFilter() + { + ConnectorSession session = new TestingConnectorSession(ImmutableList.of()); + FilterFunction filter = new FilterFunction(session, true, new IsOddPredicate()); + + Block numbers = makeNumbers(0, 1000); + int[] allPositions = makePositions(0, 1000, 1); + assertFilter(filter, numbers, allPositions, allPositions.length); + + Block dictionaryNumbers = new DictionaryBlock(numbers, allPositions); + // Sparse coverage of the dictionary values + int[] sparsePositions = makePositions(1, 300, 3); + assertFilter(filter, dictionaryNumbers, sparsePositions, sparsePositions.length); + + // Full coverage of the dictionary values + assertFilter(filter, dictionaryNumbers, allPositions, allPositions.length); + + // Test with a different DictionaryBlock over the same numbers. Results are reused. The DictionaryBlock covers the + // values sparsely. TheDictionaryBlock itself is accessed sparsely. + DictionaryBlock otherDictionary = new DictionaryBlock(numbers, makePositions(1, 332, 3)); + int[] otherDictionaryPositions = makePositions(0, 150, 2); + assertFilter(filter, otherDictionary, otherDictionaryPositions, otherDictionaryPositions.length); + + // Repeat test on a DictionaryBlock over different content to make sure that cached results are not reused. + assertFilter(filter, new DictionaryBlock(makeNumbers(1, 1001), allPositions), allPositions, allPositions.length); + } + + private static void assertFilter(FilterFunction filter, Block input, int[] inputPositions, int positionCount) + { + // Copy the positions array because filter mutates it. + int[] positions = Arrays.copyOf(inputPositions, positionCount); + RuntimeException[] errors = new RuntimeException[inputPositions[positionCount - 1] + 1]; + // Put a pre-existing error in the 1st half of the input. + int numPreviousErrors = positionCount / 2; + for (int i = 0; i < numPreviousErrors; i++) { + errors[i] = new RuntimeException("Pre-existent error at " + positions[i]); + } + int lastErrorPosition = numPreviousErrors > 0 ? positions[numPreviousErrors - 1] : -1; + int numHits = filter.filter(new Page(positionCount, input), positions, positionCount, errors); + int hitCounter = 0; + for (int position : inputPositions) { + long number = input.getLong(position); + if (number == UNLUCKY) { + assertEquals(positions[hitCounter], position); + assertTrue(errors[hitCounter] instanceof UnluckyError); + hitCounter++; + } + else if ((number & 1) == 1) { + assertEquals(positions[hitCounter], position); + if (position <= lastErrorPosition) { + assertTrue(errors[hitCounter] instanceof RuntimeException); + } + else { + assertEquals(errors[hitCounter], null); + } + hitCounter++; + } + } + assertEquals(numHits, hitCounter); + } + + private static class UnluckyError + extends RuntimeException + { + } + + private static int[] makePositions(int from, int count, int step) + { + int[] array = new int[count]; + for (int i = 0; i < count; i++) { + array[i] = from + step * i; + } + return array; + } + + private static Block makeNumbers(int from, int to) + { + int count = to - from; + long[] array = new long[count]; + for (int i = 0; i < count; i++) { + array[i] = from + i; + } + return new LongArrayBlock(count, Optional.empty(), array); + } + + private static class IsOddPredicate + implements Predicate + { + @Override + public int[] getInputChannels() + { + return new int[] {0}; + } + + @Override + public boolean evaluate(ConnectorSession session, Page page, int position) + { + long number = page.getBlock(0).getLong(position); + if (number == UNLUCKY) { + throw new UnluckyError(); + } + return (number & 1) == 1; + } + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestListFilter.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestListFilter.java new file mode 100644 index 0000000000000..053022d9095b8 --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestListFilter.java @@ -0,0 +1,318 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.facebook.presto.orc.TupleDomainFilter.BigintRange; +import com.facebook.presto.orc.TupleDomainFilter.PositionalFilter; +import com.facebook.presto.orc.metadata.OrcType; +import com.facebook.presto.orc.reader.ListFilter; +import com.facebook.presto.spi.Subfield; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Random; + +import static com.facebook.presto.orc.metadata.OrcType.OrcTypeKind.INT; +import static com.facebook.presto.orc.metadata.OrcType.OrcTypeKind.LIST; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.lang.String.format; +import static org.testng.Assert.assertEquals; + +public class TestListFilter +{ + private static final int ROW_COUNT = 1_000; + private final Random random = new Random(0); + + @Test + public void testArray() + { + Integer[][] data = new Integer[ROW_COUNT][]; + int count = 0; + for (int i = 0; i < data.length; i++) { + data[i] = new Integer[3 + random.nextInt(10)]; + for (int j = 0; j < data[i].length; j++) { + if (count++ % 11 == 0) { + data[i][j] = null; + } + else { + data[i][j] = random.nextInt(100); + } + } + } + + assertPositionalFilter(ImmutableMap.of( + 1, BigintRange.of(0, 50, false)), data); + + assertPositionalFilter(ImmutableMap.of( + 1, BigintRange.of(0, 50, false), + 2, BigintRange.of(25, 50, false)), data); + + assertPositionalFilter(ImmutableMap.of( + 2, BigintRange.of(25, 50, false)), data); + } + + private void assertPositionalFilter(Map filters, Integer[][] data) + { + ListFilter listFilter = buildListFilter(filters, data); + + TestFilter1 testFilter = (i, value) -> Optional.ofNullable(filters.get(i + 1)) + .map(filter -> value == null ? filter.testNull() : filter.testLong(value)) + .orElse(true); + + PositionalFilter positionalFilter = listFilter.getPositionalFilter(); + + for (int i = 0; i < data.length; i++) { + for (int j = 0; j < data[i].length; j++) { + Integer value = data[i][j]; + boolean expectedToPass = testFilter.test(j, value); + assertEquals(value == null ? positionalFilter.testNull() : positionalFilter.testLong(value), expectedToPass); + if (!expectedToPass) { + assertEquals(positionalFilter.getPrecedingPositionsToFail(), j); + assertEquals(positionalFilter.getSucceedingPositionsToFail(), data[i].length - j - 1); + break; + } + } + } + } + + private static ListFilter buildListFilter(Map filters, Integer[][] data) + { + Map subfieldFilters = filters.entrySet().stream() + .collect(toImmutableMap(entry -> toSubfield(entry.getKey()), Map.Entry::getValue)); + + ListFilter filter = new ListFilter(makeStreamDescriptor(1), subfieldFilters); + + int[] lengths = Arrays.stream(data).mapToInt(v -> v.length).toArray(); + filter.populateElementFilters(data.length, null, lengths, Arrays.stream(lengths).sum()); + + return filter; + } + + private static Subfield toSubfield(Integer index) + { + return new Subfield(format("c[%s]", index)); + } + + @Test + public void testNestedArray() + { + Integer[][][] data = new Integer[ROW_COUNT][][]; + int count = 0; + for (int i = 0; i < data.length; i++) { + data[i] = new Integer[3 + random.nextInt(10)][]; + for (int j = 0; j < data[i].length; j++) { + data[i][j] = new Integer[3 + random.nextInt(10)]; + for (int k = 0; k < data[i][j].length; k++) { + if (count++ % 11 == 0) { + data[i][j][k] = null; + } + else { + data[i][j][k] = random.nextInt(100); + } + } + } + } + + assertPositionalFilter(ImmutableMap.of( + RowAndColumn.of(1, 1), BigintRange.of(0, 50, false)), data); + + assertPositionalFilter(ImmutableMap.of( + RowAndColumn.of(1, 2), BigintRange.of(0, 50, false), + RowAndColumn.of(3, 2), BigintRange.of(25, 50, false)), data); + + assertPositionalFilter(ImmutableMap.of( + RowAndColumn.of(2, 1), BigintRange.of(0, 50, false), + RowAndColumn.of(2, 2), BigintRange.of(10, 40, false), + RowAndColumn.of(3, 3), BigintRange.of(20, 30, false)), data); + } + + private void assertPositionalFilter(Map filters, Integer[][][] data) + { + ListFilter listFilter = buildListFilter(filters, data); + + TestFilter2 testFilter = (i, j, value) -> Optional.ofNullable(filters.get(RowAndColumn.of(i + 1, j + 1))) + .map(filter -> value == null ? filter.testNull() : filter.testLong(value)) + .orElse(true); + + PositionalFilter positionalFilter = listFilter.getChild().getPositionalFilter(); + + for (int i = 0; i < data.length; i++) { + boolean expectedToPass = true; + int passCount = 0; + int failCount = 0; + for (int j = 0; j < data[i].length; j++) { + if (!expectedToPass) { + failCount += data[i][j].length; + continue; + } + for (int k = 0; k < data[i][j].length; k++) { + Integer value = data[i][j][k]; + expectedToPass = testFilter.test(j, k, value); + assertEquals(value == null ? positionalFilter.testNull() : positionalFilter.testLong(value), expectedToPass); + if (!expectedToPass) { + assertEquals(positionalFilter.getPrecedingPositionsToFail(), passCount); + failCount = data[i][j].length - k - 1; + break; + } + passCount++; + } + } + assertEquals(positionalFilter.getSucceedingPositionsToFail(), failCount); + } + } + + private static ListFilter buildListFilter(Map filters, Integer[][][] data) + { + Map subfieldFilters = filters.entrySet().stream() + .collect(toImmutableMap(entry -> toSubfield(entry.getKey()), Map.Entry::getValue)); + + ListFilter filter = new ListFilter(makeStreamDescriptor(2), subfieldFilters); + + int[] lengths = Arrays.stream(data).mapToInt(v -> v.length).toArray(); + filter.populateElementFilters(data.length, null, lengths, Arrays.stream(lengths).sum()); + + int[] nestedLenghts = Arrays.stream(data).flatMap(Arrays::stream).mapToInt(v -> v.length).toArray(); + ((ListFilter) filter.getChild()).populateElementFilters(Arrays.stream(lengths).sum(), null, nestedLenghts, Arrays.stream(nestedLenghts).sum()); + + return filter; + } + + private static StreamDescriptor makeStreamDescriptor(int levels) + { + DummyOrcDataSource orcDataSource = new DummyOrcDataSource(); + + OrcType intType = new OrcType(INT, ImmutableList.of(), ImmutableList.of(), Optional.empty(), Optional.empty(), Optional.empty()); + OrcType listType = new OrcType(LIST, ImmutableList.of(1), ImmutableList.of("item"), Optional.empty(), Optional.empty(), Optional.empty()); + + StreamDescriptor streamDescriptor = new StreamDescriptor("a", 0, "a", intType, orcDataSource, ImmutableList.of()); + for (int i = 0; i < levels; i++) { + streamDescriptor = new StreamDescriptor("a", 0, "a", listType, orcDataSource, ImmutableList.of(streamDescriptor)); + } + + return streamDescriptor; + } + + private static Subfield toSubfield(RowAndColumn indices) + { + return new Subfield(format("c[%s][%s]", indices.getRow(), indices.getColumn())); + } + + private interface TestFilter1 + { + boolean test(int i, Integer value); + } + + private interface TestFilter2 + { + boolean test(int i, int j, Integer value); + } + + private static class DummyOrcDataSource + implements OrcDataSource + { + @Override + public OrcDataSourceId getId() + { + return null; + } + + @Override + public long getReadBytes() + { + return 0; + } + + @Override + public long getReadTimeNanos() + { + return 0; + } + + @Override + public long getSize() + { + return 0; + } + + @Override + public void readFully(long position, byte[] buffer) + { + } + + @Override + public void readFully(long position, byte[] buffer, int bufferOffset, int bufferLength) + { + } + + @Override + public Map readFully(Map diskRanges) + { + return null; + } + } + + private static final class RowAndColumn + { + private final int row; + private final int column; + + private RowAndColumn(int row, int column) + { + this.row = row; + this.column = column; + } + + public static RowAndColumn of(int row, int column) + { + return new RowAndColumn(row, column); + } + + public int getRow() + { + return row; + } + + public int getColumn() + { + return column; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + RowAndColumn that = (RowAndColumn) o; + return row == that.row && + column == that.column; + } + + @Override + public int hashCode() + { + return Objects.hash(row, column); + } + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestFlatMap.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestMapFlatBatchStreamReader.java similarity index 94% rename from presto-orc/src/test/java/com/facebook/presto/orc/TestFlatMap.java rename to presto-orc/src/test/java/com/facebook/presto/orc/TestMapFlatBatchStreamReader.java index 65a9d04a23e4c..6b33f2a81fc49 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestFlatMap.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestMapFlatBatchStreamReader.java @@ -15,6 +15,7 @@ import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.BigintType; import com.facebook.presto.spi.type.BooleanType; @@ -54,9 +55,9 @@ import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.OrcTester.HIVE_STORAGE_TIME_ZONE; -import static com.facebook.presto.orc.TestFlatMap.ExpectedValuesBuilder.Frequency.ALL; -import static com.facebook.presto.orc.TestFlatMap.ExpectedValuesBuilder.Frequency.NONE; -import static com.facebook.presto.orc.TestFlatMap.ExpectedValuesBuilder.Frequency.SOME; +import static com.facebook.presto.orc.TestMapFlatBatchStreamReader.ExpectedValuesBuilder.Frequency.ALL; +import static com.facebook.presto.orc.TestMapFlatBatchStreamReader.ExpectedValuesBuilder.Frequency.NONE; +import static com.facebook.presto.orc.TestMapFlatBatchStreamReader.ExpectedValuesBuilder.Frequency.SOME; import static com.facebook.presto.orc.TestingOrcPredicate.createOrcPredicate; import static com.facebook.presto.testing.TestingConnectorSession.SESSION; import static com.google.common.collect.Iterators.advance; @@ -66,7 +67,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -public class TestFlatMap +public class TestMapFlatBatchStreamReader { // TODO: Add tests for timestamp as value type @@ -179,7 +180,7 @@ public void testBoolean() runTest("test_flat_map/flat_map_boolean.dwrf", IntegerType.INTEGER, BooleanType.BOOLEAN, - ExpectedValuesBuilder.get(Function.identity(), TestFlatMap::intToBoolean)); + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatBatchStreamReader::intToBoolean)); } @Test @@ -189,7 +190,7 @@ public void testBooleanWithNull() runTest("test_flat_map/flat_map_boolean_with_null.dwrf", IntegerType.INTEGER, BooleanType.BOOLEAN, - ExpectedValuesBuilder.get(Function.identity(), TestFlatMap::intToBoolean).setNullValuesFrequency(SOME)); + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatBatchStreamReader::intToBoolean).setNullValuesFrequency(SOME)); } @Test @@ -240,7 +241,7 @@ public void testList() "test_flat_map/flat_map_list.dwrf", IntegerType.INTEGER, LIST_TYPE, - ExpectedValuesBuilder.get(Function.identity(), TestFlatMap::intToList)); + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatBatchStreamReader::intToList)); } @Test @@ -251,7 +252,7 @@ public void testListWithNull() "test_flat_map/flat_map_list_with_null.dwrf", IntegerType.INTEGER, LIST_TYPE, - ExpectedValuesBuilder.get(Function.identity(), TestFlatMap::intToList).setNullValuesFrequency(SOME)); + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatBatchStreamReader::intToList).setNullValuesFrequency(SOME)); } @Test @@ -262,7 +263,7 @@ public void testMap() "test_flat_map/flat_map_map.dwrf", IntegerType.INTEGER, MAP_TYPE, - ExpectedValuesBuilder.get(Function.identity(), TestFlatMap::intToMap)); + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatBatchStreamReader::intToMap)); } @Test @@ -273,7 +274,7 @@ public void testMapWithNull() "test_flat_map/flat_map_map_with_null.dwrf", IntegerType.INTEGER, MAP_TYPE, - ExpectedValuesBuilder.get(Function.identity(), TestFlatMap::intToMap).setNullValuesFrequency(SOME)); + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatBatchStreamReader::intToMap).setNullValuesFrequency(SOME)); } @Test @@ -284,7 +285,7 @@ public void testStruct() "test_flat_map/flat_map_struct.dwrf", IntegerType.INTEGER, STRUCT_TYPE, - ExpectedValuesBuilder.get(Function.identity(), TestFlatMap::intToList)); + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatBatchStreamReader::intToList)); } @Test @@ -295,7 +296,7 @@ public void testStructWithNull() "test_flat_map/flat_map_struct_with_null.dwrf", IntegerType.INTEGER, STRUCT_TYPE, - ExpectedValuesBuilder.get(Function.identity(), TestFlatMap::intToList).setNullValuesFrequency(SOME)); + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatBatchStreamReader::intToList).setNullValuesFrequency(SOME)); } @Test @@ -395,10 +396,9 @@ private void runTest(String testOrcFileName, Type keyType, Type valueType OrcReader orcReader = new OrcReader( orcDataSource, OrcEncoding.DWRF, - new DataSize(1, MEGABYTE), - new DataSize(1, MEGABYTE), - new DataSize(1, MEGABYTE), - new DataSize(1, DataSize.Unit.MEGABYTE)); + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + OrcReaderTestingUtils.createDefaultTestConfig()); Type mapType = TYPE_MANAGER.getParameterizedType( StandardTypes.MAP, ImmutableList.of( @@ -419,7 +419,7 @@ else if (skipFirstBatch && isFirst) { isFirst = false; } else { - Block block = recordReader.readBlock(mapType, 0); + Block block = recordReader.readBlock(0); for (int position = 0; position < block.getPositionCount(); position++) { assertEquals(mapType.getObjectValue(SESSION, block, position), expectedValuesIterator.next()); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestMapFlatSelectiveStreamReader.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestMapFlatSelectiveStreamReader.java new file mode 100644 index 0000000000000..8a1ec4cd7a118 --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestMapFlatSelectiveStreamReader.java @@ -0,0 +1,650 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.facebook.presto.Session; +import com.facebook.presto.orc.TupleDomainFilter.BigintValues; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.relation.Predicate; +import com.facebook.presto.spi.type.BooleanType; +import com.facebook.presto.spi.type.DoubleType; +import com.facebook.presto.spi.type.SqlTimestamp; +import com.facebook.presto.spi.type.SqlVarbinary; +import com.facebook.presto.spi.type.TimeZoneKey; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import org.testng.annotations.Test; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.IntStream; + +import static com.facebook.presto.orc.OrcTester.arrayType; +import static com.facebook.presto.orc.OrcTester.assertFileContentsPresto; +import static com.facebook.presto.orc.OrcTester.filterRows; +import static com.facebook.presto.orc.OrcTester.mapType; +import static com.facebook.presto.orc.OrcTester.rowType; +import static com.facebook.presto.orc.TestMapFlatSelectiveStreamReader.ExpectedValuesBuilder.Frequency.ALL; +import static com.facebook.presto.orc.TestMapFlatSelectiveStreamReader.ExpectedValuesBuilder.Frequency.NONE; +import static com.facebook.presto.orc.TestMapFlatSelectiveStreamReader.ExpectedValuesBuilder.Frequency.SOME; +import static com.facebook.presto.orc.TestingOrcPredicate.createOrcPredicate; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NOT_NULL; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NULL; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RealType.REAL; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.io.Resources.getResource; +import static java.util.stream.Collectors.toList; + +public class TestMapFlatSelectiveStreamReader +{ + // TODO: Add tests for timestamp as value type + + private static final int NUM_ROWS = 31_234; + + @Test + public void testByte() + throws Exception + { + runTest("test_flat_map/flat_map_byte.dwrf", + TINYINT, + ExpectedValuesBuilder.get(Integer::byteValue)); + } + + @Test + public void testByteWithNull() + throws Exception + { + runTest("test_flat_map/flat_map_byte_with_null.dwrf", + TINYINT, + ExpectedValuesBuilder.get(Integer::byteValue).setNullValuesFrequency(SOME)); + } + + @Test + public void testShort() + throws Exception + { + runTest("test_flat_map/flat_map_short.dwrf", + SMALLINT, + ExpectedValuesBuilder.get(Integer::shortValue)); + } + + @Test + public void testInteger() + throws Exception + { + runTest("test_flat_map/flat_map_int.dwrf", + INTEGER, + ExpectedValuesBuilder.get(Function.identity())); + } + + @Test + public void testIntegerWithNull() + throws Exception + { + runTest("test_flat_map/flat_map_int_with_null.dwrf", + INTEGER, + ExpectedValuesBuilder.get(Function.identity()).setNullValuesFrequency(SOME)); + } + + @Test + public void testLong() + throws Exception + { + runTest("test_flat_map/flat_map_long.dwrf", + BIGINT, + ExpectedValuesBuilder.get(Integer::longValue)); + } + + @Test + public void testString() + throws Exception + { + runTest("test_flat_map/flat_map_string.dwrf", + VARCHAR, + ExpectedValuesBuilder.get(i -> Integer.toString(i))); + } + + @Test + public void testStringWithNull() + throws Exception + { + runTest("test_flat_map/flat_map_string_with_null.dwrf", + VARCHAR, + ExpectedValuesBuilder.get(i -> Integer.toString(i)).setNullValuesFrequency(SOME)); + } + + @Test + public void testBinary() + throws Exception + { + runTest("test_flat_map/flat_map_binary.dwrf", + VARBINARY, + ExpectedValuesBuilder.get(i -> new SqlVarbinary(Integer.toString(i).getBytes(StandardCharsets.UTF_8)))); + } + + @Test + public void testBoolean() + throws Exception + { + runTest("test_flat_map/flat_map_boolean.dwrf", + INTEGER, + BooleanType.BOOLEAN, + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatSelectiveStreamReader::intToBoolean)); + } + + @Test + public void testBooleanWithNull() + throws Exception + { + runTest("test_flat_map/flat_map_boolean_with_null.dwrf", + INTEGER, + BooleanType.BOOLEAN, + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatSelectiveStreamReader::intToBoolean).setNullValuesFrequency(SOME)); + } + + @Test + public void testFloat() + throws Exception + { + runTest("test_flat_map/flat_map_float.dwrf", + INTEGER, + REAL, + ExpectedValuesBuilder.get(Function.identity(), Float::valueOf)); + } + + @Test + public void testFloatWithNull() + throws Exception + { + runTest("test_flat_map/flat_map_float_with_null.dwrf", + INTEGER, + REAL, + ExpectedValuesBuilder.get(Function.identity(), Float::valueOf).setNullValuesFrequency(SOME)); + } + + @Test + public void testDouble() + throws Exception + { + runTest("test_flat_map/flat_map_double.dwrf", + INTEGER, + DoubleType.DOUBLE, + ExpectedValuesBuilder.get(Function.identity(), Double::valueOf)); + } + + @Test + public void testDoubleWithNull() + throws Exception + { + runTest("test_flat_map/flat_map_double_with_null.dwrf", + INTEGER, + DoubleType.DOUBLE, + ExpectedValuesBuilder.get(Function.identity(), Double::valueOf).setNullValuesFrequency(SOME)); + } + + @Test + public void testList() + throws Exception + { + runTest( + "test_flat_map/flat_map_list.dwrf", + INTEGER, + arrayType(INTEGER), + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatSelectiveStreamReader::intToList)); + } + + @Test + public void testListWithNull() + throws Exception + { + runTest( + "test_flat_map/flat_map_list_with_null.dwrf", + INTEGER, + arrayType(INTEGER), + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatSelectiveStreamReader::intToList).setNullValuesFrequency(SOME)); + } + + @Test + public void testMap() + throws Exception + { + runTest( + "test_flat_map/flat_map_map.dwrf", + INTEGER, + mapType(VARCHAR, REAL), + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatSelectiveStreamReader::intToMap)); + } + + @Test + public void testMapWithNull() + throws Exception + { + runTest( + "test_flat_map/flat_map_map_with_null.dwrf", + INTEGER, + mapType(VARCHAR, REAL), + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatSelectiveStreamReader::intToMap).setNullValuesFrequency(SOME)); + } + + @Test + public void testStruct() + throws Exception + { + runTest( + "test_flat_map/flat_map_struct.dwrf", + INTEGER, + rowType(INTEGER, INTEGER, INTEGER), + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatSelectiveStreamReader::intToList)); + } + + @Test + public void testStructWithNull() + throws Exception + { + runTest( + "test_flat_map/flat_map_struct_with_null.dwrf", + INTEGER, + rowType(INTEGER, INTEGER, INTEGER), + ExpectedValuesBuilder.get(Function.identity(), TestMapFlatSelectiveStreamReader::intToList).setNullValuesFrequency(SOME)); + } + + // Some of the maps are null + @Test + public void testWithNulls() + throws Exception + { + runTest( + "test_flat_map/flat_map_some_null_maps.dwrf", + INTEGER, + ExpectedValuesBuilder.get(Function.identity()).setNullRowsFrequency(SOME)); + } + + // All maps are null + @Test + public void testWithAllNulls() + throws Exception + { + runTest( + "test_flat_map/flat_map_all_null_maps.dwrf", + INTEGER, + ExpectedValuesBuilder.get(Function.identity()).setNullRowsFrequency(ALL)); + } + + // Some maps are empty + @Test + public void testWithEmptyMaps() + throws Exception + { + runTest( + "test_flat_map/flat_map_some_empty_maps.dwrf", + INTEGER, + ExpectedValuesBuilder.get(Function.identity()).setEmptyMapsFrequency(SOME)); + } + + // All maps are empty + @Test + public void testWithAllEmptyMaps() + throws Exception + { + runTest( + "test_flat_map/flat_map_all_empty_maps.dwrf", + INTEGER, + ExpectedValuesBuilder.get(Function.identity()).setEmptyMapsFrequency(ALL)); + } + + @Test + public void testMixedEncodings() + throws Exception + { + // Values associated with one key are direct encoded, and all other keys are + // dictionary encoded. The dictionary encoded values have occasional values that only appear once + // to ensure the IN_DICTIONARY stream is present, which means the checkpoints for dictionary encoded + // values will have a different number of positions compared to direct encoded values. + runTest("test_flat_map/flat_map_mixed_encodings.dwrf", + INTEGER, + ExpectedValuesBuilder.get(Function.identity()).setMixedEncodings()); + } + + @Test + public void testIntegerWithMissingSequences() + throws Exception + { + // The additional sequences IDs for a flat map aren't a consecutive range [1,N], the odd + // sequence IDs have been removed. This is to simulate the case where a file has been modified to delete + // certain keys from the map by dropping the ColumnEncodings and the associated data. + runTest("test_flat_map/flat_map_int_missing_sequences.dwrf", + INTEGER, + ExpectedValuesBuilder.get(Function.identity()).setMissingSequences()); + } + + private void runTest(String testOrcFileName, Type type, ExpectedValuesBuilder expectedValuesBuilder) + throws Exception + { + runTest(testOrcFileName, type, type, expectedValuesBuilder); + } + + private void runTest(String testOrcFileName, Type keyType, Type valueType, ExpectedValuesBuilder expectedValuesBuilder) + throws Exception + { + List> expectedValues = expectedValuesBuilder.build(); + + Type mapType = mapType(keyType, valueType); + + OrcPredicate orcPredicate = createOrcPredicate(0, mapType, expectedValues, OrcTester.Format.DWRF, true); + + runTest(testOrcFileName, mapType, expectedValues, orcPredicate, Optional.empty(), ImmutableList.of()); + runTest(testOrcFileName, mapType, expectedValues.stream().filter(Objects::isNull).collect(toList()), orcPredicate, Optional.of(IS_NULL), ImmutableList.of()); + runTest(testOrcFileName, mapType, expectedValues.stream().filter(Objects::nonNull).collect(toList()), orcPredicate, Optional.of(IS_NOT_NULL), ImmutableList.of()); + + if (keyType != VARBINARY) { + // read only some keys + List keys = expectedValues.stream().filter(Objects::nonNull).flatMap(v -> v.keySet().stream()).distinct().collect(toImmutableList()); + if (!keys.isEmpty()) { + List requiredKeys = ImmutableList.of(keys.get(0)); + runTest(testOrcFileName, mapType, pruneMaps(expectedValues, requiredKeys), orcPredicate, Optional.empty(), toSubfields(keyType, requiredKeys)); + + requiredKeys = ImmutableList.of(keys.get(1), keys.get(3), keys.get(7), keys.get(11)); + runTest(testOrcFileName, mapType, pruneMaps(expectedValues, requiredKeys), orcPredicate, Optional.empty(), toSubfields(keyType, requiredKeys)); + } + } + + // read only some rows + List ids = IntStream.range(0, expectedValues.size()).map(i -> i % 10).boxed().collect(toImmutableList()); + ImmutableList types = ImmutableList.of(mapType, INTEGER); + + Map> filters = ImmutableMap.of(1, ImmutableMap.of(new Subfield("c"), BigintValues.of(new long[] {1, 5, 6}, true))); + assertFileContentsPresto( + types, + new File(getResource(testOrcFileName).getFile()), + filterRows(types, ImmutableList.of(expectedValues, ids), filters), + OrcEncoding.DWRF, + OrcPredicate.TRUE, + Optional.of(filters), + ImmutableList.of(), + ImmutableMap.of(), + ImmutableMap.of()); + + TestingFilterFunction filterFunction = new TestingFilterFunction(mapType); + assertFileContentsPresto( + types, + new File(getResource(testOrcFileName).getFile()), + filterFunction.filterRows(ImmutableList.of(expectedValues, ids)), + OrcEncoding.DWRF, + OrcPredicate.TRUE, + Optional.empty(), + ImmutableList.of(filterFunction), + ImmutableMap.of(0, 0), + ImmutableMap.of()); + } + + private static List> pruneMaps(List> maps, List keys) + { + return maps.stream() + .map(map -> map == null ? null : Maps.filterKeys(map, keys::contains)) + .collect(toList()); + } + + private static List toSubfields(Type keyType, List keys) + { + if (keyType == TINYINT || keyType == SMALLINT || keyType == INTEGER || keyType == BIGINT) { + return keys.stream() + .map(Number.class::cast) + .mapToLong(Number::longValue) + .mapToObj(key -> new Subfield.LongSubscript(key)) + .map(subscript -> new Subfield("c", ImmutableList.of(subscript))) + .collect(toImmutableList()); + } + + if (keyType == VARCHAR) { + return keys.stream() + .map(String.class::cast) + .map(key -> new Subfield.StringSubscript(key)) + .map(subscript -> new Subfield("c", ImmutableList.of(subscript))) + .collect(toImmutableList()); + } + + throw new UnsupportedOperationException("Unsupported key type: " + keyType); + } + + private void runTest(String testOrcFileName, Type mapType, List> expectedValues, OrcPredicate orcPredicate, Optional filter, List subfields) + throws Exception + { + List types = ImmutableList.of(mapType); + Optional>> filters = filter.map(f -> ImmutableMap.of(new Subfield("c"), f)) + .map(f -> ImmutableMap.of(0, f)); + + assertFileContentsPresto( + types, + new File(getResource(testOrcFileName).getFile()), + filters.map(f -> filterRows(types, ImmutableList.of(expectedValues), f)).orElse(ImmutableList.of(expectedValues)), + OrcEncoding.DWRF, + orcPredicate, + filters, + ImmutableList.of(), + ImmutableMap.of(), + ImmutableMap.of(0, subfields)); + } + + private static boolean intToBoolean(int i) + { + return i % 2 == 0; + } + + private static SqlTimestamp intToTimestamp(int i) + { + return new SqlTimestamp(i, TimeZoneKey.UTC_KEY); + } + + private static List intToList(int i) + { + return ImmutableList.of(i * 3, i * 3 + 1, i * 3 + 2); + } + + private static Map intToMap(int i) + { + return ImmutableMap.of(Integer.toString(i * 3), (float) (i * 3), Integer.toString(i * 3 + 1), (float) (i * 3 + 1), Integer.toString(i * 3 + 2), (float) (i * 3 + 2)); + } + + static class ExpectedValuesBuilder + { + enum Frequency + { + NONE, + SOME, + ALL + } + + private final Function keyConverter; + private final Function valueConverter; + private Frequency nullValuesFrequency = NONE; + private Frequency nullRowsFrequency = NONE; + private Frequency emptyMapsFrequency = NONE; + private boolean mixedEncodings; + private boolean missingSequences; + + private ExpectedValuesBuilder(Function keyConverter, Function valueConverter) + { + this.keyConverter = keyConverter; + this.valueConverter = valueConverter; + } + + public static ExpectedValuesBuilder get(Function converter) + { + return new ExpectedValuesBuilder<>(converter, converter); + } + + public static ExpectedValuesBuilder get(Function keyConverter, Function valueConverter) + { + return new ExpectedValuesBuilder<>(keyConverter, valueConverter); + } + + public ExpectedValuesBuilder setNullValuesFrequency(Frequency frequency) + { + this.nullValuesFrequency = frequency; + + return this; + } + + public ExpectedValuesBuilder setNullRowsFrequency(Frequency frequency) + { + this.nullRowsFrequency = frequency; + + return this; + } + + public ExpectedValuesBuilder setEmptyMapsFrequency(Frequency frequency) + { + this.emptyMapsFrequency = frequency; + + return this; + } + + public ExpectedValuesBuilder setMixedEncodings() + { + this.mixedEncodings = true; + + return this; + } + + public ExpectedValuesBuilder setMissingSequences() + { + this.missingSequences = true; + + return this; + } + + public List> build() + { + List> result = new ArrayList<>(NUM_ROWS); + + for (int i = 0; i < NUM_ROWS; ++i) { + if (passesFrequencyCheck(nullRowsFrequency, i)) { + result.add(null); + } + else if (passesFrequencyCheck(emptyMapsFrequency, i)) { + result.add(Collections.emptyMap()); + } + else { + Map row = new HashMap<>(); + + for (int j = 0; j < 3; j++) { + V value; + int key = (i * 3 + j) % 32; + + if (missingSequences && key % 2 == 1) { + continue; + } + + if (j == 0 && passesFrequencyCheck(nullValuesFrequency, i)) { + value = null; + } + else if (mixedEncodings && (key == 1 || j == 2)) { + // TODO: add comments to explain the condition + value = valueConverter.apply(i * 3 + j); + } + else { + value = valueConverter.apply((i * 3 + j) % 32); + } + + row.put(keyConverter.apply(key), value); + } + + result.add(row); + } + } + + return result; + } + + private boolean passesFrequencyCheck(Frequency frequency, int i) + { + switch (frequency) { + case NONE: + return false; + case ALL: + return true; + case SOME: + return i % 5 == 0; + default: + throw new IllegalArgumentException("Got unexpected Frequency: " + frequency); + } + } + } + + static class TestingFilterFunction + extends FilterFunction + { + private static final Session TEST_SESSION = testSessionBuilder() + .setCatalog("tpch") + .setSchema("tiny") + .build(); + + private final Type mapType; + + public TestingFilterFunction(final Type mapType) + { + super(TEST_SESSION.toConnectorSession(), true, new Predicate() { + @Override + public int[] getInputChannels() + { + return new int[] {0}; + } + + @Override + public boolean evaluate(ConnectorSession session, Page page, int position) + { + Block mapBlock = page.getBlock(0); + if (mapBlock.isNull(position)) { + return false; + } + Map map = (Map) mapType.getObjectValue(session, mapBlock, position); + return map.containsKey(1); + } + }); + this.mapType = mapType; + } + + public List> filterRows(List> values) + { + List passingRows = IntStream.range(0, values.get(0).size()) + .filter(row -> values.get(0).get(row) != null) + .filter(row -> ((Map) values.get(0).get(row)).containsKey(1)) + .boxed() + .collect(toList()); + return IntStream.range(0, values.size()) + .mapToObj(column -> passingRows.stream().map(values.get(column)::get).collect(toList())) + .collect(toList()); + } + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcCacheConfig.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcCacheConfig.java new file mode 100644 index 0000000000000..51e67bd51dc02 --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcCacheConfig.java @@ -0,0 +1,72 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.facebook.airlift.configuration.testing.ConfigAssertions; +import com.facebook.presto.orc.cache.OrcCacheConfig; +import com.google.common.collect.ImmutableMap; +import io.airlift.units.DataSize; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.Map; + +import static io.airlift.units.DataSize.Unit.BYTE; +import static io.airlift.units.DataSize.Unit.GIGABYTE; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; + +public class TestOrcCacheConfig +{ + @Test + public void testDefaults() + { + ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(OrcCacheConfig.class) + .setFileTailCacheEnabled(false) + .setFileTailCacheSize(new DataSize(0, BYTE)) + .setFileTailCacheTtlSinceLastAccess(new Duration(0, SECONDS)) + .setStripeMetadataCacheEnabled(false) + .setStripeFooterCacheSize(new DataSize(0, BYTE)) + .setStripeFooterCacheTtlSinceLastAccess(new Duration(0, SECONDS)) + .setStripeStreamCacheSize(new DataSize(0, BYTE)) + .setStripeStreamCacheTtlSinceLastAccess(new Duration(0, SECONDS))); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("orc.file-tail-cache-enabled", "true") + .put("orc.file-tail-cache-size", "1GB") + .put("orc.file-tail-cache-ttl-since-last-access", "10m") + .put("orc.stripe-metadata-cache-enabled", "true") + .put("orc.stripe-footer-cache-size", "2GB") + .put("orc.stripe-footer-cache-ttl-since-last-access", "5m") + .put("orc.stripe-stream-cache-size", "3GB") + .put("orc.stripe-stream-cache-ttl-since-last-access", "10m") + .build(); + + OrcCacheConfig expected = new OrcCacheConfig() + .setFileTailCacheEnabled(true) + .setFileTailCacheSize(new DataSize(1, GIGABYTE)) + .setFileTailCacheTtlSinceLastAccess(new Duration(10, MINUTES)) + .setStripeMetadataCacheEnabled(true) + .setStripeFooterCacheSize(new DataSize(2, GIGABYTE)) + .setStripeFooterCacheTtlSinceLastAccess(new Duration(5, MINUTES)) + .setStripeStreamCacheSize(new DataSize(3, GIGABYTE)) + .setStripeStreamCacheTtlSinceLastAccess(new Duration(10, MINUTES)); + + ConfigAssertions.assertFullMapping(properties, expected); + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcLz4.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcLz4.java index 6d4e8997da921..e8400e085d014 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcLz4.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcLz4.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableMap; @@ -46,7 +47,16 @@ public void testReadLz4() // TODO: use Apache ORC library in OrcTester byte[] data = toByteArray(getResource("apache-lz4.orc")); - OrcReader orcReader = new OrcReader(new InMemoryOrcDataSource(data), ORC, SIZE, SIZE, SIZE, SIZE); + OrcReader orcReader = new OrcReader( + new InMemoryOrcDataSource(data), + ORC, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + new OrcReaderOptions( + SIZE, + SIZE, + SIZE, + false)); assertEquals(orcReader.getCompressionKind(), LZ4); assertEquals(orcReader.getFooter().getNumberOfRows(), 10_000); @@ -72,9 +82,9 @@ public void testReadLz4() } rows += batchSize; - Block xBlock = reader.readBlock(BIGINT, 0); - Block yBlock = reader.readBlock(INTEGER, 1); - Block zBlock = reader.readBlock(BIGINT, 2); + Block xBlock = reader.readBlock(0); + Block yBlock = reader.readBlock(1); + Block zBlock = reader.readBlock(2); for (int position = 0; position < batchSize; position++) { BIGINT.getLong(xBlock, position); diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderMemoryUsage.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderMemoryUsage.java index 999c801a6739b..c72dbdea78555 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderMemoryUsage.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderMemoryUsage.java @@ -38,6 +38,7 @@ import java.lang.invoke.MethodHandle; import java.util.HashMap; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; import static com.facebook.presto.orc.OrcEncoding.ORC; import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE; import static com.facebook.presto.orc.OrcReader.MAX_BATCH_SIZE; @@ -49,7 +50,6 @@ import static com.facebook.presto.spi.block.MethodHandleUtil.nativeValueGetter; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; -import static io.airlift.testing.Assertions.assertGreaterThan; import static org.testng.Assert.assertEquals; public class TestOrcReaderMemoryUsage @@ -83,7 +83,7 @@ public void testVarcharTypeWithoutNulls() break; } - Block block = reader.readBlock(VARCHAR, 0); + Block block = reader.readBlock(0); assertEquals(block.getPositionCount(), batchSize); // We only verify the memory usage when the batchSize reaches MAX_BATCH_SIZE as batchSize may be @@ -94,8 +94,8 @@ public void testVarcharTypeWithoutNulls() // StripeReader memory should increase after reading a block. assertGreaterThan(reader.getCurrentStripeRetainedSizeInBytes(), stripeReaderRetainedSize); - // There are no local buffers needed. - assertEquals(reader.getStreamReaderRetainedSizeInBytes() - streamReaderRetainedSize, 0L); + // SliceDictionaryBatchStreamReader uses stripeDictionaryLength local buffer. + assertEquals(reader.getStreamReaderRetainedSizeInBytes() - streamReaderRetainedSize, 4L); // The total retained size and system memory usage should be greater than 0 byte because of the instance sizes. assertGreaterThan(reader.getRetainedSizeInBytes() - readerRetainedSize, 0L); assertGreaterThan(reader.getSystemMemoryUsage() - readerSystemMemoryUsage, 0L); @@ -130,7 +130,7 @@ public void testBigIntTypeWithNulls() break; } - Block block = reader.readBlock(BIGINT, 0); + Block block = reader.readBlock(0); assertEquals(block.getPositionCount(), batchSize); // We only verify the memory usage when the batchSize reaches MAX_BATCH_SIZE as batchSize may be @@ -194,7 +194,7 @@ public void testMapTypeWithNulls() break; } - Block block = reader.readBlock(mapType, 0); + Block block = reader.readBlock(0); assertEquals(block.getPositionCount(), batchSize); // We only verify the memory usage when the batchSize reaches MAX_BATCH_SIZE as batchSize may be diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderPositions.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderPositions.java index 7d9bbe7c52803..26df6d3d82dbb 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderPositions.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcReaderPositions.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.orc; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.orc.metadata.Footer; import com.facebook.presto.orc.metadata.statistics.IntegerStatistics; @@ -163,7 +164,7 @@ public void testRowGroupSkipping() break; } - Block block = reader.readBlock(BIGINT, 0); + Block block = reader.readBlock(0); for (int i = 0; i < batchSize; i++) { assertEquals(BIGINT.getLong(block, i), position + i); } @@ -216,7 +217,7 @@ public void testBatchSizesForVariableWidth() rowCountsInCurrentRowGroup += batchSize; - Block block = reader.readBlock(VARCHAR, 0); + Block block = reader.readBlock(0); if (MAX_BATCH_SIZE * currentStringBytes <= MAX_BLOCK_SIZE.toBytes()) { // Either we are bounded by 1024 rows per batch, or it is the last batch in the row group // For the first 3 row groups, the strings are of length 300, 600, and 900 respectively @@ -270,7 +271,7 @@ public void testBatchSizesForFixedWidth() } rowCountsInCurrentRowGroup += batchSize; - Block block = reader.readBlock(BIGINT, 0); + Block block = reader.readBlock(0); // 8 bytes per row; 1024 row at most given 1024 X 8B < 1MB assertTrue(block.getPositionCount() == MAX_BATCH_SIZE || rowCountsInCurrentRowGroup == rowsInRowGroup); @@ -297,7 +298,12 @@ public void testReadUserMetadata() createFileWithOnlyUserMetadata(tempFile.getFile(), metadata); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); - OrcReader orcReader = new OrcReader(orcDataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); + OrcReader orcReader = new OrcReader( + orcDataSource, + ORC, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + OrcReaderTestingUtils.createDefaultTestConfig()); Footer footer = orcReader.getFooter(); Map readMetadata = Maps.transformValues(footer.getUserMetadata(), Slice::toStringAscii); assertEquals(readMetadata, metadata); @@ -363,7 +369,7 @@ else if (rowCountsInCurrentRowGroup > 20) { private static void assertCurrentBatch(OrcBatchRecordReader reader, int rowIndex, int batchSize) throws IOException { - Block block = reader.readBlock(BIGINT, 0); + Block block = reader.readBlock(0); for (int i = 0; i < batchSize; i++) { assertEquals(BIGINT.getLong(block, i), (rowIndex + i) * 3); } @@ -372,7 +378,7 @@ private static void assertCurrentBatch(OrcBatchRecordReader reader, int rowIndex private static void assertCurrentBatch(OrcBatchRecordReader reader, int stripe) throws IOException { - Block block = reader.readBlock(BIGINT, 0); + Block block = reader.readBlock(0); for (int i = 0; i < 20; i++) { assertEquals(BIGINT.getLong(block, i), ((stripe * 20L) + i) * 3); } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcWriter.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcWriter.java index 70c8d19c3273e..b160e1000ce99 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcWriter.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestOrcWriter.java @@ -14,10 +14,12 @@ package com.facebook.presto.orc; import com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.orc.metadata.Footer; import com.facebook.presto.orc.metadata.Stream; import com.facebook.presto.orc.metadata.StripeFooter; import com.facebook.presto.orc.metadata.StripeInformation; +import com.facebook.presto.orc.stream.OrcDataOutput; import com.facebook.presto.orc.stream.OrcInputStream; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; @@ -31,8 +33,10 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.util.List; import java.util.Optional; +import static com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.OrcEncoding.ORC; import static com.facebook.presto.orc.OrcTester.HIVE_STORAGE_TIME_ZONE; @@ -41,7 +45,6 @@ import static com.facebook.presto.orc.TestingOrcPredicate.ORC_STRIPE_SIZE; import static com.facebook.presto.orc.metadata.CompressionKind.NONE; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; -import static io.airlift.testing.Assertions.assertGreaterThanOrEqual; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.lang.Math.toIntExact; import static org.testng.Assert.assertFalse; @@ -95,7 +98,16 @@ public void testWriteOutputStreamsInOrder() // read the footer and verify the streams are ordered by size DataSize dataSize = new DataSize(1, MEGABYTE); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), dataSize, dataSize, dataSize, true); - Footer footer = new OrcReader(orcDataSource, ORC, dataSize, dataSize, dataSize, dataSize).getFooter(); + Footer footer = new OrcReader( + orcDataSource, + ORC, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + new OrcReaderOptions( + dataSize, + dataSize, + dataSize, + false)).getFooter(); for (StripeInformation stripe : footer.getStripes()) { // read the footer @@ -120,4 +132,78 @@ public void testWriteOutputStreamsInOrder() } } } + + @Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = "Dummy exception from mocked instance") + public void testVerifyNoIllegalStateException() + throws IOException + { + OrcWriter writer = new OrcWriter( + new MockOrcDataSink(), + ImmutableList.of("test1"), + ImmutableList.of(VARCHAR), + ORC, + NONE, + new OrcWriterOptions() + .withStripeMinSize(new DataSize(0, MEGABYTE)) + .withStripeMaxSize(new DataSize(32, MEGABYTE)) + .withStripeMaxRowCount(10) + .withRowGroupMaxRowCount(ORC_ROW_GROUP_SIZE) + .withDictionaryMaxMemory(new DataSize(32, MEGABYTE)), + ImmutableMap.of(), + HIVE_STORAGE_TIME_ZONE, + false, + null, + new OrcWriterStats()); + + int entries = 65536; + BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, entries); + byte[] bytes = "dummyString".getBytes(); + for (int j = 0; j < entries; j++) { + // force to write different data + bytes[0] = (byte) ((bytes[0] + 1) % 128); + blockBuilder.writeBytes(Slices.wrappedBuffer(bytes, 0, bytes.length), 0, bytes.length); + blockBuilder.closeEntry(); + } + Block[] blocks = new Block[] {blockBuilder.build()}; + + try { + // Throw IOException after first flush + writer.write(new Page(blocks)); + } + catch (IOException e) { + writer.close(); + } + } + + public static class MockOrcDataSink + implements OrcDataSink + { + public MockOrcDataSink() + { + } + + @Override + public long size() + { + return -1L; + } + + @Override + public long getRetainedSizeInBytes() + { + return -1L; + } + + @Override + public void write(List outputData) + throws IOException + { + throw new IOException("Dummy exception from mocked instance"); + } + + @Override + public void close() + { + } + } } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestPositionalFilter.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestPositionalFilter.java new file mode 100644 index 0000000000000..76671ce348e4c --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestPositionalFilter.java @@ -0,0 +1,87 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.facebook.presto.orc.TupleDomainFilter.BytesRange; +import com.facebook.presto.orc.TupleDomainFilter.PositionalFilter; +import io.airlift.slice.Slices; +import org.testng.annotations.Test; + +import java.util.Arrays; + +import static com.facebook.presto.testing.assertions.Assert.assertEquals; + +public class TestPositionalFilter +{ + @Test + public void test() + { + PositionalFilter filter = new PositionalFilter(); + + // a[1] = '1' and a[3] = '3' The test data is converted to byte[]'s and the comparison is done using testLength() + // followed by testBytes() so as to cover the double use of the position when testLength succeeeds and testBytes + // fails. + TupleDomainFilter[] filters = new TupleDomainFilter[] { + equals(1), null, equals(3), null, + equals(1), null, equals(3), null, + equals(1), null, equals(3), null, null, + equals(1), null, equals(3), null, null, null, + equals(1), null, equals(3), null, null, null, null + }; + + long[] numbers = new long[] { + 1, 2, 3, 4, // pass + 0, 2, 3, 4, // fail + 1, 2, 0, 4, 55, // fail testLength() + 1, 0, 3, 0, 5, 6, // pass + 1, 1, 2, 2, 3, 3, 4 // fail testBytes() + }; + // Convert the values to byte[][]. + byte[][] values = Arrays.stream(numbers).mapToObj(n -> toBytes(Long.valueOf(n).toString())).toArray(byte[][]::new); + + boolean[] expectedResults = new boolean[] { + true, true, true, true, + false, + true, true, false, + true, true, true, true, true, true, + true, true, false, + }; + + int[] offsets = new int[] {0, 4, 8, 13, 19, 26}; + + filter.setFilters(filters, offsets); + + int valuesIndex = 0; + for (int i = 0; i < expectedResults.length; i++) { + boolean result = filter.testLength(values[valuesIndex].length) && filter.testBytes(values[valuesIndex], 0, values[valuesIndex].length); + assertEquals(expectedResults[i], result); + valuesIndex++; + if (expectedResults[i] == false) { + valuesIndex += filter.getSucceedingPositionsToFail(); + } + } + assertEquals(new boolean[] {false, true, true, false, true, false}, filter.getFailed()); + } + + private TupleDomainFilter equals(int value) + { + byte[] bytesValue = toBytes(Integer.valueOf(value).toString()); + return BytesRange.of(bytesValue, false, bytesValue, false, false); + } + + private static byte[] toBytes(String value) + { + return Slices.utf8Slice(value).getBytes(); + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestReadBloomFilter.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestReadBloomFilter.java new file mode 100644 index 0000000000000..90bc838454bd5 --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestReadBloomFilter.java @@ -0,0 +1,137 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; +import com.facebook.presto.spi.predicate.NullableValue; +import com.facebook.presto.spi.type.SqlVarbinary; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.units.DataSize; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static com.facebook.presto.orc.OrcPredicate.TRUE; +import static com.facebook.presto.orc.OrcReader.MAX_BATCH_SIZE; +import static com.facebook.presto.orc.OrcTester.Format.ORC_12; +import static com.facebook.presto.orc.OrcTester.HIVE_STORAGE_TIME_ZONE; +import static com.facebook.presto.orc.OrcTester.writeOrcColumnHive; +import static com.facebook.presto.orc.TupleDomainOrcPredicate.ColumnReference; +import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; +import static com.facebook.presto.spi.predicate.TupleDomain.fromFixedValues; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.google.common.collect.Iterables.cycle; +import static com.google.common.collect.Iterables.limit; +import static com.google.common.collect.Lists.newArrayList; +import static io.airlift.slice.Slices.utf8Slice; +import static io.airlift.units.DataSize.Unit.MEGABYTE; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.testng.Assert.assertEquals; + +public class TestReadBloomFilter +{ + @Test + public void test() + throws Exception + { + testType(TINYINT, ImmutableList.of(1L, 50L, 100L), 50L, 77L); + testType(SMALLINT, ImmutableList.of(1L, 5000L, 10_000L), 5000L, 7777L); + testType(INTEGER, ImmutableList.of(1L, 500_000L, 1_000_000L), 500_000L, 777_777L); + testType(BIGINT, ImmutableList.of(1L, 500_000L, 1_000_000L), 500_000L, 777_777L); + testType(DOUBLE, ImmutableList.of(1.11, 500_000.55, 1_000_000.99), 500_000.55, 777_777.77); + + testType(VARCHAR, ImmutableList.of("a", "o", "z"), utf8Slice("o"), utf8Slice("w")); + testType(VARBINARY, + ImmutableList.of(new SqlVarbinary("a".getBytes(UTF_8)), new SqlVarbinary("o".getBytes(UTF_8)), new SqlVarbinary("z".getBytes(UTF_8))), + utf8Slice("o"), + utf8Slice("w")); + // Bloom filters are not supported for DECIMAL, FLOAT, DATE, TIMESTAMP, and CHAR + } + + private static void testType(Type type, List uniqueValues, T inBloomFilter, T notInBloomFilter) + throws Exception + { + List writeValues = newArrayList(limit(cycle(uniqueValues), 30_000)); + + try (TempFile tempFile = new TempFile()) { + writeOrcColumnHive(tempFile.getFile(), ORC_12, SNAPPY, type, writeValues); + + // no predicate + try (OrcBatchRecordReader recordReader = createCustomOrcRecordReader(tempFile, type, Optional.empty(), true)) { + assertEquals(recordReader.nextBatch(), MAX_BATCH_SIZE); + } + + try (OrcBatchRecordReader recordReader = createCustomOrcRecordReader(tempFile, type, Optional.empty(), false)) { + assertEquals(recordReader.nextBatch(), MAX_BATCH_SIZE); + } + + // predicate for non-matching value + try (OrcBatchRecordReader recordReader = createCustomOrcRecordReader(tempFile, type, Optional.of(notInBloomFilter), true)) { + assertEquals(recordReader.nextBatch(), -1); + } + + try (OrcBatchRecordReader recordReader = createCustomOrcRecordReader(tempFile, type, Optional.of(notInBloomFilter), false)) { + assertEquals(recordReader.nextBatch(), MAX_BATCH_SIZE); + } + + // predicate for matching value + try (OrcBatchRecordReader recordReader = createCustomOrcRecordReader(tempFile, type, Optional.of(inBloomFilter), true)) { + assertEquals(recordReader.nextBatch(), MAX_BATCH_SIZE); + } + + try (OrcBatchRecordReader recordReader = createCustomOrcRecordReader(tempFile, type, Optional.of(inBloomFilter), false)) { + assertEquals(recordReader.nextBatch(), MAX_BATCH_SIZE); + } + } + } + + private static OrcBatchRecordReader createCustomOrcRecordReader(TempFile tempFile, Type type, Optional filterValue, boolean bloomFilterEnabled) + throws IOException + { + OrcPredicate predicate = filterValue.map(value -> makeOrcPredicate(type, value, bloomFilterEnabled)).map(OrcPredicate.class::cast).orElse(TRUE); + + OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); + OrcReader orcReader = new OrcReader( + orcDataSource, + OrcEncoding.ORC, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + OrcReaderTestingUtils.createDefaultTestConfig()); + + assertEquals(orcReader.getColumnNames(), ImmutableList.of("test")); + assertEquals(orcReader.getFooter().getRowsInRowGroup(), 10_000); + + return orcReader.createBatchRecordReader(ImmutableMap.of(0, type), predicate, HIVE_STORAGE_TIME_ZONE, newSimpleAggregatedMemoryContext(), MAX_BATCH_SIZE); + } + + private static TupleDomainOrcPredicate makeOrcPredicate(Type type, T value, boolean bloomFilterEnabled) + { + return new TupleDomainOrcPredicate<>( + fromFixedValues(ImmutableMap.of("test", NullableValue.of(type, value))), + ImmutableList.of(new ColumnReference<>("test", 0, type)), + bloomFilterEnabled, + Optional.empty()); + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestSelectiveOrcReader.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestSelectiveOrcReader.java index 51a1f8a35cd42..95fac30794ac7 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestSelectiveOrcReader.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestSelectiveOrcReader.java @@ -13,45 +13,90 @@ */ package com.facebook.presto.orc; +import com.facebook.presto.orc.OrcTester.OrcReaderSettings; import com.facebook.presto.orc.TupleDomainFilter.BigintRange; import com.facebook.presto.orc.TupleDomainFilter.BigintValues; import com.facebook.presto.orc.TupleDomainFilter.BooleanValue; +import com.facebook.presto.orc.TupleDomainFilter.BytesRange; +import com.facebook.presto.orc.TupleDomainFilter.BytesValues; +import com.facebook.presto.orc.TupleDomainFilter.DoubleRange; +import com.facebook.presto.orc.TupleDomainFilter.FloatRange; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.type.CharType; +import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.SqlDate; +import com.facebook.presto.spi.type.SqlDecimal; +import com.facebook.presto.spi.type.SqlTimestamp; +import com.facebook.presto.spi.type.SqlVarbinary; +import com.facebook.presto.spi.type.Type; +import com.google.common.base.Strings; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ContiguousSet; import com.google.common.collect.DiscreteDomain; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import com.google.common.collect.Range; +import com.google.common.collect.Streams; +import com.google.common.primitives.Ints; import org.joda.time.DateTimeZone; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.math.BigInteger; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.function.Function; import java.util.stream.IntStream; import static com.facebook.presto.orc.OrcTester.HIVE_STORAGE_TIME_ZONE; +import static com.facebook.presto.orc.OrcTester.arrayType; +import static com.facebook.presto.orc.OrcTester.mapType; import static com.facebook.presto.orc.OrcTester.quickSelectiveOrcTester; +import static com.facebook.presto.orc.OrcTester.rowType; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NOT_NULL; +import static com.facebook.presto.orc.TupleDomainFilter.IS_NULL; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.CharType.createCharType; import static com.facebook.presto.spi.type.DateType.DATE; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RealType.REAL; import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.testing.DateTimeTestingUtils.sqlTimestampOf; +import static com.facebook.presto.testing.TestingConnectorSession.SESSION; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.Iterables.concat; import static com.google.common.collect.Iterables.cycle; import static com.google.common.collect.Iterables.limit; import static com.google.common.collect.Lists.newArrayList; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.nCopies; import static java.util.stream.Collectors.toList; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; public class TestSelectiveOrcReader { + private static final int NUM_ROWS = 31_234; + + private static final DecimalType DECIMAL_TYPE_PRECISION_2 = DecimalType.createDecimalType(2, 1); + private static final DecimalType DECIMAL_TYPE_PRECISION_4 = DecimalType.createDecimalType(4, 2); + private static final DecimalType DECIMAL_TYPE_PRECISION_19 = DecimalType.createDecimalType(19, 8); + private static final CharType CHAR_10 = createCharType(10); private final OrcTester tester = quickSelectiveOrcTester(); @BeforeClass @@ -64,21 +109,130 @@ public void setUp() public void testBooleanSequence() throws Exception { - List> filters = ImmutableList.of( - ImmutableMap.of(0, BooleanValue.of(true, false)), - ImmutableMap.of(0, TupleDomainFilter.IS_NULL)); - tester.testRoundTrip(BOOLEAN, newArrayList(limit(cycle(ImmutableList.of(true, false, false)), 30_000)), filters); + tester.testRoundTrip( + BOOLEAN, + newArrayList(limit(cycle(ImmutableList.of(true, false, false)), NUM_ROWS)), + BooleanValue.of(true, false), TupleDomainFilter.IS_NULL); - filters = ImmutableList.of( - ImmutableMap.of(0, BooleanValue.of(true, false)), - ImmutableMap.of(0, TupleDomainFilter.IS_NULL), - ImmutableMap.of(1, BooleanValue.of(true, false)), - ImmutableMap.of(0, BooleanValue.of(false, false), 1, BooleanValue.of(true, false))); tester.testRoundTripTypes(ImmutableList.of(BOOLEAN, BOOLEAN), ImmutableList.of( - newArrayList(limit(cycle(ImmutableList.of(true, false, false)), 30_000)), - newArrayList(limit(cycle(ImmutableList.of(true, true, false)), 30_000))), - filters); + newArrayList(limit(cycle(ImmutableList.of(true, false, false)), NUM_ROWS)), + newArrayList(limit(cycle(ImmutableList.of(true, true, false)), NUM_ROWS))), + toSubfieldFilters( + ImmutableMap.of(0, BooleanValue.of(true, false)), + ImmutableMap.of(0, TupleDomainFilter.IS_NULL), + ImmutableMap.of(1, BooleanValue.of(true, false)), + ImmutableMap.of(0, BooleanValue.of(false, false), 1, BooleanValue.of(true, false)))); + + tester.testRoundTripTypes( + ImmutableList.of(BOOLEAN, BOOLEAN, BOOLEAN), + ImmutableList.of(newArrayList(true, false, null, false, true), newArrayList(null, null, null, null, null), newArrayList(true, false, null, false, null)), + toSubfieldFilters(ImmutableMap.of(0, BooleanValue.of(true, false), 1, BooleanValue.of(true, true), 2, BooleanValue.of(true, true)))); + } + + @Test + public void testByteValues() + throws Exception + { + List byteValues = ImmutableList.of(1, 3, 5, 7, 11, 13, 17) + .stream() + .map(Integer::byteValue) + .collect(toList()); + + tester.testRoundTrip(TINYINT, byteValues, BigintValues.of(new long[] {1, 17}, false), IS_NULL); + + List>> filters = toSubfieldFilters( + ImmutableMap.of(0, BigintRange.of(1, 17, false)), + ImmutableMap.of(0, IS_NULL), + ImmutableMap.of(1, IS_NULL), + ImmutableMap.of( + 0, + BigintRange.of(7, 17, false), + 1, + BigintRange.of(12, 14, false))); + tester.testRoundTripTypes(ImmutableList.of(TINYINT, TINYINT), ImmutableList.of(byteValues, byteValues), filters); + + tester.testRoundTripTypes( + ImmutableList.of(TINYINT, TINYINT, TINYINT), + ImmutableList.of(toByteArray(newArrayList(1, 2, null, 3, 4)), newArrayList(null, null, null, null, null), toByteArray(newArrayList(5, 6, null, 7, null))), + toSubfieldFilters(ImmutableMap.of( + 0, BigintValues.of(new long[] {1, 4}, false), + 1, BigintValues.of(new long[] {1, 5}, true), + 2, BigintValues.of(new long[] {5, 7}, true)))); + } + + @Test + public void testByteValuesRepeat() + throws Exception + { + List byteValues = ImmutableList.of(1, 3, 5, 7, 11, 13, 17) + .stream() + .map(Integer::byteValue) + .collect(toList()); + tester.testRoundTrip( + TINYINT, + newArrayList(limit(repeatEach(4, cycle(byteValues)), NUM_ROWS)), + BigintRange.of(1, 14, true)); + } + + @Test + public void testByteValuesPatchedBase() + throws Exception + { + List byteValues = newArrayList( + limit(cycle(concat( + intsBetween(0, 18), + intsBetween(0, 18), + ImmutableList.of(NUM_ROWS, 20_000, 400_000, NUM_ROWS, 20_000))), NUM_ROWS)) + .stream() + .map(Integer::byteValue) + .collect(toList()); + tester.testRoundTrip( + TINYINT, + byteValues, + BigintRange.of(4, 14, true)); + } + + @Test + public void testDoubleSequence() + throws Exception + { + tester.testRoundTrip(DOUBLE, doubleSequence(0, 0.1, NUM_ROWS), DoubleRange.of(0, false, false, 1_000, false, false, false), IS_NULL, IS_NOT_NULL); + tester.testRoundTripTypes( + ImmutableList.of(DOUBLE, DOUBLE), + ImmutableList.of( + doubleSequence(0, 0.1, 10_000), + doubleSequence(0, 0.1, 10_000)), + toSubfieldFilters( + ImmutableMap.of( + 0, DoubleRange.of(1.0, false, true, 7.0, false, true, true), + 1, DoubleRange.of(3.0, false, true, 9.0, false, true, false)))); + tester.testRoundTripTypes( + ImmutableList.of(DOUBLE, DOUBLE, DOUBLE), + ImmutableList.of(newArrayList(1.0, 2.0, null, 3.0, 4.0), newArrayList(null, null, null, null, null), newArrayList(1.0, 2.0, null, 3.0, null)), + toSubfieldFilters(ImmutableMap.of( + 0, DoubleRange.of(1.0, false, true, 7.0, false, true, true), + 1, DoubleRange.of(1.0, false, true, 7.0, false, true, true), + 2, DoubleRange.of(1.0, false, true, 7.0, false, true, true)))); + } + + @Test + public void testDoubleNaNInfinity() + throws Exception + { + List> filters = toSubfieldFilters( + DoubleRange.of(0, false, false, 1_000, false, false, false), + IS_NULL, + IS_NOT_NULL); + + tester.testRoundTrip(DOUBLE, ImmutableList.of(1000.0, -1.0, Double.POSITIVE_INFINITY), filters); + tester.testRoundTrip(DOUBLE, ImmutableList.of(-1000.0, Double.NEGATIVE_INFINITY, 1.0), filters); + tester.testRoundTrip(DOUBLE, ImmutableList.of(0.0, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY), filters); + + tester.testRoundTrip(DOUBLE, ImmutableList.of(Double.NaN, -1.0, 1.0), filters); + tester.testRoundTrip(DOUBLE, ImmutableList.of(Double.NaN, -1.0, Double.POSITIVE_INFINITY), filters); + tester.testRoundTrip(DOUBLE, ImmutableList.of(Double.NaN, Double.NEGATIVE_INFINITY, 1.0), filters); + tester.testRoundTrip(DOUBLE, ImmutableList.of(Double.NaN, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY), filters); } @Test @@ -109,7 +263,16 @@ public void testLongSequenceWithHoles() public void testLongDirect() throws Exception { - testRoundTripNumeric(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), 30_000), BigintRange.of(4, 14, false)); + testRoundTripNumeric(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), NUM_ROWS), BigintRange.of(4, 14, false)); + + Random random = new Random(0); + + // read selected positions from all nulls column + tester.testRoundTripTypes(ImmutableList.of(INTEGER, INTEGER), + ImmutableList.of( + createList(NUM_ROWS, i -> random.nextInt(10)), + createList(NUM_ROWS, i -> null)), + toSubfieldFilters(ImmutableMap.of(0, BigintRange.of(0, 5, false)))); } @Test @@ -121,18 +284,27 @@ public void testLongDirect2() testRoundTripNumeric(values, BigintRange.of(4, 14, false)); } + @Test + public void testLongDirectVarintScale() + throws Exception + { + List values = varintScaleSequence(NUM_ROWS); + Collections.shuffle(values, new Random(0)); + testRoundTripNumeric(values, BigintRange.of(0, 1L << 60, false)); + } + @Test public void testLongShortRepeat() throws Exception { - testRoundTripNumeric(limit(repeatEach(4, cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17))), 30_000), BigintRange.of(4, 14, true)); + testRoundTripNumeric(limit(repeatEach(4, cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17))), NUM_ROWS), BigintRange.of(4, 14, true)); } @Test public void testLongPatchedBase() throws Exception { - testRoundTripNumeric(limit(cycle(concat(intsBetween(0, 18), intsBetween(0, 18), ImmutableList.of(30_000, 20_000, 400_000, 30_000, 20_000))), 30_000), + testRoundTripNumeric(limit(cycle(concat(intsBetween(0, 18), intsBetween(0, 18), ImmutableList.of(NUM_ROWS, 20_000, 400_000, NUM_ROWS, 20_000))), NUM_ROWS), BigintValues.of(new long[] {0, 5, 10, 15, 20_000}, true)); } @@ -143,6 +315,584 @@ public void testLongStrideDictionary() testRoundTripNumeric(concat(ImmutableList.of(1), nCopies(9999, 123), ImmutableList.of(2), nCopies(9999, 123)), BigintRange.of(123, 123, true)); } + @Test + public void testFloats() + throws Exception + { + List> filters = toSubfieldFilters( + FloatRange.of(0.0f, false, true, 100.0f, false, true, true), + FloatRange.of(-100.0f, false, true, 0.0f, false, true, false), + IS_NULL); + + tester.testRoundTrip(REAL, ImmutableList.copyOf(repeatEach(10, ImmutableList.of(-100.0f, 0.0f, 100.0f))), filters); + tester.testRoundTrip(REAL, ImmutableList.copyOf(repeatEach(10, ImmutableList.of(1000.0f, -1.23f, Float.POSITIVE_INFINITY)))); + + List floatValues = ImmutableList.of(1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f); + tester.testRoundTripTypes( + ImmutableList.of(REAL, REAL), + ImmutableList.of( + ImmutableList.copyOf(limit(repeatEach(4, cycle(floatValues)), 100)), + ImmutableList.copyOf(limit(repeatEach(4, cycle(floatValues)), 100))), + toSubfieldFilters( + ImmutableMap.of( + 0, FloatRange.of(1.0f, false, true, 7.0f, false, true, true), + 1, FloatRange.of(3.0f, false, true, 9.0f, false, true, false)), + ImmutableMap.of( + 1, FloatRange.of(1.0f, false, true, 7.0f, false, true, true)))); + + tester.testRoundTripTypes( + ImmutableList.of(REAL, REAL, REAL), + ImmutableList.of(newArrayList(1.0f, 2.0f, 3.0f, 4.0f, 5.0f), newArrayList(null, null, null, null, null), newArrayList(1.0f, 2.0f, null, 3.0f, null)), + toSubfieldFilters(ImmutableMap.of( + 0, FloatRange.of(2.0f, false, false, 7.0f, false, true, true), + 1, FloatRange.of(1.0f, false, false, 7.0f, false, true, true), + 2, FloatRange.of(1.0f, false, false, 7.0f, false, true, true)))); + } + + @Test + public void testFilterOrder() + throws Exception + { + Random random = new Random(0); + + tester.testRoundTripTypesWithOrder(ImmutableList.of(INTEGER, INTEGER), + ImmutableList.of(newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11)), NUM_ROWS)), randomIntegers(NUM_ROWS, random)), + toSubfieldFilters( + ImmutableMap.of( + 0, BigintRange.of(1, 1, true), + 1, BigintRange.of(2, 200, true))), + ImmutableList.of(ImmutableList.of(0, 1))); + + tester.testRoundTripTypesWithOrder(ImmutableList.of(INTEGER, INTEGER), + ImmutableList.of(newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11)), NUM_ROWS)), randomIntegers(NUM_ROWS, random)), + toSubfieldFilters( + ImmutableMap.of( + 0, BigintRange.of(100, 100, false), + 1, BigintRange.of(2, 200, true))), + ImmutableList.of(ImmutableList.of(0))); + + tester.testRoundTripTypesWithOrder( + ImmutableList.of(INTEGER, INTEGER, DOUBLE, arrayType(INTEGER)), + ImmutableList.of(newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11)), NUM_ROWS)), createList(NUM_ROWS, i -> random.nextInt(200)), doubleSequence(0, 0.1, NUM_ROWS), nCopies(NUM_ROWS, randomIntegers(5, random))), + ImmutableList.of( + ImmutableMap.of( + 0, toSubfieldFilter(BigintRange.of(1, 1, true)), + 1, toSubfieldFilter(BigintRange.of(0, 200, true)), + 2, toSubfieldFilter(DoubleRange.of(0, false, false, 0.1, false, false, true)), + 3, toSubfieldFilter("c[1]", BigintRange.of(4, 4, false)))), + ImmutableList.of(ImmutableList.of(0, 1, 2, 3))); + } + + @Test + public void testArrays() + throws Exception + { + Random random = new Random(0); + + // non-null arrays of varying sizes; some arrays may be empty + tester.testRoundTrip(arrayType(INTEGER), + createList(NUM_ROWS, i -> randomIntegers(random.nextInt(10), random)), + IS_NULL, IS_NOT_NULL); + + BigintRange negative = BigintRange.of(Integer.MIN_VALUE, 0, false); + BigintRange nonNegative = BigintRange.of(0, Integer.MAX_VALUE, false); + + // non-empty non-null arrays of varying sizes + tester.testRoundTrip(arrayType(INTEGER), + createList(NUM_ROWS, i -> randomIntegers(5 + random.nextInt(5), random)), + ImmutableList.of( + toSubfieldFilter(IS_NULL), + toSubfieldFilter(IS_NOT_NULL), + // c[1] >= 0 + toSubfieldFilter("c[1]", nonNegative), + // c[2] >= 0 AND c[4] >= 0 + ImmutableMap.of( + new Subfield("c[2]"), nonNegative, + new Subfield("c[4]"), nonNegative))); + + // non-null arrays of varying sizes; some arrays may be empty + tester.testRoundTripTypes(ImmutableList.of(INTEGER, arrayType(INTEGER)), + ImmutableList.of( + randomIntegers(NUM_ROWS, random), + createList(NUM_ROWS, i -> randomIntegers(random.nextInt(10), random))), + toSubfieldFilters( + ImmutableMap.of(0, nonNegative), + ImmutableMap.of( + 0, nonNegative, + 1, IS_NULL), + ImmutableMap.of( + 0, nonNegative, + 1, IS_NOT_NULL))); + + // non-empty non-null arrays of varying sizes + tester.testRoundTripTypes(ImmutableList.of(INTEGER, arrayType(INTEGER)), + ImmutableList.of( + randomIntegers(NUM_ROWS, random), + createList(NUM_ROWS, i -> randomIntegers(5 + random.nextInt(5), random))), + ImmutableList.of( + // c[1] >= 0 + ImmutableMap.of( + 0, toSubfieldFilter(nonNegative), + 1, toSubfieldFilter("c[1]", nonNegative)), + // c[3] >= 0 + ImmutableMap.of( + 0, toSubfieldFilter(nonNegative), + 1, toSubfieldFilter("c[3]", nonNegative)), + // c[2] >= 0 AND c[4] <= 0 + ImmutableMap.of( + 0, toSubfieldFilter(nonNegative), + 1, ImmutableMap.of( + new Subfield("c[2]"), nonNegative, + new Subfield("c[4]"), negative)))); + + // nested arrays + tester.testRoundTripTypes(ImmutableList.of(INTEGER, arrayType(arrayType(INTEGER))), + ImmutableList.of( + randomIntegers(NUM_ROWS, random), + createList(NUM_ROWS, i -> createList(random.nextInt(10), index -> randomIntegers(random.nextInt(5), random)))), + toSubfieldFilters( + ImmutableMap.of(0, nonNegative), + ImmutableMap.of(1, IS_NULL), + ImmutableMap.of(1, IS_NOT_NULL), + ImmutableMap.of( + 0, nonNegative, + 1, IS_NULL))); + + tester.testRoundTripTypes(ImmutableList.of(INTEGER, arrayType(arrayType(INTEGER))), + ImmutableList.of( + randomIntegers(NUM_ROWS, random), + createList(NUM_ROWS, i -> createList(3 + random.nextInt(10), index -> randomIntegers(3 + random.nextInt(5), random)))), + ImmutableList.of( + // c[1] IS NULL + ImmutableMap.of(1, ImmutableMap.of(new Subfield("c[1]"), IS_NULL)), + // c[2] IS NOT NULL AND c[2][3] >= 0 + ImmutableMap.of(1, ImmutableMap.of( + new Subfield("c[2]"), IS_NOT_NULL, + new Subfield("c[2][3]"), nonNegative)), + ImmutableMap.of( + 0, toSubfieldFilter(nonNegative), + 1, ImmutableMap.of(new Subfield("c[1]"), IS_NULL)))); + } + + @Test + public void testArraysWithSubfieldPruning() + throws Exception + { + tester.assertRoundTripWithSettings(arrayType(INTEGER), + createList(NUM_ROWS, i -> ImmutableList.of(1, 2, 3, 4)), + ImmutableList.of( + OrcReaderSettings.builder().addRequiredSubfields(0, "c[1]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[1]", "c[2]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[2]").build())); + + Random random = new Random(0); + + tester.assertRoundTripWithSettings(arrayType(INTEGER), + createList(NUM_ROWS, i -> ImmutableList.of(random.nextInt(10), random.nextInt(10), 3, 4)), + ImmutableList.of( + OrcReaderSettings.builder() + .addRequiredSubfields(0, "c[1]", "c[3]") + .setColumnFilters(ImmutableMap.of(0, ImmutableMap.of(new Subfield("c[1]"), BigintRange.of(0, 4, false)))) + .build(), + OrcReaderSettings.builder() + .addRequiredSubfields(0, "c[2]", "c[3]") + .setColumnFilters(ImmutableMap.of(0, ImmutableMap.of(new Subfield("c[2]"), BigintRange.of(0, 4, false)))) + .build())); + + // arrays of arrays + tester.assertRoundTripWithSettings(arrayType(arrayType(INTEGER)), + createList(NUM_ROWS, i -> nCopies(1 + random.nextInt(5), ImmutableList.of(1, 2, 3))), + ImmutableList.of( + OrcReaderSettings.builder().addRequiredSubfields(0, "c[1][1]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][2]", "c[4][2]", "c[5][3]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][3]", "c[10][2]", "c[3][10]").build())); + + // arrays of maps + tester.assertRoundTripWithSettings(arrayType(mapType(INTEGER, INTEGER)), + createList(NUM_ROWS, i -> nCopies(5, ImmutableMap.of(1, 10, 2, 20))), + ImmutableList.of( + OrcReaderSettings.builder().addRequiredSubfields(0, "c[1][1]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][1]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][1]", "c[4][1]", "c[3][2]").build())); + } + + @Test + public void testArrayIndexOutOfBounds() + throws Exception + { + Random random = new Random(0); + + // non-null arrays of varying sizes + try { + tester.testRoundTrip(arrayType(INTEGER), + createList(NUM_ROWS, i -> randomIntegers(random.nextInt(10), random)), + ImmutableList.of(ImmutableMap.of(new Subfield("c[2]"), IS_NULL))); + fail("Expected 'Array subscript out of bounds' exception"); + } + catch (PrestoException e) { + assertTrue(e.getMessage().contains("Array subscript out of bounds")); + } + + // non-null nested arrays of varying sizes + try { + tester.testRoundTrip(arrayType(arrayType(INTEGER)), + createList(NUM_ROWS, i -> ImmutableList.of(randomIntegers(random.nextInt(5), random), randomIntegers(random.nextInt(5), random))), + ImmutableList.of(ImmutableMap.of(new Subfield("c[2][3]"), IS_NULL))); + fail("Expected 'Array subscript out of bounds' exception"); + } + catch (PrestoException e) { + assertTrue(e.getMessage().contains("Array subscript out of bounds")); + } + + // empty arrays + try { + tester.testRoundTrip(arrayType(INTEGER), + nCopies(NUM_ROWS, ImmutableList.of()), + ImmutableList.of(ImmutableMap.of(new Subfield("c[2]"), IS_NULL))); + fail("Expected 'Array subscript out of bounds' exception"); + } + catch (PrestoException e) { + assertTrue(e.getMessage().contains("Array subscript out of bounds")); + } + + // empty nested arrays + try { + tester.testRoundTrip(arrayType(arrayType(INTEGER)), + nCopies(NUM_ROWS, ImmutableList.of()), + ImmutableList.of(ImmutableMap.of(new Subfield("c[2][3]"), IS_NULL))); + fail("Expected 'Array subscript out of bounds' exception"); + } + catch (PrestoException e) { + assertTrue(e.getMessage().contains("Array subscript out of bounds")); + } + } + + @Test + public void testArraysOfNulls() + throws Exception + { + for (Type type : ImmutableList.of(BOOLEAN, BIGINT, INTEGER, SMALLINT, TINYINT, DOUBLE, REAL, TIMESTAMP, DECIMAL_TYPE_PRECISION_19, DECIMAL_TYPE_PRECISION_4, VARCHAR, CHAR_10, VARBINARY, arrayType(INTEGER))) { + tester.testRoundTrip(arrayType(type), + nCopies(NUM_ROWS, nCopies(5, null)), + ImmutableList.of( + ImmutableMap.of(new Subfield("c[2]"), IS_NULL), + ImmutableMap.of(new Subfield("c[2]"), IS_NOT_NULL))); + } + } + + @Test + public void testStructs() + throws Exception + { + Random random = new Random(0); + + tester.testRoundTripTypes(ImmutableList.of(INTEGER, rowType(INTEGER, BOOLEAN)), + ImmutableList.of( + createList(NUM_ROWS, i -> random.nextInt()), + createList(NUM_ROWS, i -> ImmutableList.of(random.nextInt(), random.nextBoolean()))), + ImmutableList.of( + ImmutableMap.of(0, toSubfieldFilter(BigintRange.of(0, Integer.MAX_VALUE, false))), + ImmutableMap.of(1, toSubfieldFilter(IS_NULL)), + ImmutableMap.of(1, toSubfieldFilter(IS_NOT_NULL)), + ImmutableMap.of(1, toSubfieldFilter("c.field_0", BigintRange.of(0, Integer.MAX_VALUE, false))), + ImmutableMap.of(1, toSubfieldFilter("c.field_0", IS_NULL)))); + + tester.testRoundTripTypes(ImmutableList.of(rowType(INTEGER, BOOLEAN), INTEGER), + ImmutableList.of( + createList(NUM_ROWS, i -> i % 7 == 0 ? null : ImmutableList.of(random.nextInt(), random.nextBoolean())), + createList(NUM_ROWS, i -> i % 11 == 0 ? null : random.nextInt())), + ImmutableList.of( + ImmutableMap.of(0, toSubfieldFilter(IS_NOT_NULL), 1, toSubfieldFilter(IS_NULL)), + ImmutableMap.of(0, toSubfieldFilter("c.field_0", BigintRange.of(0, Integer.MAX_VALUE, false))), + ImmutableMap.of(0, toSubfieldFilter("c.field_0", BigintRange.of(0, Integer.MAX_VALUE, true))))); + } + + @Test + public void testMaps() + throws Exception + { + Random random = new Random(0); + + tester.testRoundTrip(mapType(INTEGER, INTEGER), createList(NUM_ROWS, i -> createMap(i))); + + // map column with no nulls + tester.testRoundTripTypes( + ImmutableList.of(INTEGER, mapType(INTEGER, INTEGER)), + ImmutableList.of( + createList(NUM_ROWS, i -> random.nextInt()), + createList(NUM_ROWS, i -> createMap(i))), + toSubfieldFilters( + ImmutableMap.of(0, BigintRange.of(0, Integer.MAX_VALUE, false)), + ImmutableMap.of(1, IS_NOT_NULL), + ImmutableMap.of(1, IS_NULL))); + + // map column with nulls + tester.testRoundTripTypes( + ImmutableList.of(INTEGER, mapType(INTEGER, INTEGER)), + ImmutableList.of( + createList(NUM_ROWS, i -> random.nextInt()), + createList(NUM_ROWS, i -> i % 5 == 0 ? null : createMap(i))), + toSubfieldFilters( + ImmutableMap.of(0, BigintRange.of(0, Integer.MAX_VALUE, false)), + ImmutableMap.of(1, IS_NOT_NULL), + ImmutableMap.of(1, IS_NULL), + ImmutableMap.of(0, BigintRange.of(0, Integer.MAX_VALUE, false), 1, IS_NULL), + ImmutableMap.of(0, BigintRange.of(0, Integer.MAX_VALUE, false), 1, IS_NOT_NULL))); + + // map column with filter, followed by another column with filter + tester.testRoundTripTypes( + ImmutableList.of(mapType(INTEGER, INTEGER), INTEGER), + ImmutableList.of( + createList(NUM_ROWS, i -> i % 5 == 0 ? null : createMap(i)), + createList(NUM_ROWS, i -> random.nextInt())), + toSubfieldFilters( + ImmutableMap.of(0, IS_NULL, 1, BigintRange.of(0, Integer.MAX_VALUE, false)), + ImmutableMap.of(0, IS_NOT_NULL, 1, BigintRange.of(0, Integer.MAX_VALUE, false)))); + + // empty maps + tester.testRoundTripTypes(ImmutableList.of(INTEGER, mapType(INTEGER, INTEGER)), + ImmutableList.of( + createList(NUM_ROWS, i -> random.nextInt()), + Collections.nCopies(NUM_ROWS, ImmutableMap.of())), + ImmutableList.of()); + + // read selected positions from all nulls map column + tester.testRoundTripTypes(ImmutableList.of(INTEGER, mapType(INTEGER, INTEGER)), + ImmutableList.of( + createList(NUM_ROWS, i -> random.nextInt(10)), + createList(NUM_ROWS, i -> null)), + toSubfieldFilters(ImmutableMap.of(0, BigintRange.of(0, 5, false)))); + } + + @Test + public void testMapsWithSubfieldPruning() + throws Exception + { + tester.assertRoundTripWithSettings(mapType(INTEGER, INTEGER), + createList(NUM_ROWS, i -> ImmutableMap.of(1, 10, 2, 20)), + ImmutableList.of( + OrcReaderSettings.builder().addRequiredSubfields(0, "c[1]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[2]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[10]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[2]", "c[10]").build())); + + // maps of maps + tester.assertRoundTripWithSettings(mapType(INTEGER, mapType(INTEGER, INTEGER)), + createList(NUM_ROWS, i -> ImmutableMap.of(1, createMap(i), 2, createMap(i + 1))), + ImmutableList.of( + OrcReaderSettings.builder().addRequiredSubfields(0, "c[1][1]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][2]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][1]", "c[10][1]").build())); + + // maps of arrays + tester.assertRoundTripWithSettings(mapType(INTEGER, arrayType(INTEGER)), + createList(NUM_ROWS, i -> ImmutableMap.of(1, nCopies(5, 10), 2, nCopies(5, 20))), + ImmutableList.of( + OrcReaderSettings.builder().addRequiredSubfields(0, "c[1][1]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][2]", "c[2][3]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][2]", "c[10][3]", "c[2][10]").build(), + OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][2]", "c[1][2]").build())); + } + + private static Map createMap(int seed) + { + int mapSize = Math.abs(seed) % 7 + 1; + return IntStream.range(0, mapSize).boxed().collect(toImmutableMap(Function.identity(), i -> i + seed)); + } + + private static List createList(int size, Function createElement) + { + return IntStream.range(0, size).mapToObj(createElement::apply).collect(toList()); + } + + @Test + public void testDecimalSequence() + throws Exception + { + tester.testRoundTrip(DECIMAL_TYPE_PRECISION_4, decimalSequence("-3000", "1", 60_00, 4, 2)); + tester.testRoundTrip(DECIMAL_TYPE_PRECISION_19, decimalSequence("-3000000000000000000", "100000000000000101", 60, 19, 8)); + + tester.testRoundTripTypes( + ImmutableList.of(DECIMAL_TYPE_PRECISION_2, DECIMAL_TYPE_PRECISION_2), + ImmutableList.of( + decimalSequence("-30", "1", 60, 2, 1), + decimalSequence("-30", "1", 60, 2, 1)), + toSubfieldFilters(ImmutableMap.of(0, TupleDomainFilter.BigintRange.of(10, 20, true)))); + + tester.testRoundTripTypes( + ImmutableList.of(DECIMAL_TYPE_PRECISION_2, DECIMAL_TYPE_PRECISION_2), + ImmutableList.of( + decimalSequence("-30", "1", 60, 2, 1), + decimalSequence("-30", "1", 60, 2, 1)), + toSubfieldFilters(ImmutableMap.of( + 0, TupleDomainFilter.BigintRange.of(10, 30, true), + 1, TupleDomainFilter.BigintRange.of(15, 25, true)))); + + tester.testRoundTripTypes( + ImmutableList.of(DECIMAL_TYPE_PRECISION_19, DECIMAL_TYPE_PRECISION_19), + ImmutableList.of( + decimalSequence("-3000000000000000000", "100000000000000101", 60, 19, 8), + decimalSequence("-3000000000000000000", "100000000000000101", 60, 19, 8)), + toSubfieldFilters(ImmutableMap.of( + 0, TupleDomainFilter.LongDecimalRange.of(-28999999999L, -28999999999L, + false, true, 28999999999L, 28999999999L, false, true, true), + 1, TupleDomainFilter.LongDecimalRange.of(1000000000L, 1000000000L, + false, true, 28999999999L, 28999999999L, false, true, true)))); + } + + @Test + public void testVarchars() + throws Exception + { + Random random = new Random(0); + tester.testRoundTripTypes( + ImmutableList.of(VARCHAR, VARCHAR, VARCHAR), + ImmutableList.of(newArrayList("abc", "def", null, "hij", "klm"), newArrayList(null, null, null, null, null), newArrayList("abc", "def", null, null, null)), + toSubfieldFilters(ImmutableMap.of( + 0, stringIn(true, "abc", "def"), + 1, stringIn(true, "10", "11"), + 2, stringIn(true, "def", "abc")))); + + // dictionary + tester.testRoundTrip(VARCHAR, newArrayList(limit(cycle(ImmutableList.of("apple", "apple pie", "apple\uD835\uDC03", "apple\uFFFD")), NUM_ROWS)), + stringIn(false, "apple", "apple pie")); + + // direct + tester.testRoundTrip(VARCHAR, + intsBetween(0, NUM_ROWS).stream().map(Object::toString).collect(toList()), + stringIn(false, "10", "11"), + stringIn(true, "10", "11"), + stringBetween(false, "14", "10")); + + // direct & dictionary with filters + tester.testRoundTripTypes( + ImmutableList.of(VARCHAR, VARCHAR), + ImmutableList.of( + intsBetween(0, NUM_ROWS).stream().map(Object::toString).collect(toList()), + newArrayList(limit(cycle(ImmutableList.of("A", "B", "C")), NUM_ROWS))), + toSubfieldFilters(ImmutableMap.of( + 0, stringBetween(true, "16", "10"), + 1, stringBetween(false, "B", "A")))); + + //stripe dictionary + tester.testRoundTrip(VARCHAR, newArrayList(concat(ImmutableList.of("a"), nCopies(9999, "123"), ImmutableList.of("b"), nCopies(9999, "123")))); + + //empty sequence + tester.testRoundTrip(VARCHAR, nCopies(NUM_ROWS, ""), stringEquals(false, "")); + + // copy of AbstractOrcTester::testDwrfInvalidCheckpointsForRowGroupDictionary + List values = newArrayList(limit( + cycle(concat( + ImmutableList.of(1), nCopies(9999, 123), + ImmutableList.of(2), nCopies(9999, 123), + ImmutableList.of(3), nCopies(9999, 123), + nCopies(1_000_000, null))), + 200_000)); + + tester.assertRoundTrip( + VARCHAR, + newArrayList(values).stream() + .map(value -> value == null ? null : String.valueOf(value)) + .collect(toList())); + + //copy of AbstractOrcTester::testDwrfInvalidCheckpointsForStripeDictionary + tester.testRoundTrip( + VARCHAR, + newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), 200_000)).stream() + .map(Object::toString) + .collect(toList())); + + // presentStream is null in some row groups + Function randomStrings = i -> String.valueOf(random.nextInt(NUM_ROWS)); + tester.testRoundTripTypes( + ImmutableList.of(INTEGER, VARCHAR), + ImmutableList.of(createList(NUM_ROWS, i -> random.nextInt(NUM_ROWS)), newArrayList(createList(NUM_ROWS, randomStrings))), + toSubfieldFilters(ImmutableMap.of(0, BigintRange.of(10_000, 15_000, true)))); + + // dataStream is null and lengths are 0 + tester.testRoundTrip(VARCHAR, newArrayList("", null), toSubfieldFilters(stringNotEquals(true, ""))); + } + + @Test + public void testChars() + throws Exception + { + // multiple columns filter on not null + tester.testRoundTripTypes(ImmutableList.of(VARCHAR, createCharType(5)), + ImmutableList.of( + newArrayList(limit(cycle(ImmutableList.of("123456789", "23456789", "3456789")), NUM_ROWS)), + newArrayList(limit(cycle(ImmutableList.of("12345", "23456", "34567")), NUM_ROWS))), + toSubfieldFilters(ImmutableMap.of(0, IS_NOT_NULL))); + + tester.testRoundTrip(createCharType(2), newArrayList(limit(cycle(ImmutableList.of("aa", "bb", "cc", "dd")), NUM_ROWS)), IS_NULL); + + tester.testRoundTrip(createCharType(1), newArrayList(limit(cycle(ImmutableList.of("a", "b", "c", "d")), NUM_ROWS)), + stringIn(false, "a", "b"), + stringIn(true, "a", "b")); + + // char with padding + tester.testRoundTrip( + CHAR_10, + intsBetween(0, NUM_ROWS).stream() + .map(i -> toCharValue(i, 10)) + .collect(toList())); + + // char with filter + tester.testRoundTrip( + CHAR_10, + newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), NUM_ROWS)).stream() + .map(i -> toCharValue(i, 10)) + .collect(toList()), + stringIn(true, toCharValue(1, 10), toCharValue(3, 10))); + + // char with 0 truncated length + tester.testRoundTrip(CHAR_10, newArrayList(limit(cycle(toCharValue("", 10)), NUM_ROWS))); + + tester.testRoundTrip(VARCHAR, newArrayList(concat(ImmutableList.of("a"), nCopies(9999, "123"), ImmutableList.of("b"), nCopies(9999, "123"))), + stringIn(false, "a", "b")); + } + + @Test + public void testVarBinaries() + throws Exception + { + tester.testRoundTrip( + VARBINARY, + createList(NUM_ROWS, i -> new SqlVarbinary(String.valueOf(i).getBytes(UTF_8))), + stringIn(false, "10", "11")); + + tester.testRoundTripTypes( + ImmutableList.of(VARBINARY, VARBINARY), + ImmutableList.of( + createList(NUM_ROWS, i -> new SqlVarbinary(Ints.toByteArray(i))), + Streams.stream(limit(cycle(ImmutableList.of("A", "B", "C")), NUM_ROWS)) + .map(String::getBytes) + .map(SqlVarbinary::new) + .collect(toImmutableList())), + toSubfieldFilters(ImmutableMap.of( + 0, stringBetween(true, "10", "16"), + 1, stringBetween(false, "A", "B")))); + + tester.testRoundTrip( + VARBINARY, + createList(NUM_ROWS, i -> new SqlVarbinary(String.valueOf(i).getBytes(UTF_8))), + bytesBetween(false, new byte[] {8}, new byte[] {9})); + + tester.testRoundTrip( + VARBINARY, + nCopies(NUM_ROWS, new SqlVarbinary(new byte[0])), + bytesBetween(false, new byte[0], new byte[] {1})); + + tester.testRoundTrip( + VARBINARY, + ImmutableList.copyOf(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), NUM_ROWS)).stream() + .map(String::valueOf) + .map(string -> string.getBytes(UTF_8)) + .map(SqlVarbinary::new) + .collect(toImmutableList()), + bytesBetween(false, new byte[] {1}, new byte[] {12})); + } + private void testRoundTripNumeric(Iterable values, TupleDomainFilter filter) throws Exception { @@ -163,13 +913,19 @@ private void testRoundTripNumeric(Iterable values, TupleDomain .map(SqlDate::new) .collect(toList()); - tester.testRoundTrip(BIGINT, longValues, ImmutableList.of(ImmutableMap.of(0, filter))); + List timestamps = longValues.stream() + .map(timestamp -> sqlTimestampOf(timestamp & Integer.MAX_VALUE, SESSION)) + .collect(toList()); + + tester.testRoundTrip(BIGINT, longValues, toSubfieldFilters(filter)); - tester.testRoundTrip(INTEGER, intValues, ImmutableList.of(ImmutableMap.of(0, filter))); + tester.testRoundTrip(INTEGER, intValues, toSubfieldFilters(filter)); - tester.testRoundTrip(SMALLINT, shortValues, ImmutableList.of(ImmutableMap.of(0, filter))); + tester.testRoundTrip(SMALLINT, shortValues, toSubfieldFilters(filter)); - tester.testRoundTrip(DATE, dateValues, ImmutableList.of(ImmutableMap.of(0, filter))); + tester.testRoundTrip(DATE, dateValues, toSubfieldFilters(filter)); + + tester.testRoundTrip(TIMESTAMP, timestamps, toSubfieldFilters(filter)); List reversedIntValues = new ArrayList<>(intValues); Collections.reverse(reversedIntValues); @@ -177,24 +933,55 @@ private void testRoundTripNumeric(Iterable values, TupleDomain List reversedDateValues = new ArrayList<>(dateValues); Collections.reverse(reversedDateValues); - tester.testRoundTripTypes(ImmutableList.of(BIGINT, INTEGER, SMALLINT, DATE), + List reversedTimestampValues = new ArrayList<>(timestamps); + Collections.reverse(reversedTimestampValues); + + tester.testRoundTripTypes(ImmutableList.of(BIGINT, INTEGER, SMALLINT, DATE, TIMESTAMP), ImmutableList.of( longValues, reversedIntValues, shortValues, - reversedDateValues), - ImmutableList.of( + reversedDateValues, + reversedTimestampValues), + toSubfieldFilters( ImmutableMap.of(0, filter), ImmutableMap.of(1, filter), ImmutableMap.of(0, filter, 1, filter), ImmutableMap.of(0, filter, 1, filter, 2, filter))); } + private static TupleDomainFilter stringBetween(boolean nullAllowed, String upper, String lower) + { + return BytesRange.of(lower.getBytes(), false, upper.getBytes(), false, nullAllowed); + } + + private static TupleDomainFilter stringEquals(boolean nullAllowed, String value) + { + return BytesRange.of(value.getBytes(), false, value.getBytes(), false, nullAllowed); + } + + private static TupleDomainFilter stringNotEquals(boolean nullAllowed, String value) + { + return TupleDomainFilter.MultiRange.of(ImmutableList.of( + BytesRange.of(null, false, value.getBytes(), true, nullAllowed), + BytesRange.of(value.getBytes(), true, null, false, nullAllowed)), nullAllowed); + } + + private static TupleDomainFilter stringIn(boolean nullAllowed, String... values) + { + return BytesValues.of(Arrays.stream(values).map(String::getBytes).toArray(byte[][]::new), nullAllowed); + } + private static ContiguousSet intsBetween(int lowerInclusive, int upperExclusive) { return ContiguousSet.create(Range.closedOpen(lowerInclusive, upperExclusive), DiscreteDomain.integers()); } + private static BytesRange bytesBetween(boolean nullAllowed, byte[] lower, byte[] upper) + { + return BytesRange.of(lower, false, upper, false, nullAllowed); + } + private static Iterable skipEvery(int n, Iterable iterable) { return () -> new AbstractIterator() @@ -247,4 +1034,72 @@ protected T computeNext() } }; } + + private static String toCharValue(Object value, int minLength) + { + return Strings.padEnd(value.toString(), minLength, ' '); + } + + private static List doubleSequence(double start, double step, int items) + { + return IntStream.range(0, items) + .mapToDouble(i -> start + i * step) + .boxed() + .collect(ImmutableList.toImmutableList()); + } + + private static Map toSubfieldFilter(String subfield, TupleDomainFilter filter) + { + return ImmutableMap.of(new Subfield(subfield), filter); + } + + private static Map toSubfieldFilter(TupleDomainFilter filter) + { + return ImmutableMap.of(new Subfield("c"), filter); + } + + private static List> toSubfieldFilters(TupleDomainFilter... filters) + { + return Arrays.stream(filters).map(TestSelectiveOrcReader::toSubfieldFilter).collect(toImmutableList()); + } + + private static List>> toSubfieldFilters(Map... filters) + { + return Arrays.stream(filters) + .map(columnFilters -> Maps.transformValues(columnFilters, TestSelectiveOrcReader::toSubfieldFilter)) + .collect(toImmutableList()); + } + + private static List randomIntegers(int size, Random random) + { + return createList(size, i -> random.nextInt()); + } + + private static List decimalSequence(String start, String step, int items, int precision, int scale) + { + BigInteger decimalStep = new BigInteger(step); + List values = new ArrayList<>(); + BigInteger nextValue = new BigInteger(start); + for (int i = 0; i < items; i++) { + values.add(new SqlDecimal(nextValue, precision, scale)); + nextValue = nextValue.add(decimalStep); + } + return values; + } + + private static List varintScaleSequence(int rows) + { + List values = new ArrayList(); + long[] numbers = new long[] {1L, 1L << 8, 1L << 13, 1L << 20, 1L << 27, 1L << 34, 1L << 40, 1L << 47, 1L << 53, 1L << 60, 1L << 63}; + for (int i = 0; i < rows; i++) { + values.add(numbers[i % numbers.length] + i); + values.add(-numbers[i % numbers.length] + i); + } + return values; + } + + private static List toByteArray(List integers) + { + return integers.stream().map((i) -> i == null ? null : i.byteValue()).collect(toList()); + } } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestStructBatchStreamReader.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestStructBatchStreamReader.java index 635b1dac1d2df..1f2dda89099a6 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestStructBatchStreamReader.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestStructBatchStreamReader.java @@ -14,6 +14,7 @@ package com.facebook.presto.orc; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; @@ -153,7 +154,7 @@ public void testReaderLowerCasesFieldNamesFromType() } @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = - "Missing struct field name in type row\\(varchar,varchar,varchar\\)") + "ROW type does not have field names declared: row\\(varchar,varchar,varchar\\)") public void testThrowsExceptionWhenFieldNameMissing() throws IOException { @@ -263,7 +264,16 @@ private RowBlock read(TempFile tempFile, Type readerType) { DataSize dataSize = new DataSize(1, MEGABYTE); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), dataSize, dataSize, dataSize, true); - OrcReader orcReader = new OrcReader(orcDataSource, ORC, dataSize, dataSize, dataSize, dataSize); + OrcReader orcReader = new OrcReader( + orcDataSource, + ORC, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + new OrcReaderOptions( + dataSize, + dataSize, + dataSize, + false)); Map includedColumns = new HashMap<>(); includedColumns.put(0, readerType); @@ -271,7 +281,7 @@ private RowBlock read(TempFile tempFile, Type readerType) OrcBatchRecordReader recordReader = orcReader.createBatchRecordReader(includedColumns, OrcPredicate.TRUE, UTC, newSimpleAggregatedMemoryContext(), OrcReader.INITIAL_BATCH_SIZE); recordReader.nextBatch(); - RowBlock block = (RowBlock) recordReader.readBlock(readerType, 0); + RowBlock block = (RowBlock) recordReader.readBlock(0); recordReader.close(); return block; } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestTupleDomainFilter.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestTupleDomainFilter.java index e7feeb3abbab3..500dd373e17f2 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestTupleDomainFilter.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestTupleDomainFilter.java @@ -35,6 +35,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; public class TestTupleDomainFilter { @@ -144,6 +145,15 @@ public void testDoubleRange() assertFalse(filter.testNull()); assertFalse(filter.testDouble(-0.3)); assertFalse(filter.testDouble(55.6)); + assertFalse(filter.testDouble(Double.NaN)); + + try { + DoubleRange.of(Double.NaN, false, false, Double.NaN, false, false, false); + fail("able to create a DoubleRange with NaN"); + } + catch (IllegalArgumentException e) { + //expected + } } @Test @@ -170,6 +180,15 @@ public void testFloatRange() assertFalse(filter.testNull()); assertFalse(filter.testFloat(1.1f)); assertFalse(filter.testFloat(15.632f)); + assertFalse(filter.testFloat(Float.NaN)); + + try { + FloatRange.of(Float.NaN, false, false, Float.NaN, false, false, false); + fail("able to create a FloatRange with NaN"); + } + catch (IllegalArgumentException e) { + //expected + } } @Test @@ -200,9 +219,12 @@ public void testBytesRange() { TupleDomainFilter filter = BytesRange.of(toBytes("abc"), false, toBytes("abc"), false, false); assertTrue(filter.testBytes(toBytes("abc"), 0, 3)); + assertFalse(filter.testBytes(toBytes("acb"), 0, 3)); + assertTrue(filter.testLength(3)); assertFalse(filter.testNull()); assertFalse(filter.testBytes(toBytes("apple"), 0, 5)); + assertFalse(filter.testLength(4)); String theBestOfTimes = "It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity,..."; filter = BytesRange.of(null, true, toBytes(theBestOfTimes), false, false); @@ -210,6 +232,9 @@ public void testBytesRange() assertTrue(filter.testBytes(toBytes(theBestOfTimes), 0, 5)); assertTrue(filter.testBytes(toBytes(theBestOfTimes), 0, 50)); assertTrue(filter.testBytes(toBytes(theBestOfTimes), 0, 100)); + // testLength is true of all lengths for a range filter. + assertTrue(filter.testLength(1)); + assertTrue(filter.testLength(1000)); assertFalse(filter.testNull()); assertFalse(filter.testBytes(toBytes("Zzz"), 0, 3)); @@ -279,7 +304,9 @@ public void testBytesValues() } } filter = BytesValues.of(filterValues, false); + assertFalse(filter.testLength(10000)); for (int i = 0; i < testValues.length; i++) { + assertEquals(filter.testLength(i), i % 9 == 0); assertEquals(i % 9 == 0, filter.testBytes(testValues[i], 0, testValues[i].length)); } } diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestTupleDomainFilterUtils.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestTupleDomainFilterUtils.java index 92deaa1cfac50..c57e508e8eb87 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestTupleDomainFilterUtils.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestTupleDomainFilterUtils.java @@ -14,6 +14,8 @@ package com.facebook.presto.orc; import com.facebook.presto.Session; +import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.orc.TupleDomainFilter.BigintMultiRange; import com.facebook.presto.orc.TupleDomainFilter.BigintRange; @@ -26,7 +28,12 @@ import com.facebook.presto.orc.TupleDomainFilter.LongDecimalRange; import com.facebook.presto.orc.TupleDomainFilter.MultiRange; import com.facebook.presto.spi.predicate.Domain; +import com.facebook.presto.spi.type.NamedTypeSignature; +import com.facebook.presto.spi.type.RowFieldName; import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.spi.type.TypeSignatureParameter; +import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.planner.ExpressionDomainTranslator; import com.facebook.presto.sql.planner.LiteralEncoder; import com.facebook.presto.sql.planner.Symbol; @@ -47,6 +54,7 @@ import com.facebook.presto.sql.tree.NullLiteral; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.StringLiteral; +import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; @@ -63,6 +71,7 @@ import java.util.concurrent.TimeUnit; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; +import static com.facebook.presto.orc.TupleDomainFilter.ALWAYS_FALSE; import static com.facebook.presto.orc.TupleDomainFilter.IS_NOT_NULL; import static com.facebook.presto.orc.TupleDomainFilter.IS_NULL; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -76,6 +85,9 @@ import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.spi.type.RealType.REAL; import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.StandardTypes.ARRAY; +import static com.facebook.presto.spi.type.StandardTypes.MAP; +import static com.facebook.presto.spi.type.StandardTypes.ROW; import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; import static com.facebook.presto.spi.type.TinyintType.TINYINT; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; @@ -101,6 +113,13 @@ public class TestTupleDomainFilterUtils { + private static final TypeManager TYPE_MANAGER = new TypeRegistry(); + + static { + // associate TYPE_MANAGER with a function manager + new FunctionManager(TYPE_MANAGER, new BlockEncodingManager(TYPE_MANAGER), new FeaturesConfig()); + } + private static final Session TEST_SESSION = testSessionBuilder() .setCatalog("tpch") .setSchema("tiny") @@ -130,6 +149,9 @@ public class TestTupleDomainFilterUtils private static final Symbol C_SMALLINT = new Symbol("c_smallint"); private static final Symbol C_TINYINT = new Symbol("c_tinyint"); private static final Symbol C_REAL = new Symbol("c_real"); + private static final Symbol C_ARRAY = new Symbol("c_array"); + private static final Symbol C_MAP = new Symbol("c_map"); + private static final Symbol C_STRUCT = new Symbol("c_struct"); private static final TypeProvider TYPES = TypeProvider.viewOf(ImmutableMap.builder() .put(C_BIGINT.getName(), BIGINT) @@ -156,6 +178,11 @@ public class TestTupleDomainFilterUtils .put(C_SMALLINT.getName(), SMALLINT) .put(C_TINYINT.getName(), TINYINT) .put(C_REAL.getName(), REAL) + .put(C_ARRAY.getName(), TYPE_MANAGER.getParameterizedType(ARRAY, ImmutableList.of(TypeSignatureParameter.of(INTEGER.getTypeSignature())))) + .put(C_MAP.getName(), TYPE_MANAGER.getParameterizedType(MAP, ImmutableList.of(TypeSignatureParameter.of(INTEGER.getTypeSignature()), TypeSignatureParameter.of(BOOLEAN.getTypeSignature())))) + .put(C_STRUCT.getName(), TYPE_MANAGER.getParameterizedType(ROW, ImmutableList.of( + TypeSignatureParameter.of(new NamedTypeSignature(Optional.of(new RowFieldName("field_0", false)), INTEGER.getTypeSignature())), + TypeSignatureParameter.of(new NamedTypeSignature(Optional.of(new RowFieldName("field_1", false)), BOOLEAN.getTypeSignature()))))) .build()); private Metadata metadata; @@ -253,6 +280,12 @@ public void testDouble() DoubleRange.of(1.2, false, true, Double.MAX_VALUE, true, true, false)), true)); assertEquals(toFilter(between(C_DOUBLE, doubleLiteral(1.2), doubleLiteral(3.4))), DoubleRange.of(1.2, false, false, 3.4, false, false, false)); + + assertEquals(toFilter(lessThan(C_DOUBLE, doubleLiteral(Double.NaN))), ALWAYS_FALSE); + assertEquals(toFilter(lessThanOrEqual(C_DOUBLE, doubleLiteral(Double.NaN))), ALWAYS_FALSE); + assertEquals(toFilter(greaterThanOrEqual(C_DOUBLE, doubleLiteral(Double.NaN))), ALWAYS_FALSE); + assertEquals(toFilter(greaterThan(C_DOUBLE, doubleLiteral(Double.NaN))), ALWAYS_FALSE); + assertEquals(toFilter(notEqual(C_DOUBLE, doubleLiteral(Double.NaN))), ALWAYS_FALSE); } @Test @@ -268,6 +301,13 @@ public void testFloat() assertEquals(toFilter(or(isNull(C_REAL), notEqual(C_REAL, realLiteral))), MultiRange.of(ImmutableList.of( FloatRange.of(Float.MIN_VALUE, true, true, 1.2f, false, true, false), FloatRange.of(1.2f, false, true, Float.MAX_VALUE, true, true, false)), true)); + + Expression floatNaNLiteral = toExpression(realValue(Float.NaN), TYPES.get(C_REAL.toSymbolReference())); + assertEquals(toFilter(lessThan(C_REAL, floatNaNLiteral)), ALWAYS_FALSE); + assertEquals(toFilter(lessThanOrEqual(C_REAL, floatNaNLiteral)), ALWAYS_FALSE); + assertEquals(toFilter(greaterThanOrEqual(C_REAL, floatNaNLiteral)), ALWAYS_FALSE); + assertEquals(toFilter(greaterThan(C_REAL, floatNaNLiteral)), ALWAYS_FALSE); + assertEquals(toFilter(notEqual(C_REAL, floatNaNLiteral)), ALWAYS_FALSE); } @Test @@ -320,6 +360,27 @@ public void testDate() assertEquals(toFilter(not(lessThanOrEqual(C_DATE, dateLiteral("2019-06-01")))), BigintRange.of(days + 1, Long.MAX_VALUE, false)); } + @Test + public void testMap() + { + assertEquals(toFilter(isNotNull(C_MAP)), IS_NOT_NULL); + assertEquals(toFilter(isNull(C_MAP)), IS_NULL); + } + + @Test + public void testArray() + { + assertEquals(toFilter(isNotNull(C_ARRAY)), IS_NOT_NULL); + assertEquals(toFilter(isNull(C_ARRAY)), IS_NULL); + } + + @Test + public void testStruct() + { + assertEquals(toFilter(isNotNull(C_STRUCT)), IS_NOT_NULL); + assertEquals(toFilter(isNull(C_STRUCT)), IS_NULL); + } + private static byte[] toBytes(String value) { return Slices.utf8Slice(value).getBytes(); @@ -510,7 +571,7 @@ private static Expression cast(Expression expression, Type type) private static FunctionCall colorLiteral(long value) { - return new FunctionCall(QualifiedName.of(getMagicLiteralFunctionSignature(COLOR).getName()), ImmutableList.of(bigintLiteral(value))); + return new FunctionCall(QualifiedName.of(getMagicLiteralFunctionSignature(COLOR).getName().getFunctionName()), ImmutableList.of(bigintLiteral(value))); } private Expression varbinaryLiteral(Slice value) diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestZstdJniDecompression.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestZstdJniDecompression.java new file mode 100644 index 0000000000000..767ffffd742e1 --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestZstdJniDecompression.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import com.facebook.presto.orc.zstd.ZstdJniCompressor; +import com.facebook.presto.testing.assertions.Assert; +import io.airlift.units.DataSize; +import org.testng.annotations.Test; + +import java.util.Random; + +public class TestZstdJniDecompression +{ + private static final DataSize MAX_BUFFER_SIZE = new DataSize(4, DataSize.Unit.MEGABYTE); + private final ZstdJniCompressor compressor = new ZstdJniCompressor(); + private final OrcZstdDecompressor decompressor = new OrcZstdDecompressor(new OrcDataSourceId("test"), (int) MAX_BUFFER_SIZE.toBytes(), true); + + @Test + public void testDecompression() + throws OrcCorruptionException + { + byte[] sourceBytes = generateRandomBytes(); + byte[] compressedBytes = new byte[1024 * 1024]; + int size = compressor.compress(sourceBytes, 0, sourceBytes.length, compressedBytes, 0, compressedBytes.length); + byte[] output = new byte[sourceBytes.length]; + int outputSize = decompressor.decompress( + compressedBytes, + 0, + size, + new OrcDecompressor.OutputBuffer() + { + @Override + public byte[] initialize(int size) + { + return output; + } + + @Override + public byte[] grow(int size) + { + throw new RuntimeException(); + } + }); + Assert.assertEquals(outputSize, sourceBytes.length); + Assert.assertEquals(output, sourceBytes); + } + + private byte[] generateRandomBytes() + { + Random random = new Random(); + byte[] array = new byte[1024]; + random.nextBytes(array); + return array; + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TestingOrcPredicate.java b/presto-orc/src/test/java/com/facebook/presto/orc/TestingOrcPredicate.java index ea95f95a32ca9..722bd4eb6f904 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/TestingOrcPredicate.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TestingOrcPredicate.java @@ -15,6 +15,7 @@ import com.facebook.presto.orc.OrcTester.Format; import com.facebook.presto.orc.metadata.statistics.ColumnStatistics; +import com.facebook.presto.orc.metadata.statistics.HiveBloomFilter; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.SqlDate; @@ -33,6 +34,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ThreadLocalRandom; import java.util.stream.IntStream; import static com.facebook.presto.orc.OrcTester.Format.DWRF; @@ -84,7 +86,7 @@ public static OrcPredicate createOrcPredicate(int columnIndex, Type type, Iterab return new BooleanOrcPredicate(columnIndex, expectedValues, format == DWRF); } if (TINYINT.equals(type) || SMALLINT.equals(type) || INTEGER.equals(type) || BIGINT.equals(type)) { - return new LongOrcPredicate( + return new LongOrcPredicate(true, columnIndex, expectedValues.stream() .map(value -> value == null ? null : ((Number) value).longValue()) @@ -92,7 +94,7 @@ public static OrcPredicate createOrcPredicate(int columnIndex, Type type, Iterab format == DWRF); } if (TIMESTAMP.equals(type)) { - return new LongOrcPredicate( + return new LongOrcPredicate(false, columnIndex, expectedValues.stream() .map(value -> value == null ? null : ((SqlTimestamp) value).getMillisUtc()) @@ -292,6 +294,14 @@ protected boolean chunkMatchesStats(List chunk, ColumnStatistics columnS return false; } + HiveBloomFilter bloomFilter = columnStatistics.getBloomFilter(); + if (bloomFilter != null) { + for (Double value : chunk) { + if (value != null && !bloomFilter.testDouble(value)) { + return false; + } + } + } // statistics can be missing for any reason if (columnStatistics.getDoubleStatistics() != null) { if (chunk.stream().allMatch(Objects::isNull)) { @@ -327,9 +337,12 @@ public DecimalOrcPredicate(int columnIndex, Iterable expectedValues, boolean public static class LongOrcPredicate extends BasicOrcPredicate { - public LongOrcPredicate(int columnIndex, Iterable expectedValues, boolean noFileStats) + private final boolean testBloomFilter; + + public LongOrcPredicate(boolean testBloomFilter, int columnIndex, Iterable expectedValues, boolean noFileStats) { super(columnIndex, expectedValues, Long.class, noFileStats); + this.testBloomFilter = testBloomFilter; } @Override @@ -367,9 +380,17 @@ protected boolean chunkMatchesStats(List chunk, ColumnStatistics columnSta .filter(Objects::nonNull) .mapToLong(Long::longValue) .sum(); - if (columnStatistics.getIntegerStatistics().getSum() != sum) { + if (columnStatistics.getIntegerStatistics().getSum() != null && columnStatistics.getIntegerStatistics().getSum() != sum) { return false; } + HiveBloomFilter bloomFilter = columnStatistics.getBloomFilter(); + if (testBloomFilter && bloomFilter != null) { + for (Long value : chunk) { + if (value != null && !bloomFilter.testLong(value)) { + return false; + } + } + } } return true; @@ -407,6 +428,25 @@ protected boolean chunkMatchesStats(List chunk, ColumnStatistics columnS .map(Slices::utf8Slice) .collect(toList()); + HiveBloomFilter bloomFilter = columnStatistics.getBloomFilter(); + if (bloomFilter != null) { + for (Slice slice : slices) { + if (!bloomFilter.test(slice.getBytes())) { + return false; + } + } + int falsePositive = 0; + byte[] testBuffer = new byte[32]; + for (int i = 0; i < 100_000; i++) { + ThreadLocalRandom.current().nextBytes(testBuffer); + if (bloomFilter.test(testBuffer)) { + falsePositive++; + } + } + if (falsePositive != 0 && 1.0 * falsePositive / 100_000 > 0.55) { + return false; + } + } // statistics can be missing for any reason if (columnStatistics.getStringStatistics() != null) { if (slices.isEmpty()) { diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TrackingTupleDomainFilter.java b/presto-orc/src/test/java/com/facebook/presto/orc/TrackingTupleDomainFilter.java new file mode 100644 index 0000000000000..27871230edd69 --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TrackingTupleDomainFilter.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.orc; + +import java.util.function.Consumer; + +import static java.util.Objects.requireNonNull; + +public interface TrackingTupleDomainFilter + extends TupleDomainFilter +{ + class TestBigintRange + extends BigintRange + implements TrackingTupleDomainFilter + { + private final Consumer onTestCallback; + + public static TestBigintRange of(BigintRange range, Consumer onTestCallback) + { + return new TestBigintRange(range.getLower(), range.getUpper(), range.nullAllowed, onTestCallback); + } + + private TestBigintRange(long lower, long upper, boolean nullAllowed, Consumer onTestCallback) + { + super(lower, upper, nullAllowed); + this.onTestCallback = requireNonNull(onTestCallback, "onTestCallback is null"); + } + + @Override + public boolean testLong(long value) + { + onTestCallback.accept(this); + return super.testLong(value); + } + } + + class TestDoubleRange + extends DoubleRange + implements TrackingTupleDomainFilter + { + private final Consumer onTestCallback; + + public static TestDoubleRange of(DoubleRange range, Consumer onTestCallback) + { + return new TestDoubleRange(range.getLower(), range.lowerUnbounded, range.lowerExclusive, range.getUpper(), range.upperUnbounded, range.upperExclusive, range.nullAllowed, onTestCallback); + } + + private TestDoubleRange(double lower, boolean lowerUnbounded, boolean lowerExclusive, double upper, boolean upperUnbounded, boolean upperExclusive, boolean nullAllowed, Consumer onTestCallback) + { + super(lower, lowerUnbounded, lowerExclusive, upper, upperUnbounded, upperExclusive, nullAllowed); + this.onTestCallback = requireNonNull(onTestCallback, "onTestCallback is null"); + } + + @Override + public boolean testDouble(double value) + { + onTestCallback.accept(this); + return super.testDouble(value); + } + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/TupleDomainFilterOrderChecker.java b/presto-orc/src/test/java/com/facebook/presto/orc/TupleDomainFilterOrderChecker.java new file mode 100644 index 0000000000000..c361f3ed95265 --- /dev/null +++ b/presto-orc/src/test/java/com/facebook/presto/orc/TupleDomainFilterOrderChecker.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.orc; + +import java.util.ArrayList; +import java.util.List; + +import static java.util.Objects.requireNonNull; +import static org.testng.Assert.assertEquals; + +public class TupleDomainFilterOrderChecker +{ + private final List expectedOrder; + private final List actualOrder = new ArrayList<>(); + + public TupleDomainFilterOrderChecker(List expectedOrder) + { + this.expectedOrder = requireNonNull(expectedOrder, "expectedOrder is null"); + } + + public void call(int column) + { + if (!actualOrder.contains(column)) { + actualOrder.add(column); + } + } + + public void assertOrder() + { + assertEquals(actualOrder, expectedOrder, "Actual order " + actualOrder + " doesn't match desired order " + expectedOrder); + } +} diff --git a/presto-orc/src/test/java/com/facebook/presto/orc/stream/TestBooleanStream.java b/presto-orc/src/test/java/com/facebook/presto/orc/stream/TestBooleanStream.java index d13ae1e180e08..a604555e62112 100644 --- a/presto-orc/src/test/java/com/facebook/presto/orc/stream/TestBooleanStream.java +++ b/presto-orc/src/test/java/com/facebook/presto/orc/stream/TestBooleanStream.java @@ -58,6 +58,54 @@ public void test() testWriteValue(groups); } + @Test + public void testGetSetBits() + throws IOException + { + BooleanOutputStream outputStream = createValueOutputStream(); + for (int i = 0; i < 100; i++) { + outputStream.writeBoolean(true); + outputStream.writeBoolean(false); + } + outputStream.close(); + + BooleanInputStream valueStream = createValueStream(outputStream); + // 0 left + assertAlternatingValues(valueStream, 7, true); + // 1 left + assertAlternatingValues(valueStream, 7, false); + // 2 left + assertAlternatingValues(valueStream, 7, true); + // 3 left + assertAlternatingValues(valueStream, 7, false); + // 4 left + assertAlternatingValues(valueStream, 7, true); + // 5 left + assertAlternatingValues(valueStream, 7, false); + // 6 left + assertAlternatingValues(valueStream, 7, true); + // 7 left + assertAlternatingValues(valueStream, 7, false); + // 0 left + assertAlternatingValues(valueStream, 15, true); + // 1 left + assertAlternatingValues(valueStream, 10, false); + } + + private void assertAlternatingValues(BooleanInputStream valueStream, int batchSize, boolean expectedFirstValue) + throws IOException + { + boolean[] data = new boolean[batchSize]; + int setBits = valueStream.getSetBits(batchSize, data); + assertEquals(setBits, (int) (expectedFirstValue ? Math.ceil(batchSize / 2.0) : Math.floor(batchSize / 2.0))); + + boolean expectedValue = expectedFirstValue; + for (int i = 0; i < batchSize; i++) { + assertEquals(data[i], expectedValue); + expectedValue = !expectedValue; + } + } + @Test public void testWriteMultiple() throws IOException @@ -89,15 +137,7 @@ public void testWriteMultiple() outputStream.close(); - DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1000); - StreamDataOutput streamDataOutput = outputStream.getStreamDataOutput(33); - streamDataOutput.writeData(sliceOutput); - Stream stream = streamDataOutput.getStream(); - assertEquals(stream.getStreamKind(), StreamKind.DATA); - assertEquals(stream.getColumn(), 33); - assertEquals(stream.getLength(), sliceOutput.size()); - - BooleanInputStream valueStream = createValueStream(sliceOutput.slice()); + BooleanInputStream valueStream = createValueStream(outputStream); for (int index = 0; index < expectedValues.size(); index++) { boolean expectedValue = expectedValues.getBoolean(index); boolean actualValue = readValue(valueStream); @@ -132,4 +172,18 @@ protected Boolean readValue(BooleanInputStream valueStream) { return valueStream.nextBit(); } + + private BooleanInputStream createValueStream(BooleanOutputStream outputStream) + throws OrcCorruptionException + { + DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1000); + StreamDataOutput streamDataOutput = outputStream.getStreamDataOutput(33); + streamDataOutput.writeData(sliceOutput); + Stream stream = streamDataOutput.getStream(); + assertEquals(stream.getStreamKind(), StreamKind.DATA); + assertEquals(stream.getColumn(), 33); + assertEquals(stream.getLength(), sliceOutput.size()); + + return createValueStream(sliceOutput.slice()); + } } diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_all_empty_maps.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_all_empty_maps.dwrf index 68dca46a1db7da5889f8b38bf74284447dd139d7..c8a34b621e14f027d0ec9031fe6fbbfaae7679a3 100755 GIT binary patch literal 1026 zcmd^8F-yZh6n=NPw%2QFZnME4l1Xa5FmI%Eq&!DPuQcu z-3F);lUOkPhF3u7xI)yBvjPrdYQ}{`oWwx1O+A_VwjUS2oTXW(i}t# zp_mE(ZeRZRd!G`(?c35K-8eOfikqwPwMSxYTzH1}Z?BKnPhNJ$QV`R5Y$ezwsp`0msj@C;kb64K_)YVt OtW*qBi)+?BHQoW%`L*T% literal 770 zcmeYda`tCn;NlkIjCMlcH`#sp_ED=~dc zCkTOq1I1GR0}pIO3kwm1^9n1muv4*73lT)$j$RNC&%(yS*m*56hj~fLZHm~8RHJoLIl(NUSQB!-jQH|afwV6-w zf#r4^nW~lej;z)QZ8Ut4VY`R-1S{`h<@K`JDgCbg+w>$?-j%G_zP0~6;Bo}w9jntXi*yON{1=-*iC{vV4G|rQ$)WL;$CwG2 zp=i!C@JK}$Jt9UjR8et+h|%N-f$NTbuk$Yq5n^zp*nxZmj>IUEKBCY>D#rizZ?*ql z>dI3Bw|rSu_VHy!MBVs0)s{b!yY?vGxcmIovrljCzj#Y6R3~&`1*wV}bQT7cOnN?4 z&FX>`LRN(y9S6#0Y`%ZCzS!>$I{kX@%(;$L({aC)ASI2qI)s0+o+y5R|tDV7$Sd*ueew}{H NOG{xaYtpUB@Egce$ol{Q literal 959 zcmeYdat>u+;NlVD;+WuPz~C^!&jT;77^ql~i-iFQgg7}E3>YCgKte(R7(!}t$U>zI z4E1EfY=*T!iv>8?1Q;ZA6&MW|EdB!lKNk;Bgi*okmJRWoFRze3^5dEu#@Hty2Zjk z2PDFNDu>`fMg|UMflg*A86cGjOa_q}8(uLQ0I|iFBVSw2U;OmO-GKop+r%WGF2N3z z21fZMPK#FyC!Uz}z3;#YMh6BZ9u6KMMn);tvc#OyR3$bJZXqTnW+f&LFxC?1;?FNi z)+;Z{EJ-cW%S+5n6=HD-4{`%a2q8(7r4|)u=I0498XGiyVYK4q0FuDO1tdMLFj{eg zvkizdjnN92WQCX{7=W@29t@~FP|gCX3gTeMJjD2t@eor43&&$d0Tzjd1_1?T4Q7Kw GOacHC3!ZQQ diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_binary.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_binary.dwrf index 0efa10fc39bcd638002944083630c8ce1cbc05b5..7b8b203ecd6b5b6390c50f84f964215a799ca30d 100755 GIT binary patch literal 14219 zcmeHOe^69a6uxiYF8f$vSst+W0PNSUJ|={T;qB-V&%boQr@1>}wUHfwl0Z zIp==gxpxn85v6S)RJ7arp0|%%8(rbcALnB#w1VFK{w^1K_ie>J zZq2Q3q=+zeKf6eUI-z%f$Q4;1qz7?8C`U@1(21!7LKtkAXb9Tknt{*(A@UA@_DXYm zqnN6+wikKDA#`4h!lHWj4}kuAWFrxJ5 zOd_9~9w*Gk@9LNJDrA&SOlLivxCiT%%cw?cyNUJo%X+Y$wZF{DdJn>ltS2VS#;?bz z8^GDz{QVK1)qXbB;q3qK#Gggg>G5q+{YHp?|HrCM_fL=bx9e|<|N9XC$q@f$)jx|< zH~Wr%yZ$!+bMtfd=i<+nm2>Owu|9MB1G_G3{$aifaPa{>x z8)oJ4T?mnzSVGDX3OzFR(ZbJOSdNffh#?FvR1|mF9ove~8iYA`#Ud=;GQ~pi8HA!W zl!S*g#|6k`x?+a>W&Q|aL}^rsax|Pzn1H-3dIcd#WF&^(=)k1T!ay*feuCSN!N0Py#PLBaq7*ul9NcmvyE zf^#uY1U4{+iB85KVPJW0NaFnYjBiI?5S$Lti z2_Z@YW&n;t0wn00tlu(`^#oWLOW7jpK0xr9fQW*qw1HrOrR*#LqD4K)5@xE5FuLOc z{vumcX$JySiwOt?EhgBv1cU?=5Fp@e61Kf1FciqZ9V{!C9SDd^CLmarE_)E5;S4On zvUc?bAz=W5W#O;`!9yA*AXrw8ULYtYAXt|1-XMs9-JxJvzwAH&O)vpL7cln83RWl+ z5Jtvy9&G~%$Z-2tQA?UlXG|8P{C%60S7S&Hi9R}xUaDwMLhS-q&xhUJ`meCyD*e$K z{n78?S+n#Oyv<2p(7Zgo`IC0Zg?U$m)-9P?<)dNY*aVh+d=f|TO@uXN!^c7-+1U>g zfTy*eB{!fiH%ILU+iH!?zeh0GR%?s~)PRGx(;ZyS$ie14w=sEPbI;uiAXv~LfC^Zq z&ID*&A`StRgNuy-qj7T?bsHb*3>PGgkiWjIA!bg^_|@zOpgWZ&*?!?s*1^&8=@l)n?Wp8-YaD8?6!!ud06?jU&iFMskQcmPm zwYsMi57k_X-ISg>RkrEgIsdC)gu6U1OB;JkpZKAA@9OruQpN4B!)}F)S~pDp?n&L$ z6RTtPHWp_owx2BYoN#Ar`ya0Fbw&7G%uR2;zPU^Muw${(eJ#J@@Jh;;Kh@)K$WERo z72uH{uET>fFX$>NtE=bEpPzVoPD)D3!P?r|)Qyd!I>(+4O)6WmJ2-LIx;dwQPM_2w z*B;DUu5Y`0XXx77GK2Pq$oRljbul#t?d5ZMb$huA#`u(zMaVlDG8NH zVYh$R?*7!jDSoz8pWMOK`wXU<53bq?CkAV+7gum&Ex36GWrlR)bM-b{y-nX$uJqs2 zP4zB=rQXQy_hPQz#!}_X)f;j3MqIrSS8rtbF9^CmaP>x9z0v=(-s)M_`*5dG|LN1H zHVhHCq+^<2Ap9{5_q`SrSb~SHyNAhjfB8jMRg|{u(UtC+L0t<9@*>(|M=Vq#N z{RGBWMUV?Igyiw?ARa+jo_Ls#$YWW~;xQ|7vT`$X#$+#gGn3@U%}b7l29nnBPG*ie zYgsmlMap45r8{Aq6;Uv+>Rg=dwtl~t>Q1bZBsN`w@-!u%fDfqs=Ur7u>DBAAL$OL9 x<1m(rwg*L?Fkt>!p{g9C3TLWd=wtyh3@-?Nl}~hH6231STnFNzxV#`(@(+?+>FWRh literal 25941 zcmeHP3se->8NRc-f#J2R3&9l>7cc@Eby;2unr2X|s8P{U62(-A2aixe14PA~$Tpfp zjjb3BYNJi?a8T5sQZ**1Nl{EpNQ;QZ8hoX$s69qYh_x!F_s-158JIg9VP;3N%w-n# z|7X7c`(KxVow<{zj!Ff<-AnI57ELk%>Q1psBIoU@^}M&V_EC>N1o8X2y8-*A_a@6- zO(0%PmiD9Swa{Fky9Zd|_7VW~TJU55)Ng0>+xk%rcKuqP(Eyb71A}7QoiAWO0Dv~o z01fnQ2S5|x8Q`CfH_)J^w!3mY*zU?RP_L!R?Rwh4NiE|6Yk)mpz<{GdB%cTT7(PsL z_sN0m9>|{L?r(tKC=aWs`!t&DIq3c&lHCK@JGuLNAbXj1e*>~7xhF*Y5DQ!v@ekoqHPM5M{4{YZH0)uIf zi6kJBxFZRO^6or-z0;+@+TBq;u=cX=k~?+~N!*bHL=tx-0a4x^d$CF4j`AROcYa}k z?W#Ki0SMbStTpY9??eD}5pe|g;kP$;+og9IsfhqTa-__$kxIKRt+w#@$^~c-BKQ+YOB_Ws*P>b@jrua2Eb$`fKl@^dUN?nRU5quP&zR^ zFp`9rUInZjoUai*p8ymoh7W^7iRmG|hv5Tck5RVV1dj#x5{*>7H z(7b4W*t-+QpX{JMVS4!Z=T*!vuV0ZvJizIqxTlH;t;zEU`f4%8?q{xg<+ zF(`V&hIht=j|p$qYepJ(1?h%tyRg8J^7rqGFMaj>d!IKRE_lx;wl?l!UdZKnyZZ-C ztvU%$D7&yLqDvg+=Ca=O>Vwy zk@i8gCaGrN(d*l@M|}WP%hDElt9*5o@uQMk6(xtiQTKx%cY5mG$$B^VacW(9B4{Uw zgGcytKM?gI^XSh*uMkvTA{Y+U0bI3~%%|0LYU(P{DLn&UtER4W?) z(616ye%FH|U|mXh8FI^uZ71+~wb6n59jIlt-U)bKa0J+Q(07j#n~vDI0`M|W(XRx2 zozMQvm9&?0wam}l*UlY}&ayK>X!8LM=b;uJ-vmIc<2Pv4;`j|(jdA=2t;RWiy9R(+ zfeo6CRA7T7^r~boMRwYEzU7e0mnGU7)}8CHWhTMic-4` zZx|qc@Gb{McbfqPk`o! zZX;82hxx@4VDgJ@WKwR2`NbKh?;zlg-w5s63hA1=X&AH2&<(H-^& zmH^_X=r0c#I~$&Wy?@c2Y>Mrue{sg?{>4_OW7@`sZx3dE#&x%YtEVtnaF!{yKK8~3dZTRWW%H&sM0>@4463StI zaRli2DSB}LuMN=OADF3v<|lDE;j0PL1U3-@D8Ja~;-W6G+=eGW$ImfcfNugDo&e1c zo+9o>zc>PP|Kj=MotE$%8-9MHkxZ>@fCop3<>2Zzc>PHesTOAB1ief7y={J%o8#B ziuHXCfAAkv%ig67@czd z<4u#)b^rb=`CRacw+`0qbn((nOwNb~J4SeffJ45w>h2|wdryQ;$=wqA>%fM*x3e~$ z$~CBCiP6Wl5Pp9tzVOG_$5b4-b!kacud0RLUpY8wsm}1!l;*K#-uE4MZp-w=fval_ z2g)1z+*x)%$M<5w{>m8{gPO*D*(-Tb^{n~32hMVtu;AUs-tLX(2T#y0pY7#Z;x}iu z=fIR18%A8Ve!eZ@jE4a}AXDVy0(4;^(6NNYM8bgimAAh!EZ=$f>xDa7w6CUVLq=?_ z%X{s)8>r@Pq$vr%F28tdf(p4`l@N!AAb1s z8`>>vkM{OzEjk&{>bJAtUg+PJKYyXA_*8Pr^sGUHhm5^-#`VFxth`ZY8-q6Q82{#z zb8`}7H?P?=ct^$cx-ZI;p1nb&!bf0wmLvo{OA<26LQ1yQo3snp2rRuL`|3opdm#Jb zLEYbg>`Cqk5kJHNjV61D_zYxNXH51^?*1Ogp5*RtK=vf}goqzvfw_o(fDBVb_s@as z9JUY*V7tr$_>{md~<0r7N0C&-d4%D4s z|0vcw0nZDr0DHzfcC`{I+4cOl?(zR`-8}-3YF%*&;U5IQTl*N!1sa=GVY{yrru#?r zmZf`dA9%23eN_oToeT6glJ)Vr4Wwats%?85Iu1; z(@Yd;9LK+fDL3Yyln@;|lqRvkB>&HZI#X5}Mwrepns&Xt@MhiH#n4d*mgD{`?|r}D z_uluu_t@{2tLK;ge!o-(5H$7O^74-9W8G)jl7UV4XrSAfHB zAbVGahNd5mrCkOK#gHxo61WVA){Qa32gWI=Fq=eiiC0`Aic9W`3&Vy3kXbr~(UCZM zW4GP_)N6XqL^~`xhH&6mhX8d*9VMX-5vfrO_nEs&nL|Y8CaITLl=_JM#wziOz{sna ztguenDnDLe=ZLYHg8`QF)B{XP{WSNKmsi|VXmV@uKIOgK7F?~F{G`*Dbd~+Rt&}I4 z>LZ~tGhdnVFQ1Y+MLo1WDpWupvtI?2)koP_CElj`*mm}EyLWX5j4sV&OcGjPOK&UM zN7~c^yCVelP4E4(4)2D!&Unkt(KN7JM1fn6{|8qbAiej?6qJTD&53`HQx!)s6l_aj zxG?&Zku!L~zkL<4t9lF28CgjxfM$Y%S3>}=T znmSHqs(do2PPPOcZBWXu>(L_toc~x~qbe?J!UGS~ySpb_8!Eu>D4|@M) z^}@^bDRq6XZ)ExyG&VJSC4IUXeWm%}^jn5c0IxP5C4344vxWMsn!b{~=uTC9C4Jn3 zs`yItM|Y~~E92W}wfQUIqx)6$mGIGARefcAF)t{`rx#zA7;r0s!sZIaEaipwI2ZUf zz~^{Ih-nAhkhr7aWoXbFL+^S#b-RgPJL?YasylcHpS!Yu!;kh!6VK0{bbfsUdv4ri zgP8KXU@|E7(V)C$$f3wbpOw>RU@3ul8#5)iooL}|sfEQ4wGgM31#&FWTP?(rxL%Hi z$>A~3qILh4uK58Yf}qRTVZHZsUPn~lOgvfk&0F15{TnyW?c_Yfj=J`~?MI$8>tgyIK9~E);l?k?tydCUV6z>cdwf=C z@**d9W6f4`K50xGxGX5JD2187|MCJVzIIM_T*iqdFAPh|>5(0`_Px6&uU##R*?jAP zK7!VU(Ap5bo{!df5nE*ck7iTzN0(2uwI`)+ts}(N;`P7xZ}+a&OjbGU_vKCHV?IeQ zKprEKGkLLBF5OW6-fs6th;3&tw|iH2!06IU?g*!~+A3?SeE&;ozD~LFSKK6ZS)T7D zb-Atd=v%W5p&j3}!Q-3T1kgHmd3+>~Z!SKf%ETAx`sCjtteD3n`58m$h61Jcw>4d zM-3mBo`M2SB6v5`o-=Lc42~FRMZN?REwX|Qd4)gw`7SqAcNJE}CWO;VYxSHF8(T>c zn~L{}it8G>2$0Z86j=0(ZgD+{f5Z!+rZ1`^SW}bjPGKktmem-9V!~E1T0nwyJ-B#x W&+!!PAy_H`k4$-8ADz|Rll=#FHgISF literal 16503 zcmeHN3s6*57(Vyz>#{;z7R(I^)^#wX!o^3>rnM>BwJ9G+BaCKP;$WjI0-8qV8k4|a z6F#$imEt5!52GY3*XZS=F(b?pC9>e!%QiG8oFSugFAFZ*J%f21h5)XIpS?Z*0Ms9$XMp?9z%pgJShb!m(?NS> zI%w`zeE=v5cpRRvl!D@NQCu#F%R9syLX!ZXGgZ>j??Uy1J=6ya<5hYJbPSKUrkyqf z3&!12(Y7G%RNj58oVum7c$>GI7Fs%Qk+n|b64qo^FJ;CHubn>a&MSAx{-{DTqul@~pqA?>Ubg6D~l*=6J{)J97kEGDMuFc}g-N$uZ%KCK- zUFJL@7wb}wlG82qC{bEuRxf3ST&&BfTS^PjJPHC}75q;F|C56dh^K8;TWVX|m1~F1 zJso;K*Q3|tus6y)hIt%%Xt)3lf&Uy{-AqoxbYA^mLG@P{UPnqA@QFY^u!N~#Sf>4wGZ{<`ElU&Lw?k_ z`9t$~emvvbOZ;?X4~id1Z%_EK_~7(JelYs6_#u5=^&h6&lb_RmsR!B=>yTGh8-QgL z{L4QdJZk2IdAEHy`Rp-!?Xrrk?`^yA{fa-SM34Ytz;Skr_Iw&=eC*;wF{3V|C*_X* zu9~~qd@xm0Txv+2m=~H}+Yne%<4iQhZ(ek^(s8oko6_R^-61b*-S^kzLHWOy7N4-s zPER)NAI;4hs3~5dGCvx$`tB2vnUNER%^kZwAnxt;yRv4ho8~>RXS}9x;MXft>PHmi zX_oAaaD{w&b$V68g+r$LysW8=tDxEmzSk`c-r4%WD(>ahq61&WwQS-4Ew7)ovGvL9 zsgqkaG|idhiaxsXa#cY@%hbk(wEBd#dBYqetWn9p`zqT@C^rCei_vPXavW z7LVsbK}?SD;3o%;$l)O8-KDu5kA-skD9weM(PNQP5tZ<;$}1|oRK$$bjGvJYY~VHb zM=H$MD_8jW`hXd!x!-KyHRIXn<5)Z!{qcUk*+9yW3!$80%Y!8)eJjS?y5AK~6Z9E2mfDi886m2{PfZM+; zq*$>g_!nhdz$e^ZDA{0spa6*Zpok3V69tz3ighcqt6 zpR)8(JlcPve-$MUa>ru;2>c?`J5bTHfivl3>hs4 zryF{BeMXwLhPF?tJC~lR<`(|7)LNHr*}8CF-8&Tb#n#Q*qS9BV#oKM>+YLuf*$!{} zWch5zGtF6M=jYQy!fq;F9=>*O3jO7{gY_de=k3YfInA~1*wrBi5=vXm*^7^kyI2(! z(K5dA-a=PYu0XXE6=*G&Y4v8QhqC!6|8aX{JEj5RhtvF z9li)E*c<=GYufdZ)uFNQJ&*~QH0*o20Y|>LqY8D zQ)enG_TTyDU~b3)Ri(3KN2H;N{28I0F4bfRmoly!0eBd2be^GIg(QfA1-x}3VD zv=Ggs8x?mqD(<}2^7XMo$%pzSq-Oc5s(h&R>JKWRA`4xAC`-*`ziavWIA1)I@_Apr z*zf_fPZf7Jo>e$b08CFpr~-ovbhVFW|+0i z00T{{%rql35Iaa8yg2)bxTmrg!+e)TzP)~9IC^7yWe#Tr8ovLP@qgl1l;b2Xw7?q_Wn*9$xJkBow diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_boolean_with_null.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_boolean_with_null.dwrf index aa9e2ce1bc1fed6b93bc3f14d4f0f512ec7f2f33..543e25548d0ad3cfbed40f04676c477a2b68641b 100755 GIT binary patch literal 24379 zcmeHP3s6*57(Vyz-QBwjEX$HX2D%~AiiRr{J}PHbq_t2dZL*gHDNSOeBGzP0-o&h= z2Ya!i#w_1fX7tb$BT8qK9PPoBOjcN0Mvy>7vpv+?xmO-{&$+PJ#5p^&S3%s_bMN{2 zzyI<5|GE25pE2nH0IX4t2)=BZ8+a-YMR@!2vd#h*KldKv1p$}0m0Q%8_xMt|{F5?x z`2Yr3oz_U8`s1+k&8Ae&$~K$ajOR5oyyGR-DDZ^kPPk|Z3qYYEaLKFkk|e7Ql*eWQ z@CZ|20PM}Cz%|Nk2E#QkC8hwdoV~*cxJ8VWNQ5hhv|1}*`e zvN%y$)GHKGF2O`mFX>T|!p?1=B-(2EMN!KyJ*kC*zjBkZq)A@Vqgv40plV@twa6Bw zt3`35zF5nA*>vYan_CD<_7mTC_1z zv4xKO+r<_boGmE%Hwar$ZljL>+<#yT%1u;k;hZcO73iHU^e%~Py3~0~aBD$HiHgq2 zN?{>yL9>h&KLq;0q8@1v^)01XPOt3_9f9ObAx(fO2$VH8_* zwFH!kGF6Z>QD~~=ci8Pouo_?|#W4W3d>lLv?2h=d8YArIe7K8+WpARD13TiZy}f-Z zt;e@7hF5ROvN~e-8C) z03HEGItURk0%I?vj&`UK$c7p``H5uf4g@0@={!L(q8(}kvY`eyNs(-2h+qUGU2u%Z zvxGo3Q7@oHJVo`V*&rBkuz(HfN3_#{KsFuVVMD~JUVnyP6e9*WgJRU5^7O?X1+wV? z4;vy*Bl^>95R70t3hqaJ(PMwAQ6QTRaFZ0tGFt>An2z}1euOif{)7ZWL?9b#aFZ0t zDgeXC3Hzx*xHnk|F^J9zc=RC~d_=Nsex%I+V1=a7W{813Z)uq{O_KhUi`YP#MOI*$ zu!LKIgn^{l_3j3#O$Mn3H*TV~9c~p83X+G{I+{R0(&;zZoQQ(t)I6L}fh6};ak4{} z>rm(4KC&qg@2kq;3_FPJM(MP26iXHqj5&rm1BadjLBiQxH%bb1z|=-KOHn^0A$(wt zaB#kd-YSX`<-A1wn1t||If_ybT_FS|vZ)g2z-!yE7;+VH7WPy?>s+I9nIGg@L!OH%Wj%+PJkRok1Z_Q3D$^PAv*i)PKB9dxNf5g5+rhx+juij}2%t|5;KsfIs8qs~9gw;O z5kBWY{Lgv*{)YrmoF0&E=2Cy!lkkK82!QYbr1K4+DgfmWO1XUYQ-7+$M>dIIAyoFk z^9-Q~gk*C7PHd?^yyPQWxMC2>UihCOJbm17?xR%?@z+Fyj9FR3%b2Mk;;!1H(h&!j zuDoIHN4f3!c}ET;OsN&8x-7RG6>n`>oo;Du820OG)&nMhQQ);fL%u4W{MG8jsw+Ji zJ1S;QoS11YSz(R-XrAfdh?wzPuAcDp{_NbCZM&`B&9xh5UhkW9;EI7y?VlL4&EiR& zygjXH!57O%?L4xyb?B76_V3uCLW#5Sfk6ir-S})?Mf?fr#F)+V#N=VC)^Bmwvg(-PAE`Kx}&G4A%)j2_!E`DnFy z&wGF6>}>g?IwGSWedMd}P96HJsUYL$;xmVfTeH8pYKlL_?!WHy>c{t0`@U}|dA}Zf zJ3PWa@zZMK-s)!-k6gFAKCh+FmM|ZhJTb*!VQdDlyWF|9lwIcic@-t2 zZ2J*%|wF#d`MEUD2nwrQg5k9DLN$zW2%rmAbN@A;$Ct&uz(kHtH?(KY@#M_Cb7AH07aeYYZf(KkhXzT5TW8kY=Q>sC zzSiYS%PvjKs~xavQ`AX*OX7tm8@#1!e&eD8#xUtqob)M9nfeqb%}HCjT6$EAdM=Ol zhwafR9jeg`@?5t6a{!rIn9!M>6graV0EPazBD&TRd@c_qch)vteToy+>65{&MITqB zIjdAxi>{V{@o*hO*|ejaZY{K}B{b*qP`4IcE&3EEcjpYK-ljO|-No?hE;20O4fc=A znSoCDvne^gvB24Ga4m0V+?&@VkEvf#`t?uEb(Ld1jRi@ue58YAxPe@8{^H_+M)}|L zBoiOuU|B;F%gf(D{+SwQjb2bRC++^CdByofY4_$8<~z8|Su?WW8xGYQ_vIHYp10s$ z2Xna_?l;cH%ew*(+}B(8O!w=xhZ4N4Pch@-_?6!n9TqrwY<&FMjc>_IE7~sff^h@* zmtQxg8m)#;zhvQjP`JulsjP?h>llOfcX$qlZXIlhFo QbMf$i+G%hZ+zl@KzsfmR6#xJL literal 29852 zcmeHQ3sh9c8J@el!19u1F=!NU0Zj>~>e7&-l8UP+LXL{HIn^|2gAY^?6%maVA6Knf z2^D;ls5E93nuyvKO|)oKASZ3Au}XYZJVepDYSkiI6Fi6Nc4ijVy>Mqn4tsZw8}wXQ z=Kk|PzW*_gdl^zDB)kQHkH6kmTR6@LOpTkolNdjCl>RK_Z}9O04*&Ez?QPF!jgRWK zr%^DKc|nokKK@{x_Y_F9KmcI23XVOAIb9Ls#hyr)QWPbRa%HPcox z_)ak@MNV@YyPwSp17Jg$-y{wjDsq@T+7U(on-vBC*JXu0$*j5}?~-L!c>J$GqaW{3 z5)}?5X4N%Yl^y=-@f?G5EQ!36$*eE{U85gLWTi(xT$dI0Br_Y`ekO-emW;O?YNV*} zW)0IMv%*tJTrqf;MYo@0l%vs)nbHbRbliUD^CF&OaGuJE3U53z#~ri60N{$jtXm3~ zEm7g=Ds#up%v?Az9YysM9{m)SVWnyS?{Y6$W`)Op9Lt?^6ca>d+5djbU_FbDgiS8@?Y;q`F2wg_8-_W!C7Va6VbxeoyXJ*Pa}VW2N`xY%@8|C4>70 zx6;o_aK-4{0NVM5o7=xBU_+U6VRri!Hm(>7uY8z3bo-spt`#=g&*zW@nn{dG`5+I^_rQ+QG+6r{Y@}HsQR<;hYHgAL-CkDNq^`b6RkfqPs)FU+LruL+M|A;=w~A$3x&;~;DRdP z$Bm|g#;OwEkdF(GkL){S^Y}HZ$CPGfM=Y5iHXN9M5ll=-U0)EDo0lFdF4;FEW?Pjw zJ*Ruh^wFQ|cFyZDZEyIA=)j^0Dc%(?8a?VXhIi%%_KGR0PBrR26^n)1h@4o}*q|{d zN`iOfh3eB|8Z)#f=Ovcj-CFa*^<$1LIx=nDs>t=@eEO|EF~yLb?0;kMd)vKVcra>E zzZ09zMZ7UDEBJHM%kSu$e6Gb0sf%3GT#_~S;?m(2gK|>*8g|q#Thy@n@bXhjzj>`R z{l5$1YIhooGxyaBTbtJ%Iz6Cii{;7gy0MC!!7)#!kL`Bm%*A{E%-;LQ7w6QJR?od~z45`?ki?^j z_m6t@X*ojSjJFY(P8AACpnc0QNE6!TXaaxjEOau#EO;`({3IYxk=c57K@N#xHlmmf zc@Qb=JUSr-^T>QADRe!kJUV~}-4?18Hey&0bYcWi3g&?*=+;sxp3z#WEq31$^JCkcu-MiOJO6{v5X1_;J$RTz7aJ^SB1Dqx`atZRMHtarjkCx!cEd= zxb?WBZLR5)6N_TN&B;qC+aHLBa%uAyDJ)2$< z{#8gX86{-!k^-HDI!f7Ny_iUpVHeAk@Ztq6V0c{?ieRK4T)A<*EEEN> zesBW>&R@A*7K(7BA6k6!yet&qNI$gr<#}1i!U56`u5Gzq7K#Eb{)G{MmS}t~Y@SLJrD8g<2r4(>dVRvFX^e;&PjGvRkof7Tx z0|gp?37xQhq~LT_yM-bM`IneIlzz_a1M`EBzFLeQ-|Rr+FEKq^^OrKiCJ)}uS{9)E zB`JXOmpD3;Ms4S|TgcM`j2|((Nm=jYb_-bnTmHiQCM^J5(Qcs#Y4I7? z3CvpZS7u`9+z=y27?cC^m!v?ef5{^>WpBw@vs&a|k^(mWlCs-1?a;p@LEwL?*5{;u z@C7ejFu~`f=Ky?2T6Ndt+yB$3ZlhIKHuM|yqwHd1a>a<(ujbdS`?&is5Dx}|ppagx zjRCtOCchfL{i~O!M}|&IOpPCX_3b$Z-|#7W!V>()k6Ty%M)vD{X9kHmd#h)BG0AgP zwL$d0l)tfh#!O9N?FNsLd7FYKuME{E$NV|(*XLs+Yb@Pg$?Dl(%xTQJwlQsG!XCrU zA%9xAD$4#@;qX91)3UlCq3QIz zVX+OXZ(WXQ@cm{`-~IoNE1ouf%b5M|HJ4RX>YIYE9gC~`>6+%Utbw&lM^?OldT4CL za!r$P^Kx19jqEZEIxPe!o<>_ z?CCk|RKd-X+!3nOW#KuQeJ&ULWomHn@wweoayxo1d`Ics@-Rnvqul&_Plb<{<%8aH zX_S2U@i`z>;q!SN4)Vs59R4bjed?L_k0J`jaUX~z*b4r;w zmk$Yt@WWO)gkRS=^3p%_;8firEBAq^Tw&S z3TV~3uZ{@44Dfq-@1FOj8cNks<+lZ6&1by_f7|fwjr{y$cV?LGP3;w-{o6`GcU{}l zJixF*9R+{k1pd0o;Nq&w+Zyu(w9*Wle9|5%0%>V!Z diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_byte.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_byte.dwrf index aae7f616061aa1e2043a7de734043f7a5eacc732..24b54c831100b53c54720eec86b294b40db4a528 100755 GIT binary patch literal 13180 zcmeI3e^30yb3?Ytdo{qfH#8YA~I#YAn+_g+#4=pFP04eR<#P z_*bWMJp4M|?|z>T&+hx~vd{8D<EuIR5GM^oC`%Jq1ue0(IDx9NYWm$y%vH-G4kYF&s2lB@$GM353!~uYj5dauk0IKDWq|Qcn~1@Qo#er2 z0RssB4FVoRv;)fkRWiYnO;25hs$4g#rc;90LK5A=-gufbp~LFhe|DA4Wc2A5vg_P!tub=R*tV(Yh2^)g|YJXj16+|1@+ z80$th&L(%RL?s!xvguzpK=Efo!P_Y&kH;8uXf|O<2m=&# zyFkIiFnO>TAPD7&FbZ)YIr@nR0~B8y3Lb{ZgT(+rZN3O&3?4%^o)lq#qS;XJFiajS z1_*kd7GaFTnax@`3Rg_=t5pwLTyM6cWHj>F%yt<5sMf05;~qa$f9(eO9pxWgmp z*1M*;R1dx5LQn2se)c8m-JTynsWhu>cPcJ-B369}6&xA=uxe-Sa)f+w5|ZdU+EQ^f zL7LV-RF!xRq2iz)%P4rIpDJ6cgl<~Ud(BjQ0z1QjF0}Puv^(dy(Mze?Ga<)0 z^oYJqab0gf!Rq5mAbzrG2l!ZJ!|ZWzR3#@y*wsmU)4FO z@;pYum6l9i6EvI^(qc@}2w6qwC*ZIqfIMOM7Q@|7UVc^y|KN^kIO%r^!Mu@FF5(IqWOi)u3Y!sq%A9_o_JWXpt7_}Lmu>Zn7M|T+^kK)u34fcpea^xii?k%Y z;n=sW_tsyo=s0~N&vkJ250huF_^Qq>$#%Q);^eDGo(rwHvE;&nw%pTS&HUrKX&cx4 zYE$~Ao{!plpKe_|d&m5r^}WC1)v2r2`pdne>TsaGgXBqg^QsSLT((6uwPP*u{)Tm| zB_5|lH?)U%G$7Uz4?iLiYl(;J5NnCYTH-@IAP{Sb8xM5#9~-fjIITvkC5~J}%#>J5 zJo4CxwZtQ<5o?J@RwLFDkE}+lB_3IgSW7&z8nKpmWHtVeTH>)AYX6%z)NC?RYOP6Ih;B!%q zQyHepvMoner2Y?EVVWyxam|8pZ`LeYR$Vjhjrp%vtM-bymDA{e8Zq$G>YAmC7QdlN z6MQsbNrD1tk)I|G+%Fk?xv_azVEg2fG-X|vLye~oT9lFZ8V`Yx{vd!#$j}2fQ xu>Zl2@3z3B(zTs&fkw$+H!DDsR60;g?)m(e?aEr2{>Mcyh`qJ~o9}!+{x9lUplkpD literal 18451 zcmeHPYfuzd7QWN-YKGTk2BAS1$C2Rp9-kmSGKnut)R|bZjB(3VSB+Xx5sCW)-OL6O zQE^2@Bff$O?jnf^Nz}!ZQM5E_7Nce@pZIoCm@>pdBkrQ^-qXVX?ex92RmAx-oa*aL z_c!O9d(OGH@9XFsJ7gk4hHzsjUoggw99K`P)1UJ7S@HiO{M#Ay$oprE=C|vOJa5N# zq5KL*L~hp;i53HomWE74D55Pvl=%=>MMRkovBoqMp@bB99)5B}F7Y^CIJ8eid z%1Q8X0YsUIXwsxvjtq~(q(0{6`j|jlJKt#y;@cT0AiH(+I7z`MGE^cP_XmX@H{N#v1-5GZlzWJIJY=F2aB4~D#@{kXLFdQa481FN_ zOiuuS+W?>ifrNv6Wzuk(@O>#O2mq10ofQO99`bUMaFTFn3Cn1Js1Xq~qe*$l8!ZAy z2vsu}Mgzb;0MLvk;ow&pF`|vwDBR0oMH(QkiwIhzNqP9m5hHCN9w==3C(Aj2I42@# zMw9Z8mt(^=p{9i896%Io0|ZTul!v^W!FaGxdze)pKz#NcE7+twxa$^m;phnkLg_-gr?``kEJ@MGWevCQkuhW{CZ$`-OMiz~# ziF@7le{)Yf=a7Sc%5%(mjx0QrKM~)opNxFfQ{}ta@*Pi+&$Hzd^(@T#Ie3osocx?= zKjIz->-R)`rhVLv?vL_DhoAp6=xO%BO#MmqnDPmKwtd`e{=_~SR{e>1wtZU0w@m!_ zS`W=W#Jo?_pQ#_Io@gHif2RJ#d{X}VIB(-m*X=N%;im^!i=$key*K$=`Dugw=N)fq z-&}pE@ba{jZ7JtRUmJa4bGNPKKj!_qw`y4SDwJT9xASlBQ-(>bz! z&CN-}YhO6J_+G`5gnH+V%M+?o^0Sk5?j@x?OQYVNS)(d9=~S33;wR-T;_^2Y_3yo4 z#!ppMMMb@PztzwFzWL*od)8e1b<`Hy&|Gf0%6z}-_(z$ayxyME@RnL#--XkDS^Fx! zf`_in4Vye?!w1OU_9qkRY$4i z0CULVBnAN10w35Z0MHQ@C9j*jI{G;i0I(!-(?9`$8DImZ2B!8jC1L3%FPQ=W5wK}- zv<*W9OkvljQaFGPu;N|Ng8_imy*vnWnv$@Xd8Z=<07}Ac)#tGUAQw=e1MKjego!v! z1ngy;>%;)Srb=ly3IKF~U5dguh7PcjU94wtfMvn|#xpp;7Da`Y!U3uY`-hdV$w8{A zMQFYNg_;x@5l}?J*H|bxC^T>{96rTC5d%d#xEBVWv{1xD(FO_?DKbLgQ#2I7K0^nE z0lv0{0z+Yf!c2;c5cp(-A`!moq2S?b0u)gLR7f1x!PqWPmxnGGfqpQq4CLiu2nYSh zpgW+KhoJ!IM}~a?y*vyB8vToji34_le`&%&KbV&U@*?enrhwQ!0@!rLmP`Cg>pMjK zU|tr;%fk@vjXxNf2ks*AM-?u{UtrCr^3eS!`Zu7)BewG}6af2CTMw;Oweo!rT>;U4 z)Ye05RjqvA!%(0x{upn--~ZJN@kddhv45c(_<3~Pq<^6);Ag!y&_7dtB+*)zUmq}Qp68Z>I24QqkU# zA=NJpd#yBPrX$8wd&4v7%j%A6vvV`w9aS5CY;pTbrd8QPYL^@tlvN$NqQO1?e=+qd zYA+qoENdvLID+fXUc20@di?v@%Q9X(|81X&4^J*ms#wC+pIGqg>BTvF-rCf#P3wNY zR8_uJx7u83dKF=0K}d`H^r(KVtn8I><93vljT^VCtZcWn@S^^~$lCg^e|Y80hBLd* z?AkpnX7&9qZVmY8H@!QTUxnKn$8>SK`KY)Kw?_ZfaUi4QZ0VMJ&zg(!ta$ljYQfx< zrsMI!+!bGL(i3S5!Q2(OEGbFm3CRlcC4DtzNdY^xD1K~6JyDjkzOJxt*&xRyQ+zu< zwy#3B}tBwyfa-7=A z@hZ4F#~LE%W&WxApEGA=P0#E;cgmdUM$ORiV~3Lm#ug72PS0F0Yu;R=s<-{Ud#bc> zo|I%doL>=|?w&09y8n-9vHSx4=3;S(vX!u-?oEUbyu|;iD+xOe>{{ePt+_eAz diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_byte_with_null.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_byte_with_null.dwrf index ab0cd8c17556fe541961bcfaba95ab2f8f4e1687..f1567711984b64b10403c51408c87ccea718b3ac 100755 GIT binary patch literal 26323 zcmeHQ3se->8NRbGhGl`vvdFk1AfT?IuvM#+8r-JFkobtw9&KtPXsV5gCJka^Ym%8# zQBsYCRU5UoP$Nf8)o7wQM3ZRcq$N>%to79#Dw2my}N00Itp8Q*JB$YX}x0bYupg-{Dkz)>v6 zY{i&Ndn{O$I2M4PGEbU;d$b%x9Q8w>U-umVK)I>d3Ov1?ke%11v9dxi5HZRKWt4~+ zMX@Gm;!ef}DP_ti$w7oumYkCu)!924sRVX@XL}zps6861%b1k~TDYBk)LE4{6WNPF z&=w4$u~KoB5gIDDGXaIFvpus0D-tUkv~lHw)L9iHR-qsot5B3ytZ;iPts1OI2U*B+ zXGf~D5@UryM=^-TO2ri=R-`?f1}oA*JG65<#;UUtV-1(Z2RTsfeFt19Y-6$ag$(jBW% zR}@wlw8XB#iiAB5#&I7NtFsan6hX9DskoA{!V2O!h+SQX>5i2XI=Q8byJ01QG+Bu# z+8o5JFlcRj2dkkdi)MyqQ5TC8WLXzB0%C1Jy`Sgjaug9#wZH z3s-Px;U=4)1~C+QmhA<5ac}&RLaZ1Olc8wDWH4I9@R%n}IyH!qh;fkPYBy1ci4nu7 z)>ko#Moi5aC1TiFrAvbt>7*OFxih;c#Kee6C>k*}XEI`VBtq>nSd|-XceJ4D~YcRn8CeuK^#34ViHP|m;@ueq!7cX(<$8%ONFW2&yP@~ z6eA{~Xi}11N-(WiNl?8tq$C+C3ubW_u2Uj514&5+4@T|0A<|coQq7qxVjPkZU%0We zMf@zI038g8e1k;idqbk!ih>=cupogofrVtY&hEZn;ibhP$86{%a zS!IR>F)}O$!@=Cj8VWISj7caOF*Rp0V!>I7oz-V*5F?!&0*7$*+bG1uh+$OKc8sDC zQ*$OG793;P**pC;h>=d_z#MMxUJ5ZWViJl*OwE~$STJJP*@Xcb#7HN-(950uBZZh4 zF^oFU*lomQ&Sb=b5yPlfj|MT)$zgC9ckTp*m>4k$r5P~^rZp=uYVBYRVkBZC;0Uhr z6ACdgVi?s*P&8s{&Qv0X7mCDLwgxd0v5{~j=f6fFCPqv`(TJ%zlMxG!G3=~ys0K08 z$vl|H?QEwI6C;LEwE>Kx5mR#}BZkKqju>&4t3ix(G9Tu1@2$XlbFIuTp=iX^oXLm< zBZi&b^lA_zog58EbEnr)h=~!ysH5vKibhP$nT%L4V#L`n4PvB|1+aj-xS2vs9AgrS zMoi6_j94&Y*x8m59mM{LPTo@NW^}COOhU(RXcGd^9;#};6`ides5hZQt-f{(*pqD6 zdix(a)VKSfPuGpJqw|(y(P$K5=O0c*z8eP=f+W_z;A2- z6hg8LMaaIqP6K_hkYT$xc1APo7ioWptUa!(T*!0_O~^D}r-8K3kdTH>Kb6@hG#h$} zQjXhdV;6aXOi7el0cA>>d7Z|Xv69C2L%Avz!YpK<+SseKLP?gYq=naMj0G!cY(JE% zWWm_ghB5_J8@ugND#_61q?OlcjFqf+R5G5C2~P;)HkMo<{l*5MP~1wM0Kg}tvsg%A z1yD}PXq)bV4 zmGsf(q=VOKz(JG@^PC8KQZRToDOZvK=`?Z?$V{?#9Nu&T(Nhhz< zfRoJ0FmEWselp-UHUJ7GeN-hAd7TClv69C2<4brl;5YPQ5>P71(z2_I*J;2-=43b9 zEoDBuyXOY?_{jhRd{iZqc%254u##aGGH{Qd44_R2RmonwP6NHjoKzlaD-U9NaL*a` zZIIvC0LZc{QW8Bcpnh7H%r4-8C4kMOwl zTD#kgl??M?-EG0x1&4Cr`oH+-`(O4gawUB%P018qr?sb~U?qDnaq2og5_)<`)tyj& zT{Q0+OSzKhW{Ap-W;`cT^;&ysY9za&Yox*1Rd$_3PYkG@pkpax^jdpNOeDRcY-@0O zl}U;2Pg13f<#k$nY-}XIqIH77`Bf$*xrDS`m-g0mUi+GXOk{XxB1iK&4MdZftX$A5l?e3Qt(LIg#WDCR z2y&54cQljjyiNml{Jh>hzq6?}c4s39V!qzPpl=M3ge{AfMqte zlgdkX*smDXJ(#FeQlLU+oU-pgW`@^k3`3ryR-t0#8GqQHV5l}W07{ur(!K+kO}tKH zO!elDFMvMJu89NVrj8v_Rq+sTi~}`1`&)f~xWC$$a%`pp%tM~Ov1D|L^@nvCLU!^#_klGZ98O$TR`c@WWzRO} zwH^NW%val%|Ne#S`ca-M^V_WDi!0t9IHYacl~+oO+dh2PH7#iZGicc9qQB;rruDm$ z*?8ZlEvY|g8k2IRX6fRnvgVTdWif4zirWtyeAm=caIGmbrS0n30-o!Ge5;fc;^sVGaGk=xS$on3*gBR2l_oagzQMT@TexwZAx)|T-t??2ek z@QZ`Xax0S4V&liy)82E<7_xlcq911sy#8QI!z&xl4@{lccg>+&bvIikX5?=^KiL2G z@%J5hV(kmm2JJc(Z=8^W5r37KIe)iN6A>x# z7b)?FDsGg{G zn8JegDRg(%)J4IG-W8jJG?^xJnCiGvR2m(wud(7Sw+3>#|o^Lz= literal 31742 zcmeHQ4OA6X9-n#d0Ruc>c))uD3O)f5HHpk)Xso>wiOQpCr}Xq8exygu)IhC0Y&j(v z`H7;X=B}^S&)N?v+k>vARwl=6Bem3|B<*?7SjDzgTUOcozmGS=z}!3Ik#Y9W^PV>{ zGr#$N|L?tXXXf4!qlP@jFcDFXNUnT%Hsh%~VlB|vyNYhlMrg5M{7-!>cLS>f#&oo(4 z#B-b@4$R>PZ2?e=HQMN@MvAOvKfa%E<|~42B{gLf$+lQc83v-Ms)=x6oQ1atUq`8{ zS%}nBf=FsAQ6QMAnm9PR@?C|R-2`ur3&aMXR$~;&n<~V}YKHQm!b`n0)RaI}H6;{X zO@S8#?*=V30oC|NKvC6H5>vG;PS5^)f1&I&A%vv#1XLqNkwQp?m?~2oYFT`iuww^- zsT8*|io{ff7@4V?cMEmjNQP@wC!mT*Vk)DkOl25&K{bkSoxt&&P&Yt5iljOLKs5j& zMNxZFWHkry1B4Y9wQMV+sA|eE5RJ?=54dob~5tsn6R7I;& z=xBq%N~X1KJllXkDpehEI;HR_!otr9wUCr1G77UKHJvgHn3B`P&fA6ZQ2`|hO(%e^ zj6@)kD{-1ANRica@=oEs*R<4>QB*Z$7~Qsm_(8&li>t@}cfZDkZGez2ot7|5oo zDUP#S_*;akKWPOmpvwQO8MI1bWTxqSx={abLWLk@3ZN=4Nls9SBCLUum}(T^ERN=* zg>MSgqe!X{BoHZz+LI!ynak%2$JP)WDya#mnzeu;ITWj@B&Ny~J3fQY5Khe}I8=%v z8AW2MLQK`RIIVb|7xp|IP=S!40P+HeWZU+nRMkWX6 zUrx|e3SB_eVHAm}3Nh6vLYjtg4&EUwC{d3hDMKX?DT>;Y+HYaRrsm7d7n)Bt?{1!V zt@P2UyGVjSS>HP`?(U}ZBc42p_oxHe#6FtFF)5PJ@+Y)SyP($aJ{dkuDTBUstg z>H=BYbT@kq&gLuF$~V0xZWYK#LD94pp+yd_!3MywY)+deTiWzw@)}%c`fChxyBNDg zllnH2pS=bfz;9*qI&oN0(S+WkWMMF{*I)zSdf8x4YSy}<*;SU;;P$@?T0gTxNi;Z= zG;3XNv$4$|dbWsqawf-;Xm&Dr4K_fq^@BtjRV{#~tjUJQp_vB2j zGE5)fHu_l!t}$+`0lo1Xw@&}2fZxg{SInLM4jb1P2H*K;`mocFdnqM=0Eg-;u22BS=qt_` z06X=S1Sf#I>MIFd0UW8XI3ocZqOUkZ0355YIAa0aMPG4t0kF>EWB_c_S6ns#^ZJUD z2XKnkohfGzrp%L3pieZ?6CU_oC= z5CF{TD^50tVKP6s_3F4!qE?~5MBc`HV`aWyZjW0W^Yhj;OWV^H>S=eIs|QKMf6p-4 zBFlI~CX4x_v~t z{2mk_RQa{+j|8M$kJw}SB>9nhJhbbPdP@5Jtp4QBkUUe2y$mVIrB$!+xZ zIn2a}^SkFS84^BeOm$vF#j6D~GUHCJ3%i@Shq;Y8J~8i*dtyqId;IK2qI>4N)vJ5x zhF{jaw_(`Sl|oKhGwk&*trp2<}ig zn%AabU;)$az)*y6D?hA{<(@*{BErY2l`xM;zxf*768MvlDz(T_5QO z4M7&~rnETEO>Z@m0nL{1!D(5lhDi8(dj~}%@CwXRgQxo-u>-HbC%RQ>8th;$m9lsW zJD>zJf`~A&4~-psoV%qz1q6b_H}*SwP(Z*AX6C5wO96rF@H9vX2t;xnfMCLe(g+F& z*x|X2;0tA%b}(T}eFg;t>|mOnZ7vEsFagY%vowT42|zHjRAVLu1cKw4Yx+%UMz-%B8`0PV&XDhHxfIO0Pp#!-&O zU|I^{FZktB(BA`csR*HBFQuVp!G%ilST0RVRANgguEZBAo#sEJAcmz^2!B#Vp&B5s z3=>SNjr1W5Gc^%5Hnf=%gErFy*jQl`0>`4!W@-{_T(Gggu_&~e=7imDup0^+JM4CW zO#<3XW8qjN?DDW14}URmECT+DgG~x-!eA2)o33ax%>jQUqs5@-K-w_OEGzwaA6)Kazl0K9X&3)^Vs`NDlVr z2U5Vq6h`?$5&+xJ)Z|6E>mv(*JI$2+Kzb&YN}IM_~m1C1wxhx{kaC5&-8fv2`Z3+8uk`NA4bA`w_F7l=U5a+(#C`pT8)- zNjCsDcllPJ@T5=*jAy22W()ZJA;FBkWP=p=wd)oA$^x z1;>BLyK*M?E@m*3%{V{uoSnuUycj!a#BIsHOt{^fn7psz`10)1DYYSqPnEHESf5P z3nJZxx$A}gXZkmM&}PmC?W2^~C19->!JHw0CsYXTGuT*7jUgQd%@` z*!iflAC}kNSv10P{`=D}%&LuC(zL&7=IjgQn?65!_Q^#bO@D8nb5YaQsxKUug;UoC z)jqbM>Frq~j$h5Idc9^>|EguS%QeN#M|M?gpSiAN)AnVno5Z)bjI0`OU6E3=s0VtW zD(0}Z3*X=SddZb9eJL-@-u_CrC(fhY+B9){;mV4-3-ed^9Bxm%{gW--)Bcf{H)GEi z|4O;^&IEqdfsHBR!XBa159Ti|{UH8<(M212CqA@h^_P8mE_>?W!=(*x{`2L1X@{PT zA5n6>t9@KA-iMwD23H3Ot_~7h9i;V|HZ4n~1Xl-XSu`lHERVeIN6Q4zsF`N5@y zKQJ}rg!R%?;}WPP(%P-Y)SeW%Eeoy=63AeA*$-@2m319(&dD(F5jjm1OM0M=gj652 zURW!*I!IupG;G`8>L7uo=mMP%Y2goAg&={b6@r8UQsfFjxu8dzHbq)uA-Fn-KX6-? zzEdnTqH0@}1wC5ZHn=*7`T~l$TKHA24ibEUJNN=O@x@hm|1JaPs zb~=KbSi9AO=sTPifUvF1L%*>*RS!HK3ltfzc<0U|dloH?1?o)*Hu+Xkx`5(O9jbbvj9$ICN^k8jO8!-FLZt-+SME zo&1xTE+ao~j?eSFpMBr=ZkPRz->;nc3jk-LdxYFL&j$ge%M%oNcrqUSa_=wgK``K< zq0x~z^P!5bJ#P!bHVK>)ow1;PxF^UvvlG{9Zc6L+b-s0H5xb+*xw1+3?!pxKjP4npA)3U8#y+2TY|XD5yew zDGD-lVn|UCRE39UA_W0FQvpq?)m(~#f-1z9qM(Y%P7Emug2e(-GM17xsiAG=QWP|x zLVPI-GIU}{QBY{PfRqbMx$sQAV=hHOK^5XlQBXx>Cx#RSLA`)f3YJRIq$)ejr6{Oe zh4@kwWaz|@q9E@|0Vyw*^5U8Lv$+%n1yzVIML`vjoeWYn`l}xWh1P|Kx0(Oy=fP4Q zyr;U%r6_1Xh4@kwG!)s1Aw@xrs|2Ldu~fPy)$u2DDGJ)6LVPI-GIU}{QBZb+fK(=y z%G9LlL*`N>q^wXOz7z?8p%X)jf^r)Lq{d;Xahg=+L31ezDpw)C6a^VNF{CKy<^}<& zEG(6!NliFpE=590h6?eeNC*s_7*Z5;s98WN8%t$tQbQk@OHt5(3h|{V$k2%)ML~@l z1*CGYRE{Qf;;^|C2`NWah%ZG#VCckU6OhWqQn{K`#}RWW3fiJVd?^YtbYe(R zQ1)+FQV~}XJ}l+aq=Lu7ls-Q7(xWGQ=@D_gLO~TOq+hQLUwR04(xm!`OAo3Ow1$Tl z?JPZ_)kG{c@vfATe%xG&gkZc1@uf(}*oh%ULeTprODf`^=V7TlZKmoz3{(2}qFyAD zkW!;U`t%OZlyE1;@*_LJjh686qMas1%v3&>%GYLUMX&ixkr33YkbWeFrGz^%W{T=m z(Z-soh*p!Z)TFyop@k>RrASD5MuqrNBxLNwkRl-{dP_iRGM1XGEx+=U=28??sX}}y z3NmzJNKugYw*pdAu+)^hQh~x#=29f26sr(liiC`v7*Zqz@>T(<0xVUa?Wv5@=29dC z6I6&VMM8?96GMuGpkG9)5K9$mduqhT=29f2B&!f#iiARjP7Em$f=}87W~vBF6={2F zKtPIuh6JQ2$k2%)ML{9;Pdp}XzEiQ(RBcc7oiU#&5>o!DLj0K`p@5+iLyCl8dxyYG z`LUE=+fzr+noE%ooKPXY6bUJYP7Em$g2vwoNKM01)3iPHhfmCv zq(}%>Y!{F!#!|)Fp4xKGT#AA^RERG{L55BYDGFM!LqMtoOO&n@f=p)Tt0(ii8wHCx#RWf$wbrsZuOes_m(&3+7TJ zq*SXAUy6i6hE5DA5(3XIEcI`6cP!m$NB5uHqtPugw=9Fp?P)v^gU;eZ-RK6E^8u#= zy2dzpPk)ieeENPwZ=3COkCIQuyW`a2M}LfJ_hbcOH0o+|#LCWuB$wLH?~OEL6b7DIkfAAca`~6t9D$plAw3sKD~53ZS?|3QXk{unCO7&n|$b zL7>1iP61h@Kryd?Brt*!b^)&}QeZl#fJ3A}>2QIdzLY-(bT=9e04*elftcuP87~A6 zyMjZkLOQbo+D!qGF=R0-pzS17A)8%6604BItKb!}$YoY=piL=K!N;rM5~(neT|ok& zG2}5T*w8D3$QbfD6#_yPCh;oBB4e1$tbmq-P=zU+3SOZK1-uIACDwdz77kZX^xfSh zp*Je@A`b_Y;_dw#pg$%)dFalaYrCcK?br6bdu`u-^!e`dpQ9g_RXuTbdDYpLe%G1# zm*P+mLwy@u`*=Bk3(#BOug)j!Aq%5Dd#{D_^umh)JOF6a=-n~Z7P7shkNO=6eCnV9 z4f+R>gWkm*G=A&FFIV2-LqZgHp(FA&i)XTnqoeXQi;o*FP95`mpGw>i6(eZ=%Gh5}1rl!iD4t5;d_8)zrbB%NG+bJh-~< z{Ov&B#h+b|`KbQwx<6hxoA_M8XMLSx58f{Be#CR(LFe_v<}X)XeJQj1wf4FhSD&15 zee2ShJ|)vDAU7&YK4uFK?~BuVc-f^@Gp1o&M2tIa4;aJX*f3qq8g6)Ouyk zNJHGqN{fN^fzH^q>k{n}!%B;>bA9+?5VXHV7&2U}TptNpX|breG{*9i@!`Xel@
    EI$ge(qd6_X^iD(rN#8+N5-*?87{{1BccDhw3v0r^qYCd zbb<|JyKC>h7UU@_E^|s9|`k?zS2XFs<&4(>gU?@3rl$_wUNscr} zZFNm;T1@zBerBvZ!fmtJGi`GCGdBD^XN+^?3(KC)dvV#bwbje=mOQn%+U=M%zj8Ku z!X5F%&#RZ!Jo~~Dw=~Iz8u}CEa9eb%vTbf~(;WR_Pw%YY?;r7xkyl@camS+!;&jiO z?R&$&xVb+y2>zV#6Z-B1(EhUrEz3Fb+XZFlSNHTi6udDT6+tbul;o}EWc0+^-nd}9 i6lk0mL~Sc$prx?W|CmGGWOF$@isJIxv+cf4zw3Xp^h1vT literal 23434 zcmeI44^&iD9>?!I5Fem2kQf?g8Arqs2?tSu;oqd3&fqRZlv!JLthBA&QWmn<$nCAI z3R!#5L@h(HB^9a-Ty^odxl_@qltuQ3EG++xx$Q>ey3ihM&)$3I-Fa{BoqHdirRJPH zU@01i8#s6`;UghgFM^z;=)0Bl}9 zWKT4o&Q&VmOLcnr-aCZ&e6BEG7vj@h6fIIZvB-B(8sE@73HbU7)%f)ldihG9GUj7M zE-VuBF(S>AfDe%P0X2T54aR(o$c05>K1QT@67bmwpH1bv^rkT%BkIQ@F&`t+Jkfl* zRLUWIIbJ^J$&V19&sD(jL5q}gSmaAuE&qn*Nx+vy__Dlw?&u+mUs|NZ4q^P#BAq9i zPdBR42w$4ozdMc?_b(&bhDE-hYw;VJCjp;K_+*lQFBtPNqJAtA^D&}JK2J2CE-M^_ z&*9}$I=hYYj~0RVut=PLv`FVkz-K3Xb~S#kqsDxUs1%FDe2hr*MDyvg0s!^?41o9k z3?Fia&kv4B+`k+Vx+npknZ(bm#;^2cAfG>scrKo&UI*j>fkunjB z#C)^}Xr5?3-T25Qe7Rme_nwc9^N$uO2e3$-f3!&FNx(Oo@Xhw}DHA_1=A%Vm3Kog^ zXpzp7fG>maWvG0K?;7*bA}|??#C)_!(LB+7y74=f@QqdTuWYw*{xPBwEE4A*Bhoz4 ze7gM0BYb%(-?rn%@nb|gut*#~Mx=S7`E>D1Cw%EDU-etY@uNjbB^HU}M~i^wNx+v( z_>xKgzHZFNh)S?X%*Tkze4c1NUH{G^e6zfK&aC&1`NBBc(C#POp=PR)~mZwBF;;pJ1R zx{Uc~5m%K0be2EEA;ZY7xozQ(IRCD7K!<2k7l%HXdn2#2@HBU63 zZv0vZpGEE8(pM3m&sD(nD@NqPBJucOM4G1*0Bn1wf4AuT%_RVs0Wg6x-D6tQePx-CVzA2405sVzZuT;J&i&c>Zvsn`%w6P5mQ(qw-DJbW&)sfOM zx*mR|n8U#*Vc^udQCB<}bFq1nM42sOvsv<2JA92dz(orMSi()d)-N=_FHv?0IAsI7 zS!4$mS=c^wuki)&Z!+|+i2XYa{L5nh5YHmKKBp{+>`*R@AM75dNDuLa@ln+7kM(8; z(|_T>C_ZK(KfFAlf7re_K8m<~7@u&_^+WgK_~`RHK>m=?!;23LuMxkH4_+RQkDz@a zKMbFL`) zOgSJ2q=KW#`N!V8Cv1k|%xs8$wykF8EiF5p8#4A}N7il_vmiNc&vB_~?YkovUtMsO zef+41FpzTZjh0&<{hhs`WVh`nDYq`}e;rsmii=mxz3!Kd8~R7>`l-D3SC7tmCr#S) zVB7iF;-Y!scdtC{occ;t%PM8#8&!9hRuuJ2SySCn{@B>w(dQ4!?Q0%Nvh)n@pAs=} zeR*N_z?>5Y&K>OC($@MnY180+`~Q+UFwS%E*`6EM4^FL^UOW(c_Kt$yiHE*>zy0Ba zfupCl7WYhlqI$AJS+gtSrP#a)vZE_{DtxT~$u%E`jRHkAH8tkg*vVF_wW!G4wEgSz zhx%XmYT)OxvsNmSW|pjYBxc`#&V3bG6IUb^nT}n%Y-7@j_R6H@u>A3}ZaB7Y-}=4r zFPFq8A89yU9FwH(j2?@$182>rk_)hBmcXB}`*t@|Ci6<smnmJx`?JF`Kn@^~QMg3=$-~ z#Pg)#(c502Zl|$mESc4jRcXsgV-dYT5Bi1>2>PQ8;Py?%fdB-LZ2E%-jPdCGUGQ;} zv54O32D^6fL|W%E+q<@BjIW8=z5*+ZJuz$27oRou#B9{Mm-9SnX_QSuo~_RF-FSBw z-6P2CyHO&u?=91m5ozkneWL^)Mc&zW{epRWKs1?_H%f3?4kzDej#9rM>NeJbCQ<&D z&8h~?p`P;75DS{T`df$}G)?kvjabl}(ceN^L$j{Ht)Z!!zlOAi=8}OeXjQ=9Lj0h~ zSYSVBg6yv$e$d1;umw%U16$C{&fh`^&|Ee!0a|(ptU+rC{urH{Y_XqClZLkQ4Hiob=lh8Au7TO$^aP{=@wn)ZoF`c?19`e%#T+8&$)6-oppL@#Bsj-l!Vp^Bw_!nty^P;P?M%M)}79 zQ137N0RIp-Zg_v;0g&;*eZKIXJHvS%jSn6G>R)G2@6IOR;RE3K1@+|xzPot%0N(K% z+}A&|NAUjX9lwGE;3dW|o`>U?2Y}K(^)C+q z_pbuO8^puIcaGyX%#SO<(<<v@mheNr92f;+fia?ii+%W4meUyeZS-=N-JQ2+7(NdNNR zAOv&2;d<>s{mTRJ_Al@5{llDx{mXL%+I>6h^-&<=1IMH~(*_We*XRC+Gy3X1whvC( zzD!$eU3Db+)Y;NG&&03q$oc|)Bqs|@1ve)@`S))+o-1B?_VGWIpL=}#(sar3{4Dzo ztK>0duUU7Voxe_6D%HJ`lbw*UG3~A0%DD2(&WMq7C%Nwb=W@sKMfNTCw;pdB;p*J7 zF`~Aqu{2|;(>g8g(1*^yKKFXn?cm=hPbIHwy0Bw%xf~xEpP5*CH2=>%(>GSXP_eDF zcioZiChpH_8njj{JDmSbds5=SoHGSAy-8;dcD)+CcCfkqP~5&<)-<~GvX(n|8?T+Up^N0%-o`w7jtjwZ|wXP}Tdb=F*7A@3y#cf+RCf$OsP35L zA6wxp$Uo*YIa)O^KmQm}sO}hq>W)eNu@%mO{G&gqrK<+!=N~N!)g1#6syin6M?Omy zsyha1v2v*H7=TdSF&Q6hg|p!POn-`7R}IX+KhvU6-7x^6x?_@mY=yHR|LD)b>#BkI z`A3UFb;kgN>W)eNu@%mA{^{0-n9>)m8d%FedVPoy{f~9WZr}24s6Z-IAO#D=;2-sU zKU5&4{z8gSffV`87oh?vwR9v@Af*N(R3N1W;{UrqDzwY{z3%c(lYks%9;}*Rf&ZPf zC3KImoRMs;Uz+Uhe_fO9e*41K&%bT`{1IpOm+Dx{ zzV@fuKd$XQVR9Zww~mrW#hFYKMm%)q{mYkFBjo5fNeZ({GCCqsuZ)UWR&jUw!xi^E zxTGR|>75TOi3=}UPK08MU!K+4F9#mjnb|n*S<=>Z!cCj zu9R2zjEIYrfw>@IYf~Hg!OeZ+T)>fv48FQ;Uumy8V~CkL`x^M8ttYQ`?XGavUFU+W imqmh;wl}7&$vKyMOSrsQGKVK9irEro3$vfgHUAgd$HmbA diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_double_with_null.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_double_with_null.dwrf index 65e199aa3d77946aa7b490842fd7d431fdc48178..2cd8e86394d38a176be4a175d0f87ff234ac8868 100755 GIT binary patch literal 30744 zcmeI53se->8OQG|3l8fe>xxJSNf4E85Mjj!3JPmA22Yy!sI9iuN`0JYt$^>MT+9?ik1$@b1Lm%TIJ&Tw-gvpsb@ zK6w52-fw>Q`|e|A|Bf6pbUs41_KvoE(TgtRuC3{k#q#N~aQwrCcPv>*AulTIOv^Pt zmE<)?u>25*Z2fFD)cniw9sXKa^b_Ia3mwS^kCP8voVz#`Hhn9y=#$TRiL8tpO z3D=6k0k-yNRi|MHE#ZD1f_BehQc6t@uysJMKN*iuzGYGg<)U7Q<}F zl*4SuYzlMy@7+vcY6uy^)KF@MIjcELpmhI4?CedAF#ki*bQ7BHrZBVDiOLOt<~fn( zRUP3dNm!^sld(x9%_22Tb!X8`v9k;`d7|XyF!RgUXle)5)?(kjFQ7FC}#^32p391hA@GWYEnN; zptLX53zM2RV@Q}Wa+ufO7M0Qn(w#RogbagLL#a7v{qrVJmi|uc>}3s_RG4uj%s4sB zgYUA1sUc(tQ$xuRCQxR-suw0TiFGGoc9+A9IUp)qo;QKeAqg1JM6=tSAyY ztCgD+&3F=Kd{dahNAI!G)DSYz)KF^D^p_yD5>55$g-OwLk}#bz&4Ow+nn1Wv5;D*P zN~%eHnm}n^trsRmGl7JeAk$2JpN*!5kb$O#l7XgLiQZYG7bZnBk%XBj)4Wr|MiU4x zNkRsiKuI;JPZKDo7wd&d(R7h8U2>S?KNOV`6ly}@z(!0%C^OO9h-oP0ap2#GS;0mO zDA78xvkXi3-XzT4@}O;hge^=BAw!rNN`^4iO0;`DW0*-K%p^I?jfX^~)LV)s+*WG{ z8FEuY$&j0BC5rfy*jcR!h1xczkT6r^LA&p$sFYHS3X?;%;(n7#Y9Rbb5=s+pfYSrU zCh6R?B0<`3`nT1f&21aR&T4I|sod;C!tB!&rh8DWsBAemHG~XeB>D{tZT&{YaSon!>EQCH1ytni@g|ni@(5 znrbDgkiul3*`I{jU#3}ejEyD`9+QL&G=Y+8QlBPJzO#{$W-19YRnE<=$3*3;)GN69@|=Ap=dIq?**H36yiB^Erk}JCKAqP|nQ> zC)j9e2pMQdy~l_+X6Bh54tW|~a1_enOIK$sy38E66})np)={?h}XY|sspnm5x) znCWtEI!>|C)DSYz)KD_eR4dU3TNr5$B4G}aXKvS(m&K)Go+xYHkj4cG;q;zz~w^$uK zkL-@9qRLQw|Fo(C-vG5eX|tleU2J;azMiD{?I*B$^KhG^6Mrn+(MEdl!#~8Y{Xu*d z>Wo{pSi^W*yAF1T3nAy;VcbkWhs||-Z{2Q$r_E~%Gf&~dTef7Rd-{P-*U-|C) z8hwU;Dg>)x(yrIo3ou;9k=@}4LIVUO#9D>QP!u*I<$JCSYj?hTX=+JYF9L1)SPU%| z|DfA~CZ0GvIWk*p>%#Hb|9eaGHTvwiPd|5QN1K&CKUD_5wg?oKATpnKoFNX~uRL)i#I^o zhTVKZ3=M^%Q2nEF>g4AO5DkRCZL}jkE_EA7fydSlr5fwH3 za4Zuwmy#MdD=;$AFj4_7vy3g;UjGzlM8lApoh8?LE2#y!KsG*VXUA%49Ma7-GOr~m z=QU$T8r&NN{T$M3Y;jRhvmk*f*HVA-%?)=>ckPCTe|@)Qs&joWz2D z4(T=aOb=4yS?0CAN@~XT8J@&E;^&ZfyT(MVpORWjD|1-Wm=3C#3-2R-4v7~dOw{@- zsloLD%!h||;2i+C5bO=3x3$Nma+R*CkeC{N5VGYn|5QIU@vf-yHCvNUU_3!z`FjHQ ztF=b6AT-~q*~{UFj`1_cs8SfTt-*sDLc=>g@cFL2#q}t=wr|(9ef#mBT{ExZ4|66? zI5ltLsn;6pC&$&diI-nu`q&o`vLpNm`IHkKc1!o&j2(htxIAh(_)0U(KXb?>u|$~V zXqx4lx-8XCm&f|k(|k(+UI-ZcmH-t{J5J7T2{cX#6t3)zI3@B|v!Fk`WboSs)IcrY z8fbSvQ0cLvAfLhn{lL>>>;(W5D82#vO()PKxopoFI}4#GgfLP3pjf1($|4dqc0mBQ zs6oH+;4$_P9}@lMOCmCjl4t)KGMk_M+ZEFErn`mwtPW_)njr~AOGco!g)^44c`9#3 zWL|CdoxM}<ew2{@{gB4rWPlw5X9 z`r?x}w@&#y?;k0B7Cm!*cuK+UrDa2}J-*24>BNuwXYrhy&pxP3`)q#r%?;JQt(!`U z&mVpM!Qvl3_|kFo@72ENYD$t1eEQ+tsC)bDA?V7H+$^jHXE>HlY-asYD zZ@mSSrfaU!4=svZTGKT*pxb;5xjss}UD-N9&*V@;Dce=}O%CC_36!R5u5!=6)v|P2hb?8THb+ zX}adh7Qd!zF2W;-au@?fw5DsW;;2LIU@P10GB}KBC>b2a)JoGeR~)pIU!2;A)^yEP zPPo9q)^yEP4j{Gnt(!9H)t@)jO4Bu0m78iK+A7mESK&4f9BfV3TxD*my>H!=QLlb( zs+Fc|t|~XxMzmF?YpxQ8H!#Zr-nBn*v4(c1f znwB$n#_X3JTpt&1Fs%bGHjCW2;jFqH{?8ll4axd)8<*CF|7~5UBODt#XGCo%+avzy z9~&aGP+CvEWM^pee}ssT{6DJ7Hrds?`VPTv;PoqdX6=(+1-Bx5PTyPA5x+=5ZJVqz k&Rz6k7H)Y=C_0;7o;lvizh<#pqXfYo8{!Ocm1o-j3#u6ql>h($ literal 36769 zcmeHQ33L=i8m^ff+ev^-h7baVWr!F9h7bsZ$dL@EAqyBl7koh-kW)d!8?sJI5@2to{UC`J%;Sp$ostg9g)8kaoGR@HQ+yQjLk+eez-7n}D+p=-YS zzy9j4zpDS~s=9kv-&_E~8#^PoWkXVcd)d-vSsMTRt>3@b2Ex0Bhl9F5&fB@~LR;Pv z=D(2cjv{|)f&Vh=q&vmtUfK=v=oa1xEC?F|FVD1r6KN3my$UcN17AkK^zg=@plLq< zp0_<51d68UHmOARKcda{glu~^Cd+sG5RokmAZrI_5Ls>DY6e+tBr>T)Rw57}WPzV7 z@T@~*u|N`K?cf|DD`^r6n1g}1NfeaTXHto*}5 z3S{+})F4}xDfnzvrr&2rF4E@)44@7-U?6q4K@w3mNpOR!W%}HJ0)B&mI^2LliY7Ib z>IhvFQZ%XI1_^|0 zf}rfN5A-R^0P0Ydfn+Hwaf4ojY%f3AlCm?1EVe0$2K3~c0YoM>q5%VmOe!fWi3Vwe zY?|N(W#x#hb~K=Xb8BN|Xh(WHhO#1pdd0@;K6kg`}H$#6?gz8OFqM$~g% z3?wpHW(RytZFTLX+KSr!wa@)pd%gCi|C2Y%FtO`3Sb!5qWiu50N9t! z8S_M?P47%zgJY&YVwgL{*w33Zx2bDqufYQNRW@%Dhi{ZOVfI+3FcR2numCtN8{Cqb zQI|J6WO)r9|BKP+XSOTx2D_4G)b-XHJN%u)a}1Mxi8s45c?}l88vRg5HhT2KaoOOO z)a;E7ekO-imW{V2#7N%YWR2!6c@3UQ;)r2(SZwffj3PApX?AIYCpry2^LYl(F-)Gy z@dhUzHOC#V!2;ljp&7Rf4qLpz(^bukn^*I}Nwc9;+X;_;2HU7rWdKV}4_RJ=$A9e0 zH`Y*^B5H>H-^WbO><-JD-I=_aj&Nh+KaLoh&-n&7!|&r4DQomIxHI9ntTj3r+@ES2 zd`|A7m6L;hY;;b}Ka(>#WSE}dHu|>`95HTe0KM@Cw+8>FfL~>k19O9a!p0H9;3pr= zHf->jPwNRAB{__K!p16VW>XBFPB!?r5%bVY&U*H{@m!4R45BIlfW&t?UiWZA2Ls>$ zzy^ZKn-nDrR~?w8Mp9z2qfOuGR{nI03Vmvld1cv&Hy1=5J zVSJ<9L5%TGgQAVboaW-H^2q5ybZXyy4tVo_{<(qZr3OWu3(!6-l0|wT%0Q22EU+&cR+$_ z<&d5zkHY=&f<94xbrtTPSLg3m=MQgB)jp)>QsWQhx2u&y^Xl~jE&dMG`XRkwHT#@~ z{*b;ZKjPgi{euCot`D!y-who^mA{t$kf0hKn#c9C_`~LLtKs48$?J~|@(-(7T(d@C z{_5VhF0-8kOFy?qUb98^taAU5)AohKE8<)19-JNa?!4UeTc7w>yO2zf2|9wKbKB1^ zYT5b|U-a9yt&5Iv$G%Nmf3)$~-g73M%H7`PNLbX$?BoT(Q_?_r>4Za`4jFy>wrCVO z7bHv^?z(T*k^xDs?Dbo@;otLF4?oqc@5sTa^O7foj-NWJymZUZIa$M^wx#DBo!jh? zuoQhg%tx#-Hq?|0;?pRHVQ;Cj=u zkst5PseXO;8#SRV((a#7kQ81pFlO!X;KG_oyZ5)Rj`8h#>Fn?oHSMN#99SLo^#d7| zoA{ctsjVlT?XkFKz|>EYDhn@WBvh{AcWm!n9F@`~xRtjlo^61*{MA98@tY$$7hDWE zK4`{)@y@;8wF|bb4V``P*D)I_o_)-fcBS3MD+?lfPaYcG@!jG53wDFM&hUR2bbJZ-04iLCb78GNuta}uxQads^R;p1IEmC{<0KCf@%c2MjTqO$DGoTiP0gYxPT+odV9zQwaYP`s@rGlb8c-AmkhQgB(>w5p8dVV(xc?KF zxJ3;pQl&OKn^>quRcsE9P$hev<;_4qR__pj=>(WAtRK$9o20RT;yqFua#g*ec8{XP zXkw9U7sNfUMpYCiaJLr_y;e;ekt(%m6nI9Bst63+4-mxG-D4X7F3W$PtU{OMi7T3 zk*`p=p%bWjxtrj+0@#yGl*djoQMq^UAxXl6uE#!(WTNgJk-ZBfI*3u!7|K5p#(kR->+DL(i6%hD77LlA$CySLT-n!MtKl9H&TKU z8uAiS8_G<`Ya^^twn)x^)P@pn%C(`ql)N@#2PH@3C8Rc#4wBbKN>Cn7UP8E`%!r&D zv4gTz%63o!Nx3$ZTU4$M<@A(mL+L|#ZG<&S5h|CUG@x<`N_;Appv0WKglK>=hjP}0 z0ZJUoF+?4d4^_S&l&|w+de{I?0iiUISs$N@VZ;ySzfAr3R0v>x@aG1Y1u*U7Q!$SC zp%k8(AD@bG#1Ewq&HVV-#sT67GlZsod@2Nl_JtncHtmDjmu1}FzA&_)Jg7+@*uE^| zxP46{Ch&ssk3|skV`+P{uA}(EYG8l=fHz=b7l!?VMF8t(YUhQ0*T*J+`I$I$z?S75 zu>Ug0F5G|JrcDj-%)rNu0k20Uw)yzkOl%F>{LQQ!(yuUzGsP3hecG9kwrv0M@Ub-Cf1ol?Mtm{xU$AAImv! z)^(qXA=JLi-lKx6PYmoIjPA=}{mkthX#8b%Ptt!?X4ve(yS^R^Q2%8S!2OpwI^*G6 zW%RC}avytofc0bcZdR|a=j%Q;0e}C6{hRdwa5Jv^RE!Ais}jMPzFTh?)V_?$g#K%| zi?KZn`U>p7ECQr`*(0<{ZmGUzwWxhr1pMvG>fJZzI&5E-A>b6=Hc8~JM}XC*GUAun zRs#5*Ny+!_hz`3-$~VOxiOWtr*e7AwnnC@hy?ERCbwBn3JwY;TWJvL#5 z>%W_K&TD)3kZgD9tSmUvtIzJtgzEWU?^{;6;PBoy)dw&9v3unUAI>f*er{3C zfa*%mm$Nnu`FrfbnzhAcj_T9rzv_H;#0$?X%**^@ZsSr!Q4)%=WaJ(Q4)cU{O{0kfZjZCe7a2v*;n0 z^LmG8Ut8Dzy;j?Xmt9V8Qg*($?c8gphhE5wTT(VP{g^ZUsTJQ%iu|nyy#3biEA5Uf z*~*#vb<;W!PR~(tXe(zWWhHy1Q@@Q{2g1pe4gBd?at57}vXZ^hsn<@d1L0(6HB#?L zSO>z%PF$p3BC-yIlb@d%_;!wUAe>wu>SZ$V3_7LIB^eZ`7j4AT*`(*M(x-${Ksn61922()mKsb4u1ir7wTL;3)CNk7} zsn&sTvMF@nS^?oCIUQ&JFpxUV{-IuL7EfnWj|LRbIuK4C4X70Z)`4)cX*;!Q#5xd8 zrVq8`$T|>C-oXM_omdCL$@K|b>f#p9pwpNlQcI7-)7jJq1q%4PI}pxV`Dv~Egojqc zKY?>HYvm^=E;O=Meo~8##RX$j6tz}#UWZ;)+4)3ENuvDRwLBw^3{5r`W9m zf4-Bf{Ip&#`2TsmpsO8lL5`JYZ7p5!|9nzx;Y8Qcpv0o@Y$-*5%S^A_v8wRxO@}{z z)Ll5SWjwc|(B?SQBz9e@YjIE_-24J=eUV=KbII!;{j;{>7n^%sYJ4NEk<(^t8S?n( z$uq~rhjI~4yFDo0&Y@3e`{v=1k4<|h`H5+hW{jJb{OIT@IDZpKQt#{VXr{w?M1^?Zgn}0UM8O8xeM(oR5-$Y+sQx%g1 zy5GTV+!<200Y=@%3BnQDhhFc}8?NxKSa?U)m6JC2vLRW}z_2jDCoS#xS+5HB$Y5@R S-4Pti^A1-~Vo*wj+wni1Ux@es diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_float.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_float.dwrf index 3d50c6db1ac691c7fbab6b99dbf740f64584f361..5a9547a36a947e61ce048688c9054d8ee4250818 100755 GIT binary patch literal 17420 zcmeI3eNYtV9mjurFR+}7dxtM*v^@#pNfA#}B#3f~#iJ2JX{M&Fww6lUw4+hb)Q&M( zX=^1`2el?vqn;5Ft+7bU446r{IygqF4I1N1LJZ7AZR5nbPK`K5W1m}|=UAVIXZPti zo#`LTh;Xy_dA`qQpM7@sx&84}X~8c6TmwA)A||?Is7@#Ys2_=- z1Ex?dfwm;g0yv#3o|J`>c8*K~xHZp{CNDSZUAx`TDUy$!BKbW|B^v^!P7D;(AY&&6 z(k1bn7zP$ZduQSV0WwYj?6lL|iGhL|WbDL1CA~>JCx(H=0!}hoZc{ZVxXV~89OmhNpBL*iD6*1fYTu2GzdGDHkvyzP_YIX zJ28+hiRZ*H-YNknFLCmcaeB+#iGhL|WbDL1CA~=xJ5|Qcdkho)U9@+njn8|@#3`A~ zrzUeJ2D+_5#!d`$uQ!S3#4t6#5pWtooJL@$x)yUM2HLGb#!d{ROX4{(jIUb2DTO$t zV5ik#b0->7S89;46AghbiRZ*H88reU7K$paGVwkoq0!}{SZYYPJeE zr4y%g>~!{oxf2bkr!>geiH1Oz#B*Yp`L7B%We}$f>{NHs+=+p9Yml)M1L=}@P7LGw z1J6mfZ158&KXwY9iBhrdX={zz=vqU!Kw+Q~4T>#Lde<6)NinC$!)pyD2zEw$*Vlcy zE@=XBn$Y8Ym#4z611)L@mr^!7|fhq5qJJFDur$NR}G^9`BInfNrdj*`biBmS7PovJ8 zJJAq~(I8_d8d7yhJSUogh=|h^;xq-%r+)97JJFEp)*xdi8Vc)@cuq6}mudyZDTg@a z;Q4f0z=?tG2{>C{DYCmMpY8f5H5L#i%`=R`A5vtPhzDsh^M=hI&B zm&9|T8CY2_;FL$4^6-4xec9ZJf$B8K*olF3NjxWpS$IIeX&P~whUZiD2j)&Rq}FJV zu@emibV)oXnt|X!0jKH2X*!-yl^>cr(GaZCAY&&QQgul@Cz=8O9|fH9iBmqFPYXKC zooGlc*C1mj8Vc)@cuq6}$%ly3ztPjL5iUD=zT_E<9)Wpe85B=)O{){#w}+e2^DWm% zmjjxHx{Nk|mdV}x7^3$Tx;#npxdczV*7&J!qqg5k3&LQOYIDTNuKtON#}D9Z8h?M` zYmcu0@W)Fap?|Pt)|CNLA>Onw@eDwY8c=MYwmKoMC~J;?x+7Ii4NcjdJ1zsit1Y7( zzzYxvNbuI;D{cmtg>aRNOCsqZ4i2i4fk(%UHt;m> z5aT;JJeo5Bk$gX=|7g+(@c|w$iSZ?T-Ye4og&f|2CY}hd@_9vsU&rGm5XxW5;caN< zi{#gE`G63=jnB&>`5_LEhF^%U=ki`5zJbr96Q}w7YbLxJo4-vGy6B+OHnggF^^r{w zaVEI$J$P{APtxex8%Gb{IQk~~KK%1f(XSN?=3iW~;Ntd(a$(-b@u+A*yQH;c6ay## z60~lJf9Azl!*597*V+Zt?l<-A9x`l~T*`iZbfx&r$6S}hs zcFLnt7FCva*PYqABm1+o9ku7b<;sIRNQb4P8)sa*K190w`?8e8wZ~@FEOJbnn>4K1 zce0?PYt5*i9{SpmYw6x$lU~bd3Uyt7<#^eVlKQsow%V$-d;0xg{K1n)YpWKg9|$$> zE%jfJ+KTKUY3yFPYS)Qn+tUiyu37q>L8FVVOihg6w(eJZ{O;5OZ&6$zYu%hi=jufr z6Wgaf-7&s;)zy}#lHN~)@!`#QTX$jQ_=PvW_tBrPzP;pU+5hZpeEit` zyyn@-9ba|b9I)jJ+h6aewu}jOmwxj7nK$>InD$BYOT)Y8F8??qQQcAB)v!i=^5u}d zA?X!R6t(|M<*M}hjk0|!#vOk8;_+|X`0o4zhu2^DJW}v!zk!DyT<-9H{`D^DU%?lB zZ%?o5OgdIDd*lA%%|%&7DMhK(sqW`HnqHWbv3Y!E;;Bt9lr8gaY#X}Xxh&p2_Djc0 zU%dR4W50b+-Mvf+%Bc>IEn&^C12fx8Q>*v?_yhYx6>{ zM;3F?vzP;2nH9hAj)ts)F1+`wf-YRXxtc7x4T?Sok;BL;=)zT%qzcR`=pyr}DS90B zaW3zD5SvVnB2sQ<6?9nzU3fl`Dln^{3(qI3psVNlWEFH-1zq?YL8`#4g07x(gjLXm z&y`j|msQY(&k>{w%qr;WnHyF?7d}T=1zlD_7d}VeDln^{tB=yL*ypfR!Le1)Mg9Pq zDVfrhS;anwrT-{_fvkcqe0{PCx_Z{RRzcVQs-VkyBKrUNM0B(bWV>?y$>Cn~<8F3x ztJ}NH?yI^b`I{Ri=X@0E3UzP%d*jMLt2-r0?(eZlj^U2w<(12aJEQ++rNqhoJT{v> z#U@9;ozb7^LtO)(tN2moFDjm0US5&8B&`3Iuvl06LFB%>=YzWEsHG!a--E z`I%B>6Mosg5WnMxHGbh#m|wHT&vZ~UNIil<;e#^x4K9;_Ur&jyUr$MhU*%eBehkQu zL1KOk$jBt%2Sh*6^{d=!&5r^3F-Xjh0U4PD{IUqYES=x^cdYp_pgs%|^J73pCYqnA zlnMyHf)GDn*WWaL;Zy_84;rL?i9z9_HR^A0nFRc%5`I%d`~vZVn7=egbq`|x(jZeN znxAP_3y1%>Lwf2_*ZO5Rn=!SlS%Ov2Z5Pk}&zZb0eF`zyS67yp~=fjz3ex|A@ zB>W0P{L~{KTGt;9g5ww@u0I-N$|T_DCH%a)e*X8Z`7xkM3=;EWKt?8-pQ$PU^z$=7 z==_W>a+cQ*4oK`T2c#X8fS-%#=hF47{G)aKF(5w%iTN=gBa?t%CgGQ<^PA9S&5s7D z<1k3fj|PE}iRNdTA4P;;QHWpQz-jCHqe1E$7$mMg8f3~O;5UWvn-b!ujyq+|j|Ra6 z3=;FBL8eRsetCpnp3ZOd2iE*(5L}5tVtzD8H8RosO!GH|@JrF_uWG+_{V|{l3=-EL z12Qtv{7m&%O!yV+{I-8&tsev0g+XHd7?6>P=4aAxBH=et=U4llwSF{6U5Y_s{b&#v znFRbY2)_*C?_1XV7*GWUiTN?0s&FQnpUL0Jgx};4Ki|}o*8b8UwHSlM{?Z^*CYqng z-&DdcRoCzQE7tlkpgs%|>&JkMOf)}}e$xoQX}W&(A6x53gVY8L66;5Ud`2b#ziSD< zYeW3hhdZqK(I8lXL1KP1$drlZXYzM6;Wt|Mch~FI{xYEL7$o+W0U4PD{L%@(be-Rd zfHglFq&|#6VtzCTj7&5?Q~gaR{HE*vF8##XUm66p7$o+W2B}6S0l(`Azw31UDo$AQ zV?b3HB<9C}j7$Q4C4^r|h+p8=PHTQNNL_$IVtzEplu5vE65%&V=Xb|(Yko9H{S^j@ z`O%<&k%{JKn!g^x&!hWWc~IjQPBq~BD+c7pAo2WQKt`rafUH+?zn|ZmT>;<%aKOLU z44;=1-nsz5gPw~@9q{$tXWR}l(32e+*o+3g8fc)0H}EYz(5vsN%g4#lKqE-VVCnlbYXbsThV(56+F?7@ZP`KbJ9tT zJHe+|jJrhR;8BG8N9hfI1LSvD=2yk}eHQr@aehS4BRoEz;uMV|xKKZ2k57b0^o9DU zy8Z4@buj)#gOxrmA%Apu!u)9eVtrKc{*gY>qT@&D#rl}*dw}{Ovxlb-lGlpAkRLic ztdC&-LjFj;3-|x3GjwHK7_&AGR-GzJZ(NxMM*&R(1wh@kvT)T9N7?!7D_CJHha&hgx|{Ausjiv3wPX5LiY_ZFnLm6tzp{jB>NH})ko->j_p&GN~| za-5r%?K|r(Z!52#e`Dq7tl%ZHUvYKRq<*lXtaHMXwQWltP3cZNdq`=0^1-p5&U3F% zi0Plbq$I!pCx3tA%R}AI?Q8kUx#`@Z*Z-2;e`)a0md-ir&s|lMTi);f;Z<0vz3$>PrLgOgOmt%a8Fi$8bh7ina@3sv*Aq)`nE~eoL9>}1_d_aay~vr*`>IKNbeJv6{}EsLU^@do z02y}B0Nd$7vr~p0G{APckexE@paHhC3!Q9b*g*qqC!pCW!wwqk(B29KkexE@q5-zl zhfcUM?4kj-v=z-#8FtYCTMB5F%CL(D*isc)D#I=sU`yN4iC2bQG{Ba&AWLP~MFVW< zx5!c%cF_P^+K8^DGVG!OwzLDyQW&Z%CLzB*wW3&P8oL509)FFER|sw4X~vR$Wj@0 z(EwX|7FjC8E*fA0ysx3DY$p{LP zYy8k!JQ0ncfPlu&qpimh!3YYH>;Afs#1RY4UtV&JpSE04Bm>P49sxW*A~<#N%tik4 z&K;Pawp>#rBPd87sy}VPp2!8NKW=hdf032X5#;-a{f(%2NV}kbfTkaJ_V8xa5N{9i z31I!WvxhgUhIo5WKtQiQ!4>d_zcg$0$04AfU-$z57$fc1;$%?rAB3GxYq=5J(g|6D=A{by+Y3JQQ%7(=)n&0ih?tRHuF z^Je`JZ%6gVIlU;vU%?$Lzc_~KmRs|eM*#b)B6%YiL4IFdJ!MfBkpJ^Kdv1VaAuug|>e_Kk1aImq z%htub5C^|VDb8qgS4JIeEOt1?dC)(;nc2K@a@q3obx-BgXUzD0S?1!ZBd6xX72>sJ zYj*BD^5Y_n@XR9raKcaopGir zDzp_6TEFt=Y);5)isa$ySFbJTi1Dc&En$j z_`}nVw~pOed$=`s*{2O_&h2R(mC>EscJP7LKW(ec?)vTi+IN@a#`n8F8_jE_8Pd4YZxTY;U>j;dD(-xhFCU7oKD092>($i;@wbj+7y4|@kJv+wcZhH08 zH`vT2|Md>YM!hO9h)V4raF>%h)@7)gz@}BsS$7Nqcv3 z%Iw;UZGw5RzmEz`HqA73IvlX+9<#uu(Vmmm&(suwS&7tWWoa}WrJH&(AlWn%WYbKt za3|ZIldeBgQy6-&R8uQUqyFfow=_sL%>Z*y->NSl_kIaXpn502_TzhlKLY{smrFBpm*AoO)~*x(@Ziy*!G-)^E2JV z#MH`?e}1MxvS}uOY??{xk8RH>s6V<5jH#6+zy4^DY?=umn`V;wW7~7m^=EoN#B_c# zT3H(PN53CpK>ugcOxYStwg%Jg@b#8c`mMff4W>6zmaV}^$7b0YOmCVbTZ8FB$kt%G z5cX~jCO`51z@K;voS;OxTEBFpjYI$M%1YgmNaYWbJLtA2NN{`&f}b$gqS{iomI z+me(vUiqZJ)s~pr`n<#QnKKLZ97P>R^Y_1e;)$JKKf8ao!-oda6O@Exha+v+gSX%F zz=HG`B|h2dj7oPZS`5n`6*ppW&7Bh;s#&ybLCwVbZohXya&+0;su^e?*);Irf|{j^ z7T=fbnB;w|*HM_Hp#OSNgVU~S{C!;){(J!aJW5&BIV?F=0oU}@ZB6^MU)|husUHfj z)&|0zOT*1A^@oNLS4}C^fAyPB9lnNHew3gp7P_(yUHw?W=SB0PmF-SfbgHVlJW*Lu J-p`9%{{tux;dcN4 diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_float_with_null.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_float_with_null.dwrf index 8986e2dbc1eaadf7b0bb8e2e215e2954f1c5842d..21b7faeb21b79c5b2aabc2668ed46098bb3461fb 100755 GIT binary patch literal 30608 zcmeI53v>;49>@Q4a}$#*xUNT1qP?4Dv+=$NQ4$%g)u=~OZLvy4)vi~uLDfM!!qKqK zO0L!QT7n>+jXi9GTJJ}TuBD! zzTr~QF%G#px?HIIm*;1Eaafb4;rKu5;}4#}ANV+-xT(}kqALQ;3x8Ys?jdSQg4G?-&Tv72z!k zOXem?mf1R%`o5yEa$gx(@>t2MF0=C4SW<)xEGbHJmOwahG9ybjcIn0}H?y$>!gNK* zz!E4k$R-U~0wJ2t$g&}J*-&*Ext)zAMaaOCqBLg-g#0WM^h zMyoFGekv=~;bSmgQiKd6mZGGM*oyfQ2=hLY`}!$`CE+p#yNpp?9{MwzONx-eB}HlO z5(qPXYvhucykfD-Sk-0p0aBBC4s>?wqWTk?H*m-ea=S2}JM2vS{6s6i^g-fS?=LLjlx!hNV)p|SZvYk3& z>zrb9Nf9!*q$tf@>V>Fq1*6M0*kv2l<(kiBr5Yp#s{)FUAucIO^SIOtQNyKjUuiQ5 zu}y4`UA9+8?CvjRr4nA65nCYdFNuH#!jpJ zfl||?0ZSm{S2MEgh+TFpbGiRtN@vYkQiKdFDN1vedLb%OTr#ligk5%0SspvX#u5n6 zC_)C7K&ffcfF%%qwuX^qB6gXm#^r{yveKL-MaaOCqBLiz7ot&X<-Rho?2KJ@R$0#Y zij5@@W-CGlmO!a#(tsrpPE=0f7;@|j*yRgqTn;(M#*!jrU`bJ$v(yVw)H+6%U9ih8 zD$91~*;oQ$H$}+65-2rI0+tnL1wdG8=#rQ(yJDAJ)wuLrU}H%UGO(m5%~|S&==gd@ zmPy!UlFIU?;?jKW1BCY!Ap=XG)HG?p5(x7)7`Y@^CS#Y$YFr+>$i|W)WMD~AnzPgk z(fb8PE{U~IH|(;T%5wQ7S!vD^2sbH029`jnY0`it5T=agXQ>yW_)Xa557OP!Xjh1I6VlU2x`pcDdF1v)1?4wU5W8qDE4y4rdteim2=M_z;TU+4fP!isz>w z-JG#}Y+mB3F0EUs-?gLpNQAry?eET^tz)j%SCCoTVK_9zAPp5v zEa1Op>5v7;+$_kBlgO;?FbrlfIKf>`?3&+FVviv*he4DTs!?`YLbkCy%7U2{r3)tp zB^_4~3;3j2Fo$v$n%93yQ&*KnY^|{MJt702yzD3`f~@O_agu1zy)` zysTmnmOLb}fZy5+0FiM9i4jNHZ8FNN?J&%OqV0OeL^OjR;Ob)Rz1{P$-vBX&(W+tCmS=(cn2t{j`*)Xm6D@oX}n@_EmspV%@=MWM^X#pZ3L!l_t z=I?1dWhZF=<37?=(s1~1E>6l3?5Wy(U z@KWDLF7=r%EX*iREj(-5v6YUQv;r|?hS&N6UhB*AS+JJ?L7z}9KDW{{W zSta?Z8p8@KQ)}A66=dc$%8fYAGRQcq?rpzW60K>+AUbAF>1HPjGm&Hl&IPOtG6|>% zmsZxMYrnN8L6Di7g;_SqEXb+xeoZ?e)-&UfVVvRlY@)=h{C20cqYUnaf_~oUw>Gip zm`STZrZ~$XD zEc00&$;{dw!&8?Ue%>f;&zP7Mkj$zz5x}qlYr-0S-pC;@6SG2+8C(^>Sa@oO+Lh#e zG1wbKf0d>uhEb_qnd@?+#7ugy(fnM$2s4xKd1_y=I_XpnC+KVa+{*h^U1M1k%WuZ) z8Zr?5m4%U4O3TMSw1pA<(p zIPHh~r6qvH5BKYEx~0&pB2G7n^NABYh_9;Us-QoVWbuRldYq0b+Ut$XR{8!bPRB8SV`dM|ZwHE*al_Io?KoNe>C_kj^ z=jLYcKd+1a^hWm<;;g=C$?5~ezKBDyl9tYKaQJ8uu#n+#2-+2DvIXTyV(!L>o+WXvwX$#)m5}P_=sI%oZ zN6yG^k8WH%Z`Jf?7am)?nja8>-iYfLxuy1|aj$k?{KeeYXSX~5@$jfyD`JIT_Xrul zCH0u=;F|pXM#2OxqLe66I`Ke{!$pLXNww34Pr zA5AJ*dSvfwSK7{-{dY3nIz zwCJJp;G@`GFAMk297tYp<++HGwOP|c4$R~-)84=G(roeDg4IJgF-)jG85K6@At1L; z$ct$Bec0eNy}A!bXnuNroNI7yukO7QnjfEk(!ZwZvd78Z2OC?@SsEQ*+R=L$r9GFAM)PWw}gRlc{^&0jTU5W>m!Vg%-bBEe%Cp9q~n!6qpp9|C@puuusgjL9KHNa z;Krlm;*(2%`DN0i^-DXPS~2=&pAM&@dUf3}<6y`%yQb|El+msD`ChxGZCPbnyQZyX z`0)5=we6a=a;<57ePTDhJh&LHUDH-Gd>Z2GQ)wApkbNquWC5XF(^gZsEDB#(yQXbr zHQ=N1<haVX@IYwSwAGAQP$yPXV3w#vsL!!LXxFsW z+}XxFsW#3d+At0|*f5tnvN+p?XPUDHGNTP*1p2Xt`U|%KYu}w=OAdoQk@%i^s?mMp>qe_j(rN+`>*rg|D1d7txDIaJMW${1EJV>Pn@fKdL|OexAn?5`Sq{;{P!Fvc5G}c z>ip*!>blsy|A3g_fgB-0{^4f&L+L3Y(;-|KfO%xc_C%{RPD4QmeO_Ru$)$l3H081&5Ls$ zQ&P65j*lz=9C9=NP_+pj;7E&Nb%rZ#Dq=~ZQWwPU?Fz|fBrz`;E zP!>Q!%Bo^^q)c{XkgOPgjE^h;shHFws~RRk(%t}};NAf5Ub&P7fE>yK zNJv=~*$i3P3@K(`d5CQZU% zh)i~fq-^~mK4k$Qhq3??QdSiP6J@d!gJi2~8wpuzQ-;0zi(K1&~n8s=^>wCYvjf-O|LTECbv{fgH**NX4XyvZH0Pql09H2@Qm- z`2+)i91{#cLK94tXT366ucWMRKOt+LXBl821#;wB2C10TQ&vSbRVJG%kxf3xN0tG4 zD3F6JgH%lFkyTBwu`=1QL9*3fe#jRF0FWaL03;L!DzbSp**qy`e|nmaECA#n3m_q~ zs%$VuCObw70}oZ!d{JP4$rQ+u4H%?i(j*L0WU?s|*~9M;WvM`_<(A!i13;QajPG>; zNMf?qiNyAfwvIn`eAaQGJ2u6A7aUq!17%Xa-?7NCKsY^B0vprRG7l@%re`@rc{(+ek4M;ZEwu6|**(P78{_gbE^{9#`I zX)IrxqfA9KP5XZzJ2|T-tZ4OQ@|y<2wXOd&W0+p&huIB(AAgauSHCb%CYqPcR;Mt} zr;Z!`o?NqiPY%a%xbMjYcXAe|49jnDhx>0OG-F)b0($KiZa4fl1%fJDoS1L;D{PuE z!hGdp>cboU&Zm8aO>T08`wE+?Ykx#}emml- zFFI`+LT-c{sH3b$QF`amIS6@i$0P^3I`2Od9C8Or3wHQ%$JgqPXsP2XEy5*rAl%_7 zR-H;lY|(`~>VA+pQH1uZMth7TijwW`wEkWY0~ z+aaGy(4I=}K+re}wWpHDQH?uv$06oPQIaD~gPQo~@K2&U8l8?pfA~euKC63Q**Vy$ z?RFY=do}2BohN?@KjSJokl^Y(`=556bZYvYCjH18?J&W+jrx(-Wr%m+4;bOyn4nQV zrl-iGGd@w$Cn2b=&iF-R{(>=oJUm1Dn4Z^&KbGHV)Q`;@pC6g=cN?A`(~CB;?=sOJ z(>LTNM>nK@G!l*V5smo^I6w^fo9T}U8sV{ddVYvMc8_3$$HNOfzb7iVGUmIk#-XR5 zpES7KaSE;d*xCF49K9#B3SSha{`}~=l$-ZXD~Q>%Vn)v3$IcDwmWT4t2voOx_|ujB z2YwJp{EcJVv-Pg}3u!Ob#Xm4CMG|M9|^Cyu;&*E#nc!w)6)nEyoO zvZO=vOJ}aVr(U>{-*3z86{*t;p2c`~sje*Xwp+s|579lYA- zOz-1w7q-2z_gC%RZ_b)EcYS*7`aAk=J{G;Py?F0|;ca~b@4Rs4o^|cR7LB;0E#dR~ za$B~F?X?RB&O0-Hb^C1#KS*!c*ql40;K)bpkp=iOV?rur4RQUXvrrcEjAZpQFK;}u}ikAs+KLAuLGEbZN))3$xQ=qn>1*&|}h^k_dc{I0hCm&Gzm@IPn`G8D? z2>%WvpyEg7@zJVs0yOs{^Q^RgB>|eNijZze3iAM99&vPSB^IFoP{yb&MSvbyXEa77 zo=i! zuNnars!Xx6c^i$WD(-N}ROMP_B})hpnjH!-M+W9d&;ORtB4-d#86#6rtM!*gV^l0A z$`<8vKdz^Ys4C)UtF@rS%|_xVRGFerXsHoZ1*om(hmtzSI0&IHL;t?mN)CnI$fAcr z@4EQNr$TQc&`*W_*_|E={o2V{3Jm!oW)K+iha>u_kPo%fPlevi4FW^X?*~pF@}UD1 z7<%Jb$09wXBTneu7kaJ`@_!8S3ORo@;iW@DN3T$!O9(LZ^puyW5|vbm5F`bLIxTo8 zH)+2Gl@K9~vg~TpITGM&Zq1$t1f)h6dDKKI|^LMS=#@U4__Pc8dfJsJp5K zgzOdx8c=r$cy`$>5;UOh%Ea!nTO?>e-Q~sZvRfo*K;4C~yX+PT8q6lwT}QFI>=r2+ zP?w2#cWAdr(SW+F0nakKMT!R0WmVW^c8e4ZsLK#ejCP9@4XDe$!n4e7k)i>0StE9t z-6BN;>aq&#GP^~J2GnIDcA4EGMFZ+GyztupB1Qq~vQ2oF*)38upe}30F0)&tXh20nH$eCyG4oy!C8h~cGhkYq5*YTGj^BVB1Hr0vNG&4yG4oy#pF+%)!1cr zizE%G+XTF8v|A);K;7oYZnIk?X+YgpiIb$=B1r@4HZc$1BDGrtX%I}3c((cN7C{bdhwEwK8X1(FsLu7atE85Rm+9utJA_>p=Vq5TN>59DXPO6A1bj zPai9{r~w1xLH`;ga44~N;c@I=hyc~Ei^JU@+VvL-vi<@f$`1<8>vcR}V2Jn^oIM7( zuEgN^0Zd;P)z8}5LDpYzddl zH#F&&5S+hwenS@k*W-A=z=-5ugA5MFZkJ(*e}R)p^Ed1v21gi<6+C|-0`@8t#m_mn<$nYupiDHmTaUT_5I<hcl|l(Vw3#qI2I&J~Lxw`}-w#HeLO0)y5S|i%SN^v~0cb&CaDm z7Y}@?ef!>(o1(;BxEky1sCBng_lb47kr$zE?$>)Xyt?+5iGObX@bg{WP zKe%<@K<~z{K8qW+_uhxHw$z>fe$%4*Ufciw-Q$Z+Ex2<>Y3=znLyMy--fnyO-8FCQ zIr+ts#k2f_8hX?;-S>k}nm&4T-^f3$`aFBrj)^m*I;`c>Gm%iftyeyZjrYX4z+eB` z&wtN>V#kKma2@Kp*uAr)DpUHe^b{$JcVPhLksaF;t%|uHe^KNS2dWt08S1)h04$@x z76QzPjYsSIOhxD^$NVVNn1zviCiTc(Qfz9gKZ@D!60(;xWS!`BLe?C35g==hBqsI9 zssv&RNBU?vPAZT}StqI_WL5ewz(NY7F40kz&!iq%)iAY14y$K=M3kihsmMAQWdR^{ zi4IvllO|-Dnyf|7@`V8a?3{o*^qO7(`v*7uJPgwxSp)7!elvTy7wos5%Nxd*or7oth{COG%=BbMTUZFsa)WskblX_%T!vspp zn=1G$G5pFl66xKC%q3o&q_@GDyXw9$8fwFt5}2Vh>-;0zi(K z1&~n8s=|PIzsZ&+K4lr;E(+vOmO&~eO_bHXQ$?82K**X;FaXFg!2l#Q!Blycd999b zKOt+LXBl821#;wB2C10TQ&vTmd3Q|mK|Zn!&_jV7WErGlQje@^f@xpLQvKzJd|?0p zIl=%yLSdjHt9>bp@YB9~=~CQ7l{H@!7+^95a%2MrshBhg zgA|!;ibVGCJ49J3kZQSQH{Sq|rV-=i7C;h{_B+w+ccS4t?Ai^P{Z2HO@(Ld9Vrst= z&856biAjX^JJFQLVeZt~??hAHrNvxc+wVkErd5u-OR(RGhLs=j{R(a8pS$TPQPW6s|dvW{j`j?Ey78NgYskLuD z8uxA2zrWV4^J@&9_h9pad-|F;#!SL{`v3a$ru>ihIfe4+`PjhSF-S~bJ)&V^qfiv> U@;lwpNuucXMx{k%HVW>)0B~?dasU7T diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_int.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_int.dwrf index 5395f2f338043a31adf638c59f0e2003043d40ab..21f176e97770df87a63ab071127ef0a1b39e4467 100755 GIT binary patch literal 12318 zcmeI2eNa@_8OG1uFD{z}50?**6zB@u4IA2F3oWBzCwti>6V7!7J` z$4;{-qqG{ch&4z|LKA}M7{ydaf~jJ(mRjf_CdM%g5=^I7Q_CbwArWfNdl$C5_ug}M zXZ)wrS%$Bj^E}V{#lEgzXl1W4u5kw2Lrqi?xvxt5b`(uRYkR%=q9&er+ zuTLyR=yJTKKb`ttPq!m9kt`*M8-n6{^&X@bkU2e2KN0H3h&rqzAH`1rd^-pr5di#x zJ`l(RAebW1On^UO3PGX>{(zeT4PY`Duz|Fua~MF-3<4HIXo8vn#(VWV1`{@s@!1>( z5c~@SEQZhoH3N*d8+i;O7D;arhXDj#AYd_sCa4);yeWpqV8&+BypY2Hf<_Rq7(x@& z3@}~`Yl?Y3V2nxAMh*iAd>~*kgeHVCtf%KBjJLr_-OXAT7HlD7%Q*}n_yPnhhR_5x z1B`!e<}t+Mc+$O!!vKQQAYd_sCa4);{Gw%&A(rkB8G-#F3HAq3l(0lbw!-q2cPMMn z29X3CgkLcvD`W~#OzjZb8EOnDvKR9(l5i5~+ReiN#c^4|!cb(WF+eb$z{5zvDP;d| zc^IJBCM#GNiVQUd2!;}Q7z7hCe3XX)imS4Mg`vn$V}PJ5nTL^zQ_1<$JPc4Ak`*ir zMTQy!1V^UxFw$@uY3Si$fMUEG6f6uyh8hC|duA{&7Uq=yUP}ENF4vHc}jR6AR93Dm?E+j2G z;2#&(_63T5S;3kOMTQy!1Y-^!h6}sM*ghTxDB2o9!NO2vs4+k==;UD(;UaRem4^X} znu9zHMTQy!1YHF@jAC3&_MhZofa0pGV9kaiLyZA~wn8382`(Xb{=mZk#o-IQ*-&Jt zF+kAZn#A}@et@B;&o888`V58M`y+be?mGkj)ONPh2PE|If&rbFZsraDBcD0^E$V)! z%q%5}eX;cF4&fJ{q}g3|4@#z0bq14Yw!~w}fl&5|ITIE8vo<2+h!K!LKT&<@K%7uE zIaCqfgOJPX#yaFZV??Hf&Lxf#gH5zGJ^W@-9(_b22fS#03?XbX3F!H+gcPM$jKC|3 z46hKruGy^VMP93c!@N#rb~ps|oSGSVad@xO$@1#L)-5Q|_R@(C>n$u~c*XEumy6{U z!qzP+()L=#@ZRELhSw0@TT1v3;Oo+Bwh{T+l$}(HSW2dEQnhkV0g*RKREHe&o&dk&{3n+rh85x>p z9bL)cE8A=gvpWK_-Oe(L;VWlnYntg{j=-Fg!!TPTFz4p7%yj1qx2p2<{pMc=mgKtv za?lMIok7K|!^_aHF*afR-o4vLh3_1`-Eri0$0_=C;5g^l~XrgqKc4UGPLSM>G-sYbl}^Vz`zn zgJCA_D_Eyv1k zYj9mn`Ao4qt|)7ZRK$DYbfI#khv%FmwbHO@Kor;g?=YjO@Yz4vyx zll-^%{y%vzz_FjUcCt~pn0%ko+5_0KzAdHJrX zWBZQHWiOs79$hr_?|GxKLj!UBS=%>%I?=iArGin@`s=se$Q#{wM)z#t-v{4Y`Qb!q z`{QZ-S(ZD~>NhVqJ5jps8Q-1Q-J55;|EhKL(v6I&{`r@Ff398dHng5?tM)Gbb(6j= zaWBHSqPEU@O<%Wq)x~|z1KkC4|GH$~6RR3mKQis5hSOhnj%>eH+I``M&3ts(j}n$` z`pjn#bOwX?{dof?9&OrsW8LMI=dvz*w&ai7%64vjZP$!l!|!&DKG?Z-S>uXdjJ>(( znFZA~?(bP=`mm?2o7e=ryZXSQYx|(^wj+;jBGTO8A_i&>#A%ZT7;Z~Lxm z;FXr+H?B0+ofX_eI5SbSNIJogYS>V@enYAezB@Bb;scUSr_a=hfv=qD=4orIR_6b# zYW0T7s{9vLtf`a?rOV69=ztV7@YBkw^{dyuAPI9Fv|v%32xXC*7WTYe5O~=(nCIzm z7EKqo^&6#F+6XQoZ!{kRVZ(5W2NewqeX0P;g8sm$_vN`)eCBan&whX?bRc$#hIRHyI!D zJi#Pb`1U@F2N79~k`cGzk@|4q5!Zo^=!AP;NW8RJ2Z%T;B51lveaQ3lBE5tiJ6Sx4 zNEZ<_9;pu(9#PvtWRP%dz0!F#wV5A8)QAY${78Ms^9&(FgsodyJcu|bB4|8PAM!jQ zBt)pmXYn8+^BqLcc%(jDc*J$kgY*!}HnZjj5qm`hZGNOaTzJI!2_>Pzfe%^ZLBv-g zf;Jwh4|$$25+-aq!s0W64-y z$37MhA}U1$jYsN3UNm3gE9A#9L_@$E2%w23@gP5#Xc8^l&tVM)5k+sXE&x&=@`Lds ze!~43jKLt_9s+2Ck$8|N@hAR5#R3Ki0zO3mjYQ%>UNIvv3a1KKiXkGckfoT^hl=Bj zV18i!V9{W{V57l!u(4o~U|wJZECNgoW&l%y#ej_fYwCgzTk3a8k0>}M)CsRAZ$uB38M{GD9B5a2^dBlW$#D>!$Q`{j=Ji@wO z)jLGk4sqg{9oN};#5C!U;eeB9c$WDnym`*CZhwK_%z8Cu+6Tk{xC!A{MS(#puto`YT|8u-cXF zw-Z)>kjKPoSHc9c+F^O?neFZLhvmcMS94~je^|clwM+fOryTL*=lPX>+~?Cx!Fd5w zm4;F9Cu4V$IsS3&@dJM-e*94R(#gw;_pOgw{KECr@mvfyg3E|V^X(f}oiw1V_isy& zKRM^~seP3Hx_z-0t+QTMGMAWD&!MAwT zktM~)7eqak7dCRP*89c0*rehM+q0&2>-DGSo}8m2Tf%j(tTCKay)&$v@9x!#%y8r2 znZau#&nBB*SLg1?Rc>q=ykd{OcFWOm%4wxx7}BWzXL-!(2l)vxF;~`Syj5R*a%fZbn*+wY zb>p*d8aIF2H1GSDUcPqyeV>ZD_}j64CI{4Nt1o4|vJL*3er#RN;iSqg>g_>mjX4|N z`mWB>Dq-Zn?fk}s(5q88lVCmPI zsSV&$oKn|R63pb*U}Bnzz|>}TGz)8EVag5Da7K9TU^&OZ&lTcN=ignA*jk-2&c8n# z-q!1(e?G2&`!U}0-Dp-iIfwD#BegrtN+%cN!$*2|nw3uO&<~#~-DapCh1?b&J^oG+ z)=Hl}F+P)fmmg=P2Vh%&_)P6agY%affbIMxF#YZ`SbsEf`}|T?CvDYs@Nt-5S^&25 z13&Y+(X4cGn|?HPJL$NUI=#f9A1qKv`?$$aKMJ`mKKlKyQ*O*~{iEIe#rWK~^u(2C zr3WC!M-#WRj$0W-;`Pt1S7*H%)*tN(ZJ%Gt>ST5hKk5@8#z%X2=&a*b z29Z6!PI?lMe-Oa3-aaOXCM-DzdolW9v%qnYs?&8{&PEYtAj7IA~1P)bw>z@|Pq9 zuTIvDc;A$Q=T<^n7)e?M7=lpXxur)IREp8kvA8ZibE7p zin7bkrcK(EXb}9=!*4DaI4kY7No_XIYBG1+n}0CQ=hB>>hEMp4E}7xF_#SCvbQ9hv z&MQ9EJIiVJY<_f~=A`{I!W!bNx}8-)TT&++2p`n&T2)|X>iv?Sehp82`NPQCzSll! z%>Vt>)w%H>3$Hd6mi%4QAXNWgteTbHv}<|7mF9^h+boA7OET0AmgFBV9@?^h*^bnA z_h)3Q%pc+2nZf*5Z%fxef?H<_^*@v=7R{Wb`_0zZ*KbX{y)!Wc82y zI^S=DaoL&7~X9EZRjP^n3MrkrPjOmlX-+Cx4yr;!mxC`>A-BH&?|< z3{TCRQ!R$Qj?hyxCs#Y^shMk2b1GICN$cI5bZaS=@zl)8=_ajrlhe&pGbgQXme#vD zDb`Zm?5UY+eci)e@GY%(a}urP^z_usxripMcasy%Q#03cZi;sSX}z1AVxF2gmt`ic zcasy%Q#02pn!Qd~Qt<}gfzkRE7xqfy|KplD&ugKc*Fxo9E%m$>YQI});2l_R-t$_h zO%6_@;XSW~O27Z(fB3afPg?UAnb!1H!ke{Z-%{-W_~(dL$kpn%t3rz4%cRVWk44uv ze7xsQ*3MfM3RA9DtLK}e$d|lO8mix>3V{*=C?JSFzwuyYcHQ^8W|&GtgZ+3vjY6UA zvTE*Q9sR~x|qJ{SM?Y=DOcOza%TAY%YG-B!8=My#Ri8H26f(DIK!>Yuj zt8zL@~~&re7X`|`8S)1UKOG;TpmXL1MG`?EdT%j diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_int_missing_sequences.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_int_missing_sequences.dwrf index 64bbb4f374414b5e775ac8eddb50dbe31bee106c..4d394b93e3a96f069be97b44f7ae380e44854bf5 100644 GIT binary patch literal 6726 zcmeI1eM}Q)9LIlmz0!wL@bM^eAjJ!-QXOg|M3E3v6j+^ak<4kfnK%r|3{K_68Rz1v zTcS8jP|!HzW;Zbd=9E7qqVuI1OxS`6a}IDu)XhW!j&;#YGfUj_TxnOXcUKetTS_Q! z{e8dR=X1|J&+$ChWhD!?0vO}@I8?XP4i49EvCbDcydF!$?!$}|wuIU}&4y6B;-QH* zBFWxY<%DO%oo0*y>5J!4i-C_pfI$F~-((?S=|%)Kv84c{#G~fWdSjqopI8X+bAqDZ zLi}G?rT~m2@#9f-NPM@>2|5OhmSFpdw0#5OFefxw0sxaj01_#HU(iSe5&;zi2ow|O zDFhIBg5VG6){y{J285B&ou|P-1)Wr&#xOP^oPpNe1}z4RvEbgU!9WFdRG`K%HeoD7 z73l|BZ%rTpU250DgiXTJl^P6G@Q@1B7{(@qGtl~d<0wNc84u~f@nELo0Upj^ju{aR8e=FD%3D!8Q~aI;I?XESg}>;*rkO* z6$6q&4MUa@jzI<8q83IL&Jt>`Xkk!AyQEOVkY$8pP=Pm73&W1>LhBtZ463**Dbz4z z8Q~aI;CWRGBM0XQH+)(cRIz)6D%3D!8Q~aIP&ZQxBNyiiPd}&sZGzKLHbkkS^)OYa zVaPJVF{t3~EG>+DoG%5riwhlDTkuI_c|-2gfO(TTqk7tTUOKTFY0pz+Ihy~XrJ;W@Wwb(w7vm&c|&e6Q- zWceZmR2Y{4V1yW0|A?6&^MXs{G`yQ6uief-xsn&N5xqG%%3cB;(VLsA>=h9c(VL&| z_d0`p^$U=tDzTEj0zK%JJIjB^s^(fSrC-X42Y4Hi$6KWLM zY_qADT{yDYYE?9oQ+|P#v$Fi=?ZHd+9RqeUFyRv9H+h$XS06jx_x$rpP z&V5f_C(8%PpOvfM>D|1#_plfDth{3g40-=m#8@#GvoD!zHv4m3LEbioOhXCSMS%B( z0yn-8uz!vMnc#Ge%7ZEKO9gYBKRBKB-x5r^=?JdxooaRO0(*9Lw%6t8GA$Gn3g>*A zt?wOvzUbID4Id5GWF9^L;DVvYdiDI#l=eCL;ngs_;&DG$@n`99QNokb{%y7=+i#cO zz3}?&w|)H$!$lW+pY)fWD45r_cyIIZk=nU`PqrPvJ@awL?@!~8**c3B;yuVbnRic} z`BTF{+_HlDxo2%Z%=fl`mAdECu1z)N8w!@UWWE>kVe`dJTfclzkyUYR$6GC#F(plx zPh55_+T+%_6ITHi)^@pT9)BHIT$FsZ<)Fv(@f(xc*H$;3y?LRrqod~0f+g#3&Ayd& z`Q|iF^D$q>Ukx7#;yAi2jhRItx=^GfrfooywnTBoe*uiQ2+x{(cTTokvV(T%Ks zd85z|(T!|$Bdf5jk8WfE{@YJSH?pC<5#7j!_C|Cg8`>NHr;TiMohe^ujxd09xXY3% zl3zW%a1V+5S=-LXjNR=?&mDaB@XoKNJ)gbixQ8+lk%?y+E|se;tEx`b)1R|4Vo)5< zvbqcw1z*#WjpH^}uAQ-^a$R*<<%|vQzF)?3g)2*nNr4Y3d{9)A)}?Ofbf9C3Qzt59%m+hc3)#?NX4NcaGUJ9>(CvJ;lyc#oyEqBuj}pF1x#xGk z*E#o`-#K4rBu}1?kY9k@pR!EXAf5H8pZ-7oeOCNstUtof2RVM^!PH?NIYlXHC9~_Q z2%9V!ge|K4D748p6`>F%MjUb?+O7b?=0tp?NJ40M9KQ~JI}FV*Eu}(s0MW=INOrh7 zy4hj9RnaQO`WwevM?3?a-U6bMMRY{$iibxBznv&LimA#Mcg{)|JAk;yB1m?)I{59V zX*F}tCS(T?mV3eku`&YXmI8r5!d;OlAQB%+N=uoNLC#PjI)$QuP#gdo0N7Xn;EaV&1HTcPrWtFFfDr&# z06;Rr(ZHXvG4vRwp-m`bfY|s%Xe@Je@CQTD6jK*35DWln06+?cqk%t|1UiA)ds5<5 z;+i-+CK^CAvItW5b9M0B(b8Jx@y9}T08zFZ5F|TX9sEg?(K6=o9H$bVVFrNv03aoe zqd|g@42u>Eg(U&Y7%VgvDHaVDge4k_8cQ&iC@d;0N-WF0+QPLy5gOzCUJoDAaCn;3{+wmupN{W7xpj%sIiY8=J<}Mi}BoLQc)sT`@=)m#a-HwjuLmM=WE9bgxc}r!un0LpnlxUJ|E-=ue*OZ zej)qL^pKtxAr&R`ycj1yxZOkxwd3%0LhVvyQVF%=?%}q3iygghlU(&dqhv$Eb?HsW z&^S8d&E}@P-Dh!!lHeJ4V*3VFC ztcZK5sbK9$W7B-~?_<|w-Yke##C}^ZyPkeNw)p0`1&P?kD@TwJUfl#Rgxv<;{lP&c4+K;MAg z0fhtV2ec0;AJ9FZdO!iS`1s*Hgz(kCUORv8e5dpM?w=iZc(3C{cxr&x-;H7?8%N+D ztPbuJJK4CyKeGS5l2|*r_gMe%+6@bo+r%zl9QxPI-HYsG2ORwi^uUc`CmUz|bE9jl zXLhm!?fDZJ0KG6OasEgSIPyno?WOy6qI)XVzh0R{$a4r~Cp*CU=iavind|}%!2IIY z#RIP$b^!+*{*hmx-k5l>e?$ksKliS_NM{#t0Q_^~(#FmZb^!<2{E^?EUSSS)0d6O4 z-{_S~s(n!MmLU$m4BSQbmMcc)<|jVr|VX4{koN|I7u5aw>IrrHS)ylT?<}|oBdhR_?%U_ zMrz~Gv`MmYr#64^@-A!gHO2hAsnzRC%%R?U5<*8P6Avyqzpy&~_$2R`%@Z1B@`owo zXKuditBT8?=Eh6 zzhHU2*D0(2kb>JAj%2OPpA%w8GEJ6hQ%zpmHid^QN<5V(jkK6Frj19(iyYi4AYan@WF^sacI!F2l~bOxTke-!{>Ry^hZ|#VTrYA|f)IM*jT&T> zgubK=i@<+s1DL`9Ri0Q~hQAxu{I^P5+ge_67}DPs*$Z4j$*{RCG} zw8u{0F1KDO_#sY*B4w0JE)s=#tyq|mlNRYs`OC#(Nu-z>-rrB2X@TrL-l#tnhX7cNVaOBb(9&(xS7iQ)q&e8uPNwvtG_d1>W4@%^d1CNH@!{w~u5 zW}ofYu9i?eif6V|`0Jl8(OIVJF)-N|)dn;K?hM`=I#=q25^mf}->`U}a>nK6odsg* Tz6jS;OeR_-QA;#?L+O73E!9+l diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_int_with_null.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_int_with_null.dwrf index c95b96ff754831522d0481682df76218cd7d65a7..62cc3f4e26a0424aca7c2830794ffa2758286275 100755 GIT binary patch literal 25486 zcmeHQeNx@w&r_`JQQ zA=dlt0bi20|Neqt2X_j@k{+B7i^wHleg<1RBw5iU!X_J&HgpbpTyf zV7f6T%@hOb;>!T|h4AADa5hE>at{;(w%?Hfz*^nI2Jo)M949XHnyfj(4n`x3M*3J3 zk1dK6bxUj7(|f7)BVApd{5ds4<37eXn2?modd)0T{!mY756O@*$I8lJv!!YK&17moW$uLF#jztcwEP*0_C) zq6ig%e2l`#VBUu5@z%z*USpJumwvnk1$2HZv`PoNFk?K!D2i(sh4Bo-s85sC3`>D2 z(z7Sje2k*VjDZ-VFrJSQ)a@K8Nrp;?=~CA}uwm}C&r3=Ka3?5?YWmjWiGB?8H<+)K zN3fK5Lx>0aI0em!7w&*kX21;T;yN9I@uZ|Eu9OtUlMqu%1~p>25nNu`)Z6~)R9(I#uS4E zU<{-BGSwK%gL%@QKT%_hqPUDv7>_ZG+LEQlSU${`syo#f!>B%j;`*3kFh7jp?JF59 zn`0PRD+^$O>vTns?!@q zQ*5US~mH~p;$D%=0C$=Yv8edMZg zuxn~CmsF)*sUlA=SA2~hPvc8lYuP{z`GLaWD_^krVD!@S^m4^lNr@}IE~EXtrpeDw z?vvZK-Jdb))CMz}{P?FrFtpzQoP@ZKY!Hn`0lbbjh*&Uf&1KKya)|YE*0M=Dm^oTj z3MLLhFj1s}X=^TfK9@tiMXhC{fOV@|=T)p=f_j9ROfJDliNjBt(lSfQ+U}^?gyPVa z0K2Co^$7r_GTLz()wn375DaZON_i3H+SMl}(p^34^V+)nP^(HQ1VdX6u3)Zhy(G~H3-y{m)XGvy z!34fwt_Vv?@`^BR1B%&{4z;=zD_ANoCAuOkIhjUSun+cXo0_3k=uilzN58pL7c4cE z3dWrL*^RFn=Qnk13Hh&19op7K==1M(muEIpk@n6~D!*)!y|L!7?(aVA`6?p8P{z zOLS;kOJoHT)z`4hOe&bR=CXB_Azn*F$FM;NmX$>X3u^qc0mYothj=X!?KuV^SWXTV zjNJyXQQ)rzc&pYvIn)P12fMBYvs|Z0iAXSXL!w%>J~uZknCCd7v0!UFzh2oZ02{jB z7qo!!jeRpfpCKylJEybf4Pn%to_NhGL3 z4dR#|^3qAIwyUx%dy?z{0X2&mdAZ;>RB~WroAaO_2`BNd^_F?5w54C zel8kfK|#WVoEpP45Pc7L&-UE>`6JUFk&ddT|J!=zY3axp| zIzRsK*vaAz?{{99Zpb#uS@FAujV-TTIbq|Ntj6;5wUxHBLXW&mXYq`%M3}oyzaa9> zcV}h3wq8FwSJ&$JdbjY{>uvWnT{gvCO1ZcH%jT-2{*RWo+j@5{Kbh0J{oi zXuuy0leQI~PP_E2*jzku<6NO~_QW}x;O0@sDx#L*AEj|Q7xT|_Pc44>^sH;ws^0$o z?-nmVxugHeTPJp`y7JErue?-v{DY-a*N(d!pW@SPEZnRt+^o#~u(0=2zy=U*R_1Pj z7ms@`MT+5OWtfF&C0^f z%Dj8ly4^z2U(Blk3+YQdaI-Q`RMIYF+?F*I#%oy< zZdOKpEZnTj<72c78P~@sjOSzFW@XgJ!p+J&K1REcaea)!cs>?xRz`j7f2>(qxQP3I zRKy*o17ZYh+A1WCME_nLEp;tQJrR+)_8Y;rZbyFMmyLV=_|f^#H?|3-U5l*oVzgN& z=#%s-DweNEGT`rdR--t~tkXqUb)x4hYq;sQC6)7XpQv20qM|Z)@gs{W%=)sK<&)6` zv)6^6R8%fsuw=1WxZ8#%EQ%HJw4fACZ131eUtc?sW4~S}6b%=jYBQLlQ0u%BX-5NU zb`46m^d;Is(e2{S-2)e@55J0A{C0kxV1I|a2aW0O<@c|;IP{k4Q<3%tp|s|HJDR85 c0IrTco;^!1R_oAz-XO~`H6k;@c03#Y8<(s<@c;k- literal 32987 zcmeHQ4O~=J7JoB*IN$)!;p>UWj>wmYFd{#Gh-hS(P-%$uE455Be!G&Uf@a2Uj^-*@ zx+sX4n@$Ewy6I?|NMe+hA7ww%KFm=3Xo}1qV(Q|qtL!~@KHj`x?tSSnyp|n*U4r}m z_ndRjIrrTA&U^3PM{^#S#4z4Is-B9Hu^J}5q|!H2FGV z2w_9GvpZ!PS^6)15Z4HAB~$G6EW*u`Gpc7gqAmd{(%XkA_0M3KMY23MCOWMn42aof zdKs98vY}kVhrqO@Bhx8}+9flp0;7H+FttO>E>ql7HLK>%od%}34^};uS@o1hVSSK# z%EN$|U8Yz#j^#LArC3h^)%rG|s67=Dv&$6sG@K3R-ftG`DUYJ|l!p;}ikXJ7VchcL zz?9}lJc{Z_JdEf_*hM1P2(I~tMR7u#PDYM|jQ}Y9KLLf^&5lj&5F@KMijCqfyeb!Q zBpyXoFAt-uH;4`5c9n~%mq$_6%fsmEjbUTBl2y=C?3V=f0_q$_QPnFXMpkbu8_TWS zD5hQ>MO7~kBdQnMK8}s!YW^YCQ$Urx3n*$&g~Yx?vA3Tv_NpNEdMX%};TS{=_aNLR z^4egqJ^S0nT^kV5ns^ulumN0ak!2WISrdhNOVkC_5sae7A|WxdHRIWMuC`XpnmmeX zO&$j7XhMn&7b?~t6x%}~XXV^CK4#7Kx6!OE2&*>2R<*V$qHeMOXjlz*?y#8u0IL3o zsQ(Cw+3hAWXb*)u>(1%>n7vWh%_%B>m_?1CY5tOZ3>ES^>p*+lNguf_a_M^XKUhtd7Vll9~l4-o051w`$p1x4&8_Map+i7UD& z=0AXH#VBh05fZa&O>A@ptKiNhi}(+K8URG~9|0+HG#J5-;4W=(L)1)4v#d6)*I+z~ zYE2#n#K=sO*<^0@RWTRhQB~G1fZMa5QyqR0#am-EX#8BGetN8NCzM)M*%6a|D>|1+=17{ zoR&vX{fCFq{l}N}XS#-?$n)V-F~aR%m36$#n{YO=VS%cfR-gwuc?=kn^|X8 zx0AHp3R-qD57E(~byDX<8Jqd)oIHrJmb*Si2< zOzg|Qd1N_hU`cLfos-Rc3eSGhE-@Bron*b=JVtd+RsfC|5-#F!$k0kXm~_9{9`|Mr znVa$KCvh*e zQkz-lWCdVfmU@cU-rJngdA8BDcOET!q>Qtlb9so~SZO8ARiV(ObFu>P>?gG=Y3q#XzBL%emG({IU^-% z#BQI(Z@>{lVl`_WTqrp_?wpDk&7L^!OqEHD!B?6T&RKihFqr#yQib}{rhH>P-Ajg^`)`1RFa9oC9hh=cQ z7mmx|xE>s*;CL?_*QDdU_&rc?Y0?;%eh;>A+$SBkrsH~WycgX=2FH8xd!XRBM>$Rb z=^+KJGxj9ho^r@%4f;*1c4tnzG3Qq){a^RH{!V|n&1F{PvRv3Y)k6GDwA9fuYK6=~ z{nvL+%5AQz-DP}xJJ*?Xk@gDF_Hr73M4rwc2}o@~=jdA}kvQY&MbgK>k0pnn%DJU;)N-92=> zM`>|peoS%Nz~{V!BgbEO>!LUoc+1IPB2$@Y3c@&%Gco*)cS7Wqq!OQ@Rf-7*aZ>c%X9ZMBUQ7iJ#n9 zkp9Ss;bj@?icQ_T{^8`7 zxMS*_b2hjwe`7Ij(M_#9X}w(<6kQuI@!`E z@7>_?^1!OxktcTPh6OH82>I&bx4w@bTk&vWQtZ*tSXK0#S=(>aM{n3!85y;6No3sM zqZ6`D$F&}qSg@pZ_whc1U(x^QIc8zs3kzQ!Gx57GGyQUg|1A3hy6uGBqE2|B-J(7_ zFUw>)){M5d>&OhW(*t~fd{cwI->q@WWQZvInQx*AH44#E3pd*6rI=y0feDoq{^C4= z#&9MQ3%2aQn=^et6AM21uTG-Tf-d5HG2jzrqFlfcc4k=VflySA@V)<$K{Q&x5pRFs zNkaig_>li&1&!nL2nFB4FY!jG-84iM5w(qj<6Bb zsZbg%kRHCN*TdE;MCm~jup`q~VF*RlJ#5!e5liC;eT0ofD*O_O8SLqh>ayn9?AjUz}8+x9d^Arw`5*c5Jeu!&YbhPf(yAk+#~!cR4I zP$f*{pTj6YOWaVHBE=}-TLwn*f|p2*Rf3lFn7p7RMhPE4V)DY|w2_tohEc+&JyS4w zK}+0FAXb~5hO?oe5M|^jsKX6~`obuIx^7{V@G+5*R!8Xtinwh=3F>f_pg(tTl^r1j z)UOz630mSRp}uBgeF=!^p-N!0CL=9DU4tpJ3K`iDR3ICxK?O3dkd=jOEM#UOQwtec z$k0M27c#Vv*=13IEG=Y&AyW&PU&snWHW)I&kO79QFl2xsBMjMJ$oxXa7&6JJJs4&> z!=Rskvih0V;dg6{^;PS78tbip8hKgcJN|BsH`27OvHU(v2KN!t76U~9kPq%e zr7Q+AbH=DYBp%#ONK}yiNX)T(a3?NhF_4)X^(U=)83WnB%=&|yQOOGEkH8$uC$V`M z16cs0{>c7G^4m_WN8^_?Kuq?9T!eH5*_Xr|%ZKuKS}BWx!W`ls(K|_A+sXCF{|LiF zVYDx#(j+VfGIJ~+(fTB>=;V3>Spc*CB=tbhA3*>tpM<8xmNk$CFzJu7!0Ce37t$X| z0F(X*+9hedlk1WGNCIH_q#eDC)<6*e;$NpuR^+*gG>5qx$O0JcOR|0!)f(*b!lAgq#Le2^M}e5A>_tLhCDMke_vJs|b0 z^Xfr9k^n~ewCJ*Q1<6Nb4)PHL6FDvs7v%Rs%Rp%a@)1{u6u;=qHUByTA2EI-h|{lud?Y6MmWRrJ@nMp= z(deOaHlsYKQ66k8kH}gb@!qt+kI(e1D*N=YC7LyVol!MobNPv}12UMAOfs`-dg#SP zZ^VV}8&H>}Q#|Rj_LIylmup8U11mRchQBj=Rn?ce!$U)&;)^^_E3;RpuV*T!jXD3f zk-e3kZnB=cdq<9cX^?zHL0Wd{(*DtPSuxdK+dlgD*YeCQpZ3aF`%=;u(~^GvIIBPv zld1g1qv~XNOl`*IIbqdVqsP8nn*LFq{!ZoiWf9BcN4JHatXnpBN3S!Jjud(|pS`cm z_lr9Hz9*)%{N?K8y^YV;?Hv<6>A8~&l8;^=6SZ#FeTnyPYF?ZXb*yg4zP6DqdsY?> zyf}Gl&HV6|HLuJ{Y8-gJu4ML=Yc2B{>sDOJ-Cj8DgZWe2w$9s;b7pW2z+t1rMRtcZr!NchL&kN57vSx~gjAj3M*I`4v_u4tR$aUC|>#!lysb{XkhS0%; zrh_tEhYdl9Hy^AhJRgV@XHoX=bU!(I=bLS9kcFtx|C{=9xAWB}++y^XWE%bD*p#b_&YgJc+L?8I({Ja8 z^i%9Cld%`QI9-BzzgrCYf(w1Wl~Pl^VNdZP{oa6dT|&50p;XCaAs#PIoiQ&r+*8q0 zC6~K}%N4l927CL>%AXcLKY#k%-2C_#rq0Y&xo13)^B}sQvbykMZhpb^Sudz$Lo|!8 z%Tj$6=%Euk!HT7g9x5+%BJ~JvL*-6@R>V+5ZC1Q-y2* diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_list.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_list.dwrf index ecb33e38e4996e5962f119801d158c56b8f6c39a..ae689d89e86f3a61816784214f315984e9d1032d 100755 GIT binary patch literal 20307 zcmeI42~-qE8i1>225AP6X1H2JBuJ2_5fKj%d1OSK9esvniL1s$qehLg@j~ML+yT*` zQQsKRDDeOnl^E}MuqetyU3n@7-NYjgN7i_-iZE-8Xmrii-#tS!Gu6rNMn=tRp9fQ4 z|NV7WRabSvU;aL|-)w}`o*H+?m>h+)Wv2G}|K{U)H*#>%4@y0<2{&wM*F0SEBOhLK zmV}oQyPw(>3BNRIW}7QH^TVO!H2!J^Ep{7&kY5lf{~XHetmP|xk`U??V{^P8IevY+ zaD-leQ{%>Dki*StSs+)@ujxh8NiVwIlD$Y4{Nj@=SJ4YxMYhATFkH8_8FOG57l&){ zcyYK+{kh><3=rpoXDWDB6OLpzZSgyh1u*G9-(DoQf!qd`ch(nquzTsc=~c2mXropj zr14>%`e|Gl#HzR_ovmwxK(!K)Vjt$Ql|2B)uM>bh>MR}C*q5+sh2@;ZE|vwu4Pp%B zjlN)Hm{zQsGw;GyWm`e%3rYcA;vw$@NSyRYxFAsqr>*gml6tT#cl}ERT?(*-hZLz@ zq^%TSPPmj*!}@cV4l3zVfa`cjk=jMtN&)WfB_-w9Anwc|6W&)YsTc9FJHfM;e{qz?7li;d%qU%_`KD2)oRwh|!aPAQRM{Q`Wr+FHL3 z)_7LSW&d4Gw*owM4j{!!i4?5>mv5D{CbBy2;RScP72x)708*@!NYM(g@Svns&ko}1 zzV)D60WQA`kYc4oidKLnCnc@PY${h{@}yeoh=O-Jf$onuxkQt6)AhLJ~ zOegR?vF3f4j~&mZaS!i!)2#q^*8`+jDUqTTU`-y)%1^Ar*^ykuO|RxBCgqli72v+> z04X(0i4e-78Ff;d@b^A+ zE5J1m08*@!NYM&#_aRB^EOs8Zq0yIa1-SG_fD|hwQnUiRblPfdAoE;X{)Nd8-ih3p z`D_;E43FX6hHtwd^4?P!rtyQpzMmOjbpyUPlwf*vuv&#`7huAP-5Oq0C#he8+2xo! zp2gAi0dV%M!m(;JrY(D00FKbKJ3;6g7z!behw4Gky6+0yL zn=rctb7xir(e(l@UImz^%-vhpo~{>gZUJD5-XYpfFYwxWNqq@s zk6`ZW{1Cccz&V=%Q}hnec6xyuf3oVo=SMvy`H2}B7LCbdc1Ns+GxDpMvSJ1KE<`&r zZtV|H{~W#>DT&*X=*>DcSsb5t;}^#o?$i)+=@sF6)Cr-8{ayb(>djY1AjBbs3Zpj? zO`h1cycMISmV2~jspT49e`+}w7(^`(4hgj@PvNg8D!KGmX8`UmRdf3LK?Mv_&X!1Bxvb!MSv_8n3~5bhHj{!1;8v2^ZombX0`5;q7#^6Ys`*>1ZE5fQ#v<1Rusn ztjLc%l?oN<|D>yvw@Bgs*L8rZam9!nh0)3+^1OVE(4Ud=SN={!N}WY9s&yqou?8)x zK!&3(sMUbzK~V{+FeWTNW@B8Jma;L2oN80Gq)p#6a(#o4)Lk<_QKGs zuFvGPn#@ghSHiwHSE6`6bwU<4hNjlpkIBM@Zyp0_o)yiTr?BOlhjJ;KheJ}at|<}@5? zOhoah2imo1a6!iJJhp1TQ!I4z4NaM)Ex-%AAB+5AQ~0XT+G&Bh$bnfwx@lu_%$L2Z zU(4UsyLV8x)rkcuu@h7=$_XovZT2=lGQB@@*Ok)A3564V68cQioZa%qfuj>$8)h5& zc;M`T-uGiYI|RJiSB*BdnemoiPtSAOjqyzF%H@l7Z5rb$)2AGZYK%@bbq=^;>OY|V zkDBWyRqy)#t-muJi>U9ZzGk|1thD3&uHBB${fKE;X1>yPlCEdrC?T=23Ai)E_A9~{v{{KM_fbJd zqrw{@?=`_;@wKgUyH_Z0WZ%>F?wgR3Z0wlpb3OBhYIc5n-{6S%jjpHMZe*q>YS!Mk zns+zg%IoX9{xG0AXY{|u)_!L9FZOc-*m--*ShOQln*$`jp{T|D+iO(+l9qpvMbi2ihB z?1a$;{mb%K))u%d@b?%#$~_}(yxY95#^}8!sXQ{z{-bL1Ut${0nyUNEoqn;d^2+Bi zG2_QYY%H9U=4F$!FQ@!FkN3A*MUr{E;w_7r$p6pI6j|^B=d;U= zY~4U5*-c-%9MTrsWA+=aqL;ss~DDtBjoTkE{;|C2^l5L0dYEobqoO zLgbWx%LWoT<)5DP?vS)^yW>Dk`Ny1`@=s0smQ((1HVoJ$y~`>8q;_72lz;gL#_|u0 zyD1Q(WFNd6_zL-FM=V#~AtYNBk#!$O)f(T6Eje-O_Pzr1+TB=N-XYY7@z5x+OQ0%a z-1Lk<7x+6mv=!s7Q7Dw53Pu^~!l*S^#eYdYb_!5?O-X+*a%TF(jB)9alSfY)r%@#h zP1O;gX$EGDOP@Y*%47}h9!0!yo($NLmiYAzZ#R9cDSKPLba-3=#6A0=J=X z4;;A0?5{`UI0DG0imD`i+03FJz+W-In literal 38111 zcmeHQ4O|pu9-eo1)nNr47SR#WHbSJ$Xf?kwNkEVA0>AD^%krm{o)U>6sHN2c!Xcp{ znwcMoh8&pJnVKnz68cLs&kyp_43LN|e`bImg_j@Q`=4Rf-I;mU>}F)I8)3&`p7}rT zGtc|Z?CiY%-~LC99X5fYj6SCJblylS6=5st9&?L7Z+HJUFu%XC9mW4M8R?Dfa6?zd z;dVM3X=4S3h{4x;gv7`FGPN$NmzZ9Eg+S zzm3Tg`_2=OPB~4U$fVpT@Wi82PLpyzDaWMT(J9BI+|en=oq^w(dq}fxkx!e`nwq@XKjkga!bI1_<$g$1Mc#E%*ULJDOq)?6v!Z@NPI9 z##wGePQ z_2PJ``jO07wzkGch!?2`w5bYEIrXlwQf_aCVe7Z*2;6{{a;O})Ypm3n<%n5{*zBAD zL7S*wdHmQSz~!6~$4k|ln9l5}kMu&k$QeLabEuqp*I23g)rcuT?72nV2SLH#_WWsYAzq{&(8XT@DyQBxR?5vXf$ZU31_C#rH%bAO<93aeIcoT3rXTH$4;98fv+uCY>X3lqi`A8>P#tO_8|5|fmw(bIm6@G8vvJcMjW3> zQLj<@|K@izrT_0qMkzt!IZ8a|a5PV(W7~5)@f;EJW)!?Ac_k8@PX@-_WQhv*KB43by`n-y~Nuux8voKdvqe0xdp6qjUMbEUqew= z8x_Im9PAfMKG%ux2-+!L7DqSvWpK)N3x38uKZOOj(}(#D?r<~hULTgn>G$Q`l^j2X z?RUqI^&xjSmgkn=Y;dm+QSRFpMh8(pHo%=e%ul<=Bi!Oxo?Ct$Hpm@6#u@jx%`J}Y zGrNm7-~r+e$K4;{4#)9s`SI{_kKckH$Gf(VQ!=c5`{Ct)L4)zq(mM<T|hkH^bN&pd~h zb6=j1m&qx!@p9_41iVa~F%vJJd}?BQih9+C(vc5Em7?8f53FoM+tChKc^_>?TVSOa zZA6=3Wj!iHMX<6Oy@Lv1WjR`bR>Ddi%16s!B?slA#jvsfWua_jX}SJfzUyhMpF;Sj zA)>~|vAy}`_6yG2Z+CxBJE?rynmSyOp3ljZ8CM=2F5L`54MDT{V;L+4bDw0ZApL2R?=s|k0cBV>D51YONj1-|L8{}1|OnQ_5FADioZW7@!p41!ZRQ2cCdF=(6i}*x&iuW zek#D4nvP0%g``#O!njaZ3!Y}@baThYj z1UwpYIA+!X#_O9=YZ4FM{Y^!`Bl@RfyZ4Vu{@m&_cly##(t^iWKGD7XWv3%Q25uSu zdf6>m zC1Zc6>^baE$A*4oM~ig#V&g=BNJRvQR760a8tOa;UZulx-24OIykB6_ z4o;MZoaeCd9_?tDR?H3|!Z06O3sdm&(J-yIc>ocHxvf*tL>T7Jcfib&JUPrQ*~ZDi zUxIR&VA(zu!!X<*#s}ueLcs?{(~y0zBM0VMfn911KTB}7LDm|s0hp&aJe$}6qY0*N(Jx}IOR2h02c=C5Fni;`#o|CFkNtExzK_%9uh}3?;^*; zB+uf%3Gq^e`~=yGBV>h;K62_aatknpSNM_8LOAXuM+yi&LLtYaN+6I*;AbzJ%48*Q zHpAR(=>ee$DFdGdQhZ)$!^I|!0cP!wog_2xbIS++E@+Up!RJIAyGCvUvNFE?j@$;H zmvAMQ+y*~CU$92lLwX#LcXh2j; z6!%O9p3Ah>%T8{X0)6mIs5M`9a^o%f@P}&nVBJbDtdG<jdS{1`ShUdWC1Px ziyz6ugV$Di;r=Bx&g&yze@NFQMYqHGK_G1DU&81ljar4b>Pmd4o>~;Zr#HL=irP zgHIjdlREg+5kBR^Idz2M5<>XY5k6^zPZYtaBTnfbj{FTrMu#Jj!;$R4XLR7y4o4n{ zBQwO2E8@r=;qyth>L{G?q5Z1ifaa|luJW9kx0<)=IiMc1Hf}X<)pI~SW^LSR-m2$- zdd%9m)x1^D0ri-7YM;y0{8*~ltKQJX%$mKLy{bB(D!Dd#HG5TcKvi;W^lJ91>VT@` z+UV8nRn-Ai$+gj|*{iAps*-D?SF=}D2UI23Mz3bCst%}1u8m&JUR51Xm0TOWn!Tzz zpengGdNq4hbwE|}*p@G_GWpmU>hbRJ{1;g1>wd9%$32}NJR(0fEWStGywDB(ih>rD z_X!?3f`+|{u>#cR815M-dGnXvdSG>0E&X~NAjmcZOX6Js|ds&;l zAKtw>=lH6ZPg8MWFYDSztm-`@FlO0^KI_JINiSJhSr(n!u&7N+5?VX&?#RZT4OLgy z?VL0BaM`Hw4gRZ>|MC2#ns4hTZap_`>G`!6Cq6!5^?Rk?9y*)04zH)Zv1F1D-Bk!sD%Gxk->h$KYoW}U6 zsg1Fx$`(G={Ko~QT^=f$*YTdTj>TijsYqRZFnX~UQ$YDrl#VI1(J396k)9EiQIS!xt|GD`^5e*_Bh!{OtlD$#^ZvhFK@il?_B+>zGqX0?p&4> zh1iX`4Yiw=R_}aidtFAqgz)ZbcE6kPWX2=$=Pz8Fw)NS#4*jQ}_;k!&HJLg3xDU?^ zc`G%?=;h-RzW6{&a^KIZ=SLmwHe}k>@(FD%)BEHPEh}4*7`wPbQ+ZskLgYKnY<_lV z?*TpQrcIjk%ryGP;TQXCT-!7OfA5J_!qQPJz;nuKDpx*BIi}xSh)YsdQ#mG2JUZnx zc_Nc1_TBH>r{A~Z${n3@Ov;TXPt@dzN2mEjQ#T7zJKT^K@zF|Hx_H8s&vY$AiNDB? zuh!U7xKk@(36+-Pws0;`p<)JjQbusR0ElZV=lb0z#dGU*9;9iU=s!29b)`I1wOH5dk6<5fG@PUvIBfk-)c4S``V- z0%_R+t%}6&xQc|9VLFP|GoRQ|z}@)&DhqqvY+h^#-FyyNZ5PK4+LKk& z^mX=$g0E1-b#p)veQp$UyaQX8j?5*7P+ap1SNs~ZefP1FufJdUL23lv9^_4Xn-B_U zGkbDk>Xe|ibbFIdrw`K6`k*$n(S!`JYUyI^n3Vi%*qr3)DN~ZeW=wuzipem1+}LOg zn4EywQ<7&*Pnuyu4_ebNqrn|$yu_FE+VTl8xz(fITyAo ZygqWgfiBeHibpmZW7dc2t@V-2e*oN?*H-`l diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_list_with_null.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_list_with_null.dwrf index c77e5e3b7ab97dd6b6a2f76fc1b36bb8f04ac609..24036612d5054f1e0adeedc82c76dda1877b975d 100755 GIT binary patch literal 33379 zcmeHQ4OmoF8a`)cfXgU$84xds#DiqWvk{wShFGH@jjd4J}PKAl5~P%WxJ10lm7F|IdUvjFzN1I?%k(peyKvk`ZTGr{Npj! zXT!1{%LXX7!3(<$UTl1l8@!wq#jtMpy@QtuCYybT_SvwaUY-p*h=hExHv3*|~(T8*&jU}7mxi^odrOeb>*-+pvN%QsCo6puYmd!ts+;Lhqu=S-5 zAyXK)J=_$;A>JV5cO{crk4UjBmfukxtLaX#9XjVVivq|sOUN|#G_e?Eq!z=e zMUD&PjY3g5cA0EzWG3Z7ViN|VNKIz+OKOx}wkn)lb%B@B9n@$`o)@YX>*zf>R*`x{ z%EP&>F7VVM3SRBY7Sejej6yZ3(JvaAvdgxoSM@T=n40(qVb(l7fm!2vG6g`9sTb(U zl!C>b$W=k60v|1$nQfr=BxOp$J+mkXF!ieG$&`UT=v5)6F?^h`q=ZTiFaF!ie0$aFjHNEBQ@iBj=m%DO{5pCFVK2f{`v zxM3;<0i#|uJ&jUu`3rzi){zqVBw=-75QSY#Nx9Q208ngRFR-zBnV(W{-55%RF#W;X zPrL9bLe+KJf`Cx~6paEP@zZb`95{}G$EH#D_qNK~HGqU ziKCK6qE;!mbv_F^5Z9i3hOpyjBg_>5MXmtIxH78@Tm@NWTrGSbq2gi-m@5U(pQ9ka z6{sn4rQq&QA*-x2W%5}(}9(ORdp0> zbY^B%uI4uCuVgfL)Xbktxzf-tiA}lu1mW^-6U-F=MXmrSwlD>2KY&`8-SO*uo={v9 z26Lt0nOzhFxB@jrt`t15(Z%&5iNkH#e5o~^F9maN@%dP=Zl_Zl7Ko$C3}jB6$p^Y7 z-50tuhoB@ByRN*$#rF!a9L+38(0P%OE5Lja7A{w{b}?>8OkYnhMNoBJa9K5l8D|O? zWD!`tw)IT{6NO#1R~>NicDGJ=8@Z{NpN@s%jS(*5^zuZ=2-++%FtaWX7gFnlx~ulO z6SCY(Fh2_mN7l4~xjW!pl@tcJgSAEO4*14pS?=#)ejXOeJ`!N=AS`kRVUfE7K4zs8 zQyRbATb?Pm0P{<*u%kQ@=I(&6en??}J6M}>XD#onJp;1Nlv{@RLX^b-12xIusr+O{$Tmw7l(r7Im1VQ<*$q$3zp}O zp8%G>{zjg&Jjtookj|sXPj6XqKE~*|)_MEL7~tp*tsZ?1q2vNwh>HjmVE%6wS*N>0 zQ*kk#4nrl_hG)Uhd$C3ObUZ%G#=^E(Is1?_3m1d*mBe^F^V>E|bP$a>MwnJIsgUsexOhC zBZhb`e#t|NQ|_bkDoakI98ObsKDos(Yj9 z^4`!(y_eU?d9Rw0nKZIhl_5b?*KSj`0h%r^5S-1j@+wPC8i}oH4ES#9R@;=Fcr;yJ zzX7g0r-|f+Kc>(W@H9+2bnkKA=ES$&mdC*4M|sy%t1DfKaF zFKoU}=qG(Koh5dV$R+UDe~>I?WUMzM17+27kW zH|rVN_EppMHIN7Ua{5a4bCh&mBEITx7TA>C!Sl!bndr;Q>8tE)O1hh(Jzcm}|CNrv zWO*d(zsmMr=~`3u{b!r90m~o$tMlQ%%CeL0{HebGY?H}5;YnF}A%CeL0 z=cvB_Y*Tjs*-!d%a^tVE?4*96?LXh@_$yr>k>#(fzBgOHNF!?V?s#eY&&{E4tgOEN z9BZV}xLUHEVpV$qZu55!_FSR(BR-SqGkNhDetx9sdmHNeR^#)*k1$Fm-&#P`D7E5J zUvw!TIP|*Rer_$kXZ5*FYtC)@jQqD|dOd@8=O)TL`b)t_d}RCd3=8N?@VYkG@MgZ>{7 zUs>wXg`w(?I%X=nS?nkAN!Y(d@$(IiuP%28*%FF!${xROY!HtI56%fh^iO?CQ3yGF zV5IQ5ZYh3nTC01~Q%c{pOh{O|=kdBX8@2>xpq{7;da`|FuZ6?15{{PEbgZp157E8N zg~vSA-+X+}y02DM zCw}HePFy?x{#Oqd?9H5C@X5=iKTSNm{mqzTrjj)C($erAGyY}C>1)z&3oYWpata4N z+bi-+DTgQ3Wz^pL{p^b2v(_$)OSqEq#JDl-Pj9h3cVSq>>H1#1FFa#9Rd0CsLa$aQ z>uVA&JeYap+KDR>>6_;iU8?6Awp{vO{a+^rjmqd=er@60qun~4uW$L;g`G*~>c3ib zp)&Do{nW%Ge`$TWK0ouy+4!T~dhgqkWjNZl$KEZqCy!j~+Gk2$MQEM*bDZ!7T6%9W z3LT3qodes%u9z0bZp*q>ciQE!q3^YWs@p z-zN3@VeO`)E1&LEnpFSig=t-~hpyPavs=!>Z?ZoMo_)UjaPaISD-Q=J9$GmdEb&n9 z%CN+bpRNqcUjKAuhgqXHcdBL;Mm)ATt;_zQD@Hs~F?2;;X2p)oZ>L92AN@(j&fvOk z`*+&oL#GEW9nMdEX??+ZSpKS*pwFT@w>DBl}RqeVcYCj@s9=-^#j(0mWyh{W@oV-yg9@ zE&(kSL9=E*@OjR$czm+k(aI#@Leq{szxffa&3=Hi#?tL5?B>+c=gHju={`P9s*o2z z1(SD7!}`=xV<{fH3>;tY-aO)w=On2OP50unVO2Q3UM-W&K9cxsSWz#x7^@x9Kf;-Y z$-Q~R9!oM=XbBg{`D@PnR(Hbj^**OtDue>}j;Ey}kfcX*Zyt$X-h8%xC-hhqS!$^W z*ddxUD*!o8S4&0Uo^(M=MWCf3a7!5iBorV=#yayK0HAo>Hvo#qeQT)*Xi^q-Qag}P zKnzM+DgtLKX{iXDNj$Vv1dZv8fP?~KYyv2&%y9jA{A;EX10 zU_X+NiEWH`jmdo}X=3in!YjUF;;n8}ddFDrupCO2Thu-tJkj~Q*kosx;!lvW%y zj?Bco<;%_D=U-Ozvz*$5Q`&HaM*_&hBE&x;EGqw){%YB!2n!+=7k?N)lFWa<(QNsE z{9+_`bP7*4s1eOfl0WA^G#lIHLW0GUYY-dGv?0Hi(iB%kwLzA0Y%a;P5W86c=tS!B Y9{mm6A|1)FX|?jP`UHLQ@*e#E0S5PNF#rGn literal 50664 zcmeHQ4O|q}`kvWE7*^C_`5M6wLKCz!Dm61TZmF&1C+elL@GC70Kd!gbq^m0uA)%sD z5O8rTGBh==S%8XPP+?j>Q-5Wpqy}Bmu7q*JUsvUS&Y2yYGkeaE?anUwpK7P^JoCQi zndg1anc3OnJL6&>oWd}s4(9e;=GbT^CiAy2ODli8(c?D*|D8?knd|?UP22_VXf~!I z7W5Tr@?%!{&H(812F5&9!VW?#%+!I|5HOr!RpXSauO( z83beL%Pr%RS;2ZBP$J&d9!wP&GAF|rbtL&T{wd+y*PSHqic}$>GFfnRAE4eGn1Q3sAqTsq6IVTFv6V97V62FFk^?3jX!)QPyG@ZP*hNGoI|{|ey=s7!bq z#eAbtIEjB$I9c04k`?+sfW|1wR*0TB3(=MlEczzcV#=VPX0m)1E1cRMBH1f~D&kci zK%5j*rM)A2Kb{v#HyR{+1$&6r6hM^39=FBVqqw}r^0};#vrPc@Xl_)*m59sYs>Ct& zpa+gK^T9$^u2J%@h${h=#T7>#eUIg1g`;(T6yF3?<{L*bWl&u6S>DD9m4%%pWfgH1 z@yqW*oD^53J;okomcVxr>R$Dd>=ki^=#eallH!WnV(d{|-(>kstWfw?D6l8Rm59sY zs>Ct&D6ZZ32*H-wPV%paD*=^hkE57xq>P0hEfm$arziu_pDsX@qzrD0DTCr##PVBN zVQX%fWUq*;h!?#HaZ+5B_Kxg(@=-!ghPPy|h$}>QrbCp(9=FBVqquHo`5mlK{CXE) zPl_uMm&H|yW9(5}`|$mQt*Jhee??pgs4T8H>gf9cexy)&$wcu@KxMvh6jKJpbtlW0 zvO-p7S4mk#Tt)n34#Y`uRoY|hQC3Ft{e`MzPO?|T6{7i8h?3%p+hXidTz9ejURKz% zwi~b~#g&N5;;O_k_9(6c`N2Zz703_ySHzWo%CyH(%r{bI2p=O{zU)g;2BImyLX@No zZu^OWvHq{Qsky%SyXJk(%da(GZFc(~t)A*w>er^R04Jf!9+TI8mL3nYO_Ez9wSPt` zyFGjHv+{VDZ5j)38LEt*v->mKs!f|p7+-6%#<|qmHOHoLW{ER~ zXD@#HtQx0Mt3kABEC8A@e)q`INDZr}2a~PUI5-!EI&}5(@L5_t9cG)x0*Et)htJa5 z+PtapwApOaSO7F*XfzT0?gX(qHy)k;LS@tV+Zf7a=V>~}(%P(XF17Z})~4}bS^(!| zPp6XAxga?oBU@`>a54&|P-Txiyby1d(Z${B>A_@kK2WN060d$9K1+{>*`~1oNM%nC zSgUFxxM4oCX*`=>E~tfTZxC0`KEtb zgk}s6KX+{Q_ycaOe~~(>-!Fst%suzZ9!}<){ue1UV|e)DckQNsj3ihnwUd%T}=IlDZfQ(L! zn|l}(Fy9Jnz^w-cq=#)!?)sJ#*g5iv8vkx z{;Vp0UfqAVm-_jE{=L-v1A0gwe=#}^`Y|AUYW55ePz?v&tNLddb^lWK(D(s+POX37 zUX>pU0-$DZXdUO(?X9Xf2w%9GJqS;X8tw%G+zPJhpKHZG7#`~OKtQa2ABGt+ECy@` z4jK%$(RcL++Xse^1lvBh_XFFWy`sR@^6+S|?G_#ZwnAtZur&t;gYDSyv0&@hkq29E z&IGoT{`x4`G6pZOoi^hsuwAnBMX+7?+#;}@{mdM&O`Nv?Y#)Cj9ux#zW@VGv4h*x4 z-NWvMJ3HAA*;2T(oh@c}z?~v?3%eEWyvY`_o8V49Tfo}j&TDKAn+ta`+4XD|+(~CM z*fnq`g-vDC*eLt_Q_*!YV9v1M-|~6&sZ#UD*PHjtns4mr{nwrBYTXINH>H_E@W#5N zIMK38if>9Yh2V{yCB-+TS<5bOjQ5QltvH=rFY)5NqhnVy^KtcKk(og;Y;=$5Lh5v3 zU{pX8WhfL25>zF9aobYBZ zpNMG_t%-)?P27e9)1txDelpPVm+ykSQ-YQU9k&)v^gd+CE&6m<`W+d^27PMoF>i@a z!=w8n|M}sA-=7;2Fyq*S4Hq}9h^X#==GMGWpY_j73SOCT=D7mHjO@m@W{+Bv-x^4*mD(x%GY*-I+Q=PzjLxc&5n-1@n*n{vlQ=Iu!O zI`MD*o1QHgZ92axyj{?>5lnH^=C#`1Fiox+gzwBBZ|7CghS!e*CcKej$8WJFoqgz~t6nrg3m zFOFEVq~3qE&x*lCE2`LJ!>CSwi_A)9{Taq9+-Bu!e8!I3STJPh+@-%YHrD+3$RbkNB4<771?WLxn&n-V0GWWZ_9Rt$S zI*)yN_K~w0#)I8n>}ZU*rGNdHE)A0#EBxQsI? zOSQ8OAKs9C&mHy0+ArOD`@Vm#8aip?_I0-`DO;U#_Ok(ZrQA`nZnE{@;rMk`J#Nd1 z{iL|>e>Vhu`1wm;{7V@0V8rt2BU8Sz@VZ0!R38sAy9U68s5-VO6Az zy8)c9v_>#2hE)p7qdCCHIjS7=7pikun&->}sx>SymIjOdI50l|sloC!o2CLd1yH*W zoLEa8KF~ME!2)S%mtl=B2Xn*?){iNkZ-*VtVc9v`%cu|*aPO$By~N8AqOyn|EW@(n zA0&Q2IRzumFhpM0{ z86bwRI^LHhM25&7Rms^R*$)K~XsMnD6qc&zRnjTi%O7p0Q-lSUJ}r~Fz@h*sN>%K( zd`TyXN^eyWJphtYF}dP%be3Xys?3damawSXmtPV4KrF?wYu6g-EK$+OABY|hOR@CX z8LPwrQvw}fFA=s@I^q_FX~2I0`x!Rje?4|TYQhdfN8vx+oql@nB`?SjU;eQGoe*Y;3^DKfkRjegaF3eszX~K$#aN&w zOu?(D3FkG9A~eRd2^nJAToZeAln`ex9)(#FNGx!SqiBdZD~3nm2_PPYwH*cD;}aMo z*ghlM&T-fmkC7iKKSqwqkCE?-$BSEeySOiJ7hBZZTaLMpWza)O_+ZjgpIUmy4CDtN zihA->OYa=<1J4xUb48DRYU!OLe(<5NCqE#*Gy?YcdMx!&=7IRq2#ET@r_LVy)Y3b* z$Jet11ddW?i@uxMYpLnSBYyBl2~U1%>77gY2cMgJ>;vk~N4q(6S9`5WM|OU0x|1T`NI`xy0+1iTJo>3s zAt37KQF=Vkk4gaYqYZ9%wb!b2WRGtQE&lFkhw-HmK>7iBT|A}TkiS}$&h7kMcr~S8 zOaTpIKad|00PT<;_1m$^yyQxoS`}xAAN~4FyKcL*y_Rxw06Vesqh4QJdLg4+#~?o% z0dXA?*oy0x#Pv+#YA10Ol(>3HTq7l}VG>tLiEEg|^-|)hCUL!#xaLV*H6^Zsa$Gef zu9O0+rey1%RF*WgEPGO{Z32s&T9!t&ES73nJ|!-wT4`Xwk|+H?XB5yibVfbVR@YG1 z(0K)%_n_yYuA%b^IPXEvLtR7X6>#3eD*gV@6}QK_jLxSb@Sw}6%jgmXT;fE}MqNgi zDBuz&dN%4Zxya+eV){nbd>_G@^jK&{OVZ8paACA zC*z{5?0CknB08jgc}`s@sDPE4@b(Y(MXn2lktb`}{k-;`{c#c_Skh`~95iJ0!FJgrNUg ze~fq}bLEnT^gB$)2Q^GTKK_;bo}(v?crx*je^6A&%7py5?gPiX5qx^``e=V+K=hRU z%Z67C89C+F@fC?*_^jVwWSQyflXdZ>kxygILVZ++zO5uGe`e@}ZV&2l8ICTah7!(O&5eR%Mcg!%VXruA95 z<%_eaRgrJKTXJf5((a8tf=-V3cFjX;jvPu`SLMIzr-qVyH{2Pq?w;69i{JS5e`CM> zX5fpRX2$>0Yi8C3zOwh}r(fIIH^pXtaqXS)Nj>g9<`?$F0w$a>#4ujg*|kA4`*3^S z$c^N#)Qng?>)6VBuH0HQZcK!4dLrZZm4En@nsJZh?D#A{IMrn8@vjGRE?v2&z|@C} zX1vG8aFqi`WX_NMN4n1xP}&66#>yNU4XVlf7G`PXk2iY!X5hawURTS+UGN6=oho8M zU!f*HW|i*@P+akO17n^Fik2YkAjHB<9heOP!x?4;JIBaG4v|1;NOb!Xbo;rZ+iAe^ zJj6;UZd(l9>Gt}G%0ysgA|QciB0zp|Ta84xo@$N@(dZh-OMXi@piAm~X@k7F}%o z)J)+`KxMpf)X_JrI9Ydsr}!qIGT%6g`9|jjOF3p#1W+`AXk8UVNt)ocW%4qQKohCV z;{jL&6a}-ZJ(xi-m}D3-C&L(ZBsr|(dG6~@l6OU_5Kx&cIJyr|?`krF?@{Jp4#*;v zk8CUrl7#67xWi-#=0LJYc+NhEBWWNG{Zs6>fgVD|9s=Om`)~}XVVc}qA|Ga#h6+O8 zBoLzx97}a8GfILb-;f#{L&30P3NBJ7;*P?@+OBDZs3xT}Qe^#R05K~>s2vPU&@ zH=>H6k}@*(5UnYID2Y98i?K&>5i5P>Y!iSz8cvG15^-5vl{m&8^uVE#zFE1jPAF;a zg#~{Vs4T8H>gXF)Tdnh>_$Ht--#CgXgW@VyIIS$~EGesqtGxp1yAUVERcVj0N127n z6~F2w*(6|2Y=z7-1WNpU6OvbZX7j6I4gtfp+sg!Ph1aV4NK z?Qs#g&N5;;O_k_9(8fQ1#YSAIZPs0!Tn* zam7(b->8K1B@@Lr0hRg2QA`;WSFu8NR%Ta8Sw&p!6|{38PKvA29%GNP5?0u*N(N7B zMSDeDA)0T6C@HSEEyf{DttNUQxy8+34bn z69;{g+jQW>!P5FcG376W_T)0Qvi#9b!s!)kc)eE?c(omPvE866_2-XNfBen5|FAIt zi16e5%q$z)Zqf9_=VwKDbM4IrgE7Lu86(CT-7R_4t_k{WL%^ew0+*HD>A2pRsV`kbkSc)?yF&fm? zj-6&zMrmX1BGw==$tEF)j!{f>B$x_DYp8_`Vm=(hAi;Eos~@y@C7 zy4VtgE=TM7ld1oeWII9=@p6o`F(BR>;X(!hS(5$rW1)V8Bw~?#96beae*i!%0QdyG zAW#TEFh!u70Dr<1f=m&70Yd@}U@{o7ku)cB7(mbn0v1DXf|dcsdkj1VGd7d)xf})% z{0jsuhTsG(1B|ztcnlJjNKYY$0R){OU@-(IXc=JK8^L3+U<+wn%wYgQ0|;0Q!3kOh z7_Wgf#XKJ{#>Bgc!vF#g2v`ik3Be2-=s5}FEpSrTv(|+bTgli;4g&~ofq=yjoSerhG-m3x>j=-KyU^GEQa6&Edz{Sv`#Wa()}UDzdyvm{vb&*mdVI=SiZ^*WeeCK z;$VaDDFzjVLIH}Y9YQ}tivdOUVID>tjw79Wco?8Kp(t1wsthd#2*zW07zsFm9C)3F z0g4@pf`y^V&|-jKFqVfwFd@Uoco?9#q9|Axsthd#2s-0=7>PKMoIk_E0L5WN!NO2w zXfZ%=bS4iY2`7>IZXO0G#=Ahl!cb*sF+i|)76U_B2R3XY-Wx$hlxsO1oLmQ>=ui~? zwe;VWqL<;v7>}pdd$^cFgd47euz9K%tP+umQ^{9jJPc6m{U&I>R2ZrZEd~gNQWzM@ zzLA1c$gR(VicHY#vP zNKvp3iYmj8;Xf!K=&>0Y$-@4ONB~0|Xs#6V2Sd za&a!%cAJL*iW7>0g`vvOVt}AIn}?By^T@~JJPc6OOz;khDnp9_0?#}iMn29bO)tVf zF0Abf6n%<C9!4Q9Bo~``7@(*= z#KTZ!XfZ(0na9H@!bRl3DINwWt|$uDY^XA{7$9iL=V27%VsiUWJPc4AxxkwZRfZM= z1oZ`z7+)(7Fbwqhg&a?xq0oDOL~q6oLVbJ1I%mY^2L6wAp53nQ>@yYO20jK+~w4lA>!C z=}Hb=IX#_Wc22`=x3kPr=*k%xx@LNqr(w>_WSDK!FlS}4%yj1qwW@M*eCA*Km*m?5 za?lMIwWFfu;T34u6d5yq_wLPO!gr6{Y&&|h?KJ&5`pgabYt@rK{%q5epEV5QPgh=x zn7V}cRx8H3x%gglWm!L0yhq-+IA5;YKMyAbdbw04!c8ZlHgF;ur!^72Ybl}^Vz`#7 zgJCA_DOeO4!E`F_DcDGNR3OeUOx{$Oe`rOIf8w0uHFcxA^kboApdPs^9~@m(av*hS zHLk5L``Kty_a9!)Uwbh7v9IQsict}oi9!aj?AJQhlx_J^_8Xc}=bDs-bqx>Wlkx4|e6v|Ld~-kF9Q4^YDxp>d$=BF|zY&N!Nwz>6T+FeiE}{ z^A{eYAR3L*4;J*Fe8ju``ufWi=h7~GvFyVgrMtGjynEK};kP?SALv-OqG8pq#$Ma} z)S|7`&hOb~d$6mvi=+#9&(?!Wt{Q@xZsDHYR7)=0vx|^|J8n9?%W%)m*TW6>?0oYO z?%9QVb~+u*aL-P8$I0Zm5$@R`6z0v)XXF2=XBU2=_CNVV%^)JlfKNS?IGg@Bokj++?IlCTwh_TGR!RrtfPtS^R&7|b?m1bQ@1O+>qa-McoV2j(l|28;fsP-! z+HwnLN;~>YawKg87n4^TkAtvfAlrosiT=Qy2dRYyPBx~$t%LV+*7QWU8U?3kxr-Jl cGojOIz4j-JQng6m67F_mn<2yC=(Xej09p}Wy#N3J literal 19749 zcmeHP4OEm>8vf=-_(0Ul;LqoX!-z`AqKwE2PKtBVJm4`g&UUQTTy;h9Z21FdTiDrI zMbvVGlvPPd+(8phz1|w0 z*AlI8;-~^BEkM1qFO}Ic%(jDctln7L@(rjDT&sm+aQN3h$t5kZOXRmCgVe% zCzu2acki)y5Rua)8F2?5sSg((aUJN0PPp@l#7l>DfQa)Vf~K3)hdfVz(qGuIi^YS8 zOc6ojk@|4q5w#sgh6yLuDV^p9Ua_$9o67<)Qjf`CvXn!bxv1N4PHgt+eOW= z-Eoe}^miGFI5Qfe5gOL8W(E?U9ql9Y~mSTw5c!9Ojq&`l9V!C$4 z0#Gq;UP+X~rQQz7f}A7^K*?-?iwF=$g8+wSavtOdV;}~h_&RGah`1plXwxtCA54t6+^5e)$a2%w23@gP5#ND?Wmu49RY zhyx;mHW;Z7`N4P*FQMfZjKLt_KM0@=M&d!9gb*U+uV9cMAR7TR5{UqWKbEp&*_i8UkKH08KQB2l>IokQm`!E^9D|D1L=?0g(ETAB-RI z6YkAq3Y*;x2LG5!Usp z-X+3zi4)K4xX#Xfrb(9!2b{#fv&_MW*10Eo{TY5U>($)HD(>dXegPZ%G!zY0c4#J@ zG%KaOjcv~J;E@WPE85%d7JZ=X(9W57MH{0<+qIUL2plZFXJthZ$n`w@p8-zXAea_P4JtMt`2wUjYk*)voNk zov`|YJSJAV5+;z<4$E85Z11E$EFUJnnlm%~!}9H{UFsh`<%lOg&#&;~K77m+obNwF zX&3{4vi3Ebr$4AYdHDAw4<4ypGIeRmp>@%VpShMkk&ERrcb&g9+Gi9xbnl`Qf@YGcEs&fw?<7a!VdEaD-j2`n*@J(KI z?5UEI3!@*&4<9{G>-}tgTuR9Y+jD00>i@?lAD*itnSc7$>G z6Txeu&ZnAQR_E==Q@+qVeEFODy3NNYD6a-(rYAQ}xp^(4dH%8lzHZ&;a|)Y3ueow$ zd&1$}=dOIQ;6=r@=7Gzyo~%=CY2KHz^wGL~l@o?vP1uw^uxzWo@xm=#O3lPSr7v23 z-B6c%JJ!qmcH#R6{G;?=D2fB7DhOxh#*rrVzba!_wG<@8#(uOeYiC2{=?9v(y*gyv z&g<`e)wJnu^Zaj~d;aRRH+-t{@nw=pqIMPO=kTMP?<38Q-k76KEO7EA_ESuh(|+Jl9_>{q@w3xQd< z>>B(^Z(-!Za49w=jj}NO5604AWnj>cTV`rnxZXg()>m zcVXHLQ*4-8!?cBGEp6^Dp(#biD4 zi?CMu?1}N2+`IfZD?I?)`om{xHyWJ3)BtSfFM;WIpTYX0k=y5&vN~z2wu_I${L%uj zoges_*NtYSliT#8soPD*t<>oy4*g((LfXeohWb&+ZSm3Xf8BCphU*{g?k~pY#-%5& zJS#l_F+Q5O-F4i`AQG>CZoN9|)v*3(S7`hEQdTEpT+|i)1Ny-H(gG0U>z0#%eh#Ru z^Z;!7(bVmx<5ueQ>K*#I^$8XAqXZzv=ia*;XSOl|!1LFwA3osO8T6w*0b+c#hllPu zZe|3jE@!o#z$-GuH(4=XaU;v!w}tPpdXbS!v%I{EZCtqXi)9N58+&?mJz2+)BB*A&EUc>h&8S zKY1I*ModN-{En&T$OEC?r@7X=%E`mV<;=)jJFVcU zl;BmVx(VC_*go6;!E3&|&Up2l6QXXaJw9!=VSnacQW(8w>hbssiV3|h8b;q(XetR+ zL@O$;%+8pyG07nKsYl*eICNIVOH(>*p54ygac9B1@jjR5_A|W0SM|t_&`s}~F-|w> zm6H6DGXrv*cFz{X3~Ws)n;G61Z`JLp3EG@K>2Sob#+PaWveWOC1_d@gbnDyEb%Ud@4#sPiVJ(C@&9qnoaePr&ugJ_ua^H1H0rH}82Z z)Fua~(eR$vLZ#pT@jv`ps3)!YvrKDxE8)#ra%hQG5C0s~3VB-nc2#IGe3_K_?#!6K z?YXn_(%B7V-ljaQb_m}ZO>Xf*d6<5yDikURpnf2xY}Y?eWmmj(ZKJ|e9v1A!`)L#k zZI6}no?4LsbmmQgL zz8DrmWE(^Xh`5YEK!J>)gFMiHFbHG_DhR`76pSoE-ha3B&8?+f2B1e;cSh7!BZ4OCqySz3|#&cd5hY z9Iy9npjHfDdfAwTwnO%Ka}<*PSBO}}6%kmke&rk*y2{4%e_0c$b2k(^^c-;E{D$V0 zJ|6mbh=<%?dA-u-6?I<4aL9azIa0jku*92wj!%g;b5;vyPps_-$Go58A@WXhBlL4T zdLRz~Hv)hf0dPYAB*TIET1PaUYe#EBK>U<}%DE|ks5I0iDT{ne^k&Cr0YY!(`?2V68B+iPV^i14JmJQ6H{6)E z0W!%7nFOJk7$9S}7;MhOt3v@)+fiiG2-S8Jrb&W8CIGo0Kxza?3K0Hbs5num&9sBN ztV=UFajql?d;$$so1vjHxX~sPaseR^n~(%y6Vf(ABqV{@gzQd>!R}@fk|9h&STpqr z9Tj^>2AvXm=sju=d4Z6XNysKan1p0ZVB|@JWDuK>-3EjlOhPh*NeF9(L`Vh&A>;u< zJ~kl7K6jiBqT$ags^5vgk(?3mS`W!X2CJvb$0j5}*o3sr5D7^jHX%2x)NVVIkPKlG!kQrwl0iWT!RqODu?a~K zHX&^@L_!iM2q9QK-Ci~!3Bo3%ZH7ol0tF!itEb1tCL}@FgtW~N2}vL}ArGw79w(EK z3}F(&njsOAK|u(?+Tii92}uw(A#F26LJ}wlA=o;+?5)EiLD+<}%@7GmAT}W{Y#m>gi>#)Lsd~CZuhKNJs*)3He~9 z_OVxL8NwulHA5mKgMtu()zimbseKZJO-S1ek&pzcBLv4Kb7!*wA4ISihci zpnOvh4PHezyj0>NE)yJ^si6Y8c=s-9C}C^ z^2c0bX~6K~M~Qf<1}s3GfTRHn;4kfAYv6k*O&%4JgW&^msg5-ss=1Toq(hP3qTtVkTeAPmG7l!=q+jRm)v5_B@Dmv6%kLJOIUz90ckE_ z0qE9eLFUqbDRZfxq~WnD18W?R7GHQu#8bxs3s5H@jRO{d-aSYf2i08ar;I~?NkjFw z5iAY77N37X#8Wk30qO)K4Ojr$EZ$|&uLm?h8Ha(AhG#ArSsF0B^@@n6YQO^22}l~S z0QAA3AmcDl*5J>UG@zPDmIfZf*NS+m20TEOfTRHrC_fxTL$)#wgCq^}ZZ=|S(Bdm1 zq69o$gEl~&fT%$mfIj{QTZ4a)tf5@i5Ep4;X~6I+V?;bv0~Vl8K+=E(puI=f8c>d` zfzOe~p*S|0r2)f_Z!Y4g8n6I$0+I$S0Db-?TSK{`!JjK>NN!*FmN?XOSG+_9+b|Rjt0SizkAZfq?(9vUT4gNgkdXP0(+r_dp zVEB{iBA%)N3s5H@X}|)|3Gp2)#{Mu^xgG{f<4~5~n56+}@$cA0JXHf0piV&2fCZp$ zD%s;uu5j`Xku>=2O;{Q*Jl|EsQ#D`#>I5VWSO7ZvPqqerh;ls)l{D<}HDzhQ@Y&r( zJXHf0piV&2fCZq7C)gVNLzU}6)=-)p$I^h|XEB*joF%bvr|^rRfb;pAFBL-fwI!i zq5EDs-#%IkD2K|EP*4U15SM{aG88Kf;D^d=DETzNc&Ox|%+mt=^CXhxP`L+61psVn zk_w;@DvP14qyb8y@*OB;G{9M?{1VDZ8US@is2TuQX#gVtOi*IGQ%J@^WeX^YG(a*` zrb1~=19XB)E0i=EAPXw}P%>$Neo&bWC65MhLZufU?o&8g|eIm zSOt};O8-6WV-Q5@pnffi(*Xv{q!Z?YfeDDRIm zAbw{gYShQx&w9Tx)s(u@v!ZL~+m+1}+HBly)0TntB;u2ex2q>f+2TT*%@9IuFeyWs z%@o>fTnM!#n?jn+cssMfIZQm1X!gNO13PPJME|lp)+nEN$hf*C}2O&;p3gOtnWf$6X$stY$XQJD; zBfL(=?Vx%L~*Z`*i&+XqlKjyVTU3x+>*WYX{> zCDn2N7+Mwes~oFgZ_+S{`|U6>{CoNYZfy9v(b>eojg2@wJPgk(>fu>^`-g`>MHrvo z3KbZj|0R_r9Nc+w4dc-laq03r$!$9v-bJ$6tsLs#9d6bev`mN76%I1p9aw|Qo(8D2cVJ9cieqYb+Ff=x zhhA-HV~Q!Yd2K#n)ou7n?KZpJ?g$H|KD);bYj49JQ-|GQb-2-$1n7V8(yDKdAQ>V-6cEY*0;jg_er_1FE zi*v8bg@rHk$y4`M%J1ovf9;?UZ4vXv5WA`{bVKI9> zZjZ+k7WXn1?Rq16{Aw-{%JOUWPZtN)70fL74M&2*0}Of8-KiV-ilM6qwV;;Pi^d7 z)}s2&!BvZGNmYNI!|fdFA2zL3%gFC9e*I$A{Ta2#cRzRO_~%<|#vVz!G5L`vckZ8j zWXqJd#=X5arOo;=PbM_|>*48heYF#_N;eG|+;h+&XaAzqo`Z}T6_>m1yWkzryvOsg zXVxFyGG_3gs*Pz+Tz9+rAKY?RpZPmF_iT3MqG`tRaXmiYRrNygvT=X?##uGFWX`q8 zyR84cWy$D4JGQ5pZ*J~>_YY%B@?M>G?~=p&#|eP z@ycDX2GlGWnX*#({Ha`V5gIlFJvc-7+6f*rQK#u>!=60<&FSgsJ68Pj#Y}_gMe{3`x6QvNc75vm2}iFLKV-fz)ssDS{P~@!4|Qqw?W;>)d*q|aLTj^5pPim( z`s0Lxl#ljqU1}QfbeG#NFv%&V6?M;s<7% zZyb2$WZvroT~pVuTbp#>;RCrZyAPjTH)r`1SMqufbRElkeQnaz^#^iKxBKG95pVqK zKKspt?HS98%(GW@|4T)7T51dPr^s5G``}-)v(s8w;ALads17|(b0gj3ZjC+u)bY*7 zHy>U5=?LuDqs;M`bp^qQff!5fbGO@z@bVvkgDeuMl_ZQh3Si)13vFo z|H_~m<{Z)uq`+f}-9z9wQqPBqbd@yX`{0)7$r)}b3n=>}31Lrv)_bAwfgZXgBg5W#X(?jf!76TS|)B}m{xtY_0kX@+Fn zA~#fL+@kS7m0H4_pKwCDfs}3_#k}lgcc{Q&2cJW_fs}3_rOY9U`4U39ffUjWqzFb{ ziOQc?l`XAw11XZBn%3S)H;{rq}55`^;*8C(>Nf+B@k6Qgw4lW#pwBNC|UE z?e9dIOImv;-9U;kmw2tEm2MzKm`l9Ya7s6jBFrV-KuR>ARr@;$b4g=@h1oml22zB% z#A_|BbOR~UT+$7s;2@wINCk|sRc2&byC${26KO7K@w$N&tV>#hLES(~+~d^#PK3FH z**ob5QiQpr8%T+BN$u}Mm`g}8YvN_YDcwMdFqd=#DRC~T{hbJN3A1+!8v`l*W3Kwg zT*L8WuCE)ADI#w7AucHe{{L=V;<1O5uSTZLxWZXi9h%8`zncE(#@bo?j&9@p#~w~; zZi=-SxJF6FiK8b>Oo|k~+EW^tVk`zjM2f)_Aw4%n!t*F7!v7KJ3FfBb3Lfb)rQq?2 zqYJt`HDc^&i!pO(PA{mis46CpE|~E6xTh?f!wT8EHZ=(xkY8Yt_339^*Cz4{C9|ew zeY=nAnqZn!8EJ_IAGrGym#=q6)kL*5@Y797K6`~ zcx8@Yp|@>Ua#qRkuIWNuQmbg;{Ziqd^pnnV$!;%s!j#7a?4F9utQ9^;2#jP`yXy;@ sLB+g^sI1kTzi>d7z%nNieK+pRq+v$WAw!%ok>}%*Bhn(QXC}q{4}8>f+W-In literal 56476 zcmeHQ3qX`r+CDRY4j_so!lvbjrif_6+_BQpw7?sRDJ@n)>MC9mMYC4Z0ZXNFMJ3m? z^$KY!Y8OjWbIi-8>xH~uNKt__O<7UEAB6w?ZZqGU?>lGk|3Tbsb~^~)`@HY-ocEmX zGV`5FOzew85%Td1@Kt5>*C0#f`SwwwKWATX`jfRjEdCRS-`uAm;{FFj>l*^m-B}7C z3i45*bniD23Tz36FBroWyJ682JrN2sIeGjC1;c8?RkMXtZP@xds_F2k+iE&^L=Es* zD67@3QLEK_e1rW#a`p$QX07%!cg~#dG&Q=?)@lPhzQJm>Ydki==EJi_pyOG?hkMp2 z9S@Hh%|(wIKHQ_muflPmV~FtoIgt)cm#^J+;W$=127e6l)CLkFJd_2gssclNeUy4b zfKqP?E{|g#1v^6ARz9AnBG4lsNL3*Q+++Zy0-&@b1CWm(W#R%E^Z`nPG1xg!Im^Ta z;sMT5I41cBM5YK}($SbW0EG#QKx8rjQzVUv15lW-2qY$>CO~P_QkZxEnF)_TWYPhX zfyTrEC`?!c5|iEx+0#&%cmSCRk3eEF8lg^R3KI_?GvN`4OpyUfQv{8P15lW-2vR2g zywk#&tEZj09KiX6b2@Xe2vR2QyfXpLq^H!02axN8M<6o6)yQO`F>wG26Ba?r#AQzh zS=L8Tvd06+*~249nfUBMFE`VAxrqZ%>V!ogF`412Wu{&CcmSCRk3eLCYq^XKxaR zF+gFaaIj`FhqE`SRkYCewULxo;Q{1U!6O`mRtXWcN^_=Fe0-D<&@7R`S<#U9<87f? zo`z<55U31j&NPeNzLc>a5$xq18M~}MsX`HO(~F?pUU>kyLU;rs6Wp>QXpi6#9Du@v zMIbS0;i9WG(i({akeToZB&GL^S+fXswPXstj#oqUuC z1+<4Q^<)cg_A`?;#?qAi#WKM1C-#?8ukO&tZpJ|$?Cj%#ZpHz=ih{3X@#PHuS?IF` zn-w)CAV?a(OF{!VNpl}Gpi08=xgHG}JXsR%AR>^LWEsdyvhkAqy;L+w68uVE7EhEU zaS##6OG5pHEX8nHdgqoWO_Bt!xy#~-k|Yiy0)nJiUXnGIleDCfH&qgjPkw^MlO^E} zA_93yHo?O(h?Asl=0lSt!CP9ec%meUgNQ(0l68>KaIbKZ*0u1ZNs{3G+pu_|B#DEF zfFP-_&~W`YNySATR7p5K;U^L zga!*)8q8&BTbLhBk_5j+%i@WWBn~11f}}n|U+T+A8huGgm4xFn3RpZ@67C=(ke6f= zvSb;;NwR1g(^LWQ`WG)SJsv{uu>Lk_2zdWAQ{u5(g21yd)GQ^rdJ{(yO_MCP{)X zKFQ*Vk|Yiy0 zKn?B~H@H4RT7T3Xjwd267C=(AV@L`lDcq`^27g4m4xFpp)8&(33m_?$V;-m$V;;J@*wo-d(ElHe^S7EhEUaj-fF2$H%Ak}RAgb>Dkb zNjQGMODvu&33m_?5G3^wB)!N~>p~E$1}IAnQOcUc@~;63#MdAb$Xb(Ffby(?E~~l* zC+$i))l;%@~oj28__{Yl5ngE{G+k zzOzC&x5s;;Q0Nipd>8cTU$ZPq4{x8gnv9i?{Fi_ws({?Je^X(~{2yOs%Eu;W8hdL_ z@kAOcTr4twC#Kju;}05hmsr zZG<`fE%S>`r`H+amq=ZtK9c=K6A6Lfh}mQ|n^~|?XVRIOBYLyJ z%tSUvn6+kx)MPeAni*2?*61UcBLGBN>; zI-}mmkm}7q&crclj1fkL-C#194Gg;>!k{%UTn4?tU|`tw2E9>l3ge%s%38gSP%u)! z|Nm{d=%@H^KYrIy-&-}QF9}~;$4U8r*!YiIoQKt4TX;MVdcL3EZv|gZTB?=M&#RC^ zfuQk|6lUI-mj-j4EeKh=!Ru0s5*?Wp)G6GieBN%(MkoOx&mIV=pfu{C0LYDJym5d( z7@I)}CIO1OBUBCrkpO4Gm=EPzcLIeCj6XriB>`FjzZyzg62&_3-U4MCDMAJqmq1xb z0wjac3S}+{5DNU^P&6bB7VsVbC4m&dAB<`!8WO+;&6N)YkpLDjTA{48V42KK<66_w z1EHQ!qCn&xacc^IzF-^xV@&gXY<0D)B*152Tn=R=2`~bTqoIr=0TzRC36!NI zKn55WLdhfnlEF9`$~z>$hhY2|%HK$U_rW+D%3Km)IvA}`(gXl}!N7j$&?E4vAatN8 zIW%L=7KQGxX7A3Zj1kI$mR(f8_e}cf&!YmGHk}k7zc4c|GxK`e4ku1D9*0JwL^S)k zjI8Zn=}(yFKh=5Y+*2rK(bbfmH&PE~2ds&nur=ngzQNa_bJ~ZdZa&zh@w^<5qqbRR zx-Q7*+B9CBoE^}{W3qR+@wE!G--(Z|bn2d$*lcsGd1Tv0xt;Q(EsG43@8sv_TdWDz zO{u<%?oCz{x9;CN*|2O-QsQBc=&X+We%yE9-SC}FBKyzVz9D@~_j$uZ())~Tz2rdp z{9VP{PsNX&XU<;L{G_?v!l=0BX-TU~&ib}`^Y253d!+35T|UUaFl$Tm1F3B{h4(cb z9e&$31jUr}>d`;yMn~l)!|t&;%XUv`a^%up+tQf{;|$$~_Ayk)h8cGEPG8n7VolMg z_cz_q4=h=;EFth#Z2AJK4exYAKeOU+^b~-LG z*W*(VmpJ#pyQ_C8%FY+hjnb<;%xb{i;koxNbqNVvjYe;ez%bA4g5q$WtAHl;8W_9(Jj?dgrdo)zd0wSC{?n zSAFS%sdD#(3FXxnf4E%z-n6~dWu<=Aw=cA)>UKT5eAO`9?}amh8g6+os8hj?3E%xP zq0vX!$vO7+h4C}zc%`PcF5aTnpn~kvO_3E@UdxYY5I8p}{LKr8-Wq;S;iEsR6*)|i}bkBy6 z5(R9FpX%3CrABB(;K609|MK3rocQe1>jw|ciJx+c{b%ct#hDE&*ByI);gz4ug1*}y z&Ymnu$63-|kJ<4EDU|%{BWMro~LBywDXhNihZD^w|PaOmjUNIzYB0M zfs38lw9N}pc}g;honvK3MMEs-SxFGfKI>>Mb2jgTlbIxqpA%%4M#OP5dh9q-71$|i zc5%{Z=h%esb7=~ggnR`FA>sI$Fcci0U~U4VT^%tVH=T$QkJHCb{i5JFUVa9qOI{r} z!H*J;RKN+`Ir)q)2HVqXuHNiZ-GS|C^%jpH$j-9kHhgZ zl3n8X>Gv*i25vb8*Eq-@ED_-nXM*~|G;xCj756E$d>VXerv)=ki&O@XsSQuN!57tJiH7-<1hEd)<*C95M7QJK!Ea&qX!wFVusd0*Q8+s1HtqXcr!B*dAR(UEW&WVlYVm1q*n3qnlRQ7^BjzwV1}jC zXDK9;BSs(dGPXWTA(@=l#|&MlzXJM52Q2?WlaBm^qb87u~Aw>YMk97ThB*z(jBqBy1*y^OzVDzI`pE_J0(|nf4_KVDm@5KU1zu#C!Tf`xFu$4wyi^KFS@Cc<-(? zA4VT37^AP&FKCqOnH*Q4+(G^rtB-vDBi$Ek^>`uS=F5=q`iS?JNAbyy(MJ*h-(127 zoyYj6cu2m}AbrO$c^DcG72t-+!;pDj$1r*6*mL{%bo(H5>DY7oV0Qc1bNgs?`-pY> z5P18bcy4?;Gf-U^`EDN_Zy!Og6UNlr2jJU>{-uNcxe@(e zD-lw-|ErS>xN}=~UCW(YcW&!019cbK-EZBwt-B1=U1WE^b?3J3GEjGs-Tl^`+q%m@ z-9_H+{#;gfP%b*HyZGf=0&-M!bH-a5@dod$RJUUz!yGy`=S+}(TK>8;ZY)M;>c?{%lQ zPBT!a!QH*LPSd+fiP&Xn>y+dV8yptEsa z-NUgQYT|=uqj9@W?f*heS|01{(A5g z*6bZD)BfT`f93qeYafgKh4?KYz8VU%KC{@K%=6noTRs$KVPm=Y{@~jT3bV$v*k6e6 z2{siJW|3++doir4EG$7SwhR2?m6zrC#Vb^c{RMs-#1|H*7TbmR7O)FTKg-1zFHbG@ z7vg7tzpy&B*e=9R2D=psvp6*qzkpxG_qD)YLFn|zaqU)qutiZl`o%^_?GIVr@?p-k z@x#p>(idve=Z1Dzq}64w&FUNT+$b~>y#cF)FC3C{Wolyb-aD~t<9dCaIQ^~BN3(h_ zTItt$$^LC0zA#3+GHlfNFA4^%XqYmq&DVL|zx=A*6W$iIF51%1``*Bh3LEJDHD&+l z#p+v{4`22+k8G*x|81c6sLpfxH*PSid#WW_oAcvqU0*M0Y;M>7r2c}Uce4^p0lGAC z-oh~-ZBNaLFYNsK%Fo|z+;+j++xqPDo4)q-*qp6Js@5;1Us)0rvf-)B9X%VEG`mkU ze&$s3lD1ddsyi*8sP6p5k+7)~8<;-LS@Yym%D^MPPRw{B;BMecAC`6dbz|9%)9cGj zW&0}r5wWl0T>PA}XR{mKj_VM3afScti}SztzI{7o()!U)Pwld+dW7c}8H0Pgw_Y=- z`Ff4l)Uami=odD=nfF15Y%+iXCM5;*+Htqx+r88Zzdqc80-=;=ee!^yDl3#$>k}>o8{iIlR$( zUGKGc+Ex~`b)*6*EuMqKdrkmRdurXmhNfG>eV5G2Y!*! zJM)#Swqa}Dh<)~K*ovRcK6}*4RL``Sv*|Ca)eOwsHaO+Ry}X7A1G=a6Sa5FZ=FQ4k zmiN;ZwhLYH)h^wgUnWd=w`1RsoL86pR{dR9Z&g53RS*BDsGfhiTOGRMkjGG1jgMK% zH>1A>St`%Bj}rYk`-0P-to32>uw1@RbH_5kzN+;Nf#~im1rP=Ks8G81n+OHA1O;C( zhAVc%q9=MH6lMa0!{a|F7*-punk}4a!`9zXO@~L_R@1>FYJkT=nWLda%KooiqgJc= z_y+rfLt!f~v24E`A8$$9Ddsz7Gxc|ErhcX=H1DA*C=w({{r z6@eZBL8=Nd;3fko6#%6j8Gw8QDH9jSz^#?-9H^XS;sWsiXDJ+$d;}sBw|X~)i33oW zun0sZZoO^_69=F$VG&47{EFXXCLTa$!XprwxHZEmOdNp1ghe1R@vE_unRo!136DTx z;@2%FGw}d26CQ!c#I1ZzVd4N3CM<%Ki9heSwf3C@Ii0y2z*&mZnTth`GI8e}zdF2g zAn}>Y1DvIZ&s;nLk%?O;pTfieC`?!cDHE4H9b{P_5$qgDoIM`kEJd6>Jc5*o&z?Cz zY1RZg2Xe|D2XL0+lszm0iOCFCEwhfo!~@7ocmyI7w{HZwPG$~3VZtIvnfP|%cCv5| zB~3 z>|QmM9pS;JDiEHGp5mX30+}Zx!CcCiD+DuBpFrlhNHE*INgV8EJ$96YgEf;moV`gL z?6yH<4%STOaP}s(3cn)`xm9=oxmEB82ccD*H!gy_+BYuBiiW%&Zwt-xG&IYDKxII4 zrdjOvrHuWEU@z~;*k%1m72-EOB3Fn9kSl~oATq%%E5bxMbs{(bg$av5V$#AzS8Jp& z@c=Rt9)ZNfZ{S499^BKk_+3$aIlv>3nD{NA$V@zdQYW}=YITmRv{pDatEmVSHmk8p zwu>nB>JEMEW*qdv&ORRKW?Vpf=wf7iC5ta-@Xtb@E!eC8hnf)(B;or}%ze;+DhbEu zdNgG4WJ$P#h(KPFWWx)8FBMIa1i#Xk#Slb>MmWJ$P#h(KPFP4E!*1W`Bhp-Gb9EiG6)QIf<#L?ACo zvgydW7QQq|61;yK7EhEUaS#y@B;h;R6c>3=CE@sln=GCz33m_?$V-xJo1$sch$cyb z&ktkqL`f0{5rMoU$=)y9!u)8GB={{_7EhEUaS#y@B;lJDjlQI$O2Y9O1uUK{33m_? z$V;*bJLFh~SpEcK4Uk3Km?lYrw{~IiL`f0{5rMoUE53=+PmxV%k|g*6Jy<+ZlEgto zAeJP$jpF-kS(a8a`v|cV%x$-vx$TPXqBwr#0v6BSrX9E4nh}U4IVdEXg{@Qj)3PMN z`#;0tiCL03hzR5*Np|tdKck>YlHhH5ES@My;vgcBmxS=G$X?AwG)WSC@ktg>lq7Kw z5r`!@ni|eAY(KTiY+7HE;!9aPu`fv-90UYOl1cj&V| z&+gVnI^z(Ix0qNwSrYCbA|OZ#5w?SC$6YGXG43bPnLu`hzJOhBs+Xo7E&ci@TDxC zEJ@SipSED3k8I0y)m@I6YcU2ah&;dqOd#girB4k7}A zB(spEE?k!K!~ac{gyS`#ES@Y0cMuWCOR`G#0i9MslO(}gQ&>DvlEeWy2;?Opd>dEm z^CdJ%61>I4;)#+Z4k7}ABz&J(b>DkbNjQGMODvu&33m_?5G3^w`qGPBUrHEzpDGE* zCy!w9WJ$P#h(KPF72oOBHnfZ;NrF!p%;JfXBn~2i^Zc0O9)V87@Ahg_(Jonb+HPIB}x!I5Zk1qS?=7WNrUSf5JTf zsm?>^o^qH^T95S=jC`Dwaq%ybwNhgrt#|J z?0_~NlfA=@uT_}+PJDEwQ}?{YW}9QpBilC0?UWyFS!9@eCqF;mVok7aO7&fIZ?dAe zb^qSUhGm125)XSsXLa27l4e4XL&l?_+-e+X%B?r>y?<(GYDt_!d zbM~s{C(Z2^M#VKxOIlrW*08?j6toUbA)juaLEts#KlFU7WYc3>NRlZ{lK!U`!7Ui6vg-# zjGj?o{*p@+;Np zVKcwUIrjF2@iXUmrKYwn-lEo^g6z{xkri29%a3RfI5#Q$%?rqa6dQ)8T{fL>IOArM z%Cl?F|4;h4^6PJ$QdnWs|KwFm_84;Ry!^?Q{au46zVVl3+e~QFJsa#xfC9F~PxWi6 zQX@1X@Zhr5e|c|QPJH(1^@9iJ#7{ZJ{?DX>yACY@XF6+LEr6<@l8Aa*EVnU zs~piiA=x-RUbA`6h}2nLC0%^qx!UH^>6sVCf4e_^{@qcTD}Gb7f-mx!;l{_k;d9$( zC7w}8dZZIznmLB?c&jD@cAb9|swKl=zlRO=@E%0D#8BTfd016M4p~DABysUWC6Vay z@CITFK76Q0;X#!9SQz@i`Qbc#D^JM`-{Rlj7W+U;Z}W;mF9Xi`RVLuzWhDC&Z<`mO z@|0v0`#RB%iiTLuFW?}S{RP`x=4{>vCo@SJ|N6@=jfmr3UfFS^DzI;3?c$`-&anyM zUmF!L3Hb^VLc;NHg(x^a!Q8ugyEV{2UE_>pEV zaRzSWplclD&tMF8i8DccO(xek12>M)WjuZqplclT7o)*7FF&%&bv)=-Mzia9MsD<* zJ>D?52zByNA_U*^<|&1jt4Q*(-?o)`erNWI7kiv@c;KJP(X#D~M*-Ob|Imy+-zDcE zQiXqIgEoHYLKXfg*m~9`$04c*{*e=1{LzI6{(;SUb)TGvs2-AU!!#S{^^knrLDvt+ zafs@He`d4Z`bo}1q$*hW7^&tyMk3W37pf%R7~DN1$01UMf5TyJ-Tzr zg)00F1*$$u=Rhi<=FRw;+dh%waHx`g5pA2|!XrxHh~_!g=F&YRuLRJ>pXnY5e@BMC zJWltp;;%!|x9oSz_<1g>d4HiEF?>CCoOHo~zw|=CUv%Li846-M<-!9WNP+gBrh7=n zIiN#l=pHDRuemjrulcdFbPvf3LUipH7ajtKFiHTOJV*DCykke_&eJ{6V4)s^`FdQ< zqkBl+P@{|N*ZoW{aZm7ePw-+!zL-W!xi<>7z)d*zd@;os`m;7_#bVz1CGzbQ0T z8B@(Gms~Erx_iz)@)VX&v`@BG&G|^-|Da{?xsMg9ghIgJiDQWV^ zNuA$H96vI^vu9lFOJE2P8QvZ_DJ5ybTLB84Che}GOJf!Mgy*rP?@y2VZIjaJ?{)BZ zkZN|GSAe$)`F9Oop7kXIT~gdU3IPxd@K;*q$f%+mWtSGJg_%v^qcM!$WrjZAoz1*- z{(i4IYFWCXOBiFcKSUzLy$-g;u@@S3%G4wV{8MhvsGZ#{8U3R`OsqGmj=lZr%_*0s Yz3QnttMvB_wp#tw9-$tZ%hUY-7b2-5y#N3J diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_map_with_null.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_map_with_null.dwrf index 0469c3d904133cc3266ea771f07893df7d1887fa..5739b9fff9e0906ce91618b4b91620a6c42239c3 100755 GIT binary patch literal 55057 zcmeHQ33yc1**-JbCPTn51__H0B0}H?I{Qo{=w%TD$dVw7{zYQ_6%-ZJ(2Ce{i4m#t zV~AGxZ7P8V6p_WcKvl>fQ4~!S9AVXDcg>`RY+(7K(#9Cl z9B3I5v#h*`Go(6w-lT_Ym_9X}Atzco&Pf$(yWh}<+%x~ ztjonbmT?#|4qi46%m0k&^Rr~BRv6RfWixGFYGq;+JarQ2fkyR7G&V1B!HdI?=e@=X zy;S!^b+@nzdn4?Z#+qdaYYwyso2hSJ+_G-Aq@mz7u9xOy%|kY`RPXT2@-%zNxpj_h zDKJ}d{${LlSMYNxc+D(-dX0^vm#H@7?Bq;RkC(1Cj!hZasZN{QW%Fj1^nfGmrVKcI zbap1AO&Q8QD*%-AK=*lhfGlSU+T0JlWE%#E8hgVSbCY0<8hbusrr~qh0>U&H0Bafq zz>>xmH4POpUrA#NokoqSM$lyoXUflDhB`?F#xB*>_JK6Ek%tH z4Qgm(U^Uu+DV?z?mnfzZZ)8jFd#B1KKEK1t6VJW~q`+2~Gz* zfGOKnk&g$#>2M%QI&+kChCw$Vj?qnplFocO9Zemt-(~j(h3e=Kkvdc_7IikM>FAhJ zC7lvpI-W43;})i)LqzFNy)e?zF}!p%nY|uJ$0sz439*_*K`drzb4SOB=Z-f7>Ew#j zVL^y=NC+z(4Z}-En>#+Y%kJ|E&C($vv#4GeX6YEQSw0`66A-Gygjm&~AQpABsf;ng zsq70vI$^Oo8bqiL*$Yb@4HHG3T$kPN5v#+35bBT+mO45{sE!}1vR_=4b%;nEsuzYj zI!3IHAL>rPEmVgIv8qEsEb3?rUB~cNmR5HHPEi}Jj3)Q>f zZK1kZP%8uCsvKZJa5|h`Eb3@;NAHGbmc}axRXOMts-r_h>QKEf)X_0wb%IcpgW{^p zgjm&~APjYMOcZsXRtCdjbu@@j9kLgeIvOU5I#4S^;;I~EK?rq72umFeBUUE_RXOAr zs-r_h>QKEf)X_0wbwW^gLgKo^gjm&~APjYMOcZrslL))T>Sz$5I%F>vb+m;Z(z{X0 zAonB@hD{V=^W?nbN*l-#hmDu-DRLLCyq zQb)sx)yajboGY%%Iz*%n)eA!%+>KD3T&T*q;;PJqSk<8*40Uu&gE|kw8NIjDj*zQ6 z9CzDyNX2g#Xsv%EwNt+-sghvwmlnclypJ;hPUB~e|AjLV&8@PXUBJBWm4F&MKO?DF z$*9bkk2oA0lFlImbEpw=5aF2|q6KIiwiq0!fN4)C8C4J*Y78GTF^4Rj!}0f3*4T$K z#fOc2ynIN-pC|y|3=U^FfD*8X^nuD~{gNm?WMdB9G!F984~0I6@CFCdWQ`U;`9KDs zc`Ks$kb^n&(mAZI6Z$~nBULJ%?*kbin+9O|Kn9@VwNZTNgE@HcdiYZ4g9y*`AzA?C z0~Ijsr6@jxF^4NO4wCb*&<7G02RE+;$ZS& z0QO;^#zER}T<8OdFIuDG`96>V<^h;KkO8uH3yXuX9^`@8hd~+#dFe@^4Fj*92|Kn6ri12BCc1LR{{SsV->P(J1`MB|V? zsb1uR5YO@9; zxJKhpcmIzfA7l}p$st+*wI0X-wEI0ZBf`0#LVIVjobU&Ot8Je3&*NMdSm8&zhp*xjs+< z<^fnfPyvyVyTv|4Fb8S0#-Zp2hsXyZp5;T-0MZ940Hy5_`yh|T>)|?$L;8?Zkq;EU zZk&qe`alJk2QYjvp7T)wsGiBeDBsd`cs=MG$_6_{J_zwFAEE}3K2QPZJN1bd`K;2k zKa9caVT|U(T7L_X4~WE94pQ-4AE*HH04yJ<0QA*Ithr?P5WzV~V>J#DZ<@#lA)e(! z)Bw^4Dgf>In8m^HK^}|O!#It@sH~PEA1Hj0N5yk}paRSTuza8b(7T_oI2b-iMS7uqY@`lF9dU#`{ z^HR72d%3Ms)cj2!ZgU5SUN+e2y4bNJ)s>=lzI810{dTu}bOk~=FaP4lqK>ovgit<0 z@>P(AgzTt=+_q)nw8T8UJ!jT3go>a^0t{T0k`1+!4_=;10wT~@2N&wk0i^dwC<`t( z4*#0hQ>W` z)$ssm078cWaDoJ+>q#a8zyVj<02{Kgl1zuj47f4@U}1rK$b!bMaCPMM&)+IQG#m_4{$9sj)Lnt5`c5?Lt_}OJYEjtq4Ae+6_GvQ9R3FyC&P6I4=@cHi{ZMD z2PlEY2jG%-EihwBj@pbQ$z;d+7xSOkqv!Sy5wz(Q3*V+CAK1AuAw zQfRD#YdNonXQA;qxSr?ruo@cI!u1zk4=+LED{!sn_3$b*z7E$$+(R2!$o9l!hcm5J zy34KCf{M$g!@NoX9`J=8AM{yE+l_kDu+F`0NRn-KyDZD!r^@NCwOU(y_vR1A$#0CA zlr;I5(+lM}D@RNoz0lrAkGR!;Cw!OdaeF*AbYq(b-nY-Tp;^dw;eI6}TTh)_O6Li% zem!0{2ql3`7do|6gPK#N=JN+^D6vUW3x>jGYW7*^Lj6ibcD!^mu2Pn+$LsNV{5I4= z`o1txos$EF>Ydl`4S0h#l++Zwqu6=Eo?N9`HcfV3kJsz< zHCcAK-jEk+X4CY}=k>XLaEsC;*d-WM#i3RepU>y_!B$>>)dM}W8?Xy_N1&>TFX#*T zn##n`LC`9e*0W`=^HPzcTlO=8_qyLs3h3Wu8PU3AzJ_J*76omki(4us)erD@)Yg+t+R zxT(}kYTf0!bK#L+Q}r&_m+Q~PHFKXG$=}xRt*@@1b-Mn?`XB0#*T?)vIbH+fam!;X zz(WXLOiVbJFN6{X>o4aOwW@_zyJaM2CCb%FD0ztYTKAQST^(I3LyNEIt=Jn|a`ZHk z<20?XyY=OJV>-PX$KG_Fy|E=n7YR8&MWeypz2z}=_w>2jdpdz$z3Fz|>Wyb*29f2s zYzcu+q?(T`E&c;CD)pw}2Z3(wyqY$Sheto5rt zxMI}ISe8sVwnr1XT$RUGfOB8Iu>Hd4wbT!96}9>RKF1@&%@x>-l28);U^2Gs=yf2+ z`}V!OF{?3MuNuYQM83bVT&t|58uuH33-QS$v3}w1O5&|wdPT){K{Y>`Jlofh==uzI zS6qMNeS6dVZ1QaVg{K?D`i1S6>o2Qj(l;TEw@WuaoIG298SB@EvUv5mu^-n&n;FfI zug=|H_)T40f2rp=v0S`he%#+#P#)VSc#U!Y4F0C`)^BX@rs$_B&95$;?fRwnUpO9d z*KcfRFMZaG`}61W*cPnDSig3``i(6+`pIA1pFfw=HHDsq=kF;&vL?`(a&?@ z{`|Q-w$Gm%$zKO=|BWp>dL1-Be?Hs&mp&iC@t5mwwDSvHQP1xaujc2^=fmGDuD^}B z*U;6tntGff$NKqpK0UV#4ouHp5;RW{^?Pc5Kol=tl6 z0(3Yjwe_j``h#n1mp^~-)zt@IeGRVFQx3ql_e}WZ?&%YDKXo{L*SNhY%^z8Tjq5`E z$O>V7`cJd|Oe$|<7T&nOZR|q)I16dL{j68L@`f(!c0=vAjp@4jtS0Uc#1=FRQfw|;Y2}{0Wer&1JXTP|=nz10Z&->4V`S5N{(tM@p z{{-`6OPxL##{H#^1+jft+(_m_**{D3$0uuVC)?JhO*?dJx4V1EDR-bhwmX`)bNtl* z{;F6msF57xz`iInBHKg$GrwTLr*LLo|ednp|N5i%C;SoDejo5kL(a=AyJ=%A3ui9Bz=nlWKy|o)T!mW|*WX6-}9bQ?N`O3O^udGX3w=V6qb!i{3oA=I&J*O*r z9(}m@G>(lwO6t%FCZh=(WOA6k7PYe7v%AS!%+~c{#VAXn(ls^NNILC-sb= z5xy#itpW{jj6hxF?s9sho#d!n|9MZjx2MC{;a`uw-TA`O)$(nVA9`WHZaGl=(D;Gl zB7v96$1fQdY5($oUDY#t$-jPoVYlMpDyi&P>FqF63T1mN_E@;oqm!Q(;@GKArJ6|JfyHJr&$&OkpoKGv0K?J z&4Ct{?azqiN5y<$KUCUy!s>yRAr)a2mNl(7!m7#cnn?}W!16_ zNOk(WNe|gDeQG#EngcCEVwNxPhBPN@p0Ve~ata7L|7SF$SsydYa}!osmy3BU<1l0# zylfnn{~6QgXUS5nFs9ARX4<^e%ETym>Lkzujp~zVY+mAm7l$FwdyN%(sqTsDZebPn zM%XWnHOmmz9B2_XQ{TL}W!-E^L&0lYFU`rChiqo4-rm1usV7BD^%~<8G z;OA8Enpytz8XHG1Q*FrE$(f`cFX8XbZOX__b=ur6n>Vwh2OMEHWx(O1vojfO%24)M z0idJ@y3fl4WI0pN=6>iU+b}@X*c--}tNiV|8u-(9%rty1TR@lw17J;q09ex4qNbrD z<|}Dzq0^{Q)d;$5;mo;;8Z}sr8c?GnP}yhzPUQe#BL?6!m^~=uxW*JbzkGs_AAG`EMEO){F>Ls6xzqKW}IQ2>xr z%2SgrFpWwJm07~=B^x-+o2e=T7`v&PGN&?95|s{Uql^`RbV8Y>N;)Mt9q<6AY+FS> z9t5Yufhg(BQPLR(-GDeoHx)`c^XYUnb-aF;-5V6DqeDdMP`y~x*`%hUV@j2DN_gpb z!jO(zn2rt+r9<_?NJq!;($Qr0dLSL2&@3jzY8C~tn5E4f9V4DQ-VmgdD^7<6A<`is ztaLOCFCA^}_}nhL&nq-bhltFgdSRHQW5j0pe2`8+s16fiRfmFD)X}Cg#t5gfF9_*` z#p-Afp*mzQEOj(Y6m@c4cE3lg4hur4Lqb^U=oq0oeyGZRaaGnKB6X-<80zR4u{wUJ zI{~*)9VW!84h6BOqb+nD!&_Ne-3dV535e?s6Jkw=f-ut2G2(RmuxI(jdlnO7O^1S5 z($Q8H#&G72wp|Bc&k6)HtrO!^4$8ZvVY}8L)=eL$!|6pyM^?7b{Dv)5?}oR9>SjT$ z42Y|8fCa(naC)(*qs<+?8=hGjuOL+ApjW7l4iTwC^}URdgAm?-K%tqh5)a*zcf)FB}(bu^4voe)&zkYA{d4iTwC z^}kboQRfmEw)X_0f)PYSR>=vt|L4@j%y;#)I7J5kUMk#~blSCLc ziLiK+U_z|wP!NVXI!3Hc7^-quT$Oc*NFAydhB~+#u{uz4!{VwOW9`JP2p>-cCD0uI_N$ZQmgk zzg?iU{*Ba5{idW!g2`W62&eHr&ICA(pE>>)&O|h~%64`E^S)OCYV7=sq+%tbGG{*G zaBxUEhYZZ2M#w>gXL5)ZpmEq@aG(OFJ)vY&L2#%se8|KcvUCo|-&a{vFqWF-FIds!F$V)#I`XIs^988lnS^(t(8Gz=kh~h&I z=Fm&$u)0p@1Bs7Rsd&B*WPofMfawDnfQr{f@u3gq;KA$ROQ8=UJky700hA9^z_gd5 z_z=b%uFyD0&ci|5jf&^{Kn9oxVERA?$lfh14#s+r2Vx%vX&mIGCxt$U@Jt_~1yDYa0Vv}g z76+3LS7RRr>+9jkAA~-T_{updp6>%05HStF^nnbJk8NdfFnmDyn8Of_L;9q8kq<&V z%ZI1|R9zx7LiTN{s{A!Q1ka_R8i&disK4O+AdB!U4p9S04paat-X4X+V9enfjYHl2 zKZ<;iMR+EMXaUrEAOq0u_tcCCA57(o%#Ncp-11JHqdVVmSfyfc0GWCQ)o`H<@AYY4p7_M>H{s+5oE{X64hcg^N383Z@ zIRd(22Wu{wWEz2e7^!h^&q)yaK;kPGsCd2)WPo`9rVnI5-fjRPu4 z6#5{-Gku5_K>0ugNPidmAdkX66zFs5?j(^96u#(pDxT{D6|mhj0Lup|K)U4vu@6## z&LN_6aNm_I@ogAOLsCUPQ24rWDxT{D z6<{8~@WFV_M+KmICI_Q@OV{D`pmQi2>=gMR#It;e8bJC$1)%TLCtl>UO4I%@2Cs)P znh$IJEkr&b5??t;#dCe20?Y%je4ql*S0AzFlHo%H=Om5QI7GZ@A|Hf!mJd+_NFS&G zwC7_M2g3(>EM5=eG!CP(T8ez2@I@XK&-H-{Fb}};feJwHe!}8l_#lnL>p|ymxKk^U z4?;Z4ho}Lh4^#l!_$lUa6h5Wc5pG1`1&B8AK}naxfzn->W!1_X8YAoBjg`(z;STKO zwoXy=H+{Ix9UywyV5jS1$BtB2irV?svC#M1-SScM$C;Zy&fNTQ<}>~{^AbC9B&2`# zg1!A^@Neb4ne*1$veqQ$l>E!)KD6w&;qQK^{J%F=R6MfGCP7Q~#f~(W-Im-wvG|sI zi`yqD|MzAmJ6gEx_JnM^BSHHkC2po%#lkmDgjx_&5)s zM^<&pKY0BWJ(V`<<;aLuWV6=SOj3&Mzvf8!i*sQ>NJy_`|9%`kQ!mZko|0c>lgdWq hLz0C_sPf)zGsY)6cG}YuGi5nFDExUCB6N43zpjZ&nr2ZnR8NXLs&N69d0`3`~UCz z|8wTdBr|jV{^<`tg^;GHu8BIgpB~xPpX`!~{khrmPqy`D@de=jE=?2U{!gb-|J+Eg zupOQOTx~R=Xko}mXnkIZbi;+VBBrgarYTz9G6kVIit#}x`T?PlF_d+Ck#T!c>NW!` ze`4C+A-HuJ-b%M@64*eoBq*>Xun8h1fiaX->r1x{-(@};3GUwZ%p8%D)Mc(%@S_X{SUr>ABenloBf&>W@KKiq#;tJfX!!ZoM_u-xHNbpy z@cJkTzC0Y}jUivv%w^h0Cv?V8_N3#>+%~*gHn+>9NpPJae;qBK1ec*~5`5WZrPIh= zcA9A;bs1y$8C>7qqfrZ&x4PEq!d9J{Xq4tSozfg1F)tl1t8-$(Ne7Jz9cUFq2h_%p z0aB&|EK{6Wr;M{iTuTF{8!WqA|$4m@IMtJSH(f%EZMKZv-YYB_*%S;=4vH+Y(43IK$v0Gs-x6R@z){l>^{0i2+h3E~W&SVH0RGEDs>ZB=#U>;$ljG88(47!^#5i zn8W}n6L;L1VAe8ODMv03ARoD64^k#BCKJSDqQxW&z+(~v4#5>mN9>9vjJaY7(J1sf zg)t&07A_JBTftRCJN|lw(#){PH6*;8bLwor+`u_<$Az zNK8hU!Hl#iP8NVOi2+h3Za5g=oMzNhh64|vWKZNl%EV>Q2q!%w?W89Q@Q6v|LCVC% zWQ0?Z(L#xd2asbDd+4Y@nywlpLb_-e+-=&1uwR<&{cSDS54HieyVwsosw5cg^U`6o zchdx+dFge>cZZ{6LB68S2DqdHZb<`pNhpnzR5B%)k#sGMmvk)+7>daFj9Dx`mkGfw zscr4_FalhXJFiY5OSXZWq>Q0zTE|iGaveuAz$Ljm z&OT7c(jZQfws#0EOA>t95Ef6&lEeTbz$Lk}WE&(5Z(dSvFAXhARJ>f4Xa@3dQ2GJeK<+Es}nRPFUdYwkTisoG$67m zO_BtkVPNq@NfHB$0GH(MINK0jl9QKI*}fUAb?{M8(TVq8Sh*srf9S zR)VA=sw5g-RuaVkm*l=ALlJ_cNM2Ia=Zqxw2x6`(bD673Y|;?p?H5@*d&%dPR5t=# zlKY55odiiyoFx0k)6^`{@NP+UBfuqLLBt4>?&l?KE2CyfjCbaJCmL)1)E=x26f~1E9 zNhzEp`^M{3Ni@8yB#Hqp3G29Iq2nIrB-!>?Q9DkI&)Cl5$sH#)zzB{gk-et2y0)se zqW0_BgSE5&sJ-QX+Wkc=@H%{)fFGDF-}<}?;c?N~{no#{4u_8i)-^WE{vFC++x>J5 zx%%4S;{>o-_QNFM3XqA3)8VM~?O?8C#)QQu(sg&V!^bE8Q|!F#&kGEQ)$W%o8+Vdx9X?J#U|wd9qrklE*q8(yw2S9o4C2L?B_K5{YOEEj}yRV+221S`29yghu5#C^_x<% z*zbORPP3hMX7cfgiJhz*K28Ap-OtC@F2hPcl-L+ zqQKdY`9|dHv)_x%SoZVPYOUYDeERzLN`MBNTi z+zuMWI>MXM{0W2NVpPNbkO2K-;{dRCQxS_Bt@afmR1bI5_I! zOCjs4n?g9YGgT-CMg%HNL~my2+mt~en$G%M$1VS$Tcy^OY+0%nDbwD*%9OWF&h)O{ zd|rk0c9<+UY4dJ*FS>>f*PZ>ENTl)oc^gU-4uac2T4Pu=1u5(BuL8qxpYc6)@zB5+ zScI=esboZ0;;rzn$s7m&>J3IH*<;Wf3<*sUGMbHXMho-7U^E&{@WEg;BpCI~o_M3x z$Y2a{28$t{`Cv4d3}*OX(wht>BlE#zGMnPyLxL&6oWOiASxoUJE3-$R0FK~;HQs8q zGJdRPYn+wgvKp*LE5ijwdb0sO#9QMNtPE*i9k0V-%z!xjpFn%`vrcSB<& zY@Njbfub@Ap=+=q62&QKEQ9TbBoT$+!3m8Y!BzqQ50L^O2KeuXO;4iO3f4Pd+ef0v zg~nB|Im8q~9-V%EdD4vDJv9L`P0|b$4p>Z8-4id#mXj}!` zYLbUsXj}%{3b6-)A`2R)!S-0)oN!e>(_Rwi7~Ti--I zKbT&8W@=;jPpO)_yy@1;6$ir>^jT*tcxu(Ag5TO$_a17~HEvvXbf*DP#cQTb>GRy6 z89VdZ-nVFD<#Bb37ITbg=kq6xSiPxq@vCS1nao>z?%P<9r`)5nUMLK2@n+2OZhZz` zI($0hb>)Dor$+4iDkEi8pA~UGrtACe{%T6piE(E8#+PTz{-)oH2V2hjWma-QVfB~# z&J`Wflh3c$j<0y++>z?)wQFOtk5u>X)m1Y+b!+pmVa3X@b!F{?6~*}bdW}@E&bYVn$-5E~6`w^6uQ*aQwBq2& z!4-RVTh0$(Gyn1*^GAeNj9K||c5U*HS7#+|`LF0F7@mivg?i2oyT?`|7h<16P=T0qz%luxH&4M zbn$B4v)w=1{NkqJ&h?9G8Z|5XeM{-s?K65mHuTiYe8rBpzpc#)`%89@y#;+I_9}f? zd8p$B)%>LFAGaTyIQpJ5?@h>eX4w~~7LGhSHFJE+Gw*HJtF%>%PhV`kXXo&e(Ze&3 zMsK|sySK~UgEf~X41f4vySnrQ$RQbu4;Q46X^=H4a9#RaO6aP?B^|{HjuB_>p+Nj1< zyya>KSgoC@n)yYD9@*v`SEVB9o;bincvij)QBQ;_l#b@9$OnyQLkVs(f%7L)WBmjX zIjL?LK}F37YQ0+S{8QcWtt<$1X}4_Q5*^NA}0%PpB#6Ezf+pBuc!TKfZXFS6m!dqK$&%Ml)aF z&NZ5)r=POHji-Rvh=arW)?BraX-irg0BC0x;|_u1tk zMHU!`c!gE;kFBm@4u5q-bBEoCT&zynLvA@*z1@ZVS73`B0R0Sk40L z5ApKPXh{KMLFo_i+L(6Z5jhS_m3V_pFNf8M&^b0Q>{e;N*8Y#}~<6a_1NVMX-ZJHbh7P+LRkL`q< zg*$S)xa5v))R%Nsk_tB{>Li^*QcuR&|0_C&q^trudRC6Z9l4|$3kp3&w~!P|LA72i zB(+S?FYH3{;$sj~^*G<>Hoi&+I(3GwN>T{~m6r+@9TB?h`F=npv&-`dUT&Ain~dx( z&u6i-7(>tZPFR$OT@B~Ud!A;AtLss?2X2^e>5= z&mZ%Elm9lzABmjP7YWUN+{(%1tUh?Ika@PyZ(BK;oYTiV3+KnJoJ`K@^Q$<}R!$bc z>+@shLf?Wuk^oMh4VwM9m6OQ@eSST`G5QDsczu36xxu*&^pOOx`usTcu`?eWzr^{M zxjyjg$rsMOLVrh zR_!mR5HZJx|E>(U^9GYY!U=@$Uq7BaU~UF|L;+0x2y-`SPH*u2OTu`2Hp8C&kr}`}55Z4a-g#$h0b_0K^4Jp{&K zAcytOK>a-|3|t>J#N}}yjrAXj;31I4KpGn)0}bLNFd74CY>*5zh?Br*45YC^GSDDS z0;4gI#s5=TG>DVHXbhyWK{C)FP6DH`LDKkI;Ah*0 z_}LcvB9O@j8IXZYHdGn|nG9sI0W;80t^)I9LuGQg60vL9Hk7~rT0gKDu!UdeSF3sz zmj6YrWq=j?XO2*WC%Rhy#;yX4FQ}@h>E2u1j<2UG*N(5)D%Xy$rYhHtFQ$sO->u^G z;cKZjmC=WvIedweNZIeUD$Svh-rNnAO5eA!;y zUWm^I@uk4VEG>+;3-Q^Q5da&rF0h<`d{JNAUd87RUlsV*8}9KTwBze9PL2B);>&_|J8aD2)J%LX6PTj`dqSn*?Q?D6BxCc;u3 zBT-5Chds1^UARl%x5sChp_d;$*)r_AC6!ILwS9Wg)|2BjqtliY|a~7tp zsG73+?L?`xM(wJo?Fi_iL<8--+oq<-2yh6q&E8+I@GwZSUpIk;RDz zM(86#)#~6M2Dgrx@VoLvm&m2>u6}sH+et4Ccx#Dz*T|Pp?{G`mqK!e9Q=W`EZvE`l z1yP>SW^1NUQgLB*w#fCf4l0#>67A% z?vEYhZ25B6%`YE$Z0+%JhW;zc?;hKEeD5oN`Tk$mpN{|H#<|kZZ@g;Qbp7#hW zo_OUSE5=`m9=-AUT^s)KK-!6?$6eX9W@Om^r7z5DGU(~T;CrVHnd6+ZFE8e?Li5s> zCUwz9?|$Ov!UwGvo}ZaEC^IRdaq^qfH|Jam*`};&c5L?gF_zj3)?L^C_SV8IU)U+FM^!|JbN7M-~9^6s-Y_MF^u{k|l} zzoTQqR)LIbn)g`mSU&c()K1^O zVQ zc=nZ7XN-P#LeI0`YsbaCk@LO1$%g#BCmVhF=$5jkC9VG+-tvVvD>DaezTV`OGaV{s zuHR8|@RdI%Z#ZE*w6dk;*e?sJvaYuuwYfa#DOfU(S?@QupB~xPpX`!~{khrmPqy`D z@de=jF3+`sHR_)m!Q!%qX8>0lO(P%%P0{j}DG1F`j1NN5 z4+xEnp{(1BjN6k^w;5ph6Vvt%!L7^iR=Q=Azy^vXL4hTKO%N#wjG?SrU%GAhF7wey zaQC)nZim50aCNra0CO42=GJYBxrK3C&jIG6E_20#A7wDW>X`%|sr}Cy2|h}KkGd={ zZiRD4!_T)q>azc=0p_EF*GEb4<>4@I4Ed^NF4IOjp)-cECmmnrw&B&Xxm_krg6kCd z>uC8TxC~{J;L9#6oks4m(@Yzw%NWDY;QH2ct^LAQ%tH0T66NV|S)CILPC95*=s>F= zU28Q3P#Z%ANSO|>O#C9^*V2ILMhwsNKFd^u19BND3j`?~le`Bh6DOHpUVI)gCRqR; zlNdl`;ucV+Fv$Ttm_z_66Bj$b6goM3Jb;`%u?Hy=mpy(lb#hFy06Zo!fW*WvQ%`1+ z1>j6#fRu^P9=D)7C3_rzl0A_JDHESPZb^7@Ocpr+9+Ma#W#VGu7nLW+!~@7Ni9JY} zxC4_fci`!$kUeraDLlYa3Kx^S2PqSmy?AbcescC?0eJSr04WonJ+4%Nr(}FK5^5O` zXCofq@*rj6V&aQ7kYkbs;4z5-BqkGIzJfUK$pUaDF+j@1Wsfgd;fkF-Yv9=vDq0Y; zC-xv^;(MXU)Cl7YT)};3}dWf4vgMT(8JCGc5AWBo4M15Sc@!8Rzh9mU2L5UDb=2IEsWg zm^n2}4Z~4rz7Ye>2ZUyhL(W>-D}3*91$QX@&jBd?FY@p)^onPdB}lsl3LZhXBt&dW z0ljZ@f^KOK-QsMe)P*zMLTH~8OKc|vB-CH>I0o@k5f2~-B=#U>0>SPIPUM4>FY)9F zzz4J#Kw>g-)ss9O;g{gD0Gvq-kTP+@!NAp#A`b^1K*^rSgOrKO9$$%y5))ka8cmex zKZ+!W%H9VJtO8A;dDcuCjNfT4(t&zQyHbD0p_lIli)OL9josR^Q= z)QFZPD&8%rZUnd_cMy`IBUw*3re#TjkDS2biCK~uU<3q7;tDYO`9V}kV!R`d#giq8 z4KM;+k~^=GvMCuu)wGVI;^jJyW`Ij_cbudyi?(+NElU!7*$@^_%#y?aBfurOvm`0} zlG{r|%MukYmnE8kyd+8ep_12?G)YvvoFtk7F3H_-lF~Ih%uQ$=C&4>XSUj=gBnB7( zF3H_-PI3Jmr!kb4B`RJnOEd#q5|$-#sj1x63Tl?b__BN!PtKCq03(o>B&oJEAhIb< zk_4Y&VDUsr5(A6?m*nm^Nnx?d_RVM=N5#u^9L+#pl3iRKtt`8Ssmw&Y5}@JbB+&>6 zl6v#yrRthT~-=Q3&KEfe2B}vYdQcmZ*3+Ni+k3BuUA+oFb|u z8eUcs#Q>M&z9d7EYJOF0y*c&>Vy-H4nX5`{(h%eA7g;=e$>)|-Hv(Le`-ns0vWNDK zY;iNv@TTJ3lIli)OTvPP;fFW6pBvuxZDnjNG~wD?oFy^dna|?MSrQvy1h^zDh*UvR zZ%&eSOf7XR(eScaq8Jb)brU2RI7ukyJE|laURDyt0GEVyoVY-#eZ&vcjuYc;16e$| zLZ#Ltb)2{gt@FYaYL>+K$Wtty zoF%aVMu1ClXUQ%ua~oOwD=kY@yj+%O1_Vjs8n^b1*Qt_dcv(pl16&f;apFR~w*6Jq zjuYcEwzGJ0$B7Lvf++q;?@q1zMxZ@0&&TAxvO{4y95(og9;)6^wolgtO)ZK&HFV7A zs1~DiT}F-8jf$G8TlV9)##ty6jX~R+SG8FZn)h*fRM+sg3Ql(abY9|vj$yXXTQ1qO zvs@E>qOIwI$!g#;<*1yxuvW_1H!Se#>fJ9Aw_C*qxa2!pzLf zRUhz{zPrDg5_Mvn*}n1R z8MD9X_u|2p^M09?Tu@m3rM`1ThxFw0>$T%69yxcUx_a%}nCv6f{d;xQOi$g~JZxC8 zGHhL0`(QLAfca<(KyL-f@tBrc@yDz*)8QT-`41oHdftZNUV5Gn_Tfq#e)^IKW%ruV(FpEKMv_vSG9fHP%(K)*NQO{ z!pDDiYZtI&lvgMk*7qps3S2%~9EbFW@EG$at)FLl6rL(T%2S;CgPKhG5 z5o(oEImd2`)S8nozp?(Ocb2_9YSvR{2H$wHdexrmx(U|%BN|nHvL$#vT6g`~Z>)zD z!{)?4lvI6g@~kUsI;J+NF%@sQ+5v7XFb{Zt5dvSO<{Ve0BI%wuz(jaHq#dH32oFp< znx`TkG@cD5xXlF4pGb}M6GY^sx@80vH6y6?YPs`IrEdaoE~QznCpA(nnNX;L{+wGX zr;i$`iKlXUO@a?bJe8B$D8%^~9G{*KnH4FSdujab>Rb&R+6#hH(IbHKytNBB;hyJ$ zU_ccqQ&Dd~dERO~qIBO%{6Gz`4KPheodFD-v3FsYGR}^GGmWs2vTo_oHy<^+C0Wv4tSH|xWw@{!CZ-x;y8c&EiAA8ghv?2(ek|9ElIDu zChqPqg_qMO+}rmSFaNBDcV6zf2JgK56ES37P9OgOlUIMj(~@L=T>gZ|U%cg+f3k-X zFXxYc&dn<>j(eJrg5yRr|173!G)u=7$16Mp<`NS}ezbsS@9UKwo_#XW>>Qxh;YYoRd?9Wmhf4(Ta~F zYXxH=&aRsSiGSim%FpCD+!m7i7HC?DoCT&zd=t!;%l=?aRwcRLiH3gRg`=Mkd4Dc) zXZCR~ks~C!Ibxe8$ALvIdHlq7Le9b+xn2Bph;7uDbXAgvF;LV=I)~)Z2WS7U=p2$e zfavI1ISzN^k_QJ+=qb8|0zM zDp&+w{tvwTA9(qn`11dDB~k}z3swfTxd;CL(Z?gcT%?Fx*(hemPYV5ooGDiO#!GWA zy!QFr9~3rdY13K#zyDTfYuZJeSgeR#-zWxtQi30ptSbwyUo8IM^K};$HfZP+st(mD z6m5cE7%}nrQJos8o9L9vpiWA45cgRX44)gr7JP4OP4mf9pXv73spF=Nn%ZsBh|Ezs zRmza`UeKV!8eSMR^|^7AC+QR>ea=-yVly@T3+KP@zI;CQ^1>jGzqiBRt=03-2J1r9 zNc&L4y8N9CbX8?|Dgxm8e+0viAfJ>EOTDxpD6y^j+#D@@2Zbk}l?ibX-=28@pqJln zpSp6cBJp0P(RINooK~De*hwjJU(p!mF@US{ZK?lCvgP(m1u^L%=;-6$JavA0&Fm*s V>Qbdv6=An)BZFds^fj}!{|`U~9`*nL diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_mixed_encodings.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_mixed_encodings.dwrf index 212e8fcaac3e8da4bf208fefb1146265ec2a6ff8..e0c95aa1886a3a1b3001556b36eb18c6bba83394 100755 GIT binary patch literal 52812 zcmd>{2RxPk`~MG>Y{$w@l)W;GME1y@QIc#WSw#aCviIKWn4yf~q)4)}WfUSiStf!C&SJ(BvpXa)+*LB~N)g&At5Ntdm9LxZvqmWZ&-h{HK z{IdlMT)g0mE(_W6?JlkOKY#nK|6Kp}ha+gRki%#YY&vWV$ofGcG&5gHi0h$#;&>; zxlK@L0BBQ!{|>SHv7f+2piKc&NNs{bn?Rf1*Y9SNEqL{TYaZi}PrI+jt{%Mlc8nb^ zv>vpn$nW{~*!>^^xCpdq;uF&SgbVcn_tQe%Zu@D?CU6ZHyOc2GHsM1X!2MJ|h;%>U zL;b-0^!yXj{RA!o{ZzlO+kRTJ2|OEUQ*Fk2>~{4N5wr#LQvn>QpTI?+p9UiHx9{;E zq>&#s96SxNSlSd+Qn<$~(VC*VNkj9P@ z8U@BK^A*zAfr~&tRWu=s9e6g-rkro@ExYb%)@ep;zS8ary}JJ6=i zlpXf?Iy2Y&1g-&N*A}>|pVpI+L(uo2pAtsb{j`~ktY<70YwyA9Y0V*M0_dl*nRP#H zrZwxi!TKKm{z?g;O$jUOep-K}&5UJn<2@KW8|Wt&_^upZeAm90wPH zHZ{y3-Qx$Lao`@GXhgQh!8M?t;Hk)CcLbWUw#UcTIlk3TFE?X%1Ud-Xls>!8@vVMZ zXXd(}z(p(I9$#JO_*OqHY{U*c8)#ENCi2)ZLWe<{!a~;f_*R>K>?cNOD!89YVvE<9 zxz(m0`w3hG#x5*9bBzUCZQ9&V;MqW%ib|2�*UbV>kL7>3#wifqn{zUyt2Zn||yk zX6PtrQ~CnJ8%)`C!aQCu>;Qr+B98(+$K(F z4j8-Maip;W7lD5AsY4n&PUs|PQ(XG`b+^lY0@r}Ci%3EqyQ9#Dpr0PTKpMNF&}q=7 zgj%Gr0~dj@d)SUFcHr4SKe-el_Y)6v7PRT^1k%{?K=Z)Z6?{P&J8%)`r@Jm$NUl5Z zY@ki)wa9JagBF0XYYSUv=58zi7lD5Am`55rKIj~1Q%D=K*nw-n*fpggkDUPYG3ck% zPe@}Y0R0Nu6!R5n?7&4}>{9RSu*WyAJMe6vpNf9-es>(Y0NS(=j&zS7hZcdcdzc7E z#!PS#=%35$JO;b`M@7jUBiM^wVSqvev=m4sj{n2f>izQ8+5H_Xy^Zgk$kS91;7!y}(mK(u0s2N(%VQ7kD7(jnlvgkn#{A z`n1Ja;r5F?2d7>3V>*F=cp};KQo-C%qD@ zM<>1KSsyFU9DMbasW-awdBshlxBd=X7nh+^VwZvbXZJsyC>eRU>ON2TxN-X6%dcd; zXFH#l-^6~a?7(HPj5$Sj8F>Hf-Y2RO(}$~f=dm9vPak~ob$>5a=kv0g=x;lEKYm?A zeCoN0_IAHR>40Im%S?;0_W^e8%o3J87!fR|uho-3GI-&#I9_;~Eaz;YS&i41TGA}d z43$V$etD8~<=ErqwRSZ`Fu5T5lQtD*s`QfX*o2fMm;v!3H|O3LtKavFId1jtp-wP{ zZzPXlh09`#n|Bzy_5)#7wZ~*GCXd-Ecnn{;W9|8xMaHF9e5EUipA9^FwpNC?eI^Nu zYNCkaKmiLe~^h}exbkH93!`?xSdB9c*mT9KQx?;oj~`cWUZ>VBviJnRD+`?H7CdiNZE zd(fq8nfa}RBVcrsX)5}17lormWOFTz;cS=3Xc-Oz_JS{W51I5m?BGy9C?8RgC?DTFjCRjH1${Jl4nP~w zwu7L_!_=^psneAK7$rHb#;1Bje9$#BVBbI-_8U>rUoLux%b~qHT=eh1U-WQ7ep-ly z(mQ;f3!G?>Mt7+=$KgmTk3=>NRU*w0Nm5-EJ3bDYjbp4oBMUrEh>XPSfnrudB++Uh z@^I0bB;@DDhQ{oTDGz)^_}k+ zb)uY#Z(il;rA=NoC;_DApE@{uPnymkNPBVzR$1V*ua=eVNT_a{fD5!V6t`Aee++^MRjRkB{I#AUXUc=35!`|7j=89^m~dKD_kB zGKC#*cb;t&v3E+QX?1_ERl+h(N&Kn#k>19h(y7MDs(GAI1Z3*i>?I3xGgA{|Bg2aZ z7xrM-^}{v2!ycT;J@-g=`|diz(sEeF=KzDaQF_xMF&D&Ke=<`e%?Z2VYR}J=uMNNL zu1Z(553sre99BBr4q`BG=SwBh+HP=xUI`O;ct1~Z6l@AJE8?X2D-czMWC2*-Ffm_|ghy18M z{Ax3L_$xe|x1{`RW}5Zs0g8md=wA^K{qx0FyE6GtA9$dhea1dhC6-l6UL##O?zm;G zYmFKRu|p>h2&1|Z~EfI+ye}*D#1%@mLu(ep!xl%TmUHV{76p}=<;pJ z!cZtqc(!i7(Tf}HUYcpx?HOt*l5R_bo~@}05$u{#N3dFwrF}>kgp53z5>C0S%ylL2 zZ;TbP8yGBeY#q*4y*I2t!E>?v)=arE&b#QdRA(DsJrfPD8?q@r$GO`lIOq7&I5xqsoeNvRAP1+dS@yRp{a1jY2k2it2rqrP zENJ(afce|p zj|GGM!*hkYFx0_tp8EKe?g#&?U^s+{hB6pl5H<>bt|!PZAyK+&^Gmc^We-MfFWO< z>59_-6c}VT`v>h1$SD_)3t;~MvJbLvYXf1IT4Ie0(#1Gjq-^{|gmRNcCJN-j_Ui|! z_V@HfQ?sil-nIb^b8jU?05C) zWd(NJ%VjQ{=bl(J2a$dZWz))1OXShC2w_%<7qV^%84F^N52hEhX)_C?2WxRs$|IOf z@gle828#{Xz+l~5WRTB@N6Bkk>4EkchC_Coa;LXbFWG% zR?yn|eUW*-G+q?1akU55=T|u49J;-E1qM1#gG`1V$WxH!v0PLu>!p2P$416^w$P;J z?w3qbHq8vBNP2!&5@zLC0rPUZ3?c@(z=MJp3z!L$8lZ_{cOZ*n`7#7?C+tTnYVLea z#iQakZ-}7dWyB@t(%Vu>oXq2Xbo3$f)c*JFs}OiE0c<(Gq)%@Cn4@U$r2Z`8A5lt# zW@@A<1X7Y>homV*^BKT3JcCkX{i(TiLoiiMcPJ&?eWUnH>jNl=F?_-~&%W<{*WLLB z{lBx8_=OCZ=pY{cgbWcZkGqH)kGNT|d9unUVRJAGtsPNeejCT!bQ{eE8MbyAP$t7) z&}BfG4A)WVGCY~ty~_}a9h#w*EO}?C%`-SvA)H+`YR8;mqsxF7&2Qcm!NMzz8^NVl zRAz2nH~X6|!}hmyJ!D;mXnu>P7#0a>g$Sw2?FXoG^eb$X)u6zahh4+JF2F8UC>U*nu(`*1Fq&Kn7tH z`j1=4(DvK@<1b|RegAO_8PxCpw*R<=3_~ye!}^a~$zcE6{^M3M9Q-ftKmI}nP>Y2A z2^l!>p3Sw$IFVws#;mHOuvCVv8dFdol>~8vV8dxurqa zj=E;KPKH0xEZ4}OA%nVRxkiTJ(En7;a-9tJsB4z%WH|pns9By`??7&wVA_lMZGx$U z@D(k@O9@KAf8oJCD8wa-N2~lw?HQ~Htfwr73=^q7p%2rvU$uAK|5dy-6`ksxcFBR9 z07s{|0&|U=h!$msX#=f{oCp^shY13?jGUm0r-p%mDMKfMMd9J8fLuc-{KezLeF0X6 zPI!xa!;Jx4hEBMPM~6!TrY<_+EV2*h2IO9J!d_$^jt{W9=!CU+WY|A|>!K6p;-O*Z zfGGo~{fpGYRsp#NP8f^i!}hoIKqxvqlAfE}&o+~IYl6%0Wy$9nAOx!rLjafB8#BsdB9PA=Q zaOz}Anu9&7fP;b-6=vYXj|V;ltE5=~D1svv#$KBG#|b-3c4kPJg2PK!dyC9pnt&ZF zvxW#32@{`)qZgh#jFlUMQ%pp4VB;^nn(Tt6+qedcT9Wwf?vE)?Sh$FLVNd0+1&`A= zpNzV6zxy zL=M+Ey~mHujBCp88V`BZu0YfW_or{P5I#A!d8!VV+H_cc$;FSz(-wtlt9_$E0&b?T^iN^>6>up1Gki7Ies2sh?*oIek5Jg0WpH z=_X%?>=o5>^hFX@ZtpN&pR#oBTzc?HwJ!MD{_KV>>ipxzR@L1;*@UNwuVFl>?;y`T zs;6HBziioD8|1dHu9g%HeWUrBep2~H%q(BO0X*MGCPRscCPh%j#%F-!1>FtaL1{B= zi#-*?roR4BaeTsN*BVA6YFO?`44e4)MnoOuyI}IX+QD-S{pAn?ZPd!t^7CCkb{gCQ zdQd;e4Sl#P|MZU^;aDGedLPU;$JN3`iH@d`rIkI}b?SCppn8Vuo!S(Q@>9lUnLug< z%b^S)L_6G>)@f6$$VP)4b%E}MltP5lr&S@-`>+UC0N`UT8pDH+@SKnEr{M3LOXJ{Q zZS>A{*z0vfO+(*k_wC>yLDd@in2O z8f6{hWC3qJ&ws9tVZYl*)MnxF->+l1Y(k?D4V;h$XC)z}E+91mf2KwXQfdrrj!&PY z1;?k~<8!Z1QiAFyZ*Gftv>r^Fo?gzxq5|_?+=jT33QPWWJY9o^_=zfrBq)BKhU?cX z2iu7T&C`xK9iHN9S(WS+O<8s;5tW_a;|Taq&d(qY_B4K2JptFRTDG+l^_%Y-b)uTe zZC>T;6-{0?E)kWUf9U{tPhOuv9O!8rSY?Cjzgkwe6ZM*p9z_sMF*UCq?&V5amMEc> zoOgC8y+1iM?S$V`+P_K(7k#&K%Zi2S}mTpou69HU!GY-hl`diD+_PcIE0Z8JO7x|rRQUY z9umF%O`fC?vK=V1ZZ+|xg9DpxmgX=Q|wAza-X0k zPtr@ctq)TxKUtS22ZpP+%afprQtmGG2|J23pVdgCBGu)YXjym%r(V&O_9Oo20zTH|(~Gw0+gk}=A)5z7wLbBfpXC?CJ3_`Cl*>zlVJ z{|}>-E<27=ihnOliV2?`fAdv5TNR}I*i~ikC$FCbuIN-sKRt6pK&Hc za1Qj*ujo`bOD$x1ID1XY5CX*gR`dvl@_5M=8P`&eh5MM!&=~-V@GreTJq2~f?=Pi8 zP?n2ItZ=&mZVP8I0qmJHD1vx^i*Ds`1+Dmks7L9&X)CBRMt|uc1a`To*b1R5aA$!E z6F{F~f+A>~@tu90-AmP1_PL6>EQp@hgclX$3gX3b zm&F0t+WbP^h2o1aY|@-;cJ{1HmJAlPa(zf!aOx=qmoNmgl$=w*67>vsw|dHb=0@nj z#1mrIT*g#uTA`@-wL$p0oJE2-m8nSMUn`my-jR3ZBSvsrJ&9h+n@qf!@BF_>uRC45#ut znj#@9+jj#6&kQc*KW5~RJZ*ce@_|AZdEQY&{R+6NC0gCAJ%h6f2jazrqvW_4_jdc# zi8@43Mo4}A?$G%4E8ii4cF!&aeQVWX)PbyO`SorMu*0##?B&iRaeG!A9p~CQcm-rg zx9S>Cl!Mtwej&+yEx+$bar+mO1o*&3zamjVE4kq8@lPemLp-WN{y!l}GtnWKD5@kW z{8N&UaG_}}MWafRqDn?fynhl&)C=~?jwz!`63agz$pAx{yfdmK!RhEs$rk%xBgro! z3V5&QshvcWol#QwG`ik(@XA+{wWQ>BYw*=5>e%f`$+zqzG4#W~8YMp^B?kxk=vD+P zoFx`6c{saIS3v=sek(czX?eW(3X^N8>w*lXGv><+>^4kEVB_URj z%}^kf9k(m|5$DCT<#{f8yCo&F(QRa&Q_bfOKa+CwoW}1V&7;yFV@B+plqlj+M)C5R z7}Zt;+@bhPU&)|=LmrlU^uIVM`2{7Q=yK=}QF7$#@LE~2At6Pf>$-swRJyJkC`m3tq3gPd67m0)uIn!-0qYy^ z89Cc;Y~2kdS=lj+*VmdP?>KG~*zVYN9kD;CE4Z%Y{3Egw(SM$RvxX7{vp*1U)=)x1 zgQ9@5hLS%LaMn>WgQ|eDh7z}blz_8_5-C&#oHdl7D&VZ4ghd2J0cQ;*`Az>z0?w}! zBw*k5PbWxDZH})N9sn+*ID?G%CuX}oo7>`_^H)5u$4QsztpR=|2S;Gv0 zQ-h$BBv~ay2<N_1z=uFX1dl+%F9FSVkb9MG!$j1aoXS4|sECc@ZE6Npp zk_+J;qMp;VgwBNhwDgEW<$%P>Nmu9l3$~a5-b@D+K{4P@x58D?C%&NVA?h|A4|T@s zr=>$c%K@C&M579f*Qwz2WmsWUJXr0me2@W9$ z&Yjgcqot{?rUHOZb&EWzm#5*`azrlh&`Fy%4{+kb6H3FQ;Q>B?>kedfruBZ~!IO^D z?;fo`Q^WdwiFKD?gGbOQD@CDmqvDyC5XC2-l)m9L3+<7?>{Wa!e*3 zDtcjV`?AtR(U&rckJrqop(5fA%V+UQV+C*Wu%m1jG@frt{|OklSG!3fu%=t|8A|1D z2QD1^#NTo@Bph?9NssQSnEjnmME#e*x5VN5CmYVj9mK_TQ0Mv`%LCMwpq(%)>9f5W7(ZLEFfguK0NvtOUVB>-G|M?`R=p~6xh=b z>^_X5Q6!%IfQ>CPgPDmbT``Ks0B)4A{+Lg(8-wYD$x>cKbV5ey;7sUUbhM=pX$0V- z@@C%`8>ZpM3fc^mIeth5r=Bc!Rggn-uCn3OWm?Evc!f-pK{m%NHWL3W%0}~I~Kt3sHYy`~fcDFXunEy>k+bLL_r``;pOcuY2)i!a(fbFrEI|158C zv7O&^Ogf9|00D$`C9wjKTR7qA96W7G1TggH9>Ms#hEe`n@9&8$W;lDCBRQX=8g9H{u9> z_~l)LcS2~;SkPAb5~DbUI31B0L_ZN4AA%ju$}{`R{ErY+!uiP96+(6FAey=01V~;vXwq5!0H8{GY62a z0L!J=i1;KR`8~k2REmYrPXdD91BaJNF%h&$z`geX?NaG}L|-Cc`yQZLD#bv=Cjy4= z0kWmieF*(TK>a;Hyi~duL7NCjyax!DO7|f85&+)!0M1e=I-;=!Af5*HtoA_wrY2zj zGyqwpg#ZK<2GG%upX~Nv3=X{S?e1)AX{fKREH5j8KQ4HflbN296c-Z_78Ky)b5(VZxi50#Er8s_ojJ8)n8eQs52@w78IGE(Xi zR)iR?d>rO(q9wqH@wNRGEY#juil21fP+LKYtDT1A5t6wNFY|-%8LIOhCf@UTz$CZT z%+`?|!};ks%dZwNrDPRT+YHHr-jW8>BOgAU%PE)xAjSA2Zyu+*U(=FiJn(hsRbi-` zu_iymzAtSRDIs=7l6)k42V3)#@7bzL9DZrgJNZ@DS0ub0%lqBP`|c;iSYvhvMUCXkwx%DJk$jUwhJFF;}c` z;C|`(NdfOdtu(dK{2C2Lf9Zjvm~&sQ!S`a>SYNfWyli1^YGPz)aN)e(S=}?*ni^`V z%1R2SwPx;so1aVw~mK z0c>yxkN!QI`KUhlv}9bg2RB}x64AO$UR~vdw`Wm5-{24}n*i=0_&Dx6E4f`q1N=#; zFlHZJ@JE=Uzmi~lgjoLdx#^&0wbLV%%QiV!xW++QBAGP~0)VK~&k13})kvHBtnGN2 z8pqPrdJ^Onu}U5;%IB_A?;HE#Ml!M^{nE~_!O=FPuyx5~Mw zUI21)0C=y|^1ld#z2C2~--Gbtg?C)wHTL@Tpf&i@+O8CKUJ7OP6^X4kCNgKocPrEP zPqXa@&>ze>qj6YYy<&OE!pzk8wgT=%miKA4kF{PFtk|^~=5knL=^nDoqgD4!nf6zu znn+`SyY+llv+3_ZC|LAFK%fx3+ehiQAS4DasIKw3$iD%E+*a44vz6iCG{(@|BF5)TgT zTgu{wtMJb9%NKF-Jvr(bA0LhGM0ao;4?Vw`4}yRBIVEJil!o(f0U=1ge|bi@cG04i z5w3wS2BZM0AUuip`oU$CK!{34xOVYA&Os?7Tm#|a-;)upf$;Yk;a};~4G?ChB}Rt& zq6EUZtJ{d#T)ri4ldjD>VU#*9=`-zDyt60Q}+>mM1}kR=nW ze+vnJN(uEP0fHBR*c?Ex(kB5-l>vft0N#qe1khLpaL)laD}CZXZW+Kb2Vkw}iv#{; z0Not0f2B_huqp#6=74=`N#(A!vBbf>OE*sRxS$mm9Xs~8kdH4vkDL2pE+^;r&Kz@E zDg#4GGCh6bp@jH@2V-NXsG=gt$>(uy#!5%m8FDYY9kR&RX0xYs6N)~~Fni{R&3dVE zZlM&ywyJ;XGX62v<`%6wug|ksS^jQwFDA2tEd1o{arOL9%V!Z z-f=7~*lqi(kgqH{oCs&*SQR~mKbh^T$nmM(=Q2BPeb&Z(PvbS7<}6q>>1ML&dlu&= zM&G|{YpSj&>9Kq)bXy>}%HZLXr|lPWH8c5^?0CZEQXU;26idi`(yBKGKpspxLy8gl zZh-n2puSKVf}nK+B*p+`^ljA^O?8^7%>I#7cWFkjG|q1y%a0VszxS*s`@qd?HCS_6 zaIpft?<+b2^xMy38(i+3dlyahR+lnLrz?WIQ=2SY>rE(Wy9P;!dTS6-iz;ED3baX? zz+b7s7w@$KuFtaDvt7!|V(7xh8M}Sh1r;xf3$rs4V?zTy-EKQtTN)eaXe!Eyo#f|c zVW6QP#M}3UmNqVynmRg)f+Ax6^v!JP_&Ou&g^xp5^3PndXLS>aRxqA5JA(hMUZp#A zX#f11%G9uHCei{2FoxSVnwnn*D+)qfjWqagM`R>E^W@6jfd|kA!qQb{@n1V zuPyuuZ|@Rr6m-0KkB%PA-MBwCohhwhTbJ;AaonYF@cxV5NiFYZ zS{W^)!N*YT5^j<(R*q$?+QW<)6FCW)tiD|fxW6MIajP0pkZNo$J9TF0@MTP|V`FObBbwN(TSw{6pVah!_l5oqpkdV-RTbFQW5|;ehCESgK z+q;B2l5ptfwl*>nwuNtN!R|;xyI;D5e=`Ze(e@qhl0izs%^G=wgel&;R>$8e@g@o;}gt({(6i65^ zpV+ty3AdeV7ZTdDr0<}YY$svuPI}2sB-~jq*-k>GUj)A0NZ6j7u#*_Ijf9#z=p}zM z3Bk$y9iQxsgoGOjA=okeM#5SeWP%Bp6MolB?EeXZ$y^B>b{IT!aD%cT#hAAYuEX2brlUD3EaWTgtv833ocz4kWZ+dyWn&Boy97IsTP| zyDP_ANjS8_sKZVqY|mYLj1USW+(kM5n@I@XSg_+gO1qPAGa=j{;TTfI)C z+$RL__czYa{7gdRGc-SukQ!C#c#VY5P?3(;NQiWX=4TQ=RknQIU?%F?zq;eTL?*@#The4E70;%+UPYC$#@Da*aGE ztVc3K^D_yN&(QotLj0YFzJ4WP`_qjFs{Mw92D{JD{7k|#^MCma4X7N0ipjPocUOL& zp(!DJ%?PPg(g0^@Gv;_zs1mu^TgH~we0MQ_TY7CH?b*5i+mPK%-0Q+m!rwJEJ{SkeKfstn5 z`XC^* z4FsY`0ft#;qE%-}1XmJpzj|FWSuDMe=%WKhnt)@20OgXi`wFcDP#OT79R)~dov~M) z#SvVIfK2teXtG#(57BoBFl_?J1_A6P=esNMFyL@CPzq3016bgXfM0ZWK@c1Q2-c*7 zI@=<&J>TbDHqy6mH%)9CrCXb(nkGMnyDYVH|C}28PU%6ej{bAA-zN&!M_zvI>M^Cl zjvPr&qNDpJ#6+K$I85yC__9F%jd?+_@PT`!dN~3&3)Rw8)ANfp=>4UVjxKzjxdxA^ z3k^fizDH$3vDMPdWMCQ`#{egDek!@a=Hs_a#`EQ&rjEBmg6~2R1ZDCFg7bd0u8z(A^sr z7uDp2c<3py(PziHnqL&J3r1B%2Ay2vb6cC5tu^Nu11kqqFju}f6ys|roI1Tem+`#` z|NY*RZAqqLZepNkRf$12*GInYa|gX&^l@rOKhw=4={2}P?jYB6 zn*0eGmGG6GBu09^?3)zWRfE~n7&Hm6BbfxX6C^tj_Sc;2-YI>n@R6&S(0AQ$+L{__ zt1HT%mlQqDd$@BxyEa<+p8E>K;-e)__ z8g3`rFGJTqCpkaMDQn~t~ceBwFZ69@8y&=VR;sm7I!V3 ze-@SlQIJzWMIz|1@2lNQ=bwe;FQEF5tcmztSpG2xy+i5z?QWTG!g8DM@8y&=VOew+ zIRzB4f1GQF()nj$*?!B=_4X#>_o4opEkoBoCyc+&NN*jw{wZPHn&0_t+$Hd9!uabj zf1CG~iS}(x#P7ng+|NVTKPQZTg|NKsM%ul9d?W1==eIjkR1d{Ig;0{h^@Jg3ruSk% zR?dC<6;o1Gxc(~`7NRC!Y%irsvi)P^mNLuQmqjP`rM4`Zpojkr^!N zyrMMg&QfIrT_*cg1!U3GnBh;O7P3o$Q_`T-M+RNVTB(_P__;XGXV zoW^=4=W2`UABR{*nzOX>)TxCTyTfqnCgi(9BSU%E@>z{FO*lUK$Ww%Nh2*jDvKXtG zsMaRcCxnPao@BYnW2-i7^d{v&jgK5fa92J2(qKEk@cfP8Lp;q#PN*`$o-!7j#3SJ6h9uw}`i}e>mUPL@&zR7K?I(+d>(t}q% zQWV}@UU{Uvq{gf!EVXL&Dj|sxvCJp9c~wm=Y9&3W@R6kO?DEJXAG~DGWVgW3#NfZldNWEeRPc3FZIp?Eu|hs957uSe7*ZQzB4;d@ zE2|fHJ^+5`*+Ki0Hw4FR<}BLUM_;9TH(yfiND5Ell**OV^E~eXhd#r%r@28f&NIi_ zra3B=?%i}rr6VCcfm1B^l-`~5E^x>*gx#v+;@Irx@z(Z{S83jjmy|o=!s9qk<_haM zo_ByxmrUCAIChQYea>vv9Fa=%Zn&h>5fdK6DVTd)&-(mT`1_JxyJwEYV>X{HTHA+T zrFy@vL@%cp04-VEMU(qI?s7&*rCnYC4OSSr=K{*ppRM0f-zH~8ad49;JG zmz6xXyY1*WHak1s(mwPG=3RH`bVpct7$-+AyPnSZGw_EcId*!EI%9dWnJt<_?K#iW z?Nl8V$D(F~S{Q~%VQjTH@?;?$!8uG^OnOS^72#1O5q8{;9Amn(YAvwA&=j^B967R} zj=-G5T!;0f&r89*N<8cc9r4B}XNg)E21!%cs&Qn=0y_M2Xt`+hM9!at-zaggn{oIw z+C9_K4Eqq8%vOaXL+01vn?uD#r6+KnA8t`%X4mZSdNgV#sF~pdX);?Sjx?E1hj$Jc z7nvUCc@DU4$r-yx4%wr+GiuGSfzTwjS2$8+ULBq}#9YLBjOQ8P@+Go%feyZ-lru!l z3BWR1OQdo`y!u6v{B z%jt~MW1(%KbJRljhg1Zq?^hZeC&4dGwCcpOe`#q(Wg^ivO$m2zSK*IwPc`5p!HZ8c z?ZoAOX>3NpC{Z&_F6917g`3*l-QXw*u5+S(C(f60Ju|Xl@#5(NaJNDg4)9SF2J9p_ z`ia_|*oEbqW~8a&8Pg;}ZZH)VYBy;EW)f`CMCDE_`*KAyViWPmX(G65s0u@jE2F^? z5-i$8nNCdpaw#)HMsd$+sF16h3LUj8hQT2cOdvtDbN`nYC(Q_k#T=*c;rHxSXkzXS zU!W%0-}13a?_YXccoMasqea9fRgNWPN41F^QDZ&Z2#~n!D$Tm z9ex$Om^&sHa7p&)Cy;fb7nYHl?MpqyGQC&m4uc97^&RO8m?Y?;2?U*J_GNfxdrVGI zOryhHNLBX5xG-MWOM*t5u)h<+U$)N-jqw!LG(^Z{p9&ha3&sTq2_!8FZt-Q}oYq0A z`);Mr^?I@`wZ4vbW3XGP^d-b6h=`S*x_!sVWeKv1m|I<39Q*wFV{6AqRhoaJm2zKP ze4NP1QsLW&1WS+KwsyL@^nRsx?%87T znC<6FtsTQvss6966#Js$qeS>i`EHv#nJqP~G|a^=Mvn1*W^2_RmPz%mw^Hbfh>sBA zE9&%S;!@N~#2oh`$Jp6f^_H~3 z@D#2Z0y)~CzQE^)MGxPWc9L51TJe}8T*Mornk8;w8YD~MswR-74e0ZKPAf`#Tg2() z(v1~|xtWDeqi<(go6|moCv#O1$k6)r`97x-rMfNP#J^;*Vm8;j@Om_QCb*gD16eXx zC4n@pPoMX5GEuVIoK74|x+`bq9xY^#o}E!|P8$eM;(A3OMeEh)`J7mk_%@>x!;<`p z>|Ef2?f;#05ASWM=!*@tY=nN$ll`gq^^>Q2!ehgQxSn#_>s>oHUNrHz&9cipqcyKiE!_A% z48Lwtz9%#`REX^PK-3gaA zw64|kwNjVjMk~E$@9&o}Fg!c5;=*Gn-PPN_Xyq&)M@UzN@K}TZFOU_JAi~XAW z3vXuL>=tYn{1i1Fbxpxu;VBFb>#?5-84tN8XD|0e)-v!)K=+}ZgM}yIf)=)Rm)hQp zR;K$lTdKZIicS)cdLn7*dBvj}+Jj$61E;XywPS138kJ7>ZL(B(n-HBKAok>x<((@o z-H;wc!D{iM`DfdYt#3vu(|j8(mEXoi#|fN#B5dh+#i4up-DE*eahG|%ZC0z+h;*87 zgQe2jnCKV*!6(Npt-+r#@%~+J!L#CGb6eX>t#5`aQ+;1sD!z@1juPO1!e?oI#jLyO zT|+@^ailq~EnBPBuym?#y`{q2i0B9b?k7hr4X#}1E_?U9;C8X2`DdGtEpLV@VZL>i zr{9J}hY4^zVYk$|a;E#?yPN{OVjc5*o2(YCp*N3v(+gCK70shuzFbVac{>Rm(ucQJ?gSsV41)WNV6jv72Us-U-lj}IR^Y$<(3s@v=R|+#|3MFsw-qsKMu;2UXfg z5Bb2r1iFqqlsoJws{y;d#`81qbh}5FRl6?MBn7>)ks9)XdC8O3k%Y2@F|(?&D{I7^ ziAv{t#ICAyQ7$Q{!bWn)1Lh%5SO*QI3p>Or#xAPibmm6-+(WR|Frd(+D0Izq)fLec z;fQ684O0wNJe?|^I;1w#GFuxiA1Ys$Qk!C!QYkAV>l@(bPf1Hd!!INxU~$>bsI7Xm zDBZeQRW&)uFsV{fM$*^A?>;2~4IV$G&;bi}yCZF~qsP;&n^aYj6ATk7PsyC}b@6ki z1gKZ}mIS|=PuLE&R*w{=SvRUGC&wAaRSL@p`#Shtr<|hx#MdqO#{7Y8N~`S1@iglO zRi)$@!@Q7Qb

    upD?8(^>x0<_$K>GQ&my= z^84{onp2zdH5{)s3$gZTCjWq$OjtQ1^}##YTbiuufG>w1JEab_Heb&1OfwB@h31F> zk0ioZGm;-XlRc%0tB8CV{Ekq{QOof89rrRLvBqsCAHYl^teBDba6kFJG(i=22wD3Kj~K(yTswt{Rx;3+dC%O{$5> z2Fc~p64Guy?%vpBc%-DPl#Hwje3BX-W)A7oO-(9^i3W+~;u7L+uI~4+_v7s&#iqny z#pT&pYzWMKidSLORd5<|pUUNXC>x~pI( z;>MGRPzbTu@myBxG^$PssJSZFACwp*&hYYxo1D8Wb|9`l$w3NA7Cs(!wKGPtDFM}2 zW%~mX1H|cG9&!_NKZSh<7kuym`4aOS_k?PvVRdpq)m52(zeGQAnwQjWg6_w$t#PeL zdda()3%DPso-ve74ye2;-S3m=BTn&>+>P7)D7FEvKFM=(II|u1W!27$)ky)bu1fWL zC3=aIz9eyDac9O>#Z@MWBadR{<7QVqb5S-apyH}zzel2nIN?jE8=d5-0K- z%yUO4R5}gFHOcv1SzRebNJa2sv18p8-4rEK#Z&v$`dg;U@5{T%zf37l(FcE;fT-;4 z04M*YgR9hE1ef^7FU=XYRgFGLw`x{ajZe}~dM+s{dE3M3{u2Hw?iY$B(s8!A!)-F7 z84PJ5e6T@{pwyrk@fMOGb(tt3n_5|QX(dwg%xV^TMM6RTlnlBwS3ZV8jeyjLN;C%rdK34l2$ZFHFv$NwedFebDerwTUTjCh=2076rGil~xx}YB$TQ zWT8}SKF*(d$N{)4t1f8U3YSH@3%WUC)^)$g|Z<(~M;k${s!U+MItxUWrXm+gkTk@!eMJhW2PJtDJWQ zHn#1ln$c_m*`4RI%?pR+71{W;&2?eL;;q=P+atA1bLtCtY&BCgBiVSetIzqG^AF1_ zuyJb}=(-i7w_?|~himEQloZU`w8J#R**LR{&RLol4#}Tp^S%3DXSCdOS`Vs-=;WS`cN!01FMtV$M!FCuY8HXk0FfNn1%*u-Kv{vxYQSOCcw$ zK-UJA5*n0sBs=IFow?iKxNO#8ZE0PKV%C<-YSKV0$sG3rN*jih(15H%*>}&OneQ7M zm&u~l7SUZS8gI_5BK6l2&be05Z4FBf^~<8pwmH{t=JsJ+I*UqMK)0(XvpKVp)K`ly z$D|<2njtyVCyPA$;<*=Q`#y|IWszxf>J}GSG-tjd_10p~(JIijh9!l1Wszj7ol7)x z8yJ_&BGzWqjVxkq&a5Ey)H;$QT|jBgkQC~X1&ff%ZXN$D;A3%$J%P zS?O%)S}NMg*>O2Bx`KNAg%sZM6k}K!#4TWoot&1e zB-1Dz>l^1IL~)YLp7RFlxW=49TjZ!`I&m|Y|0bp-DoHeo$GXM23SphxZ;yWidz@qr zw~b;HE1kIMv+_uMTD+2Iqe$%SxLZPt!t-{Mj^kr(pX*yAM?BMr8$T`Bs?u#iK~$_Rxj?H zP_gh6yDN?sV{KsW7CGbzBd+@_KN6Z2s>IgF8mk%iX(CvjDRd;Hfkl}mRxM6dC|Nks zPTWys?AEMZ%h+I7N`6hS98>T}Py>T9L#%w9oKUcEpdG#A!7<@kz83RAos|6QU|FWX zk$?s|Wx81LI5DAn!guV@9TB4|Gjq*jAG(tBtAb^i{73v6Xq0JUh2jK-Y=y1u1{``v zOJ)k1%|GZQ=U0Mn^1dTJ4HU{0u{?3yLWaTycI6JwM{mv8HIEH+CFQ>gmSXZA@oFGd zCXHo{V-ZppR<%oVh#M82;cGS@&`HX#2$p2>9Pwx%R3?n2kE0Wk5Eiq$>)&vb2TQ`L4g-%8bMaL<`Dm20xU?Z`jp+cc?aV~YSM$6ZB&PnU`g^)b&{c;VIHi690ZyNbtyCX3fMJt%;eI8nru{t z5?~4PVs)oN?}WLqqHzEk2%V*jx%>(BF4LN{phg?zp*UEa{K-1uP{%L_)*1FGjb5GZ zjDq|J>N=)!X+aG(N<%TQ7Y&hP&hOd6gZ-Yik?RlJr&b)!Cbs3TjDl#ayODYy&yNUGi$rl-DnF!lh(&&cl{y(i&vW1!WDX55hWr|W-urouwj*u7yT31If~I-i z{eIrO-tX`CdGh?@#$_!Dr_u}Fj{S1M?F8m1f9n5X_w5T{wgzuls1T|do5iT$r%TVO zR&8TqLbL0(W$S|T?YDlDa{J+m7UhmqWy>l; zLblm;h8U$Rd-2*XyJhVxTXf;hq|nL8&X%U@^`-ImqcKB~Ew9m}@{M*Y@f6ZTbaOuWBt@9fVeUxDH?-syU?`eKf{>cy)c z_&qJFk_saNM;c^;@W!`g^4IT2Ck=0%FB)uGneY47!9_U>3vC~k1*HoV!vzWBS*f9U z@4q$2LER$3EoA znQ32Fd=>c9uY3K-eckw-kDQ70VZ?LAz6|$A5+W)U!U{ty?R=@_>U9C87-nP1NU?~? z_1noXP5uy-OoCOoM-E7@_Jr}+*~6G4(t)>D8;d52GwB|>T2Cud_E zu2>o8YV=o-;r9y=-7FCjC48xSkF0a7HqYT}c(p_dUr;?46Q#*t+H+qmo^s}qmVg|B zGedQaJFP}1Er>u%fBK3$tp`q80@5%kmi7!8-D!& zMDR3HM@`U|WP{YiN9FFGoC!jM1jkg)1U-5Zk>;=X(pJ-FG)^LtgZ!H7s@*x+otz*7 zr&gQ6-IJ3Orood=Pv;~iN5Dz{keVGt;B>a0nw^+f;i+5aq3KRfSP0oQU!!{>L0N zw~yMVnIs>58f)>^C)N`*$WC8C_Y_FJK%Z>g)IJ50qv%s@AGJ>qfs<{_Tc21@m6utnkb`l3MK{Mr2UNAafrYv`<#b9xz5aBau=RD#%LXf z1|7eUPv;~h_u$F+h?*Tl@N}<{nw^wrklnz?74Dq4Gn3^64LIeTr+W$`_mN#g4K=$k zau(T@Hd3>L2%H*PY1v^KJeB?D-r}|6;Y15gn(wIDK?F_{P1NkdNe4XL(|hPRcV@Dj zph0%+$Gvjm9!6xu4oRpF@h!@!s3Cjza@Z=3;tw^N619 z_+#!IpU$b?ncY0n4Nrza?i`=ai90hnP7qC?@>^oO>vE&In>5Dw`DMNM-o=~%c z2%Pd;sM*Dlet0S}aQn_npP)f@nhW&o;)xNQjCIuP;>iF!>8#Z3AVPM=JGAUD4V)BJ zbWTgiAUr91so5#eLN5mXi^9(vDEGg9zCbAMns|=f1-F7@5P{R!MOt>422b|e^z0Ie8QEEHQnP~yoDO~LdbPm5?%#L{t}}BbnSdwF1rHr}otY3JJMCp!c9;fE z13z~)-eg?1;%-r=#{PBL=`Cw;AtFdLQ^NBR zgb6a!_GX67VE8DGl}8cg_*d(zAP&0?ria&x;FBd{Th~?+8^U5U&fW3R-s()vqrT^j!~OuFp6kv W?5hG%K92~b3I&nzWBBpbSn_Yu4s!hf literal 88125 zcmeFadt8k9|302|XqD7Dgd%o#3@hoV*x1&#%MRnT4oQ+~D%DgLO%AIFSxc-L)Cf7B zI!|U&6DgFDifEddq^z*ECdbBMO#QCcUC1G2_x;)TGx3H+T2_@_OC({krb! zd0p3Y@=uOSwX}5fy6JxU`y>mkZ&zOF`eR%Ev$!)3PQKIbN3DPO>yz7Q>Ckm_wO;?} z*6Y(_+8B?P!Rf2*d!H7a?0`!h>4#wr*3r{i-`N9)27jWJqB8_X9&I}Eh<5Ll+XZ?! z@&GyVE1j-dyzbv?X{~DK*IsL*;k4H$U0JWwd*$O4!`CP6shT}vzc6St09cRydz z&G6I3)a6&E>GnRo;aGbdd8z%6TILw(=Q^Dl2m11SpBUs|kXy@-lS@;B9H{eY;~-yj z@79NmPw&(4lJ*ASN40wwAKWqcPH+L{NtI_HG=i}rS=1}?hvZgu2Niovjj|*SUVap@!WRQ!IX8zIoYU4@^frvNi@`;V^u(r9@1N6V&pM98NNbdBSp6;r8?pLN^FO@$i}0-D zNX+Sh?bPaD#IxS=>UT9L#OhzHYJK&)8f?PqpHjAA^>HZX^y1SGuRa+mE{~C3I7-dw zhev8)@H^&o&*m1MsAkVTj>Me)eyJg+;HS3mL^XTUcQfE(^&dZJef4o9=2RNfhLQF#xQf+3|9cBhRI_Klhrw2? z{&D_^7Dn3a*~g)n)AI+8e(>2RbBfDjq%RWNu=<7u+cBp*#jUSCj>Mcs#I*E8HGA_L z8b~qHYfoEWeH@9^-?y<1bK2WLhLI-UZhiG}Bt|Mb*M`;aZ4iyszrR1Vg*k0}^OHHn zk(kq8_qJiAMg|J3{@KjdSKr8B2Uh>sxi+jm4#k|Fz0~^ZXr}BDTUnz7*t|T_xwrCDTUpd zp}cll$vR)XgdGD6_Fzs=KcME6f?98)yv8T0>7;uw(mgMzIc+-WYn0bP>#xs$X+kKt zJmz#|#)tPP>ZD&7?8BTs zj>Md19BRYr(+m!fC+aS>o4z}zDX&+bW>AHZ9(hRZrtg{4#y3ATr#SQlo+u5qo4#jG z@4i}nTplC+{a71TpKfp)Bi+80dZOMPX|wy3Zg2?qY38nU^5%cfNSocKI25bD{m3!$ zmU_=fU*D&=JVtsVs|_O^Y;Xjt|L3XJ?^7I!IsHABTK)HowAp<+*x*l$^vH|W?^7I! z`?PZBhwoErp@tY7!<=SpYvI{%w)#U1{=!HP>~87VZ?^h46swto{i}8&)5OVoo<+ZGH7|d5rY_**1)Hm_Z^||N398uRe~% zoNml-!|D$+sKrQkAEBQ8_uQwi-%>ads~^3u4XbZqkc>G^&Tf76Ee!5sqL0Ib!|LNu%xU<_xK^KiTplAma-j_) zwJ|u3)xWx(dh@^gmTI;~;YiHsuTR>r`ZflSG19G9TVH(~iPgV!xDBgsYaqg$9#Xcx z`nCotjC7}}4XcksvHFKrwe&>2e)e&B%;|}??E51Oo?@g=qgr^Pn!Wi)7@WZBC-2Q@ z{mqXx!9wV(RYQyS(ZIFT0zmU`V>f=bv>6-noX{$G}RI@k#*9Olq(%)7d zZ{a?D{p{mNtp2YFZCL$L2AP=Co%|M_sAg~eQ3fwC(&&9HJyFfx{5TY=zcZor)yL&A zr}x{kn|@f=z%X>DoitB=cLq|qg>R{!QFS}T>JvVi~Q z|5pBYFJBhG{dfQMV4RhOdq2|}(51r|de12jL$*ur|J>Wcv~Z^)zUH~}?IrzQv)A~4 z-xmE`-mbltmQ9VGRzKN{tuwVnYcGHCtr&kQI3Xb+Pio6*CoTvvyH;M_xi(Sv*KXkv zm5k56yXUO0Hufy|*`#Ps;bccg$Cxl1n*%>Mp5ewA85z0Q+SrsFoS~Eo`_B{)(@)a; zs@NRw8a1vYbg?q;YGS^pzeA$xrhn#14l3Bs&7;$dz_H6;qSIM=A8()zK=AdDi-FHEco!D+<8EGM$u-I2rUQQ6P1xY7cEGP!C<10L(GE;Z6EcdW%gYjrWxE+) zCvS@H&Rgv5I?IWEH#Z~Igjvk1^Ek;rpkJN;+>#MyyHbzE9mBRT&Y?xJ2OHQnm*fnMA)nX(|Mvfq@2gEI41=S5cq za(0%Vp$(ETIKMF_rCQev$fAH`KT2YYWCW>H1)A?yz{%>Rej6v%ZwaSe9}*FN(ags*nl;cRnCEogaImvp>5>Izym-%Q zc_+I3ylqMWQ?NF`t1OyjUzW|Wk_;2YDBZay+5O9kXxg$doL?C^T~ohgF5FsvA<&as z7UPy}U%6|_!YJ*{DPn%r2=TgI^SHd49`*$^ADJ`9oVU~SNy;PU!d0%UsOTyyaZaL@ zL{rmFdX`oyTV3-tZ`9eqWLvY7;X7hXCn#nbl|8$_J?_(Q);RjzvtOqUV-{}<_AKWg z(5{v&vt$HG*VTD(L*pHcPt)4V2iQh4A}so^jCAC)dK(vxwUGR1d+eao0-LQHS9y*e zv5AqJ+L`?YkLkga_n`AH97?&%v|3%jN{wdO#+KM!~AnHtO8siY%SySwAHeCwrdzh@+r0(84;{3vZtgVsNOmV|-$NA17SdxUf~_h8KB;!TLf*rG-8?ND!7fq>^ z1TUI^d0f0`$}ZCIq7g+h@Sf?Hw#11SI~{}qtZ z4j|?1X`N25yEq-*y&-gp+Q+AN!uhRDB zM_r?$q@m8rsM5rIQ@<>MDh_a%2wcQdX;%`u2w3Ex%Edn`0ULdi zs?0xYKR}7I+ErNLDjs(4sOGxj9^l23G_$AQW^3aTdlmf?HCq4}<|{Xs)Gq-j*{2%h zA9jr6`i-mFTIk>+9u^y?d8TkX;u>Wxt^Y}Bj!`)Ig$Y%q7)7-(YpOa1qexIxBx-iy zG=a){CH0{v+fCf|Vo&9|jtO{T z0M4?2MP}aYk5g_IHcVF+;gnH|umnvServfh>}tIaek)Fu<==1^|1cMEfbPB16YE5_ z48_u*GAASZmiJ;C!KwKEyvIK6W#7{Kob8nQdGo#4Qu3$a+$hThL$_{xRJp`<6@xBd zu&sDZ_i1t)I=|36na{M^P~g2Nnl&sY@3yt%6VdNx9^7;6&Mp^d`LczgaK=o*40f=t z`S#o1Cpsn8?P2wEDYCVYj1{d9@aCGaP0Mm=W%8~f9)n}quyJ^;H+MAKyeyBVmbc^l z&ghzgO^4UlD^G?kX6>%|p&ciGC;4hgoOX~W_eu?iqs`mt?jzsDT)3(H9Da4V>}1U- zU=p=V!+fYkftA%Yc*WA3WR5jMcyl~U<$84AvSQOEJcmz|Lnd-n+eWs$84tH&xJftH z`Ec8@2N<8Ft(F_uZe>KI++cU$ZDj42{keVan1bFT3^zU6!@_%&ZY9Bcb)q{L@70(> z8s4k68yMcHx3l&w;-boi4v9N={%3A_IKvqgjK>&gIas-H4(X*ludHczs#xWbX8lIhzF7 zcXyWOOgjEh$8S>3wEHvl#|L^ImMfVHwe8OoT1j@*^c8Pl)OtTjUe2!?^~iYNgKy~R zv7=89vyl9t8X!JPi<0+L8Sv(KmCAzY>DdYq_Rx4Y+%BBZ)Z(vs0hE`>a*nFZX9AQ!Q?1CxtJXL^E6BjXAJ|BQ!|2h zEr+^i_Ez=C#64pv@#Wy2VH76go}rJ-!#!gy8C`>W2ABl*41MWJ3apIb;GW@{*5ICD zB&Oh=;T;dS^1|#-2Q9^n9W*9$IA$lLl*p$lwJ1{Y=KTHi{`1!O)vR_0i8HR!%*p|GZ?i1uzbmzjE z$=jHM-x30^TuM6CaW6}64sIK!03gMf&jLshSx5j<0(`gtDP`wq?r)=%DhvCcp7sA9 zO!@a=3hNb@qLs2RZ@@-H!7M$ub-okN#yClO`b^#)yjc4UoU)+$shJ6I%COoG;1sdr z4FHweuDVeM-wiO>*W>dwk9XH4XIV=7-g(gFuNO8pHpeyoj*hc!Y=-*DFI|!y+q?JZ zumu0Kqnc#EgeA&xS3_CABF9ww{T&W*TuofmKMVW2iHA)-qB*OW3<_jWY3O+6VUQ13 z{2WfItg$Kcg|Rc#a%{>w6-5b}Oq?cE*~{j5!2Dm_$GFa&e*ceUl>_m&#R~H{%|ra{ z4@xI#=nPREv#@+ufX~0MIHMOemZZA>o?QYXM_$viHG%1(4;8(C%HyVzFN;xPR6ev_76Lx z^1*K{5<0l4b@5y06mG{fw*VZ*E0d)4vjJ&Ns_gv2Qo*y#P?rg_riq7DBxp`5Dw6E> z9Od@!@gli$T-~>>p>x!OgkiJA!yMu?j}%$JJ8shY8A>02tdZfzx()szfQ=TyeE6rFNqFI(MIoquyR-&B= z^SImCJzdIkHXL+rUz!tavNhhvST>d}KWm$6#uRJ|@GRrAw96zMOG%(;ozjcDg6&{` znx@SUt2M%x?5oY#KASd3eiR>oHe0WCDOz&hkUla$(Cbcgl`RL~d)zYl51jRk+Idj~ zvh)(ifMt0iP|JGKeLbAy_+pA2eLB%vvbv_z@?RMW(PN$KRhz#RkLMj&;WZW8G2`nL zY{$ICUf7Q5ceAk_GmB&D+*j}q;Kka>LV_3Ta4QKujrF{^?(7l9*|aa^gKaYoI!&?e zv}x_lQG@#w(UaFRt=3QUo)*m-5|ew#TGGXqSKz^wv%AK;ey`rn5^dnK0N}P;N&w&z zSX=$J|9HFq8eNJ$f9A% zW&Ely#Aj>gaq|ILdhxCm)Ve>h0$9=U_aHPACz4?@uarQR)uYcUMA*~=j}7&6@@3yM zolv|gn6)eVfOU0Fh84q3x~py;cXxcR>H^w|S-@_L=c&MMye=N!sEo9&CBScbo?PHJ zV+jrT4G6<>iWTr%W}n>; z_EZ^iwwGU^-H`8FSYxTuc8?Xs(4sM0jDU-W@5tIvLI&FCcs%fzjX2g^h|ISVWk!o!Q@ax8+Ff&4~!u93~#3w?iuDnWBVc?HDwL%8Cpm(?iqUe zhH~69+~soIGl2iNXXxpq~HXaQw5VY<)tS8}$|8R`rBy2gLWTkNsRxflKJ zg^ZM1W-(qeC!-JGCG)%)t_5VlOUC#D4KEqna0Xs7?8Px;Vm|I+YVdV_i-7ZpE36lpCxi(`#)S5K|7)QSC_T8VOtKGvq<3|?W>~r;*@BI1mA03W}I6i-VbjUPE zM^8{Hhcb>IE6-4!!B%@i*ng^e3%2tkip`0dXn+c~GOr}`7a*6ts_y;{N5P`ZQGX?j z#g08WPV-n%gdN*Q8ah+ygB^R3pF^7J5l-_==yvJSp}h~*b_`xRHG4z#swg0h3I18} zs(fIEoGnPwE6=19w5wqFd46G;U{R*3c|wO7U{Mk^ zd_`=6YgAE5{Vz&$jG~u+*ils!Mqw(GXME;E%urS+3=nl#lDzp$+9 zD-VqiopS$7c>j|2!OwnH#$K)W1s@WxD)MjG2NuOy?Ia8Xzms)TQ>MrQzf)0K-=GM? zgg8hOof00jN~cT&ol-#2Da(S&W*C)Of8Ya|d;|KjG;%F^pYxqkvYA%ckA2+3 z@TDW#$-tKmdj_FX3{My(KG?%DbIFrhOFD_R1bA=@*t%sGXk{{vXf=Z)aAn)^YTWC6 z{jcTI)G`!)7+nR}e0Y7`vH8$P7QXxh9Hq>*W-c@pNA5x?NZzleFYlV9FjbeiaEq%) zh^3^w#)QMihu}%PTjzK^Ng+HwCvx<8TB+QTd;+FROqZlOd=g*^47Y8m(pj5+_dJja zvv_T=S9$aS`|9jvRt$3~kP7((#9OsFi60>VQsHf!_oe(uOlJ5FOibVEF%nG72O->! z20`Jgvd^2}$AJ%Zu75Hx?uLCa#eYp0rk(@!ioj$!}KhV?9Hhm&n0ii6H_r&*bt>=?e?dU3_ z<*03N*U3=Z0Pb3}$ZKxdcu#vbXRa!~ef4?Tu30|PwTuiwV7xu=@x0N_=JdM-&y!Cv zi#I<@_OEH->sa_$XHx{%askSOmJ(2K6IonPa4|VFP;j=p7@*+ThP;d3C==58c_FwMWkRL^6r4*m3lyBlS^^4A>A@{$>+AAryl}hFN1n#* z!dw{L&wU)fs<(aNq~nhp>6GKz12HcIyJ22JEF_p0`yv|Vg<#u8y2oclsFk%0 zRKyx-%3Ko`2)NHVs~L*pV45EH*ZRlL65)sz*z|3mw`_{lrbwUe(X20G&eE+VJ#4p4 z^yJ=UcZ!kFQe|x0b&T7B+YK}LMKn~zM87gn5o3SF8|n#mlJ1O(Sm2*d72#6^c+phO zraR|JQ}vhvyl7(hEWBt$mJ+;Zlq@b@G-YRK?#t!h(B(y4lL6F!?U(pMpWu}LwlZYp zzpD&+SBArV_ecMiKWRp%Sf23weG^5eI89?V(kTr?tE1n>DVJ-BHsu=%r+99iwa#em z4F+#+nJA!4RN`NDs(aaO8&Sdffc*u5*A6j`9ruk9Ez0r<{pH?lweuaf!W|=dxe*c7 zMn*fzz^b^S0Q6IoNLW^K3XF!yOf`Wh;$fTPH3Xu#M&(@%B@iVs|E9kKfheW77{XWr zQK}zaRZtKmc!iZ&N8@Ef###VRKjIE}QBC9=4!gW?Db~0Z`N#77S3-!9HA2 z)Ya!`ps1x$jOQuXfq7k6D5KN+AgIfg=&w=T7qg%KaVDfL7q&nW-I_o zajxl-?pt@R95i~#OCicmPrFnXB1G8+I%NZDJCO|RLCN;~s$m>{c>(Q7vS|(Woz3M% z5Ms2i83P*s`0-QWk#(tAM^Cza!tJ{4cAK?2j&cjzos6-`Ti88#8@)YeRhy?C^3Mz} zJ(IGa@79ez-DTg?`Df{=UooxlQoX`w;ic+rDZxv%z>B+-{f*0Mny&l{Q8Z(F@CDw1=y_v={w5zTWGOLcX^^GZZeu`}!tTVo$iikr=byPPSjx15 zEJc^kf-Ggbg#@w`GaoKwDK6(|1pz0+f!jV@@p+46dR^1#L{Hz+&*z(4F2z#ZG^5&) zIw5BVRR-%T<}(7_@5lk=8f7}RRJEg)6_Sz&V5uf zle3C3OfUhUE`3*db}fr5mU&TxQzEI6eZz4OU!dtHYs(#(#l_EcY^4>NiAs(3$^(uE z(?1B^P$O*5XQ4(IVIe_{P|t@8xsLHU+J1R&>GS@5CYFjqF**O}c{0`PnAvIp6?mU* z&a3f6S(!faJj%+HUMEkM;@?2w6y<*zC;Zoe(tk^x@~PICfKBJ-vv2!xj5_A8HIE(U zv{REB@A>r`RiBkE zC$w)r{b5wx{vSwb=}AI z{|9xScfbng?;p$-wt>0= z82CPZuRP_z77Dano%qlBl#+>ahZgv?B2P)dxAA*FjUN?x+-xRKNw{yTzuSY7r$k@_ zrsOHT?DGn(DJI1nynZkFKnkLO*9Ubl`gD_4)F#CkAIQE}a+>iew_4*<-rbj-zii}F z3?WaM53(APo4&?HbPLJ%wy6AYzha~EL%SkJ<%i*&g31r?q8BRabm!Bks51r8X6{7x z=U*R}NlACiGQ;Cqt*5u7!srGeaSo08HoiY|aC*V&!!loNFuUBr8aQ(XL4;8Vk!T>) zqviQTe8E{?4&7*KKE*WYC;KxW4iFdah^|6COz3;J=hbU-oUew%b55o;JUb<`n z^h`9=3PtM~s1>qJd2CPUndou}C#9SzK&?<6%|flPtCa+NK|gmc_yS!J25HINF^(O) z`!kosXb;4^aO|OF3L$$k=4F>B_GFL+5MB>Uriws$wLU11=}_BnqvZMp-8YleeQXH} zLe+f|Q1|&KkJ9uws=P5ici95mF*E>N+w}|pT=ozi(-U_LoiD*1!?XgxeH6_Cz%{gz z0N|Rra{+K&3TXhiqGOQs4w$1?dRmAYYTHHE&|KFH=JoXq?Ph1^_ou5 zBMPeo`uIdwbwzPZLJLV*gyL9Jjj$sJ93>Y#N{tn7j=Ph*J*lCdPqb!q7iXJzaBWFR zOr!m68H!@AUQ3rtS24a8Op3SWEp~Txwxi!IT$a3>S-hdndqwmC^Xk0k))FmS&`huE zKGO5FFJ;hOlVVn1y}Z*YCR;apel(0LX!tUfdYBT7pO>T!*c!jefAon>46rHeFL`W_ z7`Y)`enFI?WD3Bhh@)9xQ&d(Gz_@<%xPWo?7ir^UmoGGE5klp^j8n>1{$H{eZDEVJ zZ8_*acT96aWmD(gNB0L?&pcCF(xdx>+cOrPlRM;peF(Uu*`E`^$Cu+Lwkis#9G|(? zDB^PD>ASV6#RQw+Phf^vzGiVqYUjJPw+D8a{{8fye*gW^#9^m=Him2pJ~!)+=f3PE zPL7V#Hbg|^2dD4dyEolSIa63&Oo~1ZsF#?KqECO+OQ@pHX4FflqR${uAEfA`;(_`g zMIW^fr#N4Us zZ9+F(A}LX`Ly?3_l$M09P}<=V#{OAHRWZ1PnXqEEIslhYDefK76e@y`I%M7sDGi;B z^Q5WNz*;fa-KIq8_xL5*^SN@#Ovb#T8Q%c#DW&0ygo&sg^1Tj)e9#l zULUn7PENc&nmW{bh}WlnCQd`VK4GARPL*DgU3#Gj@89FypQB5h%?V`*otp_diB2|^; z*8p0mewsQjDxA!+`6#oglhNs~k+=)TVN4&%m1=Id{A_vC*RR2lx#zaf8UoEdI+rsslzTjM?AjN@Z;_&A3<-UdWWbYpJoKiZB#a89(20z zJ=nRO)3OCUdGX%{gvtYg-Q-_lY&A)*!Y%Fod-FEt&YNSP#P zBna$Z`dbL3Ye!E%Jxrj@f9Y>g8HjTeUBfyWSC1RNC}#XL7YAqBM?X%XbB60TTEygk zJ;ye>F?|2%5fO}B!N(MTq_4ynX@2wLgT|crWc*CbjY|Oyb0cc&rabf|xxzVKQWrF4 ziD{Y@)sHe@4%EVo@fs-3?O30lc_0@+{Zk&X}$IQqmQF_qJ(Q#3nBEi4(AaVPI!YDI}xP2U8lrbT0pZ+k)Y$k4> z$xvNTZl6Jrp-h9@M|A}<6yo+#^RX=xw~uBgAPaH(oZ73>i|klrzGO@`W%p5-Cu+F( zJ6~l~N$65QjeROp|14Z$uB&>u&<&SJiqkw-1mO~P($Hy2J6yuWFH5K@!zG9~MZF7` zAmS9wZUR{@-hMLN#>Vz!)}CQRM?3odUf_WR%FS2n=L2iRtMdHAARqb0Sv^hY0QpGl zQH@*?3;9SzPB?j-Fl(l|8YhobRK#l%a2k&C-qrd*oF-{+%HC({ zjt(eb=~u7nUsC@wum-UuH^c+75LRIypamYfAuEM-fsz`*8&DR1F%l6u&xUmRA2`8On~8ZtkLtlMXsmS7`%q!_PAZJWgHE$hb9&528= z>ze0BPO&oD^q_JUy-yxYrcA3X6Fqo*mQKutLza@owlJA;o!Ky%(iX{KGG!Ph!(_@E z>J5`A-8pxAsyE&Fa?bML?&Q@m>cox&cdn5AG2h7L_}rN#o60%97XIBOT};+-UGOqB z#>@1ujIwYPfS$H)(nA)$+$4ukBnj_>u6ENn4Ghps-s5`k z=tp@>U&&MOe%(lnv!4+qhDqQ2+`ps5kY1db+>_eXw=L+o_2YJ9YQwQ3&SVYmCbc6^ zS=hzoNhOLJ;pMpiCtp!TjY+BY04Gg)$W-VI`tz<8H6j%#Q>c;|w8g_xO^F?<38}cG z5Ak*K4g46pw&N>Lvh?iOl&J|oUpbVh>7$_SD>3bZb7h(PwxOgo*txUQ(gnkL@$Rju zRLb7eG+n#6DcM4D8IoR?O3T3`YqN3B(9%vz^&J(*a z;h0VjgR31Fg&4(Va3WpF#7H-9y)D{yh!ZOK@$WuP*(IZn)d3z1?xUBtWWh&n#Z5vC zXKP;k{PIh)pyvR%eOURCdod}HSwry!$=79RDLgioOZtuv^co}!U@qkD@&L~5S7QU5 zTbL}C0q5%5=R@E^_>{hoI-%ghulhuMewd}iu?F6DTIfAHH~$qHm*h~D+qUC%`daYN z{SD@pnEZD{v!=&HJ0{~_j^wRYKm?}-Y$@;|6T5jy`etI{k% zCA$krP#Nq0^h8sF%7uF8=WivboWAQh+ZCh=?Rhh(68Xm)q>BF)vvT3x+S;g2J$v5m z(WCRWSDebNS(SuS@oeN&9#Ndivs{H4xPU_n{W#5i;EC^*X>OB0Ar*KHnq~hF5m2Ow-y#P9nsgnE~ zV2lcNQHKb_V2lboqRCT)LA>HnT0cRVg>hH-HJnsA;M}poh8gM#oF-P$kf6y@G(6I= zk#102J63#E;cx1DMj_xZi+2ald#_-n{&$-vEs?YL4d-rm*~9A4_ii@qKRY5@WDoGk z*JAp^@wHF<`poG%&~(%ZZh{B5gstb2Ppf{9U?qoRA%PQD+1#IO%Da{u=%X#qHk?aR zLhcV6Jo0R<_c|(81yYTlIpMol+-Ws7_Gf55@{#x+d}^QZ>=SVm#G<{3-KWukJ7oDv zh-vj-aVkq0rr=auef`eR0-WuoEyc@Be8>FL$gCJ6UWRZgpN#uf@}vGHkNtdB7vq-j zig(LJAbBx3mHf(u&P(|Z-xv(P3mJ86O-e>uPpx^{*BVatnkQFM`q6`oG{yIM_ICnS-eT;(}KaJ z`#bKRR|XgM(?|jkJ{{3*JUplDmZ`Y)=2vgGsH$DI-)TXgtwYz5dxki;*?PQqFWfV9 z`Duupm;#u$UHB}Rw{x1@+s(xXL>me<@oKwe9q+9S`A*_$G79(1E8!FF8O9_5?itG1 zK9c0B`1B(7?f&BPsEkc`Rp)p(VEM1Ep47zDeR(i_#=)b)*r8n8cr*W&sklG1e_Hgm z{in~xh~&LYtBra=OEoSA(P`gehBxkAiX&}#&)}@T%s}T&mYvNnN|2y6>0KNV? zs<=u4unt={KCT>WyMvLNoWkzQd!Gc%-`o1V*+fL7Zg%ph&kd^+eF=pNvti4O(?U(` zx_1O}t744m|0k&>jLN?(sr7E=P&Vmq(XFupty`SKh8!{YOjd+Sj2Qh**VA zm4LjqPFV|0?U=_T>J5LQ(!}C>mv%WGUVP_Ld3j8yAHMsc!)IfeYt*A3-CSO(DEhi# zgoO{ps81JyPlqk>B639pivAL1XDq^c7p1r2=~nys%BI%`T4#jM2Q?+CL3=TGS^!mPmgQ~i)I zD{%e5gVW~Bj_pqP~goCeHFk?Eyb=6)LvxE@SD`9##7x0nSB2mNjiEF8>Y zxJ^g%55R5u+@dKdgx)Ob?G5>8;LU*acAHjlQ%w5i$dr0a8 zd=-kLC|!oOY&CNsfArZz3rUI8DLJKH8RU0o-O2+i4yJ$D+*0S0;P6XJ(GTp$!{=rj zS0enO)wDP1#oH}dqDq%^bQ&Y^I{|wOzvYA(O2V_|J(7&EobJ57V#tlHM|B-IXw(vX z8bgFYd0Yx!Y7d~vu$M1|KN$iZeo*Xoss0|Zd?2OKfceLosEy&1GVvf3g0N9mn^=7H z6lArhAv$0dL#g7*KLDl5G7AQjDs|pmC{?QSXd374#;*e3lJ~NE^SbyzbU^QOZfshe z&ULMUBnWSlCl`q!G3RLr!n0k=KoB1LYu?3qz_oOK0aDGF#TQ%hwFf{}Tj@90`DQkF zINpbR?av+uZh4RQ2a{u;y9pBC>s>aV={|+x4J;&>9L^4iuYE|-WBV@85Gx5Lhtvw4 zb;U7^VJQgXVJ_@1My?91Kl#M^lanY4(7^ih(`=e@NP$vQSlpT(gMq2(WFilX7n?45@%`!N#pa1VFli=xXml zSNj6G+O|AW4G#vq4p4l}$k{@8yg!Mkg>mByZjHiqwr>@5C7^$|0P=2;vNMp^A>XX)iPlW9*U+)P^nq%xG^ zRhp2>)cbEBm2wN4A4ASu8c?nm(@EE$^VFXWrlwa|JDdth(8NF5oa$W&8OD8KM8s|2 zlpToy^XDI(5Tgk2cY~>h;#M}J6hm<<=oH0iULZ^EP%kJw)^;>Mrac!-iTP0t$x;DmDGeoADv9}Vepw_- zrSz6i=tirHhnQmP$ruLd$t?b~d&)VNDk0(f4mC`hzLiEUllV+>CJ# z@(UBF4&&TcgbvfxsW^AKA~s307J$M-XgV znBHeaWZJDxBRaKR2KeIp`CH!6NXjqxXU?+AnXzE_*0!oIPK2@}{oHBpanKGez+~t8_5-x&C zFoi~ADY%tHL0m%8y#14pakd>&Z8L1e)&5eeGW=~L84d0CmOkkHiH~Z8n8#eBU1g{Uo1->Ml+|_T#1uiL*q8|01d7|#sAL7)v z|F-y4@inCIaG|TM&Y?k9E8WF_t~S0G@3A*hc<6WYUfQ=48v12x&|1(eKVsyP2F6j0{6sDp)WD4+x#(c~(EP(ZOO z4V|D&1Dq-I%Q~sD$GLY2D`u$6ahlzVdkLCTI8B(Nr{g1Soo_&S=$Ewb{}0n?#r0k; zf4vz`d+L;`F0x~V`O`7k`bfatuW(4z@PI>nm9ZuDP~E)Pr!x0%fa)gHRc$8>14he= z)2N%`RO+WG9WXa;ehorZ1;&3**f3Qci}5EZ8WJ_JI8BhUVQELB>K_8OApRlk$@LEe zYTnIrkUO8k9e^*rg&YPyxvek7+Fs)Vl@%{EqdTl z{xL7sMjK&HS1S6g{4C8bb$>u+c-prT%O%%V4~nju$uYK<(9%-vWxKEi+rP}Fwc%aM zYqVYN@}U@m#xKHdZ+!$7;l&!-`A3D*4*QRh09KoSC4pl(avm3sW#fXeZrDu0IeG5= zd!kwrioO(^`q;tF9i5ghXsO8Yp1RoY$k3-8mgQ#yy}A3r%>=E=+f#KydvY=b6|7N}A0?}`X`YTDCn#=C$AJXJCA1bb^E$fs zmwjnJHl`Y1!{k7fA)+KJP-RR?c99cRMtLEU@GvIk zT5U)Fr1?Vf-uOC)!Oh~|Y>Zd9R>KI?KJU9$BZ``{asJ*DqFD%i-fk_?5ebUZ?*83J zs^)%h0~P=FF4OVv{^b8P7tou$$}P&S+y-7{$cyHBl{Za5F%#DZmfL_@ybSX)kQ2ieitavJo|E6GtqE&)WF?zoF$_uF? z=>G}9%Aw+`QV3S^payL1OYlXuugSZ-^?3+ZV)n3l^o1~=X|+AFl}05n{=CP2nhC)Q zrOI!cw05oX&AcjXIj^L(#I&#}tXn8rGKMo^3F=huRF+e>X;i6<3Wo6Dyrf zs>+UD#i=%pi_t_LZZq`LCkYrCzk;C*e8x*%z#UHhqHzW;7&pBte?FP>f(s!|IbkOBur;!Fq%KuO4~%@FO~dvI8QAF43V z!B)&tt}6}w9)yZOwcpPHEp1GusZ)ggQ6Jizq*=VxMf(1f-U<*_%vJZpxeFEdj%jLfn#sy1(oi>?CW|RGS!fcY9+}ZN(IIL~b}y*rN4M@UyAP{-yPBUcWf5D)MW9^Ez~@I!YJ@ zrA}6&CPtA3rA|dj{R*W6=BB@Y!% z1QPv-*GCx||9qVJlrSscSE!g;e=sXK*miPPf28>oEjjqHea#xPz-P|6d2V*+kD#oZ z2Q;niM>^a*g*+r?PnGwQ4cb9o=%i-L;l9j#L-tv%2fRRmq^+9!9T@-%>_5xNH{p8f z^{OrRLPbbF1+qXQ$$-NWyH2;?3GLL9eDwVv$q-2TBxIdvR3AiUkZ*tFL7>TvE#fsl zD)CHuk&4y-0|gxHIv>rfG;#;%{gW46^;C$NsVtbXkjxBr9n4GKxG zF*8)Y7gElkIxa6VnTOrj>$dpl!MdAwn=>m29#sx8Yc;I4&gX5c62G3Jou(TnRsMbV zn6EdLtbaMz^hCFhC<49eHCF}Y!7rk;AE_QUZxn%W*PsYQrPc1Dnm2}|;I4s*8+Xmi zwwA=*4P{27>&4379eo{oHY)LBfN~Jb1f( zWaXsk=NV!gBbj+rH->H$zXB-_vp5GTr|2rbYE#*)uqSrHy8c>-e!h)}8GYV|)>DeZUn1CY_WKxmg~TX(NBinwu96c9qC*#^|71Waa^O=jEs zdJ28HhZr#;-x$r@kxT9gcl)oew+?&_%&j%(mC z1jOkBOi+!1#sN(1M`Iw=he~h#jK)By4^=-ri^f2x4+SqJje$@f${0l&1393l&h8m# z3=|7YQbZa9*_ED3pE|{X{YY15E($okN0US)#|%RELe*gm-WuHt)r&BA(!Ee~4oHG@FRULAH4Eup z7zXL^Dbl?#3xK48bT6#HY3>alzGFw0+x^wy87~m+LLdwjw|)%-!j#^+BMc)Drut!= zf`Tx?&jOSbgvofZS4AKU(pBcD350=bC{9Bl3>3H0dIDjfxb5{FyyITTi=ekArIaJTiGj%d#^yk?_V*bdaptoH=nw0Gz{xgzB);8edD*Tf zLx|VoNo7;Fz81Y6uRjrAwJ+4=UAsUM_b&y{9g}4mv-$M?SD%OW=O$UBRV;_om=H%* zJy`I045RX_#PZSbd0fkbl#hMw zy58%_jySWIwMKg!>6`Qc2~_0F}Fz?j$PCy zoW>qFI~bs$TG8CT;^)#({ux6bxp=kHeTz|_3>-Il38;iuje~Ko&_`lceE3zE6&T>5 zUg2zk$`JKO@X!_zdhfLoEZ6h=%-O*Bn$NuNp>ErFVeQDGz1W7q!Fp{0Kk{>CCbzR zqdwXE-K*w6B>FflrHmNfp#iqA>Jf|KYoSpbHJ{{?`czWv_oZxpp_fD|_3W9$=tl`MtLLRmfIK>~t3W z8htTTdhPqGzZy;b((n7oBcBc5H@4OPP-^my){*#qgBq$(v~{MoXzk^Z3;sjVkR4pohAPb^uQS0F%MAf3Xjf)Uf zOQ;Jt3bzl0@Vne!6|N8Ww4=~rj`~ad`ejA`IL!k9lkb(2rS&jq|B|8d@e6}N`x__q zBViT{+HUchg9ABlxUy^-vf@NB z3}nTz`}1miv}M?W&Y}X(T)4F#q4^Ct`=0e2)p_9I$^?^91^6%wz=xp?-nsp#q$|j7 z@5sPuQ*pMGgO}uSBBvDX>=On4HD6;p3QS%XeINiCViW5>1@XdAIMG9z+}=j5^?B^# zjn=~SKDiO8pE0Rj*=1SB;jlHJweU+n%&bRlIUgCpKQj$Ql`aDjrmmm(GC|KB5qfla z4kGlJ0tCg`^H~UrQ&~t56sPpzA}H=E9J`Kq6UI&5&h)28YA8uAANx_Y9an*8P5LPvQ1>7^tg_s<2 z&u~dY>fIy{4)+ZDYT=$iZIS9tSjLXTIISoD>!>dZGm;-T6FPU@`^h z^hgmd*)y#q==3uIT>_tknxAMJY>sxFakWn-|Y*f~w+Um@(Q$DwMkv@m&GgP0$-et(- zPnm(J7T4LOfz#CP*q`q!HXqTP2ZWfe99J631Wq9yB8OAJDa1n*3yxzl@eoZ0IYD`d ze86!K50NSr7+@qkMCzqD%?pM3{P|^zz}I7q`rl}B~64h$8ZtS zL?|=>wII?&C~Gea&ZLRZ8{sSsaT*eO3)V#%>ZA+-Z?W7j3z@e#pC&-}naz( zq&tnZl>CW4hm@!v?R9NeG0}gYqJ4{}FzY99w2YUGs zFOfaMI75;jQDK}>luTC~Lv^<|K_X{j3RNm`5EoLV5 zHfOmu1*}iyG_p`E0?B$jUQw|`u^M|c@yDa@FtHEyg?-2e#0%ATSU}~a@m<}YSV#~V z*F;MNQ?yi|1zHFNpowz!^9ctT#eW8EC;0B;D@Xt7cFdW5o<9=&9H~9{Kt<{ zV>@Qsh7ix4+5-{GZ@Yzo?n-PO9@7h{3Uq!pQWcn1=&rPg&q8-4LyLDkRV2VC!00(r zYqoZ1OP9k}@yJaHz?W?s+W$}F^6!gW8floGD=iGZP5FlY=lQfgaW~cH$PUV*4Eq0@0Az{#c-eA`;tQ1hn8oV*Bp^Ej;pb z$O1OH+Yx=5-Lp6RwW=3@#{|DD653yS>z2@sg!WfIJf=t@q5Z*21C&%~e@5nB6$$MJ zkeQ=4LufzDMjya0dXahsBnfwn>o?Q&sm%V;Tl<9$B(uNz;d(`EylYrDv*2fol|@(U z*}x}>syhhn2R@mlo`BGP&@o3fB(xuNOlds{?N7{4^9v)P{iV0W!Ym>QDz!8GC96`q zqr=c6`I`~jubPa%BeDJJJGcgk?bpNs0+HDMdH}H(B(}c+m#8PP{SCN87>Vr+!Devc>Pl2cm0QMB;()jhlQH z!Z+xa5`=F&Grsl({{kA@F|81Vn;OkR6z**+38HY#=5Y~)>+&jL9O0A$UF5kNg5eW+ zyIeo3ZFEC-9+FK(h-~>}U|!|V`n%^L440|}z|ndIg?Lm1h$NK+vWKKLCT>uzhG3s* z62L%=+L$v(mFpW+cEU%F@9}ItZ~y&!@@(F!*RwnDHnR51=G#6uf`bOVnso=l&)N*# z6|}uo4D8|IcHc>coKURxkA9v`iX_at^X&-yQq0WXo2`zHdiJQRyI+6vF>{ z4oZ`4X1?F~zW%?@hw1eib$g!sT=(-l_w`)Ydy$~yDwedn0xcz^8GuFahvr(xUcPvC z0ox=HU8h+b05BDSUINJoG+z-rUK+^=Ob`L`K~uxcM&vg}!cA38Om$2=&g(+q4&76p zm6RY?>n5DsFIRi@11@(Si7+zw+97SmR)>{Vmu53-hR`g+cS*BSg}bW z1x(Y!L5!ps=|R%GhNPzS2&bT_t>Dntxc4VM8p6ZTsAsyS389H&g;L;mZq0MnG!rcl z!Tni;1f`n68 z?{vgj^a2PAOf21wI7NPANo6)xwH#~P2-gikJ5H2Z#0j?WjX_wtq|4?g;&+A3mRD2_ z5WV|e1<9{v@y7682ju_uEp#r~s5K+JU zxv%8kAwugb-Sh`LAJBFbYKTzoYfNJYLAJFBTly9tPiSoEVyL(vZ0VurP;o)n(xX_| z%DaFqJys8?2*Q?jlz}t{VM|Ma!A>B;mJR|4a8=^i6=xAfx4n=HxYBE%TAKqU6K#!A z!!#X8f)b<0N-2qu1gSPT%J((FMo%@(yFbJV8@*dv$H~zOIEgpKDt;^-@`XH#6@rGb zwm0gdp<%LeqK0T_7_Ih>$fr}FUQ;K;hUiZz%~%Zl`rprJMnBLADZjgVg`Meh^fpLo zz}FeH{hPXtz}FeH{i*NDz}FeH{jG=P`y9MjoZw@FuFFK)1T?Rl;>MyVPSCu{s~Ic%e0+ws}@f*vAej+Vg0_w)`MM zz{;czWZgtSc5$nU7vM|$846Ph=tH@bEAcpzhF!8yMt%d-9%g-!{7h`PH-D>urjOMJ zuaa98&vNIKT`fj%s&^6pq2|dA3fZ-Vz~@bSdQX~C%BA)4!50vE`I(j5^hJHbE99XL zGPVvZhzO~qa^u;$_>ULxk)+kD+PXH?Ltvh&t({ebPI%(s-4aePF&4Rw1ruYMvJ03y zqt(lS^D37DCdNEII4(_VB3KseQfF+kJ3V9DPOgwkfreYF59b0Mx5-T!s#PE+ta$d$ zs#^%=WncF70A0f5<^(jG#s|=G%bnVfre$)cWY(IRuOc0{2bM`9z}E5e=fO}^SXzfm z3A53%-WN1B6S2TRid#z@Gz1sp*?1&ieHWpMI1pVnklg2v1vd!uokTf(@ODNoJ_RPm zA z8${`8%PNFjU!sy`3N4uV?xj-}Pbz0o5&mjnoFND}vLSFwT(c{4Lwk3D0XSNVSN7|D zK@Ks~noRqluXNW4AxBTElB=49fyxKfyku6n+DLFYMEXkq^j=Cs8g5~b%0S~8 zteO1tIXAR$iuht9lOV?<${AHmXbR5fJxa@g!Aa8kPzTaiLLM};7C8&k$y?=8h<*Cm zzPPpDDA1km{;vTM(o*`DRsFw(#?Ta9{y8*e{gO2R<~6+Sy^__rQ@WGdKSV|FFz|KC zFCzO8?T>5!Mx~`o*C~0@4fkJV|G59$*7u)Ve2r{6Y*|^g!RWa@G^F#emL;^A0L@!Q zu{KeL#u7Uu7_`RHEl4nEjiq8pFp$R5&<4m5kjBzz32eVeV`(fE@(PDOqk2c^$w(lL zB`N611R{+ktJsSt3<};eXe$oTI!i;YW_dGnm=8t5c_~9AI3hw!8cl;EBE+OIXE-84 zOmYNWw;Y6+ln1(Q?Fccc1Ek~LAjG6Mpf!UKlMtcaQP5!lY)s>Fc9<=Vm~;zfTWso- zGsF$YCZ`{_(tV9lv$Q<;&>hiZ6_gG* zj8)V4vA#EOn2#(V)cd~<=|z-u&ry$@U`>n<2buC|W4!fy9nOL4GZrLu8kCwmoQ)gQixNAQm*WKIjraQ)_N*dpW^p z;9ppAn$zBwL+IcbU~@=S&n8H8ZovPEQC*#47ei^QBV%-@krG;P9Y$=w=NyVG36uQrqXB@XG} zGr15>hDZD{RA3H)?7!7@TQEe}CQt%HNQJm+1k$+X>xrlEAiRt~e9i2~$hrXX-C@$lz67C}&w%#|{}qFI z++h+>ws?P#{iS5rT6esG&ouVAu^cT}0vV0`qL4DpDJqf-R+q~<>36A@K(|S5QoOHg zB+m{Upyg5^JpR5ohSt(_ffAc^aDXOgnS%o~=TxUD)pU|Y{olIGzYCH{q19#9ErGhs z&+aP@@1*!CF-l!$epW$#(Ja*&{dd2&7L(2(h+n3em6<;yeh+m*dSh9j*O{QS{q~I| zKeB{>jJrPOmi%JhEhEL3QQ!&Pm&n3W8ycdgQ3lGmDj*bNZK9Qp_%1@b$xI6IU2N)B zN@GWS7pd=>DYp>cMQi(C`nVe%k3$BNIdlSi7a@aD8SMezMX1BXjUm2^0Q9YQM0^*a z-DH**9Aa_kIn8$Q%I2o!dm$USgRrgdK;H>rTQ4@L8Bd&BbvJ|W6T(w^{Vs&U{ttER zfTu+LR>p4JF?XrF7maEy6}#&7J4A4ItPXM`L~wUp8d^&!Ho0PMdGN7mt)>q6*wAWn zT0Gs_rj@Dvvj1xf3qtDq_Vh%dKPlO_0;-JoLq51&GnMT4 za`w>lx?dK6&^u8M2SV>?-$D?2mlF)A(4E^=1WRLu660_H3SrXbt*AK7c#3+kL@Yhf zLYLrXzM-|8Q`5&OkOf{8=dwWlic-`>r0X=_^Vo@dK3V!^@aY-!{(@sE*&q!AyAqy& zrfGWsEs2bwX};4s`VusMXfDvcPe{5BkKoKl31oPiOb6#-w9^WTvLC2!)qU$_3%K7! zY};t&5^(83-unSZ#P;%F33Qs|aRk5-`4@7YP=Bveytoz|vRMOe*b3?orvcgeZT&$H zkgW^VABvm#dR5Sw@xz;Pwze>~yjbkx7quoGZ6a%=MqUqvl(5WoTt{evOFF-~eu zbAWgBh@r%3z_re6DDizJGbuzym8i3yvl~18YG(Wj;uoMMVv@;X-w`ch_t=*=~_Cp9Y|eJhnVWSfh$01LNZnE z?A0`jw$FTOc_-m=`HY8V$`|nbRHG?n@r@=LTK6A<=i~jM8#C*@6cp*ENwasF<<;Oe zc9NgZW(e1e+*t??a?Kp+zl2;f2o#N`c^H7V1(}7Z-V$VbG&Av4K3xZG;F=-0_D+f{ z!8JoXWdPR<+N3pdShLz`$s~F{@>{Po!2K?|$-|z*Fry_#BxKwP|eY2D8@t8_wyI1-R4u zw)#s~>y`?|)INFR%ifXQ=-4=Xhgj~5AZ9=+X4nrgIbuajjv{SZ{~tumd}%KwNcn;c z%>Ta-Gd-1{=g#he{wg z;tx5aT9|40-$`j94a7BBJR68ioVL({f?mpXEq%}?M%)<@?b)>{Jcx`NCGrDMm#`+2 z>ek{%XLaI$&^TbQ?2UIm7!bzIm#{wOy!yQpM_W2goep5p)B@ImisR$_#sMFsHZJm(n$I`@oEUv@kj~&SC6LZk7!Zcdm(FG! z;HNT=ZN{}u%JOG5Jn0o)XH5`2#q3jxleGztGJ5MXd7DY($;ffY-xjAJ04H*8b0 zMfViUz{$9LFc|a2!PA#cphQ>XcSb{kp7}o30#1$=n$2tK+w@aw5wY*J?nWDYa6A1U zpfM#IMZDw<2tDSI${>i47gi3gxHbM|FXnJ21gF;qfB-LFz_n2?+Xn|#-yy-017EF4 zK#z8cmICeqfb7HCm0pGCF51Q!v}%o7_t(ujg&uLNoh!<8l` zW6hZm9gt1|=1%W6fQqyg14sm-16*~8PBN(%K}f$M#$JqwIRsMr9@hE+v-8t8Ag+zy zN)0uiMl%=4OzP>FP&eglJMyUT%r>?v&JE3nMO?6G@=t$4MuT4?xHitUVRMowNV+&@ zLStb>fY7}6ZQD8PTlS2E1|YycC={MhJnPr?0B|xDy}a4iph88n8knk3ndxOV`+-QS}m?7+rS->A6)N}6qjl>#(p%jvFCYSIj;gO5oxR^a#EVvx`bAx=mk zwNK!xA;80iTFMCzn{SH0R42h-!OX;b32`+Jay>M2A-Jx6O@k{QB?-`g&B>?PJZH0( zo_uox3snA+QzF0&B&_`2KMO}vt7BA5fp@2QY)}j5N@vET5Cy#d_(s#e&Bp$c&(ILS zUyC!n!o~jx0o=O`@)`QceCDLTegjh87J%YZ~i?ZWKama@2;yyeFI?c+sQ5zIn2c!y3b+qIA7 z$RoScptZ^Qw|JB9(0je{h!eG)Vv6^0a@{?5EHtytIf&-^;6{yZDFrubxfEz-&ym)| z_wFLH8Dt3OT`3OfS|QO>(CQoR(pGRx0E!Mmi2XnkP1pqF;x!?SczZNh9=NvYf(cvAaDPL<)_etPOCe7*^Gewwfo+~Q4J-0KJo;O0 zm6U6$r#E@VIh#q`^Trl7PU)!l``|!EC4HeDT&5RI*f$;Uockn}a&b%AiGyK+IP_Bx zm3I%Bch&EldVmLkBe<@T4= z-->(z9^F#Qvyc9W3dL?Dgt{0l}aA2gDIW zWhm@r$45Y?i7oIFx8$zOfZ-QbIajgAlEqt05bFUw8pDehNc|sErb+#Vh~;$3s81J5 zc90eVb_Mz|z@s%foiJ(Y{h~i;N*4glgmvR<2TL7N^%&z@Q6&wa^&?fq}2$qQZi#|3$t_%S; z{wFLE)U6(Y7bunH1=^;&Rsc&?spz;ezO*{BmeWL&xyc>6OAA{N%);n5N5IiwTXFj4 zDlP-{_f8Uj^JY>$^yVuD7~2e8ZLCNVbhU}*c+F-NbjzbTF~I!dwXbRFGXEDPnVC?P z|MT*U2lSi%#s4l%<};JXnKjb9q4CTO1Lu5pv`bv5c>7NZG!og6&-^enqdxPkNx_ms zwxU0PMZxx@o_;!m+s6x*NV|Ng&*Z7CqS-&PgV@ad$dHg8inFtKR%*;eEih(o>PFze zqw63$c}%$#JN63l54pbLM#oc-ogi>vDG&h;MBuOfrrk) zVeHd_I;yLi2d{v}Owv>75BaX}mt1K&2%(Ak?j40jXtK6f?xPW!vU1Xg5JD5+eicU% zLK9H46UGoilh&P#2FDA1wm_>AH_IapWy4`4(>gRqFT!EQogX>Bn9gNZ9XORe9u5YM z|AR_-7Z3HZvM}67B!1aJj88Vr>C`AUoVbo6MH+h#_!;RwoyKuxNMW*utV}0BjL%VZ z^y6RPGfljw7ix5+>@-0{)pISXIOXG@eZU!@_7`x#pfbZ5mH-^>A7l@eY+37WIF3*M z6-;3M26#sEjN4`^dQx+q_pd1Y&S>`|9jDb?;E*F}$4X31dXDI|0(1r*!hWzfP4-d{ zXb3H>0Z~$#l87TC&`_a3qZN@UX*O1{{jZ50Pgf$$>%kp>wbK~k z+os?}{vLU!391lR2wtbXR;gmJ&7e1rg@nd@Inf-qc)2_+D2LFjm>mX0USx-ZydNfdE`_h0(wE;pjkF; z96_l`SO}z0*ciDCB=&g^$Hz$vg1^OFrO&y^aLqSpY~~~P=#Qw7lZ53`yyFe1zPRPB zk3|+OKbf6?*gslc&VcI%OgxHP`sB`n0eqBC7sO`DA`^j)gYJP3(t@MmipC(7$%NQI zu@hpWrAI*A%?-xdJL(&qmi!7}v{rKC#Kzm-^82lgBz?lU7jDRjydD8l%Y7^e>?+RJ z&0t!B@~_A@Nk^#G$xnN6{iwMpm-A`)CbjN~|4PmL`{0=*n&K}bf&}LOb>HtNJ~AhE zzT`ht@0t7J(~q)@fAhOK;m~x`G*`0xUO!&}I zv%EN{$7qjkO49+ZQey1bYf2)ZgPR&1wfdT%9>Z#ymptST$F4~0I6fK=$4;MzoL${q z{dr3E_e1^+(LAcc{fPew)x5;Six&+FJ~B8e4itfBLG%#9(1x5vV-#U%0~tGd3}I+% z-O*_pm+$L<*t{{FxIe@W^S_lgadNZ@=6{DWw`Sg`llJ9D4oaGweY-3Aax=F2ghO9N z3=n>cr|w*FLo~z(h1tf$0LBQ;-emwF%xl6rlvi@*wVSdWD zIJ7=g9D>J0UR7yOTV?=jaMC@ z+ehbqZJJZ6j`4rM2~1=WFD!`hn9~uQhKxuVsg8p@99q2?VPC9Ykevd2 zCJ>yF_37Fr{>U9u;$oJ2MYIs{qgvaJ)FptJ{XV|7cBBBBO%>pX2-J0`LA-3)3c`X< zK=5{I=k}l>i@>sZ30OAcz_NLVjhB|i#g(iVkEfA@(h^gD1%!0m2I6_V{Op}dAU4tQ zm;5a92P2*U#3t59oJEt2{^S}Yh4{Gyq1!&6@`uDiXuj`M&b?jx;0(dI2{2bj8iBd4 zNpov{aUxTCA#O_2)hQJ;CG`GBh=s#p#Q$jkZHMthbQZaRoej+;0U*JFr2{l95w=c0 zSTxTNo|&|X)Y1nix#7@uLA6VoFwv5G(WvE#}a}oy{S#QYFa?466}0>lu(VwPqP~ z%L5TIpIi#~iY^9{Mgbk2UrOW$kn0At7U8;q*M!z)C|(5w=owAgI_QGB;EXn$G!0j@ z2jJv}2!S1Oq@ukbIKKJL&AM@7*Wb9M{_JKQ=PR#I0R24wQclp%6S;~7{XBCKV!7VA zxj^J8wNWNts-2VX2ESY27FX@G>B^ea&ihx(6SI^;{?pv7A zF7vbU(pH$Jawam&E;cL_K_wG;Fj9MpsI6~=o z*Kb7EH_Xoi47Bz$-E(ANZEv6CYU}?Wakc-7-b~DdTE&~_u}UZvAsWDrV856H8o(fw zilq4C#L^`qEXZ~^hQaaP3{;s^a)-|)56vJnBSlgK4T+TQd_luSR=B*5z#8gnK#K;ePL6H>IADJAl5dv+-pF_|^#6v*fRSsU!Yza?v1{fnAuaJwG9Cy-rF6Fp|;g4}g&@m*UBnT>~t3Pu}Z1688v*#OtD}dJM!~1H^w6 zx^OtiT0&UC^17%<@$B_gZ*Y(R?y>O%zl-82zl%wqXPS{0NUVSY?r6*MZ)ABOw!1I< zp=HpNkOwMp91)q;5ixE-yoB_)#n^!(7<(7?;}zc^MU3sopd=zwsI-l(g43WG%tsd? z$s_zHb!`n1q{Q=`_(3z!*s&rhbLsXov=(zhYyb^CC4Nu_?%M7L$}_YQ&5l`GPG;`^ z3=_<<8FHE}G-GUo+kvYo9buYujf#K_1^tp#U;Ohi79S{pfp0D_@Bqcni#J$2+;(lz8j=eA)3I}_9}feP2envjG_0YlZMhSH|GRDy#~U- zN`0G~9KldJ2T=vigHvcDs=zvMiVsC13wkr<VL;k5~yHKh$OJ}u%6^V7zN-ncY^B2P5Oc$IjS<72p!rBkx)%@(i#&YaLo|o^gzNNtzHFK2)Ps& z3%uSUgl}biyVSsA0bHMVWj0*RKpct0#S5kyDVB&{5#saw6RqZOjEqPyB9c7IX$H`1 z1xvVwP}${RqzGmQTG``p&D`;%-A(sXegzm>90&2dK0wLQJ*8Qomi)|B;oREB#mEbO z_)v94QIN&`?Nc>0^PiY;e!e)+1YVdKSCbWze1M_QOEIPNDyfA9Jze&kEJQH0F3P?t z*&sgA2@&Jd41t{ZF3pQaj-xxyg>41$CxK2V1)W#iKNnX{wlGbjz z1g?)NuSC87HMXw9Q`lCYQ9AUf7J#4r>J0>F7WDc8j5C8 zRBz?mnWB28J^kJqS=pnZ*#{5*d|Z0vICmpfURe;G^jou(fCyJvBvKnWCtD{urT2>TatLP1z02+NcNeg{Ne{L$v) zXm!xkOJhs8MDW^n>Y)mzgoWuZphbX_vIaOQdN%hUsS&YTvF#cVa&18T>+>;-15?F^ zDQ1&8I@7?n~s#}QBEIn+C?kCtgOw#j_ZCDyw8_i*`$OpsyZ>{H)B z{Z`cfywSp`u*{REkYl<39u_=lee9y(Spm>sv~NB*eaQ)eHrk0r?B13G5E#Oy(-Esw zX%6Douz$hjDm)dYDUBc4K89$EVWEbpYym*Dpb1c6{^^(HV@xvR(&1~Ezq&!;^;(F z;|K!gyim}bO$T@Wts887spRcQWl!=RO^1`C|U(<6Y1YikuJ zn7G<75ZUxIQi^&{lOP-ETxYitM3Y-VH2Lkttl?>5`zl(4(#UrTbp_GsJ%G-H=z!QD zMCZaEGzK?4@swc`@`==RMiqY57+v+V|B(}#Xp>?J=pOxbXxbTrLk1U_e&-0md9IQr z3&;t23zH`mJU95_*87)7@|T-LJ+3YXfl)`?onj_E z@bUPStlxs4UIi=-{AE#^)bY_Um~AqJJ#owhiYB-|g@*C-&{2vUGBR^YfX0;0C_`GF z8dx{Qj&)OXV77^k;~IU5@S#OboybEOaG2J#yc46@aF|>KeEeqf=U^Z?#jSEYIa&;V zNu)@{kCEZn_I*0_i5*Hbn}~$Ql;)^ynv)I~9Yl{6Q=9-~Bh@s1v@Z|l-C;VB zJ0u0i4oaI)9_@g`1W_j9$MPss=-?-@we=J;p0~?HGxTO^cv|BL@FGQh#0geT^S@%Y zL-&cn^X4TE$>wrS#J1?GC@v_HqkYRkkz7s?9yXYHlp@jxwE3=g6JC*hn%cf)UI0K@kk?TlOM( zST5xfAI{YWC!ksAof08xxAIOw_vw|MrzYO#(&Xg6E7_ROt*-})xSZFUJE0Gjaj;~a zT|Ht00VyA(dECvQJOgRoB-j)9PI?3~@SWVDn)Qd6-_gB#jt-$sRoW<8BLn2H(rH8mi_l>|cqjTI(Q`t{r zv~=g&?ublAvog*;wZCj&I^f+T1qhZO_SIemvF#gfGX!x(7A>P}Qbv>1rFC8KV-TG0~S#=ES42?pQ^5$Ug3FjP#44c zkaMQa^!%;MnZ0hl+?WfUg7=WODC$HOs0GuK;`rIbo@S9V`gsx_{2!mx{9p8*LS}&n z3m1?7b9nH!$x2O<3X;>v+ht`-f@c%*72t0v(zc_`JnK-F{VSOjn>KFR$i-E8;qCKW zqd>Dpb|a>H2Uu9OAt6A_56){Lg_?}mwk#Vh%jmHr%37c=$<{mK`edQQ5@se9Ib;AG zmNRMWI-@_rPW*tP5Ifcdsm+nTs76N(NNsY5Fs4$_846S$ZAlA)&QMkSSPn(B@qTjh z)8UnSciHJ3)Y3jV`Y=rg7*c!V$5JSJ;S{R%j)r}za0-&yWb-Lv{JFo5&L|_C;}OLw zcI*Y5<7i(-qoXGDtp1T1n94KVi)Z|TxKw|mu-ZAY7g(1 zdr|-f3%!H{#5g(rnrc$=_-iRXljly$EO_Mme8MHE-`MC;D>#i6hFe5r&V?EAQ=8W<1My-{e@EZG0=T$} zg3fM+(-7fr^y}}-_#4gd$o>vmSpFfnm8W_Nl;u(&2mW4PgnB5=!?tm#2qvTtD_F3roK7bji@YAn;_f9S7(A`PjRB^xPW#j*cX zfoS`W;XG0e`qXEW*mGe&R54$wM_6Z$qXLLUQIOspt{F6Q8L$WC1VLh&=z{}^=}1AO znF}r51pvbB(0#hU5>fYu*qDFcn?u^U46XB-QnnSkCviaciJ>rTV2kIJ_~IFxD>R)! zVlUF0Yb!e@;L`D|Vh-~IC}fd& z)0!Gnp#^_88aR=Z>^*ky=4urt?(28BccC_6kKhb%&?|e%FtUf;>NNO}{M+3F?h{cH z#l_U;!z(T>3yN0gA`W7IOF7`HP!lx}Gjj5Pe9)sPYTfYai*!LTRbAvk_21GD_*d1u z9LO9wbwDAg_$BrI@Un|bb)xrnxeiMImVDo5Z_S;7h>_#><#Zn1p_UEPUtFmZlh{QX zRQ@gPzF%TZ(?HsY#{FYD#Z6S|Fx|!F9nn@@{)6(rk?;Fj)np9BkEq<2?|76!Z5>{7 zaaBjmo389Zt=}^4`@gCAIFLJ{eP5xY_#<_6cw||rCpr4C*W4Ni8&PtU zlX`TET0H#Y*b1qbpsuLF6Q5HZ{eo($2FN4oj>n{mtEexBSB@>qi`MD7GkElKileVi zP2vD)MA=b3?@=PPX_#@0E-$8|D`QaebGoB{N6njo>=7+Tg}mZ7)Q`i<$Cf%pOLg5E zl>MAM?juzbG!Qj%VqDJYQ4qCim|<+CQ%qi0;-Kp1v~j<@nvQ{t5zX;qPQ@M6H^ZyO zmQO@Gbp;J7d?t_kI@RP2B#x+#%TGMYqjn6h9a}XKGtredsPj2v+<&5GVjyosXIx=I zRw~TVnJJ8^NcLcUbw_lb-ldu)he%2!z2KUk5BV$k>)o!|UZfjV;JicT(CR~O&MeN% zjF?o`G*)L;M$w!6EP7i^glhRqjXm{JYmAGwgb6q&%NQ_jmD%ciM&|kI=WdT!(OcyC z<%Il<;T^h!zr{F3??%nWLs3dmA!X5J{0+vjA+6D^{1wK2A;rc|%zTW5s`xvM?}s!+Hznm8U)Bq$`Q=cMQjnfU&CVkIu!qiE zGDlY*b@OKNX6D22ifUM@>7^DX)fz|WdDZMLG7W2R7M3wvZRXa<(#RZ-i4)bZYSv9H zNUAXo(<9cPi|}C;bj%m)AMnX&uGVx*XGv$az?h3tESvRGi<0V$qxA07>?yJgYjc*A zxv=_z+YrkTb04Nxlw!3{_k8~8>Q}*?w`+D3>4gIleR}5$ zPuIQ*@ARzssmLU(&KV76e+UH2-o)4PVR$Re!OSzN|;wXIt}OFwfP=80&Qm7VTj{y_Ds zkWP;pjv|AwVrOm{h1Cjffh>W{k{Ah52TMD>!NP&sR}r0FH9SRTVNK4WGFGdt+}c^% znR78Yq7GJ{bO#Hnt73y99>zX=SM!5@k5xm5V7}?4h=$k(!D`dsh>F+>!N;bz4U-yl z%TS~p-XSW7_=@73)hY25zM^<%O&K+}q?@TMCd`JyrGgaGRKuGMTguc($=+*7Ys&VK z&Un97dCs>X4y}gPkWYmT66b3!+#>kU^hQKoY+bEkgQ2gfa6_#{gM}|%xT4me!NAv2 zxJ|Ig)Y&kyL9UFKr0(6KLgqUc_d@N3MnzgV+0V$=O1PuetieoOH!VM<+B77hDz+-+ zv8jh)a)W-^dJ>m+gvwFAvN&&bZ%w|8xFiidb(6H(lu}bK!?Xs|G8U4scbbYBA2qH~ zy-_nh!#s(i+hUwnlTuZogz%|o@@mGXYox1Nkj*nFmMwZ|MJaWrQHFOK z_LNzY+Po!IF7RH6AJQ07?<4nSP^|XrTIZjwejVI>yJ1I}9;wKiLq(QXHr`#sU41K= zEkn$5zn*pB+1l6P-JT6Um6?$0yfG?9yhibr8kOn+a&WuJ87FE!0rcSKsIJ zLa3tVdHr*rexWvjA`@qW$U3=V-Y9kVmSg0t=kC7P_d=|~y8N8qOP@DF9W~T?s@l`k z{N!qrkg%$ls^rHe9tO#E`o-&`xZER-9o<@X*L%OWIG=6YX^l>`p48gpQWGzOv^vvb zmMCHOv}0ylsdpRqH;TvGnxCdTZGM$nlU!m#G$7aEuirheGl}n*CYxqJx>)*ti*x3- zl>X+<)S~1%lPH5bb$f~}quSghk6mE95HKV*w7>6MuPx=xzNhE&&s6sWKfhhKqgXGh z$erVuESqeAyO{g_t>@Tm#rpSko-aI8+Y|oWv+k#2lc+j(%rPT2qku}W%KZW7?%ImI zx%c#Z!I_$#(C36Yu42QeGIyS1s%)wO31SKRwa%Tg&Fa6`dA{gOT~Fk5?>fF>i>Ow2 z@ng1ZwgLTO{rlU_J+aMtWA}70|7~?o$a9Z6j$(tTVt4Li3Tz4ifntICCC^FNI`rFh z4i>(x?TL8qRmW3o7S-e~dd!N=Dxh7geShw`99xGspPmjD)Krmzq8^eSzF+tGS z4otq;rKko{1E$(6II4nFfq86p+bF3?w+fZGBQQk!kYrJgvppr3B3YE{d_l&}ZQ$l0 zOAm9ma4CjjmTGjfX-kz_VshY`#5Gm>63+y_)qXCyAqQ=Tc90*Gd*|2X+bi6Hd1!Vc zs*Y4w*WJ+V`&zi6uCJlb_l0mp-SdX$zWu^&m?AT0qsS(?D&9o(z!q(?CRgoW?c^D-(=~t~!77bGs^4mk|j_r34^NO`mW zsdfIjYHIMy+f6&F^b(5#IkaUZWOLme-0io%XZr{X)@faMu9h19(zEHODwD*zK#aDL zgi&s#L#2Jd`@0{--ef+tE;v_14Sh*y;;J%CEDPk(R+UiAO>juC*Lr{IV^)7=r*+Y} zI%?!g?#aFs}1M;rm)@#7nOxo+`7%ra)0`D+#OIc87NR-1j*j9o`H-{ZLT1H(Do{ zE1GM-uRznu zua=|t+vvk4w4a~L_((7&8dJTgev=Q2kZ|?Z`mH`pLVVR5>Nogo6O!cTFqYB9SC{9< z1*>0fIY?f9?#hc5FX$>5%NhJO`>+d1RkPN!s-aT1C-EEW1@DdCo3zVVN%u_kvwT7@ z*X4+VN0*mf@m}e@if?V)4;t&#m{SFlb{i|}T35f$_X`%joOaM`IrU28%EndkYt4V4 zpqjU&3MBnxd_wnJ^@~ea4hX*CJE*x#^J+R>`brB1^R<+1&FfM*k}$?bx~kO)`TfCd zmn9EgSa#v+5Z%zqK8D`46!v|n^V?7JztVlGSRInz8C-Oks~;hvtXj>+!{uDgsmXtd1eY9(Fe$^eGDYsJ{_q0a9(E$~YJ>GA4_{O;hg z%RC2Fm#JP&pi5Y(#c*nE*0y`=&U2i`yuv?KsgBR@3vRtEe$aNA?bUv|{*`SEPu6C! z+o1-x5AeUzd#Y3&mj67s_%ioFg=Gp?1L*=+N-{{Sb=YRNZgA&-;48zY%GIQNYH-tK z(SugYtgg1xwXe)&$XV;a{s}d>qk3RA*Do7A#+Al6mWR?E)(F$!SRO_vLkXqq zyc$X;O()CfzS@OlJu{22kRY$ATDWAaq@aYU9qw@bACDhLdw3`*h3yOrlaYON*v;jp z8w+dCAFuzo>B_R1nO*pR;11JYaQgKR9*0L`JxY{_94A5}4~vC~acC)NdBiB4;Am57 z^Dt4W+bO9e>CvN9$AJmOAI8g6+$_K0x5;;_u(%+%=`oyT{gcNz(e@t0O7A&xm2y2+ zDlzP22xF2N2_4z#8p!PtA}u0o;lAsL2cMGWPR+1%nRICjSMytx zZ7u84I8rdCMmW{_gvb5SZ61pU<@MmvqdDrG9A16=Rk5@XLq zS?@S2@LIp?czwj1KF<&ge`tTLIS)dYN3)bk^&MYcKU~F|9}tAmkWx2B@Na%pT4wGN>^=ltI2oJ})lwgS)P7oqF#fV}X zw6Iz(G1wCu+OTadCfK_5l2}QX9&FtP3<1xHM_0&|AMx|@2@nz&;5Ip?ZwbAxoN#-W zVeI=2x!7Enl~{)L3`8dM2w`NsD}jlNiJR{jFIIz~=9emz%xj{qZ(sZF!Hw`uE?!uk z4J3jJmkM|Mu{f*-QO!S9BzcdCmj1=skq6G595NME%!;X!|Rn~Nm&!ukuuA@mSe-_c$LO3=RB=eM8X@6mmx zSR3-7GrY)!11q~;mgtUl=h}LdO+hShpU3%~X9Rn2&ra6HJm?9pbHQMZ)*BHk(Un{Q zNAD_#1>L)Se#aSs9{p#>Ya*p?&oSd`a~fs$nN&w_P6{!de4+R8s6k0inUsAMQlg6bLAe*QE&+QbbD||&0dmDBo~S65R>n? z`WR+;3LQ>^tbtihro&TGD1uj0=nUzM7%Nwoun?I2goQA?W@?d=BzRFVv%?>5{Gr8P1NfICDkQ;d(`VNm=rt?kFO{%FZJ{C4G&TQ)u^ z&xy159aevj$yLwwU8&9>$UtVokEDzUx~4E0G8yqz@|I{2)cw+g$h>Cic>Bh84p|%=2euy7LPOzGRDjZuiBVU-XGWIE2(}#@B(=VKV;Zf)>}mh+JD=6`#FB9 z?hD1nkn+yBB3}-5SwUH{JKo)JYZ+S=q;4M8JI@JHaW76b#+3KO)%jx7jRcLzmH0}- zfU>()VnLa=t#_Ogpz6Ok-WXBd9arYdqpm8bN>0Eh7;2TBs>%w?^sweQhoRzMs5Hix z_r~DYA{+^$z_d=;Lto(Ugu`jo}f}jF95FcnLStd~h z8H2}%o$m#yhA)&GN#)eICSOr?D?uxAJHFj8w=Ad1A!zvahaGi^(K;a!(GfqdyEZO8 z#<2Vb-9`;p4Tj~ebWD^R6#A<-=r+-9X56#-Czj(|G=x(5s>c_SX8&@2;JuoG5$qQzve+E)!L*5KGv6NGpEQoEX{4qm#Tl#X z+1I=)yb-$TnwK;W1L=m!CY8nZKArSRw^Okuq_8uz=o*K#EWNC&JCpmStvqad#kBV+oew>o@(R~^vL>dm zC$#PwM%swp$hDHGa#H}$-MwNu_Y}{EoKAkF-+8>7`>D!r;}0#m{!EuK?* zv$XFioew)reueK;sfjP_3vIn7E^SM1>)OxMzp0Jq$=)m-JH^4^fuvV@ok}%fh0jBa zuW?H&&?~qGG6imu>c}~Peo)jaA@n|V)dBrH7;AvBC^DD8lL$AgK*pTw(F z#FfYTDf?;(XA4G|9y7FTcv6}ZYwtCz{GKOQIoE5YGQ(~L5|heE?8t7{SSDpARlWq? z7>(oV8fmF1ai;2q_6_e!Z^Ul$@>1sEA;qdFtH7IjT#UvEb?twYm(Ac(v>Yp=7|*T7NxYX6tbz2A+!QZ`(xX@B$Y4h zzCapM8B*>e^(Inu_A6QkpG|tL+pX9TQra0?8bnF#B1iwpqt=Fy8 z5LWs;w%ChXSz)&VDNrR)S&}4?=%8({^daQ7AOTDQui^ISZq)u|++FKdC0)IbFtr@hS_W>6RzPr;BdO z9Aa@6VWIHH)g(C&$@DQ6u`*h1sWCoW^jL<2RYsVlfj_I}m9v1`BcYoXLg^u4O0q@w zWS+A;5@qS&52^Xx*~JYlB4;WTqjxr}gz*H6yvZx``j+a3Xp)|3SPWyETNZ1iuwP?! zU38S5Ls$d>e5P7bwE|(dqd8+!IsZj zHdy8Eg4F(Fnyq)5o?)pE1IGMUvs{R8H`%TK)6|DIrA>gi z$Zae^N4#R5_6xsn=6W?TL;EJiQGbbN60{ROCdjyksm&8=-ztdI9FixuAlY} z4~Bl`XHtHqhaW!mj<7cC0AI{{1)G~#`;sVa1H$gNKCKxTR8&$WPLl|w-jGQq$F;Mfk zf&Q(V0zd7|bb8IaQ(6ByZde8WAqqKR>v`mad~DrDAs)yMQjPwF+M0 zNAB|RcKoXS?n#Zt4&`T@Zg6V*Qq%a`yU%15e}t%&+l zdz(E7<1?;j_~G_3{v|(Os$28xVt%as5e6r{T>9&hfB5SPOPA22=^2)M{aU~IhuBpx zpCzdL`sG{l({oqAsiL>Sw06-mE^+x$ABI{lTarw_8%D-Vjf`Hk<&t*_Ba943M*c*< zW(je_;U!BhFEw4h(ElJ6S397a|z|8>b;nCT9By2*)7A8#3|5GLu_mWrI3Hc2k}A0{XHdBb{UWPXf4 zTxnR2x)QT$OKpVTuIU*f!)P=7_!A6+56t*OVazJj6_}l8?~Oy#N8^NHwAp=-TE7M6 z1*694QCIFUYbhoslzp4Abbjr(*~sA%@|1lIMsafk|=tGp|5B z7^C0vYA|Vz820UOn2I4B6lQumHElYEbI+6x^$2E~^K9YOhLJGSp@K!MHXX_t;`uo! zWsaGCU2PZ%tNrQrm(`worazz_*Dc>teJ^|Vm?nL_E`LD%0jvJFAY+c0x_!MaVJO73 zx;%Q0nZEvVUBc;MrorKhh$%hl2E;V{`NFFYBO#`KMT=N{dQ>UQv@3Y7JLv1`!$D#7 zA7?INrmIoqFw=YP_h*mkjC&nM!c2Sn7P0!PQ8!`rKb6hD`sd(YhmjD|aBte`m&3h2 z_v)`fRl@3*cP+g7Yf$d6`uF=5vHCC+Vp{%b{?$ij3a5vemd4OxI{!>jsM`?J;9GOt zQ8Vs+7zr`G{bmuXk3v<$OrMs|zxv2bVPrbPmI3EQ#B@EX7G|27weaf0NSNs}U-BGx z)Ql(ndK3XxfAHhNs}Ca~rj7oKnCV7TJ*@uY+jHDeGw%J3C{I}ZRNs4Z%yh=R4?`iQ zkHcf;z4wur!s%hApA#0b`b;Qqi0R$zg;yU&LQI|g=enb2Jo%YWjWE+EpB7$y7zwK% zdUFvm-GXX{nI^qhc=cf<%(VI8B36G3$_G||AdEamOedfGNK9cQ#PpAlMa+~1MS<0S zkhAdWv!MK7^HYk(k0rSp9mB`D04U-L|8iLQI2S(PB!=-DYHYOP3_k|1_O;Y)1t{ zO!ME;VoFQ3zRmI`@2IJR2E$B)Kht75buG8;lnnsVFo+*rkn0mga#dMaLPTo;d2Ym)JO{u2Ebe5U+?XaJk=}y!$h-vg4 zT1=6AbnZQB8>$^<8b`=ma5sgKFw@Gni@2kaAM%j)IX8)2zrvtUf0y9J!-< zXuIk6F-`uu`kbgPm}&Go+HN{aOedfGw3x!s&u~YL(RR~WV%qa%_2Kj|)7!C&Sba3= z1DTKNP7gD^m$!(S z?m|Vw>c6_b@O26!A*Q#3XsbWVOlMrDyHKxSrqQ1lzD{8zT&GX(&VQZK7HT&t7Gj#^ zHOIX_WA%5VUc*eo1LwN;XRJO9h1JiBnd{#Fy83W>i0R|RMZ}aB)eAGdnZNMr^P=Kl z_3xD|V)bDt#PnwU!mAIbhnWsMSj0^Cpb}v9+g~rd`Y;k=db4a1tG@@;4>JvnrrrCq zT&G{3QZN!$-zRhttIv;0f|w>1EWG;sC?NW82`*j4>cdc2{iKG4S07FfF};+wh?okZ z-oZ?-^e(*mf~XW&{nQVOSbZ1@F}-q$wBWrDr-zwFmo8$aLa0<&{d#ZOlmGjt)Qmj} zMnX*g__&DG7eWofOg-xtUVRt|t6v?ph}9QCr9(_3`xah(5!4XO^zP6iRv(7K>PKFl z>yG+*@5AXKruP=J@9#r>f|-80Gshh@v~c0ohmjD|-^0F^t-j5rW<2@DP$MwY8<$e&xK6*``!Et#|Bv`ZtiA*) z2V#2HcaA%1#*<$H^%-XB6FS!&HRH(-Lt*vr#xK13aC(U8z+!gO1E_JBX@AketA7A> zA67rE;{O{Fu@6)V4AXbl3CBLrdZ0^ZT!qK}==8$q^umyuUO3Lkz`-o={4a}?5|Cb} zEzx?GMPVbWM$;EYgAEUKEhg+~Tl3=Iq&9U2po9_^2agaVqZD&)Vu2DH2e%Lt6SEQ% z2N-LKbMfaFCF_+JWtOBC>E$KnrV6pRga^3+C4`V9%2JDpGxPI=7>x~pc3AOofNTOz zRy@%4*!+V9h5zL{i+Z|RRPT+Kdoy5E)8$n8Ed=T?su#y2w@Bk%1!j1Xh>)C_}^)N2PDl&GY}5lKcb)mrLcONnh5Ly02kP)!XX`foA>+uz8|06_uaeqx$iDpvK1j?qA-v5KI%X&&uF|mnwNv|MC@zh-00a* zyZ=yJs9p7Cf?(t&`%s-5O%r#9(SW31g1{e+6AV1UNJJ$6QW6O>8+r6n{2GMFkiZ`b z9Uq(<-)CBi(7Q>h{!HS3HPepJom3&2-xLx*PUy89GG+$nH^KaIyasE;OG$G8Ukw2; z0YE@72m*-!1akzc3GiQ-Ll6i;AYhy$0aOMZ)`^{&3uNM8bf%3k^#nh^ehG*^Ws1Wg8>A6AfPdXCny~b^8yycTMe*$ zr5(x=vO%Q41`$w0l)1}KIk1r0-%p~L_|Un&bD9jA-uPO~sT zaadB&Fk~4@3=kY$z{1GD8Di^276vG0`awa%kYy+_K=8^v6bxw{Sg=L(T@RZ?dX|&H z*>wO4zoZDRrT?xJwTvLfOe%Ta!^0FJJn$rh%~QT$r3fpwil0ugFhKFj?XY^uFk~4@ z3=oW)DHzheVa8_hqxZumnR{n5yKjKvPm+RmQDhlG4D#+q;9(mF!RP6$i($i~^? zcn1pu6yAfNpkc@|lo%lJ*;p94I9KdF&cXo2n53X#$TE}|AZWI;F!FGoxbt@`3{dn) z3L1tiLx}-`iF;WX`8Z!38DL?6V&`927e$t#!~nri4hzG99b)$fEDTUwkrcFQ$TE}| zAn?PRXzKP=fD6PIZ?Z5z@usApVaPI+7$E4pkA+c)3&l$_EDTUI-(g)8S%wk=1m62u z7)7{9{LRbo!G*Scfnr!v(5fNJP-1{!%E7`Y#>L{)eijBOy4yfO!;obtF+gyofQ3*rb3kYy+_K+sw|i}9IsfT1Pl7eXpILLu+{5qab8Jp&(V{XH6TfRSm`p%V*?tl@v< zQ-{Ao+y~2yf{C1&kf%F@Usz7E`|NI%N@msQ47@QR2@4K{vQON9XLWnlW`rDZ9OB4t zv~TH1B3Cv$v^wb`Ld6~z)*#OrJu*C8Q0{Q*Y`o3);DM6+$PtMYP-FZKLfBy7(1y>r zG`W|L!prj%FBf@Uqfyn1JQf{;xmsd&I5@Ok$&5TWvbUgs=G8=;x3Ey%O9~y)TU12x z@{zs8#WXJ$ao&;=b+3hw>@6*&cy*DzPG`XD4&IZ0Mda9#OiS(!q(^%7n=){L6B5+T zx@gQvNvdYD7+jHOPD@iYlhqJ~SrpaHTr}qNbeh=`89O6G)vP1SDYDsOp_y@Hv(>6@ z=1>$fo6V|b4Oz*NXSUfWW>*wuyPam{BhQ?jt!gHhISO-bF2!t#!km{!Gn1V!(yGeO z515}1F3BSta*z!dok69Y6BTGeAD=w)<(D^(abJ7=M(@!Zy>F49qwBAef7Y)0!F!um zz1KE@|F+_TxVcA2V6|ebnv3r?R}j>5HFwFIk`l=E1nY2uBacg|5FS#9J)wo@h^i2Q zXDK2NVtAIygP{_46|B)vf=MavDp*H$R3OgP&c3NItV%%7q&}46)hD9)r4xSbq8k~< z`>tOd&bU(E{`lC)TOW?3B`4?PFME5-9~T{*9G-TfQnU#DbIsyy{jT1FokDX%eJ-xu zYaOp?>&r8iKcP9-aJKpgzmn@3s^9QQ_opqMwVyb)^{;tjW%hEe%P_X1>Ds)mA6K6G zb=UHvJsW*T){UDs(+^B;{y=*8|m{i2P-d+Sd=e_0#r*#*z!RXc96o?YNGr%D}SJ-a{; zH`cQY)FIZhi}mbOI+(GZo%D^9*?S|_vqLD>vm=LjDk~+{vkSd9Vm-UiYQ%bWq1A}> z>_V#%>)C}?Bi6GEtwyY87g~+~r=DHxMD2g`L`|zfycVB)FnuBUt2av=#g=?(DwxlKo%q9;m-oAfuAR6G+L{M*UbKCa~Bxr zZ>)VP|JmBAO_jCzPe1vSNFOfjK~U(ic|3@u#6aME#qojb;YCBcmV1~NbRg5 zo*PhE1(k>^QNXQ)phh%N5ijssJcwxUB90Pc5!T=$0g~7;OFQzmE_U15$>L>vEoS^P!pzo&~6WBjA=Vso7oEUQFH z?#u}Zro7>f8PzT!r4avfE`b<~k%5W_KM?Nt_6Q3YH@TiSWepxu0q ziuAV{2|qImq~K~-(`E(`l{|tpGeRF$Jp9ZIL<70Ir8LC=vFR#pr3rnU#fqs~6!Qed zoLDK6ayQ(~lJ&HZ%o8Lt0Z8Xykw z2-0ALKEwy(h#a}b-%thvfd2r1G#G&gaT0_Om${5W0)P|%kVpg`#1#ucA>55dnqq)h z*F-yKgg(Rv!y=X|)lvonfMNiU1|#qwKA0hB2)Az|Z7l<$f=7_1U+6=eXE++p?K(o^ z0Yo{EAn^!&h>PZeT)3=Aif91X0RWO{0uSPY2}5Dri*(vx0Fk$Yb^#Fj5Fd;ia^qf1 zp$rB9PXRz0jKG69i92%VF3qKo0N?}wNF)Le;)-dJmb;KmQw$J^IW)zDK4c7|#pZ_1 z9a|VS7i>eYvDk)V3&G}y4Pgt$Cda11CdC$xZ6G!Vn*v)Pwg7DY*u1d$Ve5lUg{><# zUug^(IyEw7Wj`i%kW}39iaIljw{48@gsA=YzPJhS0>D6-Pwv2hO+0AoP=h~d!(pJr= zjb^E^x3<}N27aW%&Lu7F&vLFyTeUMfR?^Ha(ynIP>@TJDXRrfWJEm_VK1B7eV=cxn z5&?V)*sIhy}xn#N&O`_0BV2J>Z0^#Y5gU5fzaBe zZMP#@e=LuV)-J^pNNdN-TTN|mqd#6gRDL<5r~1dsx2<-efBY$jKlxd9nH%%vWSwuO z$3&@SF#eN#pg})jnDXqgKj#lSUb1N1;`}3PLl>;L8$Xf>X9hCK!HF*211n>C6?gkX z;n^`WznthTeg5dir8gytW0x*7lj3yAt3$?*@7gyuR-bTrOc6WXarG%(X>jP^TfPri zS<%A$vvWh=%nTefOX>7pW<*T>^_{5`J9Yi*J7Z?5(AFT;`c<0qvXAvV zol(d3UB3CvoQ;yS2Ja=w?^MgSH5`ap{8sgWk`aCHL~V)pKDu39clB#kOvT8*#m`%M zPg9*<74E42Ec?_Uj}Y}Yl046G62$14p{QQ|?~?F@#;mCD@LOw>ch{Dj@7Iv_VXvXP z@16Xve#^6l**~m&|IXcyoiEi)cogA1&ZAmcS(dzhJN{?#nKkLBV#*!lJA1Cyrf=B& zeNCy?{{0s(mL1mi|8&%E`&EX{pYiZsc75^5x4!S)l-}#7+i@9JGS00`e-hcb2mYMu zrf^}eIpa^%<*^Zr)q#<%bIx%%KMEgyi$idiDoF$*ih%4$-VhoBQV#cnO^6Yg#iu1t z>%qoZB(8}hkjJfpv@{U`iOsYy8UoTqFElg+q%BJ!6+mQxTws9%4FS2YOeY!wa&Rd@ zGz6rA*1xE(O6*DuS!Mp z0g25#NU)1W3M{ar_k)szfJkAFtW-^7fjx(9h~@+KzY2UwK7a*M@P&{F7F7%)g*~T6 z4;l;Xer5Kd`G9@JN>7pxV1XUzRBsXjB85Gvb$&D!*j+B}Lh}LpO*Q>UK7a-GSIYa7 z5D+Qs>wnzM!~)4FY>?1`1Q#T$Ah88WEJ$ua0?VA>f@ByZzaY^CNis-sLBb0XYLM)L zq!}dGAh8BXHFH7@5^0b?gJc^d(;$fk2{%Z>Y4J-3OBe?KT+08n{N4J9vDFe|`TMKk zjlEv^=P(Jr8^aynhGrm>GY}sj{ot9>W(N8Z$c^!l0=`U{Rrg7_{jIa4!JSo>mTXv&&OxOrN^&4 z133UbK9aZ{b=*K9;;(-;y*jJqc>R&C(5Cq%tWL_fh%5RP`r!E`1;EGGAtwR(9FQBx z0T}fosoO!v4aDhHoAtBl6DsIO2!M~zws$v1Z=eJK=dVpaeBftitRL|S5aJ^}Jap7? z1BJ*GUk5!2h(8EGe53#%K2lpp9f$Qt3ecz@glId%`Vq+?KGMTOM;$j%i1_$$_@o<5 zM;td$NFYAq(?8|BYR^6fiZhdb9dsuXzmb7{qyTvR$oCi0eWzWI8wfWyATh;9ynbWI zFWv_6kw|col;1n^XD7Qc=gLO*PLZTCorl*HKGnGnIOOkio@vS`8Qo`S>cpfqD zlH!}w6UT0f)^Kj}0r&lSPfPq@Y^$xas+3(%=X??ATsE_d=5zLvLrRcpf>+{D)uX^EGZqU-3C(0Y&&UjL~YVTKj>(}JuOnZ0U zN{D@mBAx@^u*Y ziaCo)3{w@My<$$Ra?)Ng*Q(-_uP+i-yIJVgTqL?*hVVH!;QR6?0b0OjzwE zCYrrsu0=Fcm9U`R4Sxeh%NJaj>Wu%7E9UGkh1y>V6@Hlkyj^C0Db#eYQs6gWoml%z zp+-3vg@U!e6e>Jj{)b-*wI?-yl}Sw}DSodO9bTmDh5sp1av4hXPMLom{vxTu#x7xL z&!3;F%q{x%lth=IRI1sgQ1pQ13Io*JW&XH)pgcD0@wK|UNflK!*Co2b0ADxOO(Bse z9hT2pxGdV&k#$u_r7~YBD+LSa=F)j_%pV7?h*=O99W!vztaqao^3hW!jl~TLi-zUV zF|i94FH%SbYgW}tw4GS|Q^V-eYPv?~=L`OOa-Oyun^fVTaK>MV6V82@^(g>16}Uz) tT28#-$GN(c2@%-WBxj~bd8fw-zq0;s%h@C;e&JuQM{1eBOjFh${TG&{mS_L~ diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_some_empty_maps.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_some_empty_maps.dwrf index 95d99cb91cbb1622d91e82f13ccceab9b7cb6c0b..d859b5bb2b0d4a123d8d983cc1dfe8e191c64ca1 100755 GIT binary patch literal 12218 zcmeHNeQ;FO6~AxyC3(ASmdC!3yvA(Ye6XxB#wE}+O#@v%8Us_oGAbb2h5`ys(eRnpf`wb1O&eZhIo3e_WE#li*4c62(M$Qw~IEOapf}+6B^Ovl0ShF#-D}J0b1E z%mJC;q0S8+J{sY23BOh&H1H5%z%rCW2ssF%AP*D(qA9}Rlnp|_m?WH!0J;&-5CI&L zAOz)L;sH5qbAyLt$l*RULU#f<2%B)AI@m?K{Pz+-Fde?KanU*q1?-cCMeCpkAcxo8 z;DPCY<2#ceLQEY*QIs#-2MDIar?%v19T3ol0H(u1y$*U0$l(Pyc%VAm3p(I<%P??o z!9)kDq(gMb4Q0_BKC?NZIUryg0vLxwk|jD9dJo9qFK+O_IN*4j5Q78Fg;R9OYi0t1 z<)X_5A+(wt3TTudgjSP!0CK2vg9n-mtOp$5Y=;Oj>%k?uJz#0UwT(n4hbT0HBki%9tcwo7} z@p>mXzs8{sLC%@Fup#4KjiU}gpK-9Du5J-b| z5Jm$5|F|Ir4O)Oc(I;2`OD8#Pr7)-#ARrLK;b(2cN^m)W0TI6;-p|Z1Io6SX8 zsLvxq0Zo!XNtgnV!zMR)VD*U(_T&B-^X$iqyxf=;&Ed2SnieRbh8POiB|*;uAxr_t z;UzbCU>wjv7_~OrF=#qGqDO9dPq&Rngisfv2_m9|IYKW5Rt35U;zZ4glhVi*v*m3c z>S%mn1G%-TFhT5=KyR%o3<#w0vKxG`sz5-atkM{SXXFfa3U%QrG*KWTkx3*W=paHO z)_8=1*|^D&>;A1<9Q44Y$zX!e4qQ}90|IA34_r(J1RTk~UN$nt zO!>?^hHSv8PNQLh*e$`SPRoV?p=1MmP#SprM8J-bK*Q`5Om~}7u~c5$iE6Ok{)G)( zXgd}n)^s6)^>&ShnGZ&T?O5od=4MrmK3Wv;$WL4&;q3_=_X#{b)imQHeM=)8h2X^o zPxhqQuQzR7RDS_yDz@iEX9N2SU>Q#dNj0IIp)6?~) z6R9Z}dU!YkdcOC7>FDOO3xV%ggK1`F3{&`TfpFhr)EK6vrHW~pN_djd64!NxyIPOFeY3&VVo_|% zS=>(vsT{<)dcYQ^90|N&k};LAcnmw8elt?vw1Kx8e$O5~y+<>`aSK`Ep8Y_I+6xwCkEO?o_L#+M0E74wAASXJ9N=Lr)O<7rIo4p<6TkTF!W zNO}`Ut@KQ1m^AKMU3F|R7YXg7c}`5^NDky#XI-VN1XvP-xFiC!N*;jX)Fe}^1xU#d z>&mNxGr{O9}o``HD&!gsjUIYi7@95gLO)ZiFK8- z`eEvW>NWuNt!J*p_IgdM7hR?4f)WjSG~$h8UddppDb{|!iFM^gC^ML`$Ol3B0Z5s` z{PbN0=7ohO){CyGMT)aO&lArG$XWf@{hvO@Vk?z4Zk$QJ7hR$oPym)mNVo)&N7m*y ztjHDXk`fc^s`F5%YSOO3Qn8ZU=Cg}iI}(hAnw_m!|B}_(w}ez(gc1kANDw9;8X&+O zR|)aVnQNH1C@K?=m``+qd1V-PT`V)K*DH|l2zTQelVy|W!5%^j1{e5M{7EGr8nP|% zygvFQj2?ukQ})RH8vMk}jq8jMCHizWcQCE`E`WPK%8o7I0(il^k>bKvJS=a@TCv-e zwK9|0Un8XY3d~!bJIc@tWCE$GhbrblsXCuenZTXQLjp%?FJOEOjwHTQ*jUMpu&s*= z<`{R^ygbFl-ONQYN4EFk){5-_cIMC(wLe%%CirMsFxlwC;V|_9W*{)(p%6k>=w8o5 zAt^kD_$Vk)d`yFGOIv$lhl3<;o}5>cbU*PsKJYun{*d{h-{G$A_jaC~vVQr(mHpGV zbQ~Bz>(S7e1Mkjw>&=t^Ou2yU8g=m(5Aa0^7sniK+W8>&~t_|h6F5%I(`z=R*`;}n5efNNOV}9X_^Mt3eJk?35 z)}FgYO%>m_q>TEuuXy?i-!n_(B{>yyPVf__uJ*JBeTBl-zWwV+Qh5nE-a3E7tv@O4 zdF`7g7PsHLc5uql^|?L6505WxPpf@At^J0l24Ah5Q9pQFW$5L>lK$j*r&c8YZh~$8 z-+pxL+P&?++4A_a``&T%yw&~0j>q5Hw?40b!?HhYZJ)ehaP+cJdH=@;$6NPt!Oh#3 z*017ky=k?xRw_Fv%0#GrIy~<|_o}1k3RcfNJ(que$4Viu!NXn3DF`;5%<0Q3Ti1Qx z!K_=REvlWB@y3|UKc;_R-8-#xM}|U0UcZ_G@Chw`iH0|1@k=!AA`aDvU!wWd z_Z+B3{1Oe+pf7Jxjrb*6v>NeCv}iTrmuS&y#4pjJ)reoBMXM3NL{rXP@k_Mms)%2r z(ZhKB5)F!iUc}*n>t#1w#QpDGqQzgX{eQe%vs#E?6^|69XTi5UIq<1W&vvVCb(r(l zyp&(u@Zz^mZSL58@)8&59-b`;4wr@F)Ay&0Dh2#ZQ?vvkV4YQ3H=IT(*4i!w|xk}P0Y5`e(vFO6lt zR)3sZzJE+fn()a{-US~;fJ^g>-OnNL@EfjjQX-oV4En(Y9I98x-)+Rtc~-Y2mDh2B hn(5^bWM%@{lhfgy%?oQR@bQdbQ1n=RR)2?A{12!U4lMux literal 19620 zcmeHP3wTV|7TzgVnjrwn&y&vM5I$v(q7!w)uNOrp?n_kYJHd9jz_OI zX$9qKE2OBVx8%|W5usAlqwy?m5Up1{GD4U{JnmZioH=uL&KdWs@4H5C$7jz>&iq;b z+H0-7_d08zoj-T)*oP4RAX5OB)-8ruZAAe||K-PDi{A_K8~WEE;y+VuuGH7WaS?ok zaOAWOD0Aupqs5;i$Bq7k1P^)$Vu#s>kVc)f5h|W*$P>v&3NtPv&ju$?u@R&M4+JT8 zC*n9oTf_zL1%Tc9XR|XsiW1DG&#=$B4MPl8Kb9W(Zr# zqaj1ZL#4#}J*w-D}TGIl#^1Su}NX&|<< zqJ-VM8usx5LI@C65MTo7q>%!y;5X)NXC~Hp1~;jNMKdL5geMLB#Ie zjNLYxK@-=!!^$Xtc{J2DkD75Ft-<;6enNU<=kRF=)2KMT4Kld}IA=F4fT}Yz@C9BF zF3n*Ek7zV`@E{_GBIIe-Wi$;LDjpgHaeSO`^)Ta{Lq>41E2uRPd1eR`*JQvVR7L|` zYH%FS2^G=Kxd4qOu01JWH1M7XxP$=O8w5C)9y2Z))g-7$sAu%NUMLDsdZy55qJ5DJ zuLk6rIocI1fXZSt%;NcYVa^#Q%uX6%(sG3vBBm8HVU}%%iO&^gtPJhW`1Ah4zBpxP z&KhA&0mn$XssWqK0m{xW0BsTjZxC|oDoIWlVS#j!r~&KEu*O1HBrO<#iiCPDkPj3# zbOI8(4xnqr^+xtR0+t~_;d^npk$lfcP`QN$avi>oP;O&pGja6G*^LImHTpGVX!BTj zi;%yC;c*6;lxJxmoKdEUP#KNJa!tObFs-}NGsQ+s24wfE0nS`N@(cse?o1dTCM?^< zbmpWH283($Yap`CFs59ipG9c%gz}-nxv#77;Fyt}qk(XZ8R{Gsp)wkcG@j>$>u)L} zOF%q`H{NKPe|C@0B_u~jp1mbi|C`7Zb zO}t6S9qOdO0V5_fvhURZXFenOo&l&xsI6-AwS}zlN?RQX8sTjiS} z4WUC{EkWDy34DT(d4*YC#2ip|yavLR15(Gc2)L0*fM$YL8ybPeLkoje6Iv)V3$#FJ zb)Xrb`9sq~i-Z;qEn);A@BjM$ zO$hm|{9?qTs`|*k_!~0LBdh8)&O=7c<2O$DqXzjS2f!mU{vM3;_^tZ*8Gk2!w8C1s z@!5OHtAjQY^2%*UCM}$f*kMhcpI1`g|8t)qh7K}hDIC7D6%e@=CVPUeI%h zAyLK@fEc@#gWMNH-!_nB^9Ok)LkjF)9@aZStAY?zY2^WAxAMe5X5^LVQV05$EpmZY zQ;=Ox001~vQYV>iKd9s&#AY!NyUkK+Ao=D=c_n3_cuzxh6mK9lV6;(G!kO~of<0e8 zt!GgcdSZk7X**A@8HnY>_Gii+fI6&Ck{OQ@l6(yqC218Pl%JMYGPK~7Z%+dYD<#lH z8zi8_?l6$$BFqk)EUX;+2u)AyWdJijf2KUyx9qES`j=(WQ-q}6q{1ZqZWxGl5daC- z`;~bN?rl&Gbh5w;vAeXhf7!B{yc=LMfDs!4ZyHE3kMbJJMkJTZ$cG>55yP*5J&FoAR`pL->a=GZ&eWVcS0rj9^%M~ zybu8rZcv9ZeHYm-=Ry4{i$M%Xcq8{MkU~;{gF3WjfvIIk3#8B+VlRizqoql5c1C?2 zEAfsMfV9K#Mjk*SDoK2yn#6KcOw!RxeJ9zb9SgMG1xuru4Pfwl!0ha_T1U_j`(Xgb zP#qUgBdT;vy^b29dgzy15ci_Bv$N{!y2<8&P?&Mo8TRl_?m^f}+}Rsl$W=jz1VNPB zQb{dp(D{(*c_k?@c3bLMnfK|?rDL-SUm;xw3k?8dM;R&0q&C-8;u|#gGNCUp9iT<4 zsNXeYuBLiEprMF;zeD{5dP?l=WOfmGPa4cP3h(0Zk&!p`eSO54kK>vIkz-GFh)nZM zC&BU8^L>K%jSLN4KDt+$xdT^H_|vY|pSM2IwWRCq+dDtq(9Yiec_5A^U&nC)Pjla& z*EQ-t$v60)`}X_zb&AZIy5@9dRL0(-#F&?o4DDwgPW+&Ei`M4dKlP1WX1$Qwrhecv zV_PnGCS+dB&f(bwgWK)r8y${c*ZJb)`H}gtX|d7z?$22>49%m7OqseJABFctYV;)%`rQbyCa(qHX3cj>V7S69vr8Fr+{ z$~h~BCLO%E+R@TJzhbRp?5CgX&Rl9cR*^G2zh2(DVGqK~N*d2>{d(EK6B8CJ*_Qlr zKhv3mAKmS?C~@(*@n?>HRQS=I((!LZZ2Rl+^kHAU(e?9$j<*i1)7My5zW%`r0j=L1 z{>|)}r*e;v9GUgE%Eg1O`6Ono{vyxO5N-`|8}5)XhFE88(j^h~mFKIM0QyN4;0*@& zr3!r#Q6b)RRx^D8QSJ_bb+dB3G1f#N3zqM)YPe(R3kWQW6>@;M?v4d(K#Tm{v7ihr zMW#LNP6i^dB$AI+RktiyZI~7GFcz#OtgGox2C`rg`FOL3A+UaTDcBvN56Hlh_>5YB zxa!UxxCeUR4}j3LI_iNt0LvpE#)5kuc_w!(=sDa0UDp5*n(=`wSTxHK9>#)uW94Rd zEEr?Br??{&5So2K1nzF8hqyzarnq0Y#sUbpn&QsOr3gTv_uIp0sP8I>)#Cap>S1of z12+r5Y2;1~R21v-n|1CGXfP}S?QSlqp&D}772LvGj@vSByMl#T8y36WYKq%)Ys1~? zftupR+PNm~5Ez75r(F9KATS7_VuTebtSVuJ2^K3_UBYSaI!m1ZmyRgy)%QEqU8fz~?;Pnmq z+FS7%9x9NpE58*n%5RmwrTkWutduW)7BR|KwS4szC8PZLIUR9kil~>Mcc`_t~1}o zcg)tLcW3tIfoq`MeF!lzd-MQ5&04GRR^rd}8rn|2!xL|MY0ms)Tx+W%t{Y3~=90&E z%$cQkdPDrbzjjT77u1s~4mdVFH&(Ugi2D`->M8H9uqXgG!1PAg2Fl{k9=KNFw{fxQ$^Pu^)zr8BVm;F)W<35Dq$s#gYCyaAMj=HWy@mC>Vm z^4^r<<`?|zQD0X|{GcA)ox~e4h&*}7Jl>l$End2{0_w?oQl2jM5OQ5H<A2n<+~bmClM#fA6;*ldX=W8V2|k48OCByZn4&C-9;$1?*4AA7}E zR6EjedH)xOQoE0{&Yv3FrAwU`iz7(o(%FSG)(84c-EKAKoY-9_V`Z%!xKGx4gthlu zJ7R0zuui{H9+?r6rSE6Bp!;ypzAm-^OaJ&~k=)q$zsIJ$kh1XE$o}JA&QA0jy6mDY zY)`6gO4a#IJufG0FKWEF>bo(cdXxtIu(hdU+PL5*j*jR0*^0L9dar?FWJ1Z(R{cL2 z{>NEWOE&#j$I)xt`)wVskL~|lX|wO|rROZ3F|j1IVZG9f9rp%0YM#xTT)JfYdx5r` zHQRG)rIJ~p`Q|9X^Dw(L+CoOZy{Sv+=H?5UKC|~+$Gt6X7G^IVnwdYO_-g;Q*Q;Lp z%hXlJ4?Oj5%l&+aqLL4j&c6Bkz~I&w#>HY9@G2um zQs)y}>V&_j!1?eQBSIR%TW*N3w4PGWlZ$Q0V7gOhQyxTJHZzw#TW0Sj2Sz0)JT)AWz3XvU0e_c zuQ6~wVSjkip~1|LX@leHatHSqOzy`wvx*JJu e$h8KA5m%b^GH}!Na2mm8<1IdsJ~3CC@&5s<%x?Dp diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_some_null_maps.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_some_null_maps.dwrf index 2f97c1d4ddde59fd3f1e53dd28da9184a8f1dbfe..1b063b5874653f0820271b84c958cd400f1a9816 100755 GIT binary patch literal 12452 zcmeHNdr(x@89(>#!o4hyJzO4pOB6Q<>xu+df=Y?Qtd=NNNHUm-w$T*RI*pwejjv2( zGP7FK;wzG9qD^&%HfXJ((J)4dqEcfc;)~K`h8lyK`k+R^2N5*U^!x7Jy?gK7vpe~> zGwYxZdw%El{m$?E&g0&*PgefqRYGiuQoLCEgok)*>yyfU$gdr706uzwDqDlt zKBVIMAd2ZUUGDR708x_R1d6}{5grk6f{id*fn|w7y6s_PfEA)BQ(5j_gEAB^P{vsy zLGdCL*9aBSq#zVC5OKl|%P1^VsUnL5Wi&W|LviC-kbyyJk|D;maBgJK47p)bBsUlc zlN%<4(G8S=fP+#P8Olfo(Lr*3y}^wyghag?h`9D1A~ZLS+hg=@m^e_z`wrmH+(1BI zN*EbRZDiAId7#7K#=ix-vU&J3&p;fr3w{|a4wUg%2XH9c2ToO_*pLA-aH=v()J0wL zXI%yvrV?c!Oi3{zj7dQm2srgn7#T_dx~W^PxoWWdvH-gjH2Mn=5uFM`E85Mz)ghdMlQ{j*W8KZ1_NPo!-NQR0|AFKBe{`B^W^4bJV3Mp zyduCp5p)9)8&6kg?#USC9F9yQoh6xes z1_Jgx3^7Rc17UK*ga~y50XwoIxlu?9<@#L)H!c}x3L@5gXmH~ncG$RrZkRYw#z)#r z!3m3ii#cIrC`->6I!4~z9LWs^!sLbtVRQqxHUxCLKn81vRcfP%7RjAQ4Q^Z%VB-qb z1|nKp4Q?E=8HyXlv{=6OuSjk%5GFTFh)_2W&^g>M zgZVtdQ(iucC)99fzfeFxg#UrX2rz2E!oc=@xITtG`&p?~ZgS`~To7Q>0-O9HLIHob z!=45CU;&ZL$6sYhLR2^b;>RK4a@i$AxXctG7xQcc)C-ng2jO{b~u1T zk%3i#6h0g%t2h;kESe>6K5CF*+FKcjKtlY7029KP5tM;|CfP6JTX_B*q@6+~JcY)J zgc6wq5`qL@2{2M<3fF$v4&wwTSnY$Mf1?BXgAd8#3>|$~8%k)2T>o{X$-zLFY%n20 zZ9qV4UZiYf(oFf{&Pdr{AWSk$2%`)<^ z79v)hM})SwAGX8cGPq)yI8a8T130u53jwtw{BGP0|y!oe#rXLu0(F4zo{_(Bq255!#U-)&+BzkN<`7h^Bfsp^k4?>@}qLZ z3PNa65l5Uix%4vsr=#OPC#3c}1+ME3Aq<^|DV_83IFhC7T&Q$5hyC8ik&@8JY*Xg8$h*m2P^MahPkY!U{lRPCi zTwA6D3SUit&88+GKrk8!cHJVz8AER0hYvVXRP-Rh5+G0=uv*m-7zu8Y5QYanAo#JK zKpg@h>JTcmA>4-v0HwzmLpF?IAY=@&vFaEC45Qt(oAVX&egCwyTba_Nim<+0u>FGu% z%hges2%M(_qtI(mm%)q-bulvepcIiVAxsUb4+RCxQD7D72zVxgI*Ku4jHB@9!bHGD zU??tTG8hDaASQ!43S510MK}3i90M0gLrICM;c0DQjNwSjH>Q&G-c#lo6y@m9qNG$6 z5um7u2Z~L;z@!e?8LUcuKGku!Tuci8vJhYpBT?k$vMEe85s4x{pGl!DPktS23biP* zvW#~6y$CLGDhHcFt?kTAwYHzpGLp!V);q>E!t1^3U->xlXlP1OQ`M2ou&B%Jj@X<; zvgDn|^L#=%N%+JAKl67uKlQxZdAPZ0Z1PXHA0OVhzk2U44jmjjZOOc!ADr`-BU5c& zGMDSy?1LBls*O5WAru6(`j%J|yRwElgs?ELb;BQJL@dZ~?nv1D!I{Wli({;KD|2i1=( znfLkGDaWdZog48{^?O&J+uAd_ynjN+f!DjcCiUjXJ&WtAExxz6&uUm^eQfyhq}miX zl~I{YtaEi(u6X-Tyy5VCYMqxjq`GbE%x$msepW1|mu=k8Q`K1Xn{s-r@4r&|dfV;OPk#E? zl(&S`+E%ZtDnptueuBHSZG!k}R{h?C7cJ4d!025dUaFYyM5A|s5#7=ZJ1Z?*jF1>~Tcz0)w>?y$jUWM)WRF zxlJ^l!=rbB(YrvnHj3T_YWIo8`4PPf4804CzJU8bdjZGsM6}R#MHx=`uYIVzI@$T5 z#l5_b^Yow0E2_EIxMyAO?R8b&)yY{YVuHkT)(q>S*$WqC#NfXLS+Qcg#PgOcUex~P zI&AiN3x1jZ%7QtIW-rKpY1Z7?lC^YNpFIYHd-b)hqs0ZQ|CyE%Acp+}t z7lrWaA3x#XoB6%>v&#kwiXGxVk6EQSxOpy-Z*{Ll;E`=o87Y?84PB$43lREm!tbre rudUX!3uX13xAuuL2vQkC+J`P1Hr*=zmWOYf_#hYFsrAt^jDa<9M^DLM7$vP>6nZs7=BY9X(LC=4@rGt7A?=c+ z5h6W5qMisgf_wdY)KhHJb5W%LJJA$2xZT4 zmB!f~3B-O4xW!=)jgZz8$wqREZcuszev?25{6-^0dU%^Vvz@u^IdW&eeW#K+`wbCG zXCcDwx4jxaXTQ-hM0$9?bz!@3yBDJrS9u_T5ZFT_M0)rrw_)3Gg@x!h`)E(70>GXO z$OGPDL|j=zWy+TtAFDkQGDLcKDN$?`cXb8jw~H#dv)>R={Wj&dJsP#M-)I>kJ-pw# zvE8^^h3@<&fe`qOMu_zA_C&MMTwxJq&jtGWLByiXlsy%iX3qA|GDLcKdxF_uZsQ7f zc`bnu*h3?v_4HtSaK+o*`Aq^L&_g3cdU(IZva#H;y_Db1t6(B^%sYtKw~zAMZjHaQ z-)I>XFpq{f=20EaqbkOi_2rhfw_QG=PSdDMfXx~yfXEm=@i~^`E-j)io*ESl)p7A4 zqJ&4-r`bvkn9EOY5;8=3_#}v9?sMNu<(hjXZ2j^=hSTWjVh2g#hRuPhyxm5Ygi;?Nb70Iw&WU2lpev;fCR!8 z80$!Agve>UUjx}dE;q$FGB!2V$cO203|`%m=HFEJHN@DF%k%YF*M=}nE%^4=6@~D z|Go@i2}f*!34v@rr-CekOUBjrWg5ssFh$~fk-PZVYgsL~eY{PD#2rQqu-RdaJq3_< zS8ujASNMawnO_27Q)7)ijSv~b$3BP+;xKl9_mtkyf}m-k5oj#5FlZsr0-@=l`9o_7O$kj6O%4su?FlUonlCgCS}e34(4wIQ zL+b`D3R)LvZJ>3A76~l^S~xVrON6}k>wh~C;^7kwe*lf1_v$@s^n5t(d8nR0(eus; z_xROu&l>*-&bjB5yJxR@UgIAmj!D-0`!;nIA+I;Ggkh9&IlK-0WHK7{tc(ndxidy? zLzVETCJF+l$Vh)g4Y12P39>8#90D8V#Kgvla4T*(>qGcmlZ-r$xIuP#7YNC_p=TY! zQ22E@DP_Bhf!AT&6!M0gr0O0K18)k9Wv|JJvE@TzU<10sS)^G`3fLGi&`hD=tOhVz z_pk_9aNEhpyx=<~ql_eqfO$XD@nA_ksMd8t;9$FZek5f59Y@eCBfSkSl96~s0;kMB z#8s=WLtsosy`F`~E2UB{gFu`1Daf3s3`tSF&lpw>v_VS{T4YNDo9#{3r06XnRUnp^ z8V*vgI!WEeOTBAPFRR>JC{{-LqSO@oSZ_nnO)x%6uzj${K&${un`JV(!#df|%Xf#R zyai}?Sch9J!*+|MO(X8G7$|2)xWiIFIFZMOK`0u!(F?akGH2j(febainjKm^5 zQ(?K{S~9}}NROxEBl21el}GuP!*9{s72RLiH9sl*@`UW{pNHI-abV(+86Q9P>SM=G zyj-U+lD?!X@tNK_cXnRaor)tB7t8wYs~Kc!JtAjh!ga%Nzh^J5o7I1SZ}Y;QgU19L zrcBAn4#=5$xJdqP`PrxN)TQg(JKCXD7Pu0*CspoY4=jR`tld4=7qsq?Fr?r-^ zBmv}R`o~cV@)NS&(SBG|HcxhQ^>fP&aR(HM`;G4u^K_x9Yd7o zaeeBW%tOuuyj7rEz4hGM<$oKwCAxgU+SYwvSzMGper}6_-8XKmOsUTr7n+(o>ce4? z%Y5Twr}r3tcvs`Nt$xQ&WcItVl`a3e@Yo|Ad)z&j^3nNp&9$u;=7)sB+b6Cb9`S?s z=F1 zD=io~T&gW<3kb>lKrP7bm2mE~;38gw&XpF-Fr!~$F$ za_Lee&GvP6H4YppGSwzzx*AX{_yBY_k7&4g~S^&ajis2gJ`3_bYkI9IIOXCT( z2lH_OXgs9zBtA-kIzxI6X*Q(TkfKAH4QV-~^N?;sDi7&4q~nlkL)r~#y$EPX#UVXs zfQFPEQg=x2Aq|Lh9#Vlw10wy06d%%mNc%C+kOK6DMG?5;!sjTb&u93*VTXt0aq+j7 zBNhqH-&!O)%eVY(!8m_9pDUK;EYpItkL@P-$NCDz3g0sBTivAk!)Go8KF+JLIGOT+@R~0A4@M z)_8LISE7C@AfWwz_ZuAUms|Y*-~xd9MHHh4e9&n10dKztr+XpYXTtm!a&?~UgMJYk zE%!hGY#bcgFXSIR*$4fig%r=d1>D8>i`Ex}>kap%Z=p>2{1?!)wzURN%nGPqw5W7% z4>rGc+T_2Tp(2l_=zgun1r>eo74i04<5(EbPb_^E%| zO8!SP0Rd$C@_rriWJ`&r=hf0H#;-F?s+}z&0;B3ywjB^QPkW#!uj82IgXT{>s4x;E zY;!w4JavzLTy_`J)XmT2EKbeI$*4$d^Y79s(+4y1H{?!uGmn{}{l|Ru7XO-{yiNTt zr=L}Jo73+DW2Pde?eX-pjAHZg!RRsch&OQ+D$w zmlyWc$3F4*VXf2FM7e6^+OgC8TQ;wp^2N~|VOj5|)-L(7S8+|~ zvYAiS23+2`qDJ>-Q@hD`_2ySSyR` zAHJf?TQNjc9f`KS^4ss}Y9HQj-ijfr>PX}_et$e&?ITq|D1i`c8Kx1^dgy8&DLoPh zfgTzm(!=KgUG2l`@m35us*XfX{FRqw-lOi(f+2H4_aEgPT!(n_d$A6|*KVh9(6v6fc)f}yK@q%If@7Ywa;PAmUe>Q#^9NdoC70X^`xRdSm|58N27HZfi9cya3S^m{`AMLM^8Bewk z^Jo0EGFh_jy6=|DV3#&_7pBFvVh-NPz@uKijbrXfco`2Jz8mT7Zdep>5D;8I&Z@ z(M`m3Gy{g3FgnKi0b_h92_0pfGTLryx{9HWr6NjF*w#|AAa$&iHKV9yDJeVeUB}qh zzPVS8t!a`Y8{)jb&->}zcg}h5`R^rF^EM%5Oqa%qEsO2Q_v#tZANH~<^5w0qJ*gnx zyG`(;C&J%;eNRgG+jssn;w9diCK*Nb?cFQ=XsUpWHe)Ip`H`~4c7rX|C^#g+Vd=|& zy|f!3WPx!U+MfClLhS~WJpoj0w^Tifs%=EIJF6U_-V8L&LyKjDST9a2L})l0&HkE{ zB{qb1cu9az4WnSmk|49OR2378h!q=46)~~wl3-_JsVXKG5i2&9Dq>rb-62KS zGon*F{gH`Er>Dl7Jksljij-cTBi4t7>(WBF4`GJU`5eIK;NZhLHXj*}!iQxXd@}IK z9DG>E<|E@#_^^zF&k1}^4nC}7^O5l=d|1Z8=K?+#2Ork4`N((_J}l$la|557gAeQ2 zd}KTdAC`^A7p%(?;4862?pHle>w)^c{Z?IKU1Xg{)GsAoi0?YoZxz&UgId2hT(?rc zmtxm%FrEkaJRE#j$L1sBQTVWogU<_mUJgF2WAl;mD12DP!B+}=r5t=%$L1sBQTVWo z%_qbBknL=~ppMBG6)!elP{zRrja+tc@L?UBkBmp*!!iy&XymfY!H0EhJ~AGK56d|C zppnZ?4nC}7^O5l=d|1Z82aQ~IaqwXsn~#h~;lnZxK4|2!n}ZMQ*nDI>3Llnn@IfP& zIp&9qb!8@M>ENDP%VJ3skx8<&bn4;!>&;g+aNRc1s6!pO#?dxknL*urJG}dFg%NJW^`uG$ zHTdaji2ZhQx2%$zKjPWrS z2^GOz)MUJakT>8nX;7e3k5cB#3+?yn3dDk+d-j)2hr5R=$a>=zSGhFA*}u#L>UjziS2_A^>0T%ye z1r^Ezp>)i|wQYcaGhF%<4W4F)cg#49^39=9{!0Pb;Y1&GqFJqj51>JPs`<-Jfip#d9zKGRtFN};Ag zd~rCEi2T6`z*o&*@Z2E0+{T(exU3F@*0)1|%j$rVzyK`puJ8pu#I?Zqnp-(}iRU$U z6d+`9On@?!X(9oY5*ZVq9hO)Dd`gSYqN|L*Pe(Z|-lF4feOFi4&gM6#J@V0A?|)Ax zedCqGzq8M*k<8L7jkTX289uyle&K(f-#GNon?lXfSSA}5qjUV4~+b7F6A8hsxY^$8G{-n8b z*1V?K`@TD{eaWnUpUlckb7tPucbfYhnprzhv)N_YG<4;Q6F+c2)Bcm?`A5i5^tYex)qT+?-Tg3VXWgP2En(k24j?G8LqwrxF2Onv=qw!%Kn~#h~;lnZxKGJkY<4cz6d$(ejrx@x{zv zl_2dr6)Z{@Xv*GGMcw)J}QonqcqyEYJJ-p&h+j zUTgn&jc>qa%M#NhjiAfZHPqHOMtvVSi=mqOw=`#^7gp{w)hbFT0Lp`S&d- zvxrUa>upA(0QW#kE$~NB9_dZ>qq5s5Tl(jhgzhuu`^ c(f+9yi1wOvAYNu%J7mH2*1rvdT4*&oF literal 21702 zcmeHP3wTUd9zSz;F_?KIu999_;lQk8xrt(;c5QkNU#AFpAtH@&|LL62Zk}|7QWoeK#Eyfd&)N7}vDkDOk zmETma%1BJsbjX^{Vgq6^wep7;`idcCZ_RVJ{rptQT&FdSREy5hP-FiMYJ0jKZ|i z1g$jjtRzAlD`iM5D@}UA#IX{Ku&wk&Vew_^g-i|~7Gd*wqASWM2?kMgl(lK3e!sX_7hWiRuUnOl`e3SiZIB1q;WwScL6cPZVVbBq27d9Es)IbiI(y@hui%`_>bMWu+Mosm#35o+QM!QjWy3GD9z9 zaIC~4Y%4ucf&lnVg${!I&~zy~lU?Ex_#QPw2HGU-^_d!g44BekN`omCCJRhvm_(RN zFlEAIgxZ_YLCi#l40LHiXX)sW1`j5DnTjqg=zuTH=uAWhRMvzK2*88@OsK30mqo=) zNX~@Hn#^+nc;(0a8w|jY2YLLy0l9z7321U}%k+s)d;;Q=7JL6_@-s6h;Mn{5+3`v3 z_ypwsY4-PzIX>};uRt;YNZQf++c6(SEQe>M1`YV98k}1i9#zr$rafLI%hqwSRi34@ z+~BL>z^fD(w1OM`hTWmMCtt(NH)w<=e5LYMHoi^;tSo-09uHr|Di7&7<#n9$&HSfi zUn!?NG?1kqDAj3L`4AqLZ$oPT-ui*fgI%694_bKvShdXZQtP9aXY&iNfwAym_4D#| z?DD0|d@ZMaX?VCzd2m8e3(5XN{^4$O0K4^0Mtg`_8E9isbfZ{~ej~ zozJFl+sO(KCX$a<4m;N2ZGmG-BoJ!ohC1F}Y$6$EN z(r4ySQ@`8}#}+^NUsc1&=$+BOi``g}mDkFyYh&2=%qR7`T1CwMc*U!!NkNJE6_Hh8 z{qoAT^)}cWUwgz76J$$zFMWJj-HB1)bPTXhyo+og$Z$F{M?XqIUne(+e% zt;AcgcSFAWs38BtwVy4yHn?VONkY9k=Jw0qt-p2q_O0u;S3gko^apE(*$X>`4$oiH zamm5At{AHBtu$8(?b=h~y3TlSk9{==)V^2+t-ZHxU%F<0AoUzVoyDmX^B3@NE z_t~uJ2NPty&aU&fwrgE~bI*lCr$0G0ddC0OT==kf@7A8-N!#bvJk+S}0_P^e z^`Y7$A#i@-aDffnb7KgU=q8)JXs?}$CXuBO%3(Dudr7bXhNu*WaEm?$QR8K(Q4rNN z-*HKRm{XaMQUuK#NIY}k1*8;-@&*!z&0UZTYjEJ%3ppG_b;&UtDMd%UQ4vQhU69Hu z79oe@nBN8D@FJ$eO=Q}bLyaMEsLLBjGTrNjWEoDP>26VmK{y2E4Lh0k^#&5p1$hHW z=5*bV$_5qTB^YGhv;jRm7uZjaNajm8sN|W1sBof~BT5t?GX-8L07c=np;xG6wSrfu zWO~*sROkV?hJ%Aj%pyx$IHJT^WTw_D1;~OEuTaSxo*Na0^hgZU>!_qciKb|%(pgi4 zv?(O2B-fPNOcyD}$%}*H@F$K>;s7U#ZJOKxP7=Mu;ZGF#B+*cJ&{M<#PS40EiJGGD zCk~>La4HV0Vxr7o044(fk1sSkHowC@r{xkf?{k~8f3NdC0vwp3!I>YElR=L3L*G{Z zOinsEAo{_NH}nDLrv&w*kz@U+xwKL~z7Ok{<{RQa^l|CO*vR&hg+VLu7Kn}9aP_c3HpylF8Pn*U7FW@d>{Ic zrU3CD3jX*pIqBrYe>Cs5(0wPhdUY=U`LXxYX^{UY3Sj;GTf3mt$xs0G&wgxrxaB$N z3P|+E6a}#V{5p7sBN(V3bp(U_M|(N9)O{y|NYand0%$G6 zclVKgGzEx$rHnVsPoD?%qmm%~sG~22-+fue$pAz8QNKRv{U-g3^#_!3G8Dl2(Z@H` z7Z2@)^h54D8HFVMTIfxo{Gvem(GQgbSTlxNTl{*$~zR_4PwL{v-tEVCsCTuyn%eeSIo$p3I+aY&F^2YJ?g}+Z&Uwq@- zo!kX0*KJssJaS%2m?PtpN4`4MY5jtf;#GNH^zGcn-dMZsZsS@sO*&%Kyo4?JyCFv2 z%fZDr6UNTYkKHuBXXLf3In@hNw)e{WqM`h3znq-Gaov*ae+r#*<#N=CiJx8`I(_%h zW&P&$?yJdJw{PXXDO)#Ao>boz|>aonmq%Xi*b zoZSxyKr`U5up_lqDX2cZ`L#D*`Sfn_wnV$RQZ*guK$}4|zj>^yWA?m`YAXO`^ZQTw zHY*J5dojIn?#$;Ry1o$)LVB+*8mlS4(Y@~|IC~k`xqI(^VKPU;*`Gp$+{@OL*TGA> zV#utITUL3OG7;ji%8;0>WCn}FN`yG9G9+bI<^8tJYz^$(b#GC3?R<1^Q72Ql?rLCq zShn9**~3K0v-0m@l_9ZM<@;?FSpz$F-M=cJya$-+VcC9LWe*b}cD;%+B-U=3z|LK1 ze{x{wF8;9$k<9wLdm>ib@!WH(xa&Kxb646SAK1Cu{P$a2JN`ZQ+$y%9z|LK%1qF8Q z;uhroSs(d+TgCTfVCOFUjU(67_DkHk8(8KYSmqsA<{en(4Hyfp1IxUPtep>mW!_|! z^)F+YcOc;Z^9}fisDM_j+cUkR0seQsON^tFp;4Xm-etjf_alAA>V`RWdoI3rKHKW( z1SQHJ(47yB`8vgLTAc*kjPve8(Gt_3M5UH||9 diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_string_with_null.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_string_with_null.dwrf index 6cc0fbc39dda19654d98e04fd3c9311ee5c2af9b..f76861ec436f762c65c53de89e6bf93987e399e5 100755 GIT binary patch literal 29266 zcmeHQ3vg7`89sL($!0^yasjgj&=@0x5MXoj4v0HIad9A0UOom&P#8;jWOziz3gm)- zG^LnqBGWQ}S`c|eUdn)vPJ~wEDULu)p(FwoEWv_M)Pk`6?_>9#-MhQzK(hCiVVMNj zx!?Zxzu$lU|3BwHXa9Y2_<)xXvL>+cmdauWa-R5`#f81R=>Jl=`eht|FD_^#D919Ky&<@w7eUbyxcdbKc}zftQ8cJfe?|9zqRF7> z6hTqx_Td$k7GAb%63T>cKG@E6z=@eCvL`4-S$@C(U%Jexb zlS5~E5lLtI0HOYj`k7ur(lDCKGP!h&Do7%u3Y5gCh@@gPk7e@c7!{Fpj0Ok|&gf*p znM7p0!0617UN4RNe3r?lV^l#B8C9TR7!{pKM2gN7T2U#I6tGMI9it+Wj?n<2VHg#W zRE!p~Od%bk3X;gE0wpmjBE5``gUpm3n3<9+z-YYHkmX?E*S@tA43p(;A>(A=xv3#R z2>D4u$XBrI1#Go|?ctZ5YC8$pDU4t|KP~Jog#G(0lQF@yMiATxnK*>TFjxgGvOwc3 zd##|w>ne>QMM5@B67p^UwkEX3YOpLA*LVSFe7{d)SqXy1GRGWf%u#YIVJSIQ3&u6} z=UBp0a-0nsXH#;FVaXh0U?RsdmX6~b&^U*ZV+l*iv089wjwLK5$GM2y7z4{xgw|NXk~EfMYyoIoU{}VbAIHal<6kKuR>CT=J2=N71>+j?5Z&w$ z-KL5hOM>4QI6h6}xDYfhq~sXGk~zk}M2;ma9mgExJkCMKv5ckTIEdho9LrcbjyXs- zoSgGWo`++J9OGaT$1;|ZV@Nj~N6E2-rQ}#GI5fu+mXc#gH=LaFh#X^BGRGL0$gzZ_ zk&rQ{gW4VOpBv4o}MSS>gh z$70UoWWiK9kAqOiQO08&OXgTD7}wY@OUr^u982k+g9OP@W@#Kt)>th#w8j#aq_LDt zIY^KkWtNt)l%ql|7}waFrKM3JVd*r^hMbp8ne${UoyI`~ht$|Bm}(qnL(a>l%y~GL zI4W>3DOSi>AIDdrM%>wIgvu=xLMw||iv`(Odu5#&ipZbrGQgU+i`9g7Bw52<{;pTZ z%bnoeZx6CUJ@m^sHdYjVU^K|y*VTnO!d(rf7>l({TN~>@D1Aron%OD1R zQ1!sZ1m>&(@s~VJo;pwYEzeEQ4bP7r&0q3*-Pn6yx3&ZFjf1{`;RQN^9Kigp_LN+dkL8C@p+A0`b+ecOpE?%d> z=C!%U-K6QDkBrf66h_ZgjIQk?s{;zJPk`~+yvePp&2P2b=(h}NPq!Juc>2?lr{^(f zgxAy6&ZnvG@j9dw#o}Oax3&bd+~`5VyVJsiF}fVy3l&e-cAWdsEwat)GLyjcaBDlG ziHy<1db(`%e8uS6KC(K9@OsQ1F`L)9wVlPb+~`5VyVF7u-V2oQu5EBryEU)B_6IXyc|3h&>* zl1&|O)rN`W;+=2Oc~(&sNxxQe$%|HJ%F&(wK%FIEq3*PSB||UNH^&JJ_3Ug0ZMeeR zCsS%<5tgulHQCO6`y8vDtbiMu(_gY==$&~IBi@UKF8{q3Omve^m zb!p+21#|cj$Bmg{gx5K*M(c97yYB`Iw*afdTrRBDrNUem7cI>BX0dl;VJ^_>On9ry zz78w5=*Ekj&+o>K7q!(>!bN!5*RhRty(d%AdhG2+cqqaLSRE#FVXZFvI;`Bhz8>iU zBx#KhEy&)k)zx|31czuSd>vM9zC$!Rmt=K6#pBkNfL1BP2l_hVZ4_3=wpq1(X7$=w z_d2{=UAQ5|r+mf-5gm}%VYyCYb$NeXsLknYbk99mhi`SO zr_fkksLgpRvK0Op$&AnIy>xVIOF$$dJS^AYAbjXOH~FX})aHBUVyQ z2F~EXPU}41PV2i2a=<5EP#x;K{_-!;WpiA+pFN)DEzBde%^z-V{_rFC-8}gM{CCFK zF<+I9{c6Q!Tm4f_vEI+aNSm+)FunI~$LTyCZrzg~f@3-brZv22$K!kw(**lF8a$#Ry6bJ|L^v=_|mw2nRJ#*VSN3wWd&e*}g=OlR{M(~g{ zs_>+-mMUpTXETM}eD5heZ3$7YPjs*UR%u%cjnI&QDAA%J!NGFEJ0$M;?Y}B*<76ug z37A<1MhRhRd51$nnp!#_aR@VsZ;jI7K(5WGM6~ydqRh%qnIn(>wN2uX!CjN8hxBi@ z8*eN=S@Xg6z28@#x>)wa{&`o@dVll#^QBGYmyA78ChCbsyz+hg;40_7R~{cWXQip( z{nH&g?|l2kUVGlFTQq(BuO8c{_|lxSea_GO`hkx}Oc|7t`1tb8_Vm)>yE=b5dei9b6V@EtKXu%x zp>xZ=TV9d1bHU+)$~XVJXss*dpSvGU@7KjvbZS7;v}cB1s3@@=JbPySkea;$66uAU^e5w5?kvYFAlf`8Yna zfqDL?p)2$EuPrKduPtg!8(h}hCO6|qmyC=peMXKvd35>@Cs*v7HY=~8_t%&-1%nv!;IM0aN3@-f|rI0RY#3TDO1v zfbpxF;SJZOuQ-3b;PA}3E6;t?DQt-!ugFIK?^ZU^2%Tn8K8~@5!DNmxFp*;k zOUilC?!w6Xoh%{S7Avz&_^5>Z9RShp!v5c7fj0~$j$<558pr=n-G!sO8u#R`Mxp^( zjJDHTndFDy-=?1SCFLnU8q*j2hjA=Do?P(inV(-fUGwT^8yM%3^3?X0Hmrd$C7WhV zoH;AmEd0%-##rK6gTa_;u*Ay0#*8FuV(EQ2I2Fz@-ov?iWBU3G7c%`Si0sR9J#X zBqT{@;Rnn@G+-e*q#UwPaw`kzfbcm!a2*6LA+X??B$)+P3Ne6%7&%hl0VSlsLyAj* zWg!-@5Q|wrlzEDv1&ZJSk;cf1;2|MNvJ_aUSpW+bIf}qQIf`&2NoIkSLJz<~4>?lc z0VSlsLyAj*wX!(CLR<*fIH4l`3e1re7Y1^6>p>BoNlHo~9r!4Vv)lF2FhWpP-9WAVP6z!CJptvIVNO485 zRt6MdHp%If7|7YJ2XpjH@+U9`MR*t^+QKoSg&%Fw)25JP0l@2)H3yErtJ?^1D{MW4 zb#osdZpA@zk}5C*6_{o81q%--p#mNfsz9p!W<70|(S8ml-hK`z-hQ?hfYH$sEvFY? zpqyUdMv~lqU@t)b7CmhVsUqk^e;!hf z{^Rv@ybS$wFjD(*cj9nR|777~bd;Wsl2JJa6R(`Z$ud4LgRsQPF+L2GV|;ET$?eD1 z60N7BWf&iV$uK@4IZ5LKa|er@>4=2~lrTOXQjYP(=;;_4`sZNc`sZ-6^be*u7CCbv z3kJ&3zZ*#@6_|muHnaX8NtQPq6F#8ufXGpJ82|97?oLw51Y`AdtPB%GFc~H&Bq!B= z3z)cBFGEb`bRJs`WKRu(ti{%>nJ(%A3jiA|9m9uDgV|`cGpE$jcccC_OGs+uIsKVjsB-y zt%~dFbto)=t5C|SiI;7q>S}h>iCZCQ*Fq^bzV!-QrMj9O3U}Z4k;hQkm{nJ^Ltz0R zTUNmp3yj&W-jIO#a46iHkzW=xS%ntc6>^8Nt;eD8u#aq6#pwl7uwAw|6q1HRVbelV zR%7~06)$CAM%~fdhQ@NFPwbkrUSO9qRQ@NFPH6Mv!r*bGP05tm5sN71sOW{^Z zJ=!`H7JygE@PR~WDI<@e&gum>{@K;;Ob&%R)3<2!Q@NFPwbkrUSO9qRQ@NFPDZi<( z+jKb;769_gYCn3}T?)5SYB9*6u$m`8kHL9Jg?kR6qCdwIw%3*ww&iJP^iz0AO3JM^ z`87E-`YF7+D*4FK`i{< zvl5MdDvQ}JH9RW3$gcRB9P$`S`(45`xqbrKvdSDR6k zi-tFi@R!@49E6yGARNxqFI4vSZax$h2d*CAiUn5;xT3)o1+I8-S-@ommkDqmg>Gi> z%MuT7ao}b~fA)ZX#==_+yhXzs%7_BL%&?fn0&nn_8J0DpvapyL7Bj6OEsvZ#jJWWy;m+)p-~Lc=ZUk06JJ z^YBNY&~RNeoC1(T97Tn{`@qfgJps$WFS|iY9rvM575fLyZ+&@vXN_C7P9j?)EZwu7 z_-ZgQvQq{vT~GbT^@lZX`8o|{70ag#jA(g4o~z#jeab3Po{=oC5#tY#m#AMSQJyD1 zPyE{@$^!w4@kiNpTCsdU9+hu{wSS9#C}1Aq<*|8?%F{prqUBlZBbOJip9TgZMjx1$ zRK8BEJR@PB;N_KQd98H5L9BjY-lF-y{G{@&0{TWR?);krdg-VkReuS0v`M91H_FP& z{DN9i6`xj*9+$oS!;!^5EVh*f?dy_aEKJrXNBnYTa9=8svQmQ{%~`pq=xF49S=IKO zoX^V-bsy6b)!9Jp?<|ny&mbgtiQ4Nlj9#> zx%SPVYyLTNmJ~fPF*1vu`QU>;RxVsyxN_N)$nCxFp8vwkniGX{uG&`YUDJBc#n)bc zqetSid&|n--dy|T>a~TZAI%*5b%M@yN^|40n(Uenl828d3Ez?Gn^oHUbROVpPU;4~ zM~_ecE@S7-q@c9!hvOiPk`vnX!`hZ?K0LN&yrF3D*;9EBAF*ln9}Rv#cuboiLpp}< zPF=(lpEQrH9qH5j;OPBp+Lz1;47%?wL-2r5&Cs;;ZZT1<4h;-4(xY-JHF*=yQ#F5^ zT9TCgUUfiz?7HGAt8Oe)q*)X8SOL|>8q#d3wr{o1v3nn?UDfS!+s>ssn(sL6zjLB* zn)&Mm+70=O&tAD_{^yC&HTfBi)_t@4^m}M>8+o=EO$T%0tZd-1&fCGE5yclBxG_&JCPW!4>y$p5)2 zYr+$ojLC(Lgf)v(ZU)`m5ga4qulHjPwE#z%txZp&Zb=vp5Zyf-G}DfG4xUBEe!*Vu z1C9_mNheVf@VXDa>)?X{^)L_A0|E8AQ*5eeKsz^yB4e{579G)YoW)A8QFrt~9UBsG z_T%gNMv^%}T;sD>LmpjpW`s1|aZqyTaD!uz?qo>?vWI5Wi$X$j zW^i%{I3cYbF401C%mC82G$&?!MwkrJsFCOke(6SLqXYKvDPfzTd7J`9Ex|Q997ZPB z7WU8;nU-h9M}XBgk}U3Vh;e+L)CNfXglKO>?PO2)s&A(hb@sFlY3kTD05~`WN;dEK z*f|-zW9s-oAQ{wgQ}HQcvYW~tu_(8x_{=dG%wRR)(-UPh74weIev-{QkPJK+v5xmn z6m?0si>4MP318Qogh|3JPY5O4nBhRQ1TcgWzF$g*DB%h^A}`#8g2@YD6^Oh5hEVl8 z9MJ0tlqB4djK~Y$U10J87(xkmZhIh)P{IaHKzw-6x3*6Go5JYLF&tN3`0AFbWjFW{rqez#S83tZ2S zLhJr;c^|zlu*5f~sAnk(+{dDO7VkGsA?0G^t-0dqIG0}wroc~^lTwd=kfzWhK77nq z>vEDSrXYV{0YlZxDZ&8c4?F>@dO1ZH!1{#`nd)6Yzog2MepNI*WO+b-B-=;)!6hg) zFDJQjEI*QT8!6pMR4-E}Ke&pc+6CoDs2uU9vU({enE_UQaFJEL3&@XDIV(ScbV*Xb z!|6bNBnA+FfO)t|s@mlgsT}!d)i}e9Imx2I`UNne1vEmQlUzCOAF9p{bXq7s!W0GZ zr?Pq}Cz%0Oej*cyMwri4w)qkFu?Lhoe|=bf-u~ zEPo3A(p>J^2DAUG31ZL0OXJ08%aL%j;A|C7-0D$`bSVy@jS>M zsgh83H^D0bU3FBu;77t1(W$KyYAb%Qwv60d@%$GvW*l1ozkjTJ%GPCgQbgf|QD?e! zOYU-M&)_~(Z>kq{VE62}b+3mdkD9YIvTST*&4~>eAD1l(DrLUwIH=!+yQY`U9eytL z#;6X~B`-$WFYayC+4$Y532~#(k9uq7J#)56Om-j^<*u4<4({Yb>{cA-fo#HFryJws3u!Yj$T#j`dqv)+l}CrQ-uR!D zIhhAG^lKGb8oI*veoAC*=kPIAe)~qwy1q2+a^Q}GkN+~@^Wa|w^r@VYdLj6H`1c)p zhUJY0EA zS6a13ie^>~o7P*SEit?}^za9cZJI8rp9L@V9dWVCnqdpBMI_vAl(XB#)843PqVd4k zI0@{_Kk3+TJfKAA1s)O}QZK0p{LUYRr$dx^(I31~(O{51p%m~w1}>T(IR*v~2?a}5 zgy#@u{5XMt1#eU|s0dC9-l%A3Ll-Z6ns`VGZ&Wm4kTr65#IBhi>`c@z@_`a6z*}tU zUF}a2QQ?GP%!pCpIGA`;7!D^VDvZS~gc;Etu|OG7VcbcQ+wUoinOpn4QPJw6{c+LG z#P!eNp#I6i$03P?Va(iwp9VtGxx>UO7m_1)J~kvD-gn1ES`* zGf8s$y;0HNHdA33Got=En7IBqoGj<#7Q&3|e7sT7@P=R+QL(&H(fC2uq+F1D2s5Gy z)@_oPc0ukWrCg9OjG0>(GKCb6>c;u=cf~)=jzXJ23}V(7H1% zLynmY#oBQ2DeT(xxQcbFF0I*pW5*`iddy_>XZ-av-M-n>jOV6g8hx3TdX2`%s9~DJ zt9!6t>zqkX-=8-rD>rk}{o^yTGxgg3Lv0U%3_UMnYUZTLSvliDY!kD7o{Wtg0>b^L8s+iZuAT;O;k++I<(`0s`eI*u8c|MHm+%^d2P|DKuSn5n6+%nXUl8gpHd z6p>K^p7E3v9VCW@N=n8;2T85Srwu6<78x0@p(MMR8kObS-`Q|spY!eMw(70bTOG^H z?{~iQyL+E~&)px~`!Q+i?N4f&>w5Qfdd9?9Ew1ux@AyFeda?sQxbSjNye6f43wpLs zSN_#VuYZ%L@8ov3%c=2SZkJvZ%1_&TFF!|TSFpCO!#oK76@pEz!K}{X9xqi()OZHMw((W!7Wi}9@Qy0|;(N$rxV`DXZiF6(BhD@@ zQNq?4Y=T_@X6S%%9hwJ^lKb4;C(_D-c^WDVqhYN1UR(&Q2^b=j^CTf4pQ}aq5g%cc zF~SHKp{~2Bn_XH>T8-Vs5F=oS0-iIVp#;dsh-XSpIBOwQIT>Zlw@2XwSZhGUY08wG z!0NlGoV{o-*7Tmj3Ght=B2H7Ly9OPxdKu=)zP|NDo+<1M#TfR%>;8n15t`z@9U z&a162Dv;=fmj-xIX@5^JG8fW??Cnn#PJjo`E1afG$qB4%7UDEsS7y?g>}ZyBS{`qC zT> zTY$w|03vg1%9NbIauQpdU-G?44`XlA6}BHPJ%!K|!t!79TYK^~e=^+N6}B#|xbStq z7jQvOj`3`GIyL`}ufZh#rAl$`>L*n0G(yt|+k8>s7C8S)h1-%Xxdlyqy2X9jJnjra zGYHGrs`CSk*_OvGaNah+#(c};wq*AZ&>cl3dW{wMuy|p}B-zy{wQ6ob#1JZpoJ9f|mSQDc?cp4#F;OhcB%o(=KrKUyQH3 zLxmwvyCqwa3)+~elot|ONLb?ugaQ*VH7bQV)mimOTnMZR7$TJOBq5+Y zxC^4?Y&Z>P#~0#4U=_d+p`0fP0o8`J2-|OUjBQ5*jbK&Av&{cW)VA#aSZY8+>EB*q znUWI{Gu$d?6pdm>D`3(QC%_^DB2H7L~;F*Jr9+@x><++;w+Y08wGz%rv%&Vh6w zD?6-k0!%j`;xuJSPGCp-shlI|2v)LLQRV?`{x5)t)08PWffWu`IY-mctoSvB6X1nc z6;4y81397OyLAr{f5G6%9NbIe7{#Yr_<>y_lm*^@U_bdrzul%0^4yn;xx*id2}Az zdqS$@c)X>Y1(^M&R0A4i9+nACe>n?@jWf6tEoUF353<6q6;6N|-zc1>Ovwo>Z#LpI z?nATaES9rFI&ItQ3c!Qgr3chj&YCi%xdm1;RW-Lu=n}Sdr=rXQm|mbL^Guo2IDyqZ zfQ-|)DoHenWxu6x0!%%raGEkDC$Nl%RL-a9Q!MQ}g%e=P_X?*eQ*r{UTGHaQl$KsX zy@YvS@4;+mbE0 z1uc70@=nXB2Wvwj>v{ew|XjmC&t(y_uq@uLN#-Mp0i`vL(5oIUAMo zd_waHJ6fWU3*7jkLT<^Hvh1`-Y$pvlBSIQ3(dXTW(dWBrz+7A_S zOSU8zw4|t2ZYgV>)X55=iZJe8a#;&3ADD6DlgnB;&;LRR-!uhO4I4wW%5+f|%fAyB z0^1A>5z2Xz5KtoTGWqFU|LSPxd5TTqW2?13?y>H&~7~7Aov@6Tq4l|CFm;l!p z&``={ttnG-0&DK9a`vD-Sk4KB6X4b301>AtQ*r_;>85f<(`c6cwZaMTVza_&%9NbI z8hfdnL+B8e`kkWO16XqvATmxNrJ5^q&G$J6obs#j5Z0o-puWSpi<$qDS@Xq9sUoxqxl6;6P~21J~u zOvwo>f1JuWl}=@sY86g^I}C_8O_`Dt*xPWAMxQG)=nUpdQIuW)a}0<$O_`DtSn4Fi zX_P_p>3o(_CslN9Wi7x)1Ns+rTUl$$1gF2O1y((iJI(ia!?}nqV&AM)ltBO+QWa&8 zDN}L+OPzx_jn|dgbT(@$QOqsC4F*K!)|4sDEwJ)ws<};|3Cy=oF;0N>rHXNyGNo|> zt6YGL)0o?3bQ$~j1BDY{UA@9-%9NbInir{@E9eUT4i$fq2QSNQYY1R%y1&Hpzh)bc z)|4qZft4h*I4z}RGNH+YoqXAEmCIUzYs&my`CC|)Y^l5xv=O#8uzoG>R6z%_gk$ zfI=>C)j@^ak}b&v%~+$9ZzgmzVMYH?$OS&ysE}K-CApwg>y`36Lh}gA%Yu!vagUIf zo4}RXfQ?(Fyxc6=l3Y;lvr2gZp#_9(I;4;be6Ug>w`5ClLCZEN<;8>+6PA8nAs4u; zNg=mnOL9SdTa@y>gzhCQ`8h>(Cva(wqPnwWOL9RAjdB8A?PY|P5!QTIAs4vhs6uYZ zmgIug7br(wL1+bG7cMB|0v8+ac10GPC0mjUnp4~=7dN;d_+VTY{^30Sc36$St2M)T zdsbzc!*~OLQ+p}W_0RBUgOs>C|J2>i@h<+B;gcPB!2l;eHl06Hs$$}`7)|T{(oNsZ zS?ao5)2OBqZSj|+s~imp!J=VdSk&#tqKskDUcKaKg7Fw)V~O@~yT>4+efwh30Ryn; zuwhtq)F>=Eb}SZ+kH?}DCt}fQ)3E5AIaqYzLM%FSCKg@17>gz*V$tQxvFOT`aHz92ULnE-X5EG8Uaa9gEJJ zheaQJ5R1;5g+-Sv!J2?d14jYTvSjc!1r z*=Te#8qGta1!%MwjqXLGWoWdbB|5%k|1c-~gr<$~#!-jnJrt~k+&Oqc>}Y3Sy>H&| zZ6j|qAEgI&6bay5n%Cp}VWZ}yfgR-t;9Pu=fgGg=b`%NVTpm53qny}L@%}4xTGQgZ z{8W5awO}n+zC`B3_?;+#!}#Tn3*az*y#qRo1bUb!0I8ENjld40fgDD(K+fjiD=mPt z@%aewFg<{?@o5X}Fi(Jo@#l(lg>>Nsio+SAyTZaLe~Y518*=+Ru4%EMMC11ZqGgRc zdp%j{UvRsK7R#TW+97RJp>K-j3+~YA>a}a<_mJNdpD*8izWf0Hz5C%$`Tr)~H|w3p z?|WynkG_5H2cf1j(4T$=$B%vtUjHawBK*c?48zNZ@of8Hq9jnl%8#KOM=>*C%>{xW!PfrmQ-*NZUi0!XGlyk?b4vFm5KFi9= zH#OnTXPVZ9?;Se3Xz_l>*RJ8+W3D>SG&w))^ZLR*q1R%EeH+_6{>3xnU%Bb*!pM!+ zQctXOZezo?%=mugz9;HC9yoL4>vv96Ts-~A{*e_AZ}}|h(EluaxuM^CP4idmCytI^ zj9FGOY|BmGFL>n2f-iP0+P5q1{wZ7To3dijyNef{n{;yFB=`Mk!~U4oJf*Iw|HVF~ zXF8p`{qGBJ?|5zLuy2<(PuzEA;*m#lPAp-g_bnc_&-dK?eUZ=fUR6`Lbm^+#)hWG} z_RAw$CoQZ)La#U zr?)$z;YiTUPbYW^I))#;^vbH7(Ie=o(9n6`)W=Qe-6Qpr)i?}~-g9mElEx37T$wra zvyU$Cy3lRA@5x6a@BZ7`ZSi}Wr$ji%WQ6vw+}u$-xH;utyPnw7 z>t@Fo?aO~Jag6aeS5ep4=+tD-El;M!c=TkCK6dVrZRbAu&#cb<`u$$JFedT6E$vSL zXAP^ottxZl+!Y=9ok`O?{2#r3L5+{Bx%k~}QnwGQX}SGIEnc`|8_QzqOm?FRRrR|5A|M>_~K1 zvYQ?0cC#b9*^y|{WolUjZ;fjE^qJl4h`%+%Zgv!^+s%%kXPv2K5jJK{5B zH#-W|?Pf>q8!e^T&5qDFux>JXA=%B2>}E&)u422*t&*hokP2!yJ3?CN%1w*pPleje zj@p_e`9DbmpL?~N9l@5))Urq}e*oIej^s8;{xt-jM75h8!5T8PERxrd-Ry|ZklpMk zRJWTQ!E?pbvPgce*v*dk4B5?&TKnjFMI$A<*^#Mtai-nuh<}z1`B3OqqZg9h?8s76 z+s%&rpX!uqYP;Ey^do8HrbTicZ8tlT+9dhQASk8nW=F8KGqo&|w{~{3BR)fRv!hVm zZgvFE6;sP1`MF{@JK{5BH#-W|?Pf>tTrssQg6B%xcirt~N2Z?@VmCW#tqrA}zTNDo zwfD?r^N$3wv@4Q3EeUKlJF*njcC#bAsQx82JF-6tYkw5>SNKubPaK*aM19Yas2lj7 z*$M1ich6VB{gW?|*v52Ie?C=6;?8xC?yQHo9V9f$nY7@sq^Mx{@8IYV z{W`b95ftsvgQ9!$1A?5vAyFaA<}QAG!Obmy4sQAL7ULi|u;urFNLR;46CW7#MB+n9 z3law{nfu5Bw{zURQ^#}QZU>evNPO&}N0+$CEwTKOjSSb}@o91V5yoFx-}d+V%0I?u zrIC@5`kK06{^RvD{!?JK=D!Gs=J>kCYdnzy{YDGDHxS2mPz7vH$YTsma;-^ck4vo(zaTyci g`Jl|)=HsECL=4;PFv;J$+35=3JLL`8#&N{}ET zA`;}H5<`$Hh+O0;Mp0Cv7*WxPK{3ijLU2oNo$eW+r@HXwLPDOm{c_saf1kfjpRVbu zp6UP8c$eWHFpN=iRWoJCIBUjU9oO0EHUIp*$A5zPRYnR1|4|t!6AW=HRa0;yCPqz| zO-*OQ&V#V?k#^@!69>_eZybU6bTv5;Cq|Ql>LbNDsNP*WiNA+Hd~9P}g~mUHdPh)i zf_h(tdPmUs1ab}<;GWp{m+=jVpx!-ECisuK@)0~g=xC%5(}UJ{dZ1D&r=e+F_Rc1F zP7n{X5dhnC!r(zI_;3e+v0BiwzqF@_nRi}{7IWvr3~D>K>8N!EFm)Gz)y?wE5}qXU zTr`)v`cM>B4Ksl0rJ@t6VHSIm%s$A6%gUm}3}EspO3Y$UlGzvea;c9g%m5}mp)iX* zNoIHC&J|@)vH@V`NlG@vo+R^Hw3a&@0rSP`%U_dc026luC~*bFo@uz(bryPETj*Fu z%0LzcvRv%xYHolrcU$O`S~&hpH9NrgDpI^fk@LOU6}fUbKPqYmSJSHjthf(g^=Z}g zYOyC7`DPYvX1R!RO5_3@`;-#7t~bda!lDqC3yD%{!xLvlfTQ;U%;#O48C`EOGvm-W z?tB_0GXVa2oRS%_CmDGdi^5oL!?%>k1vv5?C30PFl0T9~ku3M}X9~XnhyOz1*Yzed zGTB zr7$eJspv07g7RRi!Y z4@v|q_9U4(#BpUuD9ivB@Sp^<*ppG^|M>i-j19*W4 zC78vY`6wxgGD=7?rtI_asjS5 zM2TG2o8%8?Q8>#L6j1mDxa>NGU)P(=%yP7x3%x|i41htGDVY&_l95NTD2nAS1X9L# z0e-ZeGQR72laYHOPcG^JC2|0R<0z4fJ;}&pSrp51*-4bh1-RrZO60oUB!2>n5?Jnd zA%$On@7<*E>w1&?Ni0fYxuhV<{6T<=Hc{pey56G9^Up!k(R406M)des^Be?V`aaQv zW6g7r*prMrg+(bW7ne+lT!0I|p+v6hP4cI+D4pdZizxg8TyTfNuj?(!41b@QkLGjn z&qOC*^K1)X$a7KJ_{>OnlFT#FOzvndg&Dxm3lwItr-+%qnv>CFF0zvHgaY8{pDE97 zVo!2Xp@;Oi{1)25Upxx}>@r~da3!7viTp@fB~o&E1EpyJ%K=P+R^&(KVFH@KMSn}l z1AuvFDR~fkk^#3sEx7Y@r2_^m12BnzMSf(!tx;<(X9s0E0bnu@^29Lns`(g-JxS&c zr~{XsL16|kfd?g+#hxUyGjir)qbNxR@cTWKB#S*sW;0~QWj>_D3}7@5O2jPoB$>OR zZd`g8WkLvG7!OJ?i#0KE540412ko+NWG)Qd~aqQnefFb_&Fi#Xt{6lMUczMwFRJ;?zj znnlqp7rm1*lnZcXC}k+u^(OfvSQNo>2|rS%e*&CcLYe;QdXxNdEQ({fu#=Sd1vovE z62GoD$)CugM3xIKrSJ=I>LUujt~bda%A!z~OHQNYM}TwEDf!X$Ci#!D=qSqt?4nE< z1UNaIGGWm5CizoYl*)2dSrmQ&PRge6>w1&?TUoS~9SGNo$0RxY$wBw17nbCP6FmBWV#rT=WWQTEId8lb{v(k?CEE zmU3lZQ_>4yQ8FdHVown>Ki_VP+H#TmC=(R`H}Igu04nw*nLD9QTzD>p8Niq40hC}C zdy>rN$eas(Mqvi9yo$ms_9U5mpdMVv0m^g&z>-)1C1MtPlFT#E4DL~q=%J|QDHXu2 zJSf2|_9U6DkQEnvi4rq_xAG`4i#|4Fe&_MG}VO3VOe=2K!8dy>qK$dP-g zraXoKn8Je+eM9UiV&=0k8_nhd3n|a(0G99||1GTehA;LMG4t6NgT`=GiIkWDOyog{ zn8ltXvkP+JDhepf0LJj31hd$aWS)wqa%F*(&me%|JSf2|_OxY~9^bt4%00XJ=lK8B zW(wweedf+~W0T;P6$iWO>A>hUu%6g^U^V$3hOt(wHSVnh_A7e2xTbX~js$)hZm+@l z=*tfh{PxOP@k!!rP5cr#W2RuMvm=Qs>%`5Jui>|^+n>QFP-pu%0d?Z)*W!x0{PF46 zogdtN>hfoC0$+nG>&ETt#Px83>hi}psvB3=iR05ZtIJ=()a{SEr~PaE>*UAXw@!cD zz3TR7b^P&*Wv19TY!=v5*o?7RVl%+TV6(>d9=5^QhGBEUHV&IBwzb&YvH4>2!8RA$ zVrV8-r~&Hb-ps*cM_Nh|LC@6}B1JdSElh)(Kl%YzW&@Z1}Y*MuBZ5wl#R-69BWH zAlPKUCKWbYVRIBVp|DAWO&n|@U=uA*m2{^{4bo*B&L`e ze-UJ{QDwf`+t$NuRKQfB-!`5BEI(?x|n))B7@UQPzyO9k;cus4C(%CUh!Ym=rv*e-9^{lSxZP zd&O9mX7ycf`lTJ;_LAM+riLRvs|elXY*{)iX^e?}`h9~3V_oNb7~XN?DzkRIodY_Z zcF(k!cBe9;Yv|kF$}6@lTaWT`PB^MJJ|ufAvh%U9@+oQA)w|_zFJsj;uEg+*)OGpu zwx_z4eAeC5`_-P*W4C&H->lf;ROndJbj)KassB>eo@5URa!H{r2J354!g7?&{RiZc{V6=VwM0p1Cw= z?A?J|GTcb?L^hb6Dyo|q&mm20m}H96AdkU*;Ft`oBQw3b23%A zo7pnk#;Tc$b*w=mxnG|?z5Lb;H1eBj+QRPrx0;`Lr{%_| zz1ue2+O=``#fu|{KRI`5-;Y1;-}g)LoxG<{FXg=`Ut03)+5D29^7s9A`$)enE1$3L za^j;cS3da7G_F54vC#ZL-nxtLoqY75>c~D-w0;ji^~!a|<9t?ij+!(!>cd@KN=pw6 zzS%KtQ262-c5i?9^74hQhvO602ZSH$x{4VYzqA_ON zYbz4c&UU%xwW`$ZnpyC zM9<5EKfiBaV78`pT+kzBw2S@V(M*s1;a!}+h|g$aqt{_+pM4B-s+*T-U`6k|dHwQ^ zjl7*PpGRU-*{JSbP0J@%ZV5=W+S7gQ zsYR8a4Bu4pQ?kBCQFFzS;t>%!dq3S7?~y#?hX0d88J9XXVR+=JjO_j^> zfn`QcjJ)^X&%-jDvwK9&r1whaSNhoBv-gkUt$z&Cku}_2JKTRe!Q(iiYNK3^%e@G7 z5OU(-#ne39<@@6Us+IrCu@E1$>-?MPk|;>{Q{l}uNfhMi@rZ_k#Js4D27MQ|%L#$C z!hzLP6_6wbrFVEoDdJ@~WKftVnql-jJ|W_ag~}2k`z6m9+@T7`fg%U!b zFo7hwP;jGQ50xA$IAmGV$f0sX#k(|ep;}0EIF%eqa>NaykwdAJ0(%+?C1y4|(wY_u zp_K4aIpT^yTA+4M`e+(CRJwXRfkrOWhe_I^Jul*t2?c7ZHc}4|YVTwQXb%uwHR#1q zZb=Q*8HnHkh0;{1(>3TI+!NuQQ@&2ua#d5MP&g>)6Ph>F(<<|zp-`MEejW{l0#ikc zX)Oz-n({uTc|)nEa32~9iaj-`5QuYz3kIsY9p)H?ek{rtprCKd?k>q#wko^3D9AT>s4Tab9%ZJcVco9C?5Hk4F$Q$aaS4&GMB2ZQYiFukk^!$VpQ@&1hSr__B0gaL{%m<6yykb z(`YD2H-g-0C`h0(=g?44US)#xUovJ;r&TSap&*${TB<>b-Y@9!NB84v+P^Lk_T zcC9;cfFQ|D4bvV#@PkS(PoKswC~nJkYuyQcLDCw}OVlJI6f~(taax8!EW=AGKvvOM zR!nc0;p4-S4iJ?zmKD<*7UDyCICqU_IDfRp`S`G+1B(_K&x+}d3-N&_p%JZ^-nb?{ zta-q?h{iG;AFXjdKCFlVRZOE|@mJj3zP8t3DqqBO4t-d{`)K**mxZmxB>+wTqbHb__=en% z`yVv`m_Ik_;DPVWIDgd93*w{PyJ`3I2H#&?M*t8X?dJ1m?JusQAx-{hq1yTN=Qe>5_PufcD)b=+^I+A5aXsoBTJM<1VQ<937YFQ!Zm zpb<@c)bS4=@`qy(AFUDmb0bt>&{ki7$_r3|!5!N)Y*3*ADm6gG2B_Qs6&#?FgEclw zY*6vR6dQw$!v@tK46qsFSLXxa{dvAL17Ds5d~(fv_#aHXB=8Y@qf4)d#3QKm`Jd5m1DHY6R3Fpd10! z2&hHuah`a&2e0&~5qU&cD?NDi2QLQUwIRGpB;e0TW!;G;lT|yFul@hInzut8xv`y@ zlwK>lL~mCz*8Tcz8#i~Z!P;e}cL!Y`J8k^RomY3>jQwfTvL`pb^kh7lg^cT*rq^yA z?(-g#GNr`7)5RX4L$xn$9$WIn<66kfSXZq`?LNE_ZOLMlRkbo$O0)93I$tWW$k-yq(a}L zlT+VOu5H}PrWryz!&?M*STHkw;a663-Wq;%z(+|-y61QcFNDJONY61(BfI;XsYab~ zn6zQVlA*?9z8-KV&o0Fx$D?(2^xV+Xt6D0$F2JvnP7BMuKJ=GqK7)t58O3#poTqk= zz1;er)|IDY+i#iqkyFH1W|a=H?KgS6Z=Y*hGNVgweAWzKoBd_4I_LU6xjEl4?Zhmb z{fl0t%s(CW!qsNh(G%BaC7inwyme9U6$=@6?&Ye1ObwXZne)3*ruI;GR3-W?p zlDE0`b6nc8Nv?9A-SS+e-HS63|MafhdL?+=*Ip5G7l+&JjM(m0dG0{_c5{wc=fAIBkw z2aX*wWsligyE<<_^0#q2FHY*7b7s!Sd!-3i?k)SSX}S0HR?m(84eLwi#sHg1Do5wx&ozge@ z?|BQl`{bOyYI`sBVl%smkF&ZoH+N{_`~E}!&S7^S?@Rol_-nHk2K|L%d(P(v!j69HGTLp3$5s177Qekc?YB&m$aUZUedw`0D|;rai#U|Kc1^iS z(j~Xa{vPUOL$b<04l{1m1xKAazSWa;uHzrFAK;sxyks!Eee!25THN^SYc~Gsmi5j- zy-%Uu5!9RD4LAik2f7QM{|768%l)7MlGyk?vEG;Qzmwod?N^cc^0xAl!T2E(7EtEP z2#Xi~6&6sgt{X2e8O$&1t66JWUNTsyjxJvEPk6OoUNRU~7#C+okX*=129uc)q%iW5 z!T71Lx;}f&k^%CP!Sa&95{v)xnnK)5ui{+}taxG&8emG@twNL3>UY#kM>%=KfDr8(v*!?OzS)V6KeC%d@=f#1*|`mSWfg zuZ@LQ#q+J%6APKJYlpeg9~uLUZKFe$|j3N@uzIv3s>pZVjYB87x;`n4pC& q^s^5c=Y)fDF=Fmciv2L!@2WpLRiS*Thbzmg)yPc0hrTu5)#_g{Fe*|2 diff --git a/presto-orc/src/test/resources/test_flat_map/flat_map_struct_with_null.dwrf b/presto-orc/src/test/resources/test_flat_map/flat_map_struct_with_null.dwrf index 69b34be2fe3776c152f34764488b418bd1defdd8..e5e91385d174523317cb5eb5f93a5aa14948163d 100755 GIT binary patch literal 47351 zcmeHQ3wRVow(ice86uQJ3?)R25fQkESw!UHaxt zL=0mVBO*pbh!`OP#t6vE7pve#MF=7i0W|_@a8aTJa_dx2rl+b-`MSwu_tN=3jH$m* z{q@&5T~(*&asHJ4`~Nl!&pFyUtn$9VFBc7~B1aYwYbiIv6r%J(!l%~(eB3`%nh)VzzRIRe1Tn3_-%SIz%91j6K- zdzd@tHGBx@Y3lvHc??w(CYsQjKnQ`~a^Hk$()2X0nvi8sT+{TE#v_fd_aPgBDJ=Ks zMi4M@Xrg|8BM1{seQj&{Npo%wY0{h!lqW6JM4UAJ(itY-gbppZ3P1?BYFyt0&xvwC zhX$t@(E8(Gy+)dzCUj^*Ky!dE53U+HBlvgr&4(<%p`HlC@OoKhSgQ`&eGUpyX0Vs&SYaK;2@yLW+y;bMbzOR( zONo#x^b#+feLq9_e2NNH%uy&}fkBE=#(sjI`3i51T42b2$6L1xou0@(&Fzma|>YQcbd7C zECo)$=CwgM`3!X=9qIh1KngOq2|rKf94`)PG#ViGU8L1PYvh zZNCiR- z>Vz2L*i{H8*O^ArXvHy&P5{e}YII7Lf=I-C+HZ~~SyP{%oh453TD(D)Mp9Zm@hazfm4?2pLY@~tC_WYN-3 zL1)lqtUa)G0GN9Sz@W1^CuRwpTk3XeFs!+a*~x1>_vZvMfi8b228L9(Cb0IvzJh>` zP6-U^1b+gSp8;c))_6YJDP#&QUJIOvP5=vD0ua$DSqeG<%gjniL%Es%Jl0&3%r)t%)zCTjEOgF9CBTV619NU12JUR& z&SKnl-=&ja+|0kB+4qg$~SblYgzRGsdHCRu3GH8U+-Hwrtt z67D9;O3Dn`rC%fo(CB5w_=@W`};gL=N>Cy`~W{0u}Kdl z=Pp}nlBFgs`9LEag&pAv4CKt=Ho1+K=V0XAWBjdB~VIIU5xSPRMn#t&oU zT=^=KtTO5PXW{iIB-ByZQLey(avQDmW8_@RsR4O@ws9xC=VDv zhLLmS>rJxWq?=A^l%uetT!97UHd?*7j$BDBBS-|@eNNq+i)ojNv85-FSc;gioB~v? zCA5e|%e&!(D&`Q1Sdb7hS%4~GQ-D1qR3TfSW36VS8Lhmd&U0cW3)l|8xYGgSgoqsw z1~XaSWmD)<+8CKXEi0bH(~tRGhHSka@+ zL?D?3@O2IbcWTbbuCOf0Qqak<^K?3sND{612sja)2Fg>dp18bAins->Fb2LTl{c1((OAm$t-}2-_azqlBJ*% zuy3zLIJwT@WH|l)NljvjfR0WH4C;h?1+a4d090BNIOk|GnpVH5;Y2`(QvwA}z$ykY zo#_2JlnkXOPH6mzfDWew200;aId(lVx9nhKS>wn!I`13M8QeNpdmxzwu(Aq3B*~L3 zfisxQ0=D~hW``zulgK1mldnng5YW*nfkB<%Pr$0~gt1Eg;6!u+SiKEE zM5kma=mhNBQ30Ky2^NbXkL5Fo&!j%~f>B^45|cX=cFuhYEF?JrGz-Se^oDZt1Mh5; z%r>?BzHDCe}Yc;Jd?~b>8w|wbG`zLx>4BCt-ykA8_k`p)9p8j z-=rrFYjoRi{udhEinXBIMpsS?=vH#Oe3Rs}B;yfHZim8-ZUq+3`8HbMt2=gwl5>YI zG075>9xKu0b~ar2iYB*Htc46GVEiCP&Le%9NtT&(-lrPnDC{U#U_rT!7SF}Vx$>1J zS!vS4Ihx$ghD#sPhU!z>H7L*5!AIHeK@{K0hXwtj~;RQeBE=OTUxdIEyZM1Aj9eHTZ$H-%? zC=x}BAc^42VpgkSYy$~owIU|$9{?5e0~}jV6+#d3&{chKLKSleMJz}NSuH@@;EqAF zT0#ijGM1ryOA@nMzzP6EvRcs&2!mNIhu~3wyhE}nwX6h^K-VmU8Andd2=Q5RR6Yst(_|t^q-C4Am({EmztlSwiysnqC1Ku4zp3OWJ% zF;SJYX;tf$SZ?91bEnCCi{rIN1TV@jqCnXja>vw5JsdG@JnDauDH^ zECo)$N-shuGL|CfftBdbTFplPm?DfW3Y-!pZ%aPSWX{#hNS#0UezZDCp!EU%#ao;yMSD z!L;BbjZOfo$~8JAOF<`K+pp85OpGCdwE+a%d0-CZ42DrR>m z?40`)Sjg^dw45J%nBGusrq^TTm?X!fWd)k-&W0-u=(f>W@Ny}{b!cMA{k_N} zi%j~-Voh#`!j5hQ7S8!LT0N7W03~wn@B)(*m~_kQK+Z#4e67}oeQyEGgB)N^j#~>E zPQZ8$M$VNlH_3979{gIP9EBa_3M?qM(VTf0IUjqGNs3Hbx=@qb*>LXTn%qvY7L*5! z7hvRE`5Kd~G3n=9pm!wJQP@$gz=Coc&F7g01bb+}apk2ZDK%+vrAE087aY|nSF8o) z0plmM$~T#0lSz;GHCZhRJIWPUP;R4z1$E^1@)pTHHOXGU&+X6lErY$GHFq;?OHW+UN6>4Dc0Jijo50BAtgl|gWcM;E!NtxBi7oj z8`j#hr`TG=we{|88Y@Dy*=%WP*tPxpW7pn%vv{rDLq-PHnwcrK+G7|p1nV~|OT5 zy9&K_9eVA0=~{bsA|gb^cKlkF7-x2Cvu0Rpd_30Lsuk9nlq9y=W9ZUF9E083t{v9e zsT0=Py*t+0tC!en$4;Luj`j9@^?hTl1w>JARk2RyBT?q1UcNuPsKeEkUo{C|zsM zPE?dQ4|b$_JYt;Lt%P8$2?oGiB5W9Zsd9E083zCG63xii+tw6=Qkgni2MR z*FH=vzW6@FNQpHKc9Cg3+UJvV%tH}|@0;2KwVP{aoT)usd#d(iZP@?xH5PFluRpvK z&^O?YjAlCxqr1=WB}Mr*M;kHMb-6yJS5#Z8ZNXJf^|%Z?6W(#D5#Vd2h^TW+t^Yt+ zBUAAV(Rn7k<5F);U*p`WLB|~Mht)A>RL?}{?wQ71%Gmt)8f6W298>EL?*z!6NhLbZ zgm+x((&cN6%`L<+oBiS4WEIa?I?sf6TfWcuPyL0D>KRkBnuT{< z>eA&4|8Y=o%nZM}_o?wyf8is0CdtxyCcNWPW3SPV?S9cw4Lu))YJOr;r6|WUB#GfAOQb#>DO#{urSC_P5`p(bveHsr83<0@8vHXg)NIR3AG7 zFwYp#Mzs2bCcFbvqtDma2SmTR|JC2}rLGdPYuJN%oogC$7{6q#YV^%6%#Ua!*RVrC zs?IfP9EUfI&$eeYl55x@KVSwbCDE1nf&NWJQ+>omma=dQz7xDbzeIBB^hUi=q-c4UkaL=}fo3d+6LwnbR zH)hlvqS2qB`NO+RNUn)6eA>GvyaQ7ofi(IvG{1T$s{vGh*`MN?RIO{mJ1{jnZuDnp zeq;kwhq<}FPO_?NEZtrc-hrvnca}dxV`+`nH9vQ_RhOXpeSUD3KSN`u0Ih2pa$Wr; z>w>Sb6Dxm12cLKbmKHST8ukV>{K>QHn((@2dhkVo7fd6Cy=T+dZ1gOwy3weLZgOr- zZSB$3<|Rc(*RMLdegpfr>YgL)e<$2NbkBXa?^#qub`Lrb>%P~Pt@L4judU=Y^4q-j zOO~pJT&WuUEe&CQAFk}Rh+loL!J7@lrM14!oa)g|*K?ykURcnOF{wTw@cDvE@3XeQb?f5V}S#`^hyD{1)C(tBFCQKTKT^-e;DI-(Z#e4SoynsMV~Y(Vw&~Q11yf zy6W%LqWX=klfTk$nKAZ9gMN&^;e4a-$kCm~#}XqWC7IP(Vm~CZH)(1ZJ!hHyB74L% zyCm=Ki&}KJpR7OdpKq8C9D21@`}+zlh!}I_447J9_;;eDvfK^G$*pA_r-g@ zSo4P2tIa`U&i>4#?vF)%l>Hh_?b9K<&5AoKmbOhhH>3Hgd(BbhBX5cKRaPvBeQ)cX zk9PE4*R}vB`A@iVSNyZq)Wx%BKIBiDb7QAJ|4-*9wk1U8t{%Vk(`|R;?Hm@r$)8o? zuWE7E+7`1ONqpkr#G2R2kG)pW`i;!i(VLUHZ5VuJ`r5}TT5Q<+`VV_{Zu<7Su_e9E zOnl;W;(^aHHyj?d?^sq%$=-)Act0*Nvc-G8uNIH(HmW9W-?6wuBlcvDXj!|a+o?6x z17F%Z@Qu}lJ6E;ub>-M@R~}zcy7#8T*WSJMnb#j%x^rRj^r*3uqmDm4YVFf=YP&yi zy8E;hd$U%2deu9bSH<1&_|D9TY;R%u+USmHsn;}5ek3Cy&TCwg8q+EAx{BDm>ScTW z5P9&~GZmXhb>9EY_p#r7b$^o?<2vtZ{`sGd?3-cE7&mm~g~ys7*>}===HsD-7w&j! z*UPmLXQsV7>FQ!*-k{1$YhHO}z}5LzKX-3!bj>S&er|g4)rI#S_+$0NiBC^YnUQ|r zOw;O#htf}H%ouQB(_>X%eBS$X#?+ckwGUkT{h8%WKN>LS`~F`p*xh~5yx*tXa)C8y zYL~-POP{Td>+<^4(HB?WHnDud_!TFHTvGmRn?4^j>px~YdH$=`EAGx)a*OBIOTTRU z-ORU-kp2&x=>OKfi#is!f85{oy1(WvyRP5zt|_fFG(B-FnZg?70umAzLdGhL;>%|Mh3XPTsfr=A>@Ivl$Fvb<9-G#5%WyOo|YYY-h{ff6bTg@izU)VSL;;W(l3|4pits$VN;d4`8 zrDUO2rCaIwSFQqj-~2mb*k`JOa`edxli^bFMt?JCf2 zrDy(yeDDS2@HMExuS<6;J;PV&O7)uIt3#cyE(g%9^eonh7QVjMt@I2rB-LvcW5})a z%x1`~^c-usm7d{Vk?J+Wy~4hd*RAyId|kR*>6x8WATHcW&%BDXTj^Q-4(&p>(lh&_ zYvdc_*YQHoZl!0t4l>(WVlB7Qvs>xet@LbJZlz~-{B$clyOo|TUK2~I*DNOOZlz~- zueg<-V|o2(x6-rEt@Mo537Ur$f_AFqD6o*!+NfLU`8)P%RicNzU)Gxdui1F4C2pl> zJhyWzJ!@)Wxs{%U7iHW^&u*n>ODg6JTaWXISptL617*A2O3&~zg;cK@rq6kq0s)=5 zmB8TKx|N>mUgeSMHH&kLfX>`XU~q2TO3!u0f2DfOFt^Uz8Qeuymj8Odcyx ztl6#f4CRnRt9!eZp4np%yH4~^ReE+mf$e?*+x-Ogul)q}ml1{)Nxu50+3F(pd&eYN zIVGtirhWDaGv(CDVcj=8RJ8n^cgwf#FjFh1B)72Qyb)$>tEfpMCr)Y=4gc+u9Ah>0 zMnps=M_7@`7qSZ?qoQM4#Z104>%Nh{3;ga9_)6uM{N4#8I{#(D-IGR6 z=sfPuu_L`veFmlXWq>yXm^^aA#Jleu=QXcLVI%9&%!2V5sceL4>%MaS{^yE8Y4061 zd$hD>e+-{lHb^hJWZ8>wX?|5gn!ya8dOn(cI%40;scC+8;rZ}4+j|VM-lX|2Rz@2IQ2*rY$p5 jbNi>U(WXZmJ0@k{KRC)d89}0`&qtCX+efBk-%tJ*b;I)1 literal 60665 zcmeHQ3w#aN)}J|vT0y81&wP9BIp@q-GkYiZI*F4{bANYFWB#-Lv*y3| z>{)ZpS!@5j+v|M zf*h_?bi5w+m4b=hx02PZEc<5fAngcws6JJVG`*mnMmyf@CY)X#ZiL+sy<^-&pb(*B*&fM zsKEeLW(;m4qpr3x>U_yFojlm5iqce3j-BRd(*s~rKa($v4jY^>17NBMz=WM+jXbBr ztGm=)&QFC=Z5PxOLdG8L3INl;1JK?T)N_U<@|>vcWt=HWGevpbdT@=7KI(aY;u;^O zo-<*@PnC%hr6^Ihoa88@z-gyA%4n?-KlNaNC@m1>jgyUR|CzArzZOQy6Mmzyi_}G4 zG@mn{FhHt2!xDK;m02Q6OGG(lHF!=t4v!fH&R7F5W6Wr+5kFOCxhO3c<)j}t$}nM7 z87++XssFK}6f4S6cRBtmaQ1zU|5|ILJRP~gQm~wz=VrTnI+NSSJb)RO0A!4LY>7Ol z%B&Nmb)uZ|ZAD_v81qb6RYnUVeyYr7QQ9oZ;YWa<(X$Gi|8Ia9J*%}w{M7#xQA!cz z)LR_?nXu}=7Dmd`k^7qTn!I2)h|72$F+g?9V-PXZK5Zklkuw%?`V_#-g`7UcmWY}9 z+(K$0M_)6reP)2F&lvoM%PBeku%fgf#FPw51uN)EFghXuqqKTEq%G~W z04YFDO9yQkPagoY_XEhd6R{<7nCf7XO!D;c#Fh>;tb-m?bwHTjVOf%8%LC0c95PnB6AN-IP;;v7dACafx>g%LmXKTee5MEOEE zCwNfcs|9D(5v8z}$S!Yb>@zo>OHKL@9v;gqt{F2@_V8(ZYzI zDw8BiNuqo@i=&JJr=Q^{qqRoL&@tIp>MP&4!3p&lpgJZoh&-ptY!{{NqC9V!7ki#z z!m2V_Snv4|X^32~kMqjR09DU1sAnD^4UlgxM@lTVMqu14DBKHkkb!yUcdoda)|Q+jxCYHH1?`0Rh4u56Necw z8(@b4W5jI84mnH*v$y0epAP4Q6AV!8I1D0YDn(7Hrd+Ur6HWlQhk}gY3R@y(>a(BZ zCue7Id}e^E&luD*za_mT=P&2n+6+)-#vo#*YSfqN%i&Wwu`z(#r*L9pY>Aku8jYpK z@`ZR#sL23TpD~D-sm~pyj&kxH&RZCOD{gb%!my>DxtY{Vjyb_m1HjG4Ici`_#7xy_ zCAE@sA8^!QfT|i8)H4T3L2_OOTreZ>W`HU)1`#vWxvkVzj*aE$3}E~!j?UN;Jy7VQ z&>~6}QBK>#333$Jyq^=~Xsr=HQQOP7NR$?d^4TSvAcqMPU95q<7S{8x6s47-{L2*% zKND8@wJ_qR`oxP;yeKCx;Diba9JP=WDrl|s{PRR)r#)}CPR`t=sh@a|{EK13uoRG`WM}aMuIQnR<5kK`mQj{V^IqM(@!0?|5 ztNv?YJ%6ewrHbW~lDy<~L2Oz*rqUt|(UKY1(EPyeBD`bb{B_bG&W27|98H0$KK8$Kfwd9N)oUjeR7z#4NHf)KQ zsT%&0znqlI@tFatK4TCuQ=c114dm>(oNxlbC<-!s#+Hbg`rJfnBB!Kq!U+bb`iw!u zOmDqW(kMCm9_RfUz_bUP_iJp4n5i1grRH+}d5#(YhEtHC2DU`ZRE^eBYdJN769O?n zRSgUxX6kdG6eydMI3W-P6FDIewnWU-=YCQ@`Cu65&6EMEK4VbN++J!g?>fs-g8{0{ z7(~oejm}bMIVzm<771X%bk18OwnWTSjbYL-`RI=vH5j1kGY0j{-K1{vz68#z69ZJ4 zF^HI{8oi`mvL%b727vjQ95t{dVy0>glm^O&*KpKefU3_J+;0%f4+`%W-YvXUc(w57 z!X1U5J@S8=U3FsHc`cqe3+5vE z$?z?bpA65U?M00}sft;Tv^u0UB26aEL|Oo8l}Ym;O(3lWX>CY*jkI9Wx{%hLv`M6e zkT#CA(WDI~EtIt06$BwgB%95!SqGa~*er+564)$&O%!Zq!e*-YnIJgx{~FGjlT-Va z0fw@070&zMJVRON3+H`!Jaq6W1NVjVK9q&NaQ-TsACt~Al!dEszBkLlRXiSjQAVzU zL>+BO3nHx*Y0XIMNLpjk>XYV2+FPX6B+Z+&s-zi7>p_}CLjBLs{^_`7!DIRXER37OsN$Kthj!r1c`L8)?Hx>r7gE(gI2A zM_Oyrnv>Rqv{9rrAkCk&TBP}sCXqIVG*ZD^a3^g72~5HuIthnODr_QQlMI`Uu$c#& zc-X9j%_7)XoDF517ql!qHXe_wC|9zi9>C5|G-aVL%6%xQ@!M+w^e7Vc_0sEG>iK&$?oEJ7!tURgz+{zwP#K{3^eu` z>U58o*0AQqil6Np{K}M(-it?g-wVsk3(Ky)?bM#@V>9z=7wm31zExq$sh+7RM|$iZ zJ7Uwj=eL~BTX1jS;-wMmD{nk9`1D!-{NsrOeJkYl3O9IdsnSIvafduemFIr$nf-&G@hlv-cw(EXVK2mnb@gocmfvsUfJlo) zI4qh%$oI5!I;RPliv_p46W6!D*>>L7izoJY{d{Ue|Dko`q6gNk^!<*j-P4ojyq%je za@UZo8FBSaz2C0h)fas}y)xXV*{y}rnVYY^GUwLiCYA2SPOo>j*R-<}p9?zCw!ycH zC;GPgBX#K0=h_Z``d?pF{C-Q*jt4f*nUoAd0YC%TPUf3-sO9-SM!w65ZVu^WF$+4*PkmHyXH{h7MzM#=|k z-|YVW+rhcx_rKEc;|g(;2X;!V8ZmU`u*K2G8+YyJ_sN~z^A1dy6n*{8_^uwycL-Oe ztqS+4e$xNDf`+eFy^*Z%R?C%tY?ZTn9AvGo&I&x((3d1}zb zuWuf2b9Cg}Q8RzuGiGMX1+hszzxDZW{P4vMYEB%sxA{+JcfAr+SZ!e4D{rn$t!C`| ziP6FPS!cwuiN0_cs4Lj|wST z9k$mGWAiX0-gcgs$^BCcUfl(gk1>>#T3421D_BA+^O9SxB{{&p09mMQQ7?I^UGVBr~Sie0zzE1t@Awm(>k5RWvH=WTsCdXV47?P#soYHN)a^OcYkB z4e!B1VFbgvy7&4J6h<&Q^9XWgWd+cTPp!DFzhRx)m=GQ}tTh`mn8giOrL29L{1M9< zt_oItjU8r1b)G3}zHT06Wk;UDb-|h>7O0f2yE9=Z!RjBs)Fmj~BFGlb3@dm$jJ zStwjDUCgEuL3QH?nCTiUud!?ag#~u5zhOmnLt9xqDPx&!Xd^MfoUYx}nMDqQz_LKO zQ0QGJ2)GSv5oI!&0fiNiGU;lJtS*L?m$G{jX67AfUCe4BlN=m_#kfwq%X17CNy~rF zN{%iWY+-S^>+f4_(YS`Y!mKRn7s_)@S*R;*p3OBx0Jp5N?q1|*-U*a-^Pf(x!j>?Tdw{YGTt(G7_Cs(KDRzWnYSRV zncjtE3qC`&{p6=vC=55K@_(lmL7DduU0t9C>%8C<)Z;Cl(a{Ab-JijrhkqIrT4)Xy?k7!#}yA zAIVOF{3=(q9R7(F{UE)>g)N7Fa;twNeTAgBxT+EVcqga+`9b=P3tJBVH*$@74PGt4Zg%4k|pF8o7=K%GO2R^F(BRd^>gjh*Ut$4aULL6|6KU+Aaft_kLv*RFYo4gRQq#^I1%{AnY(#&dWrAP zDZ&A(f4uqok?aTmcqgL$>$+D<`sB(f;_;>E=h_Dx(U0o@_~+vA$}gt~2dw?e?SJ05 zD9QcMzdQ%1elDJ3ayY>OOgmQpig*HYpWP*91OD;Iz`qio(2AJPSm(-N?X>Qr{_&rG zyys$x?$6;oeBmTk|G3Y;NAYF{{_z|j-=jk@4O<}%D53$yG(t%0OchLas-gph%{Rf56B&nNrNID9;8(!SsYU#m4jBLmjq`dz8N&_c$Ku`1cUXpx-LTIPTY5>G$Z*$n=C<^{1pk0xu-;LQ*dz z_Zmnuk_HLBko5Z&X&p(cN*W{$L-MdUX}08HNEEgw4-)}tzAw%ERg->cHn3ynFFB5; z3Dbn(?I5M&Whj61F2nIi$73A-c09(BO)q89dmK5HBORHQJ>(c70VEH>k31yOCXfb6 zMWaabC2bgK14)CVBS=1iBqT^qf}|u{@=|m1w=H=Il9mFAL~7EJnr~!*K}z$IXyy^k zMxq%?G+`<1Kb6w)G8_eR-0yht|Hh58>%^iP;uUnng6 z?5D!Q9bMjUmOXLot(8aSpIJO+<-ERIC*7JCy{h&^;S*tmu>V7;;egi~RC=-bv4wX7 z=QiKa@j{q5!E-=z!PD=Yo87R}@m4B%k5oHtWDfEd+jPO|FzE^^zTzWb#|3b zJ~bkrI}mQU7!x_6O7#va{h|j7_4>s1+Lrc(_p5h1zfrZyCaGh$pglX^uO)i_eXF6u zj>vJ71O5E`jFwghg-K68ajf3L1MqX}*?pJX4(w2G@8?g%E^BfkveLe!6Im}ciGM*_ z7gF=L&N-8#gBQt@tE`LhJ7`$=Yln+5)dmI~+q2GZUe#HN_n)28D0TO!3NM6qk_Jug z{LIqZa1qD$*8HOKy7vS3>|ODeZ`jZa*@r!6_E^+w+&?-W7q*6MXn*CM5k2!~ZOy8% z^SLF*>w7%#>p8yHgBEMPGxQJX(Wh16p9s8?FL2MU`-Nlu2AqpH z|8n)2OAcS$lGWzgs(&Uv_j%JN*F4|uUi#WG?$^E^kofhCLTSzY&JSw!7~gur`#pb} zJE38GQfR@Ep4;9l=-g+-;B!qw6KhOcAAh`uY0~mFOM@mY>G6{PgrK!847US%jVLJi zd+*C*^XCjntTtt2;F=MwCWSp8JGz5s!{ETzDvgdztM`_NDa0?(SoKXa{1Do6)x_h! z>`ZNRZAkwipN#u?{wE1dvtG`e(Ywysv4iU5%^Fkf=GO~m+^%9|1Fv4cnXdFaw0GWTeNN7Lx$Q2&6!h+!n|{qnd~?>{%twwJ zybsLX7ZJ2Lc+8#84P*XMz4p*EP2}T$jII$mIAdzY+4O~9_1ZM-=?!Owhvv^5<$EtG zI{J*c_OYG4E7m#LvB}48@7XZ$@4nmm-5%3r~h^FcB}eleK(w$bZhpNPTo6S=-KsRkNeB|4F1FYRL!G7)%V>z z`E1h(1Ahp(SoqVWEzOQzo;%{u#=QYQoqcu1j?^vhM&@9^#nb={Bd%~tm}~@o22KxvE=3V_HFrJpFQ(m z&rCY7c9DCfe519kuH_uV<iM~(&V!cpeqt||$IQD{bcUFnbZ)Mpxb&!_Wy-+4} zTkn_(ammAEG}2s)i|!;`jE86u@+3_CP2MDYV}sANVS|YIcQA%bh~#sS%oA&8?njLC zY+%%R026E=WoB3+X6hDXU@YGX#tfespz1RQ^~{iev2qstLY~3Q099rTB4&C`kh_tv z378pb0BA`Bkf8>)M9fqT$X2mLanxXds?QkIGegEm+!>A<3{Yjp;5IVqYQtwUg2|UW z)5(Kf$u^0d=4sOdU{gN=lhI*=6J`KR6#^BhFKd>~y7k~19V689{=_vtOg(49h@UE>WC&SKa+Fcvv{M{qwAP59 zdZ6S!ZJZ20M`rlXgjN5wFjAiI8;vyYYd&W_VSrS5h9&ZxDx;)z#jFO;X~*F)W6kwB(ZYzI`mdzSMcw82Z_TK?&+%Vtjg+S&7ZUce^XSjO zZBwc`<^jyO1R!I~V@u>YRYu7(O!*cPsTpI239HIzVZ=|BQBo+wj{rZTXRR5Q{|1=R zvs!D!PyJVtQd4hn{Aa?d|5_L+Pe(3Iq}&bSGM*X?P#yCaM9j2LA%Qbv5vNZ9%v{Ln zQ*4Qtsn3vE8hy>c_L%{yK4b73GPm0kdn4qVP!z831>~^c~Kn< zmAv4vhBjvbBcxo6u(B{jOXT%5R+cKuQ8cFw?iflNLQKh^RIma_0;MAoFiNYpL)y|_ zgPh*9bkLUZ^Z}48{bnPEfQxF&utW}19UudG`gpbudQ8;;VS0yY>hublU!tEn0L=de zK*rR8EfF(u*l2|0=g6-(!;Aq^DHxV|X2{u|c?8^I+;j|3WyT<4rkw!!-U&B3Y5;qy7cgVq}HlRoh>D!GnR zzXJlsgMbOE{%c{RJn`R1Q#q&ZGD?!=jSC!Q6gcK0M;Wa( z;-|_eNt)qvIKcxGR+Z7hNEv!JKnm==82h-vuW1ZW9qkxIo>OI%6nDn^wyCCz=_G^i88m}i)e**GksUSE6(qn|z8Ui6qPYWdhGFz}|^_WVFFr)*0hC&&G^uwGNZ~&Ja;=F)k zOXM(NwiZ`_JZKtQ;g}5yFB}Gp5wp>v3^qIDFdfWLE8;Y*DZ&UR7@*p57(~oe3R>S{ z11FpSa1R9;!4N5uQ%(O(Ax{3?#l z*b+TZ=%dggN)}O0+rtTR6xh6<6Xa;E5kFDe%cztgIJ<-s#K^0A^%xUOcfS3Rj5dN^OpKsGCK*6^Ct5w*zCuXKUDoFg-1l6j>LY1VL%|==}h4CB1L59!R5;0Ssp`KDo3MZUkfU3_JM9lQogStl9_c-s@0H!_QykBEW z#7xzI>QedVIcfkHPC6inoVK-dy7 zQ=e(gt1!-+DFak}#-N^=RscK8QG)@h%os$>R1K&e78TBUiv+M>I_E7CTOwwv2GrU* z`XfgT2B`XsK|M3AT9&|hbz*=jGX@beRRb!XS+Y240GOZ2Q3G2dW~v6%B|E%^qXq+1 zea2vU5%uyS>g7e$%ZsQ#OiAgIwXqebZc4sNF3ke(C3_w|8^7o{-{xTpqCZUGb#^&& zzSoZmDOw%2*AHX!Fr(LY9$n2!W;t3f{{{QL|yqcnX*QO^8Ijm5p_iqlMzZ3mlshNr4^!FUPRqCC}1KjFQV?K zMtKqSzfKYL@*)`JMKH>XVEjKVg7K$;)K8LfG7L5S$$txb$x9cAe#bl;B>XN0BwYDN ztEE%UZCV~1x7t6D0OXi!H$~}IZ8@#E-*=u3$ZCOPt-w|VNeef{|Cl;6T?|ZX?qAth z*(8cJD~x$B^y5MP9>z*0gTc+;V081ZOAt5r3LZ5*DE5N&zs;=weUTmn2a?vPhF7(b zqyO1t?C7B%4I16#gZD-ZGPwu!?AD$DCI?{5pwS-=9r=Mtd@&&GhS<8Qk!;D=u79lR z8T`Xe1N!$|^6whP>8C4@r7{Ha1DRzj=x@nl@_mB^a^P3wGkr1er!9fO|4uWseiHso zeyk!$DWV*+As@c6xfkyfoYl+@8Kmu^d}7938z>U4n^I k-MoatUf+kLPntGe?B#B}Zy?`lo6VA+TLZTMvg?We1G0xkoB#j- diff --git a/presto-parquet/pom.xml b/presto-parquet/pom.xml index 6f2926106d258..3fa5aa05c257c 100644 --- a/presto-parquet/pom.xml +++ b/presto-parquet/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-parquet @@ -32,6 +32,11 @@ hive-apache + + io.airlift + units + + io.airlift aircompressor @@ -119,7 +124,7 @@ - io.airlift + com.facebook.airlift log-manager runtime @@ -173,7 +178,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-parquet/src/main/java/com/facebook/presto/parquet/ParquetTypeUtils.java b/presto-parquet/src/main/java/com/facebook/presto/parquet/ParquetTypeUtils.java index 660ef9d6f0ffc..a8931b6164220 100644 --- a/presto-parquet/src/main/java/com/facebook/presto/parquet/ParquetTypeUtils.java +++ b/presto-parquet/src/main/java/com/facebook/presto/parquet/ParquetTypeUtils.java @@ -13,8 +13,11 @@ */ package com.facebook.presto.parquet; +import com.facebook.presto.spi.Subfield; +import com.facebook.presto.spi.Subfield.PathElement; import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; import org.apache.parquet.column.Encoding; import org.apache.parquet.io.ColumnIO; import org.apache.parquet.io.ColumnIOFactory; @@ -24,6 +27,7 @@ import org.apache.parquet.io.ParquetDecodingException; import org.apache.parquet.io.PrimitiveColumnIO; import org.apache.parquet.schema.DecimalMetadata; +import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.MessageType; import java.util.Arrays; @@ -33,6 +37,7 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.presto.spi.Subfield.NestedField; import static com.google.common.base.Preconditions.checkArgument; import static org.apache.parquet.schema.OriginalType.DECIMAL; import static org.apache.parquet.schema.Type.Repetition.REPEATED; @@ -184,7 +189,7 @@ public static ParquetEncoding getParquetEncoding(Encoding encoding) } } - public static org.apache.parquet.schema.Type getParquetTypeByName(String columnName, MessageType messageType) + public static org.apache.parquet.schema.Type getParquetTypeByName(String columnName, GroupType messageType) { if (messageType.containsField(columnName)) { return messageType.getType(columnName); @@ -261,4 +266,38 @@ public static long getShortDecimalValue(byte[] bytes) return value; } + + public static MessageType getSubfieldType(GroupType baseType, Subfield subfield) + { + checkArgument(subfield.getPath().size() >= 1, "subfield size is less than 1"); + + ImmutableList.Builder typeBuilder = ImmutableList.builder(); + org.apache.parquet.schema.Type parentType = getParquetTypeByName(subfield.getRootName(), baseType); + + for (PathElement field : subfield.getPath()) { + if (field instanceof NestedField) { + NestedField nestedField = (NestedField) field; + org.apache.parquet.schema.Type childType = getParquetTypeByName(nestedField.getName(), parentType.asGroupType()); + if (childType != null) { + typeBuilder.add(childType); + parentType = childType; + } + } + else { + typeBuilder.add(parentType.asGroupType().getFields().get(0)); + break; + } + } + + List subfieldTypes = typeBuilder.build(); + if (subfieldTypes.isEmpty()) { + return new MessageType(subfield.getRootName(), ImmutableList.of()); + } + org.apache.parquet.schema.Type type = subfieldTypes.get(subfieldTypes.size() - 1); + for (int i = subfieldTypes.size() - 2; i >= 0; --i) { + GroupType groupType = subfieldTypes.get(i).asGroupType(); + type = new MessageType(groupType.getName(), ImmutableList.of(type)); + } + return new MessageType(subfield.getRootName(), ImmutableList.of(type)); + } } diff --git a/presto-parquet/src/main/java/com/facebook/presto/parquet/predicate/Predicate.java b/presto-parquet/src/main/java/com/facebook/presto/parquet/predicate/Predicate.java index 5d9bcf337fde3..1295e163e9c37 100644 --- a/presto-parquet/src/main/java/com/facebook/presto/parquet/predicate/Predicate.java +++ b/presto-parquet/src/main/java/com/facebook/presto/parquet/predicate/Predicate.java @@ -32,7 +32,7 @@ public boolean matches(long numberOfRows, Map> s } @Override - public boolean matches(Map dictionaries) + public boolean matches(DictionaryDescriptor dictionary) { return true; } @@ -51,9 +51,11 @@ boolean matches(long numberOfRows, Map> statisti throws ParquetCorruptionException; /** - * Should the Parquet Reader process a file section with the specified dictionary. + * Should the Parquet Reader process a file section with the specified dictionary based on that + * single dictionary. This is safe to check repeatedly to avoid loading more parquet dictionaries + * if the section can already be eliminated. * - * @param dictionaries dictionaries per column + * @param dictionary The single column dictionary */ - boolean matches(Map dictionaries); + boolean matches(DictionaryDescriptor dictionary); } diff --git a/presto-parquet/src/main/java/com/facebook/presto/parquet/predicate/PredicateUtils.java b/presto-parquet/src/main/java/com/facebook/presto/parquet/predicate/PredicateUtils.java index 06866992d0789..ff5a82f1cd6c2 100644 --- a/presto-parquet/src/main/java/com/facebook/presto/parquet/predicate/PredicateUtils.java +++ b/presto-parquet/src/main/java/com/facebook/presto/parquet/predicate/PredicateUtils.java @@ -94,8 +94,7 @@ public static boolean predicateMatches(Predicate parquetPredicate, BlockMetaData return false; } - Map dictionaries = getDictionaries(block, dataSource, descriptorsByPath, parquetTupleDomain); - return parquetPredicate.matches(dictionaries); + return dictionaryPredicatesMatch(parquetPredicate, block, dataSource, descriptorsByPath, parquetTupleDomain); } private static Map> getStatistics(BlockMetaData blockMetadata, Map, RichColumnDescriptor> descriptorsByPath) @@ -113,23 +112,22 @@ private static Map> getStatistics(BlockMetaData return statistics.build(); } - private static Map getDictionaries(BlockMetaData blockMetadata, ParquetDataSource dataSource, Map, RichColumnDescriptor> descriptorsByPath, TupleDomain parquetTupleDomain) + private static boolean dictionaryPredicatesMatch(Predicate parquetPredicate, BlockMetaData blockMetadata, ParquetDataSource dataSource, Map, RichColumnDescriptor> descriptorsByPath, TupleDomain parquetTupleDomain) { - ImmutableMap.Builder dictionaries = ImmutableMap.builder(); for (ColumnChunkMetaData columnMetaData : blockMetadata.getColumns()) { RichColumnDescriptor descriptor = descriptorsByPath.get(Arrays.asList(columnMetaData.getPath().toArray())); if (descriptor != null) { if (isOnlyDictionaryEncodingPages(columnMetaData) && isColumnPredicate(descriptor, parquetTupleDomain)) { - int totalSize = toIntExact(columnMetaData.getTotalSize()); - byte[] buffer = new byte[totalSize]; + byte[] buffer = new byte[toIntExact(columnMetaData.getTotalSize())]; dataSource.readFully(columnMetaData.getStartingPos(), buffer); - Optional dictionaryPage = readDictionaryPage(buffer, columnMetaData.getCodec()); - dictionaries.put(descriptor, new DictionaryDescriptor(descriptor, dictionaryPage)); - break; + // Early abort, predicate already filters block so no more dictionaries need be read + if (!parquetPredicate.matches(new DictionaryDescriptor(descriptor, readDictionaryPage(buffer, columnMetaData.getCodec())))) { + return false; + } } } } - return dictionaries.build(); + return true; } private static Optional readDictionaryPage(byte[] data, CompressionCodecName codecName) @@ -157,7 +155,7 @@ private static Optional readDictionaryPage(byte[] data, Compress private static boolean isColumnPredicate(ColumnDescriptor columnDescriptor, TupleDomain parquetTupleDomain) { verify(parquetTupleDomain.getDomains().isPresent(), "parquetTupleDomain is empty"); - return parquetTupleDomain.getDomains().get().keySet().contains(columnDescriptor); + return parquetTupleDomain.getDomains().get().containsKey(columnDescriptor); } @VisibleForTesting diff --git a/presto-parquet/src/main/java/com/facebook/presto/parquet/predicate/TupleDomainParquetPredicate.java b/presto-parquet/src/main/java/com/facebook/presto/parquet/predicate/TupleDomainParquetPredicate.java index 7327eae2ae250..3abcedb011563 100644 --- a/presto-parquet/src/main/java/com/facebook/presto/parquet/predicate/TupleDomainParquetPredicate.java +++ b/presto-parquet/src/main/java/com/facebook/presto/parquet/predicate/TupleDomainParquetPredicate.java @@ -106,8 +106,9 @@ public boolean matches(long numberOfRows, Map> s } @Override - public boolean matches(Map dictionaries) + public boolean matches(DictionaryDescriptor dictionary) { + requireNonNull(dictionary, "dictionary is null"); if (effectivePredicate.isNone()) { return false; } @@ -115,18 +116,14 @@ public boolean matches(Map dictionaries) Map effectivePredicateDomains = effectivePredicate.getDomains() .orElseThrow(() -> new IllegalStateException("Effective predicate other than none should have domains")); - for (RichColumnDescriptor column : columns) { - Domain effectivePredicateDomain = effectivePredicateDomains.get(column); - if (effectivePredicateDomain == null) { - continue; - } - DictionaryDescriptor dictionaryDescriptor = dictionaries.get(column); - Domain domain = getDomain(effectivePredicateDomain.getType(), dictionaryDescriptor); - if (effectivePredicateDomain.intersect(domain).isNone()) { - return false; - } - } - return true; + Domain effectivePredicateDomain = effectivePredicateDomains.get(dictionary.getColumnDescriptor()); + + return effectivePredicateDomain == null || effectivePredicateMatches(effectivePredicateDomain, dictionary); + } + + private static boolean effectivePredicateMatches(Domain effectivePredicateDomain, DictionaryDescriptor dictionary) + { + return !effectivePredicateDomain.intersect(getDomain(effectivePredicateDomain.getType(), dictionary)).isNone(); } @VisibleForTesting diff --git a/presto-parquet/src/main/java/com/facebook/presto/parquet/reader/ParquetReader.java b/presto-parquet/src/main/java/com/facebook/presto/parquet/reader/ParquetReader.java index 8e06f91fe561a..9f34ca7c9a23a 100644 --- a/presto-parquet/src/main/java/com/facebook/presto/parquet/reader/ParquetReader.java +++ b/presto-parquet/src/main/java/com/facebook/presto/parquet/reader/ParquetReader.java @@ -28,6 +28,7 @@ import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeSignatureParameter; +import io.airlift.units.DataSize; import it.unimi.dsi.fastutil.booleans.BooleanArrayList; import it.unimi.dsi.fastutil.booleans.BooleanList; import it.unimi.dsi.fastutil.ints.IntArrayList; @@ -51,6 +52,7 @@ import static com.facebook.presto.spi.type.StandardTypes.MAP; import static com.facebook.presto.spi.type.StandardTypes.ROW; import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.Math.max; import static java.lang.Math.min; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; @@ -59,6 +61,8 @@ public class ParquetReader implements Closeable { private static final int MAX_VECTOR_LENGTH = 1024; + private static final int INITIAL_BATCH_SIZE = 1; + private static final int BATCH_SIZE_GROWTH_FACTOR = 2; private final List blocks; private final List columns; @@ -71,21 +75,30 @@ public class ParquetReader private long currentGroupRowCount; private long nextRowInGroup; private int batchSize; + private int nextBatchSize = INITIAL_BATCH_SIZE; private final PrimitiveColumnReader[] columnReaders; + private long[] maxBytesPerCell; + private long maxCombinedBytesPerRow; + private final long maxReadBlockBytes; + private int maxBatchSize = MAX_VECTOR_LENGTH; private AggregatedMemoryContext currentRowGroupMemoryContext; - public ParquetReader(MessageColumnIO messageColumnIO, + public ParquetReader(MessageColumnIO + messageColumnIO, List blocks, ParquetDataSource dataSource, - AggregatedMemoryContext systemMemoryContext) + AggregatedMemoryContext systemMemoryContext, + DataSize maxReadBlockSize) { this.blocks = blocks; this.dataSource = requireNonNull(dataSource, "dataSource is null"); this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); this.currentRowGroupMemoryContext = systemMemoryContext.newAggregatedMemoryContext(); + this.maxReadBlockBytes = requireNonNull(maxReadBlockSize, "maxReadBlockSize is null").toBytes(); columns = messageColumnIO.getLeaves(); columnReaders = new PrimitiveColumnReader[columns.size()]; + maxBytesPerCell = new long[columns.size()]; } @Override @@ -107,7 +120,9 @@ public int nextBatch() return -1; } - batchSize = toIntExact(min(MAX_VECTOR_LENGTH, currentGroupRowCount - nextRowInGroup)); + batchSize = toIntExact(min(nextBatchSize, maxBatchSize)); + nextBatchSize = min(batchSize * BATCH_SIZE_GROWTH_FACTOR, MAX_VECTOR_LENGTH); + batchSize = toIntExact(min(batchSize, currentGroupRowCount - nextRowInGroup)); nextRowInGroup += batchSize; currentPosition += batchSize; @@ -161,7 +176,7 @@ private ColumnChunk readMap(GroupField field) IntList offsets = new IntArrayList(); BooleanList valueIsNull = new BooleanArrayList(); calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); - Block mapBlock = ((MapType) field.getType()).createBlockFromKeyValue(Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), blocks[0], blocks[1]); + Block mapBlock = ((MapType) field.getType()).createBlockFromKeyValue(offsets.size() - 1, Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), blocks[0], blocks[1]); return new ColumnChunk(mapBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); } @@ -194,7 +209,8 @@ private ColumnChunk readPrimitive(PrimitiveField field) throws IOException { ColumnDescriptor columnDescriptor = field.getDescriptor(); - PrimitiveColumnReader columnReader = columnReaders[field.getId()]; + int fieldId = field.getId(); + PrimitiveColumnReader columnReader = columnReaders[fieldId]; if (columnReader.getPageReader() == null) { validateParquet(currentBlockMetadata.getRowCount() > 0, "Row group has 0 rows"); ColumnChunkMetaData metadata = getColumnChunkMetaData(columnDescriptor); @@ -206,7 +222,17 @@ private ColumnChunk readPrimitive(PrimitiveField field) ParquetColumnChunk columnChunk = new ParquetColumnChunk(descriptor, buffer, 0); columnReader.setPageReader(columnChunk.readAllPages()); } - return columnReader.readPrimitive(field); + ColumnChunk columnChunk = columnReader.readPrimitive(field); + + // update max size per primitive column chunk + long bytesPerCell = columnChunk.getBlock().getSizeInBytes() / batchSize; + if (maxBytesPerCell[fieldId] < bytesPerCell) { + // update batch size + maxCombinedBytesPerRow = maxCombinedBytesPerRow - maxBytesPerCell[fieldId] + bytesPerCell; + maxBatchSize = toIntExact(min(maxBatchSize, max(1, maxReadBlockBytes / maxCombinedBytesPerRow))); + maxBytesPerCell[fieldId] = bytesPerCell; + } + return columnChunk; } private byte[] allocateBlock(int length) diff --git a/presto-parquet/src/test/java/com/facebook/presto/parquet/TestTupleDomainParquetPredicate.java b/presto-parquet/src/test/java/com/facebook/presto/parquet/TestTupleDomainParquetPredicate.java index 4c657e9db9ef8..d0347acdb943e 100644 --- a/presto-parquet/src/test/java/com/facebook/presto/parquet/TestTupleDomainParquetPredicate.java +++ b/presto-parquet/src/test/java/com/facebook/presto/parquet/TestTupleDomainParquetPredicate.java @@ -337,7 +337,7 @@ public void testVarcharMatchesWithDictionaryDescriptor() TupleDomain effectivePredicate = getEffectivePredicate(column, createVarcharType(255), EMPTY_SLICE); TupleDomainParquetPredicate parquetPredicate = new TupleDomainParquetPredicate(effectivePredicate, singletonList(column)); DictionaryPage page = new DictionaryPage(Slices.wrappedBuffer(new byte[] {0, 0, 0, 0}), 1, PLAIN_DICTIONARY); - assertTrue(parquetPredicate.matches(singletonMap(column, new DictionaryDescriptor(column, Optional.of(page))))); + assertTrue(parquetPredicate.matches(new DictionaryDescriptor(column, Optional.of(page)))); } private TupleDomain getEffectivePredicate(RichColumnDescriptor column, VarcharType type, Slice value) diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 6a56a6aa7ad39..909e2036f551b 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-parser @@ -43,7 +43,7 @@ - io.airlift + com.facebook.airlift log diff --git a/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 b/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 index f6e112706939a..01970ac63b9a6 100644 --- a/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 +++ b/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 @@ -26,10 +26,6 @@ standaloneExpression : expression EOF ; -standalonePathSpecification - : pathSpecification EOF - ; - statement : query #statementDefault | USE schema=identifier #use @@ -59,6 +55,12 @@ statement | ANALYZE qualifiedName (WITH properties)? #analyze | CREATE (OR REPLACE)? VIEW qualifiedName AS query #createView | DROP VIEW (IF EXISTS)? qualifiedName #dropView + | CREATE (OR REPLACE)? FUNCTION functionName=qualifiedName + '(' (sqlParameterDeclaration (',' sqlParameterDeclaration)*)? ')' + RETURNS returnType=type + (COMMENT string)? + routineCharacteristics routineBody #createFunction + | DROP FUNCTION (IF EXISTS)? qualifiedName types? #dropFunction | CALL qualifiedName '(' (callArgument (',' callArgument)*)? ')' #call | CREATE ROLE name=identifier (WITH ADMIN grantor)? #createRole @@ -112,7 +114,6 @@ statement | EXECUTE identifier (USING expression (',' expression)*)? #execute | DESCRIBE INPUT identifier #describeInput | DESCRIBE OUTPUT identifier #describeOutput - | SET PATH pathSpecification #setPath ; query @@ -144,6 +145,38 @@ property : identifier EQ expression ; +sqlParameterDeclaration + : identifier type + ; + +routineCharacteristics + : routineCharacteristic* + ; + +routineCharacteristic + : LANGUAGE language + | determinism + | nullCallClause + ; + +routineBody + : RETURN expression + ; + +language + : SQL + ; + +determinism + : DETERMINISTIC + | NOT DETERMINISTIC; + +nullCallClause + : RETURNS NULL ON NULL INPUT + | CALLED ON NULL INPUT + ; + + queryNoWith: queryTerm (ORDER BY sortItem (',' sortItem)*)? @@ -301,7 +334,7 @@ primaryExpression | ROW '(' expression (',' expression)* ')' #rowConstructor | qualifiedName '(' ASTERISK ')' filter? over? #functionCall | qualifiedName '(' (setQuantifier? expression (',' expression)*)? - (ORDER BY sortItem (',' sortItem)*)? ')' filter? over? #functionCall + (ORDER BY sortItem (',' sortItem)*)? ')' filter? (nullTreatment? over)? #functionCall | identifier '->' expression #lambda | '(' (identifier (',' identifier)*)? ')' '->' expression #lambda | '(' query ')' #subqueryExpression @@ -321,7 +354,6 @@ primaryExpression | name=LOCALTIME ('(' precision=INTEGER_VALUE ')')? #specialDateTimeFunction | name=LOCALTIMESTAMP ('(' precision=INTEGER_VALUE ')')? #specialDateTimeFunction | name=CURRENT_USER #currentUser - | name=CURRENT_PATH #currentPath | SUBSTRING '(' valueExpression FROM valueExpression (FOR valueExpression)? ')' #substring | NORMALIZE '(' valueExpression (',' normalForm)? ')' #normalize | EXTRACT '(' identifier FROM valueExpression ')' #extract @@ -334,6 +366,11 @@ string | UNICODE_STRING (UESCAPE STRING)? #unicodeStringLiteral ; +nullTreatment + : IGNORE NULLS + | RESPECT NULLS + ; + timeZoneSpecifier : TIME ZONE interval #timeZoneInterval | TIME ZONE string #timeZoneString @@ -363,6 +400,10 @@ normalForm : NFD | NFC | NFKD | NFKC ; +types + : '(' (type (',' type)*)? ')' + ; + type : type ARRAY | ARRAY '<' type '>' @@ -436,15 +477,6 @@ callArgument | identifier '=>' expression #namedArgument ; -pathElement - : identifier '.' identifier #qualifiedArgument - | identifier #unqualifiedArgument - ; - -pathSpecification - : pathElement (',' pathElement)* - ; - privilege : SELECT | DELETE | INSERT | identifier ; @@ -487,21 +519,21 @@ nonReserved // IMPORTANT: this rule must only contain tokens. Nested rules are not supported. See SqlParser.exitNonReserved : ADD | ADMIN | ALL | ANALYZE | ANY | ARRAY | ASC | AT | BERNOULLI - | CALL | CASCADE | CATALOGS | COLUMN | COLUMNS | COMMENT | COMMIT | COMMITTED | CURRENT | CURRENT_ROLE - | DATA | DATE | DAY | DESC | DISTRIBUTED + | CALL | CALLED | CASCADE | CATALOGS | COLUMN | COLUMNS | COMMENT | COMMIT | COMMITTED | CURRENT | CURRENT_ROLE + | DATA | DATE | DAY | DESC | DETERMINISTIC | DISTRIBUTED | EXCLUDING | EXPLAIN - | FILTER | FIRST | FOLLOWING | FORMAT | FUNCTIONS + | FILTER | FIRST | FOLLOWING | FORMAT | FUNCTION | FUNCTIONS | GRANT | GRANTED | GRANTS | GRAPHVIZ | HOUR - | IF | INCLUDING | INPUT | INTERVAL | IO | ISOLATION + | IF | IGNORE | INCLUDING | INPUT | INTERVAL | IO | ISOLATION | JSON - | LAST | LATERAL | LEVEL | LIMIT | LOGICAL + | LANGUAGE | LAST | LATERAL | LEVEL | LIMIT | LOGICAL | MAP | MINUTE | MONTH | NFC | NFD | NFKC | NFKD | NO | NONE | NULLIF | NULLS | ONLY | OPTION | ORDINALITY | OUTPUT | OVER - | PARTITION | PARTITIONS | PATH | POSITION | PRECEDING | PRIVILEGES | PROPERTIES - | RANGE | READ | RENAME | REPEATABLE | REPLACE | RESET | RESTRICT | REVOKE | ROLE | ROLES | ROLLBACK | ROW | ROWS - | SCHEMA | SCHEMAS | SECOND | SERIALIZABLE | SESSION | SET | SETS + | PARTITION | PARTITIONS | POSITION | PRECEDING | PRIVILEGES | PROPERTIES + | RANGE | READ | RENAME | REPEATABLE | REPLACE | RESET | RESPECT | RESTRICT | RETURN | RETURNS | REVOKE | ROLE | ROLES | ROLLBACK | ROW | ROWS + | SCHEMA | SCHEMAS | SECOND | SERIALIZABLE | SESSION | SET | SETS | SQL | SHOW | SOME | START | STATS | SUBSTRING | SYSTEM | TABLES | TABLESAMPLE | TEXT | TIME | TIMESTAMP | TO | TRANSACTION | TRY_CAST | TYPE | UNBOUNDED | UNCOMMITTED | USE | USER @@ -526,6 +558,7 @@ BERNOULLI: 'BERNOULLI'; BETWEEN: 'BETWEEN'; BY: 'BY'; CALL: 'CALL'; +CALLED: 'CALLED'; CASCADE: 'CASCADE'; CASE: 'CASE'; CAST: 'CAST'; @@ -541,7 +574,6 @@ CROSS: 'CROSS'; CUBE: 'CUBE'; CURRENT: 'CURRENT'; CURRENT_DATE: 'CURRENT_DATE'; -CURRENT_PATH: 'CURRENT_PATH'; CURRENT_ROLE: 'CURRENT_ROLE'; CURRENT_TIME: 'CURRENT_TIME'; CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; @@ -553,6 +585,7 @@ DEALLOCATE: 'DEALLOCATE'; DELETE: 'DELETE'; DESC: 'DESC'; DESCRIBE: 'DESCRIBE'; +DETERMINISTIC: 'DETERMINISTIC'; DISTINCT: 'DISTINCT'; DISTRIBUTED: 'DISTRIBUTED'; DROP: 'DROP'; @@ -573,6 +606,7 @@ FOR: 'FOR'; FORMAT: 'FORMAT'; FROM: 'FROM'; FULL: 'FULL'; +FUNCTION: 'FUNCTION'; FUNCTIONS: 'FUNCTIONS'; GRANT: 'GRANT'; GRANTED: 'GRANTED'; @@ -583,6 +617,7 @@ GROUPING: 'GROUPING'; HAVING: 'HAVING'; HOUR: 'HOUR'; IF: 'IF'; +IGNORE: 'IGNORE'; IN: 'IN'; INCLUDING: 'INCLUDING'; INNER: 'INNER'; @@ -596,6 +631,7 @@ IS: 'IS'; ISOLATION: 'ISOLATION'; JSON: 'JSON'; JOIN: 'JOIN'; +LANGUAGE: 'LANGUAGE'; LAST: 'LAST'; LATERAL: 'LATERAL'; LEFT: 'LEFT'; @@ -631,7 +667,6 @@ OUTPUT: 'OUTPUT'; OVER: 'OVER'; PARTITION: 'PARTITION'; PARTITIONS: 'PARTITIONS'; -PATH: 'PATH'; POSITION: 'POSITION'; PRECEDING: 'PRECEDING'; PREPARE: 'PREPARE'; @@ -644,7 +679,10 @@ RENAME: 'RENAME'; REPEATABLE: 'REPEATABLE'; REPLACE: 'REPLACE'; RESET: 'RESET'; +RESPECT: 'RESPECT'; RESTRICT: 'RESTRICT'; +RETURN: 'RETURN'; +RETURNS: 'RETURNS'; REVOKE: 'REVOKE'; RIGHT: 'RIGHT'; ROLE: 'ROLE'; @@ -663,6 +701,7 @@ SET: 'SET'; SETS: 'SETS'; SHOW: 'SHOW'; SOME: 'SOME'; +SQL: 'SQL'; START: 'START'; STATS: 'STATS'; SUBSTRING: 'SUBSTRING'; diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/ExpressionFormatter.java b/presto-parser/src/main/java/com/facebook/presto/sql/ExpressionFormatter.java index 7338a7e882dee..54f46f2f40b06 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/ExpressionFormatter.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/ExpressionFormatter.java @@ -28,7 +28,6 @@ import com.facebook.presto.sql.tree.CoalesceExpression; import com.facebook.presto.sql.tree.ComparisonExpression; import com.facebook.presto.sql.tree.Cube; -import com.facebook.presto.sql.tree.CurrentPath; import com.facebook.presto.sql.tree.CurrentTime; import com.facebook.presto.sql.tree.CurrentUser; import com.facebook.presto.sql.tree.DecimalLiteral; @@ -169,12 +168,6 @@ protected String visitCurrentUser(CurrentUser node, Void context) return "CURRENT_USER"; } - @Override - protected String visitCurrentPath(CurrentPath node, Void context) - { - return "CURRENT_PATH"; - } - @Override protected String visitCurrentTime(CurrentTime node, Void context) { @@ -377,6 +370,10 @@ protected String visitFunctionCall(FunctionCall node, Void context) builder.append(')'); + if (node.isIgnoreNulls()) { + builder.append(" IGNORE NULLS"); + } + if (node.getFilter().isPresent()) { builder.append(" FILTER ").append(visitFilter(node.getFilter().get(), context)); } diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java b/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java index 146b09d74c513..1b1d640398fb7 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java @@ -22,6 +22,7 @@ import com.facebook.presto.sql.tree.CallArgument; import com.facebook.presto.sql.tree.ColumnDefinition; import com.facebook.presto.sql.tree.Commit; +import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateRole; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; @@ -32,6 +33,7 @@ import com.facebook.presto.sql.tree.DescribeInput; import com.facebook.presto.sql.tree.DescribeOutput; import com.facebook.presto.sql.tree.DropColumn; +import com.facebook.presto.sql.tree.DropFunction; import com.facebook.presto.sql.tree.DropRole; import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; @@ -73,11 +75,11 @@ import com.facebook.presto.sql.tree.Revoke; import com.facebook.presto.sql.tree.RevokeRoles; import com.facebook.presto.sql.tree.Rollback; +import com.facebook.presto.sql.tree.RoutineCharacteristics; import com.facebook.presto.sql.tree.Row; import com.facebook.presto.sql.tree.SampledRelation; import com.facebook.presto.sql.tree.Select; import com.facebook.presto.sql.tree.SelectItem; -import com.facebook.presto.sql.tree.SetPath; import com.facebook.presto.sql.tree.SetRole; import com.facebook.presto.sql.tree.SetSession; import com.facebook.presto.sql.tree.ShowCatalogs; @@ -92,6 +94,7 @@ import com.facebook.presto.sql.tree.ShowStats; import com.facebook.presto.sql.tree.ShowTables; import com.facebook.presto.sql.tree.SingleColumn; +import com.facebook.presto.sql.tree.SqlParameterDeclaration; import com.facebook.presto.sql.tree.StartTransaction; import com.facebook.presto.sql.tree.Table; import com.facebook.presto.sql.tree.TableSubquery; @@ -104,6 +107,7 @@ import com.facebook.presto.sql.tree.WithQuery; import com.google.common.base.Joiner; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.Iterator; @@ -549,6 +553,47 @@ protected Void visitCreateView(CreateView node, Integer indent) return null; } + @Override + protected Void visitCreateFunction(CreateFunction node, Integer indent) + { + builder.append("CREATE FUNCTION ") + .append(formatName(node.getFunctionName())) + .append(" ") + .append(formatSqlParameterDeclarations(node.getParameters())) + .append("\nRETURNS ") + .append(node.getReturnType()); + if (node.getComment().isPresent()) { + builder.append("\nCOMMENT ") + .append(formatStringLiteral(node.getComment().get())); + } + builder.append("\n") + .append(formatRoutineCharacteristics(node.getCharacteristics())) + .append("\nRETURN ") + .append(formatExpression(node.getBody(), parameters)); + + return null; + } + + @Override + protected Void visitDropFunction(DropFunction node, Integer indent) + { + builder.append("DROP FUNCTION "); + if (node.isExists()) { + builder.append("IF EXISTS "); + } + builder.append(formatName(node.getFunctionName())); + if (node.getParameterTypes().isPresent()) { + String elementIndent = indentString(indent + 1); + builder.append("(\n") + .append(node.getParameterTypes().get().stream() + .map(type -> elementIndent + type) + .collect(joining(",\n"))) + .append(")\n"); + } + + return null; + } + @Override protected Void visitDropView(DropView node, Integer context) { @@ -853,6 +898,33 @@ private String formatPropertiesSingleLine(List properties) return " WITH ( " + propertyList + " )"; } + private String formatSqlParameterDeclarations(List parameters) + { + if (parameters.isEmpty()) { + return "()"; + } + return parameters.stream() + .map(parameter -> format( + "%s%s %s", + INDENT, + formatExpression(parameter.getName(), this.parameters), + parameter.getType())) + .collect(joining(",\n", "(\n", "\n)")); + } + + private String formatRoutineCharacteristics(RoutineCharacteristics characteristics) + { + return Joiner.on("\n").join(ImmutableList.of( + "LANGUAGE " + formatRoutineCharacteristicName(characteristics.getLanguage()), + formatRoutineCharacteristicName(characteristics.getDeterminism()), + formatRoutineCharacteristicName(characteristics.getNullCallClause()))); + } + + private String formatRoutineCharacteristicName(Enum characteristic) + { + return characteristic.name().replace("_", " "); + } + private static String formatName(String name) { if (NAME_PATTERN.matcher(name).matches()) { @@ -1286,14 +1358,6 @@ protected Void visitShowRoleGrants(ShowRoleGrants node, Integer context) return null; } - @Override - public Void visitSetPath(SetPath node, Integer indent) - { - builder.append("SET PATH "); - builder.append(Joiner.on(", ").join(node.getPathSpecification().getPath())); - return null; - } - private void processRelation(Relation relation, Integer indent) { // TODO: handle this properly diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java b/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java index b48db09d4edab..261fc1beb9ea6 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java @@ -33,13 +33,13 @@ import com.facebook.presto.sql.tree.ColumnDefinition; import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.ComparisonExpression; +import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateRole; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateTableAsSelect; import com.facebook.presto.sql.tree.CreateView; import com.facebook.presto.sql.tree.Cube; -import com.facebook.presto.sql.tree.CurrentPath; import com.facebook.presto.sql.tree.CurrentTime; import com.facebook.presto.sql.tree.CurrentUser; import com.facebook.presto.sql.tree.Deallocate; @@ -50,6 +50,7 @@ import com.facebook.presto.sql.tree.DescribeOutput; import com.facebook.presto.sql.tree.DoubleLiteral; import com.facebook.presto.sql.tree.DropColumn; +import com.facebook.presto.sql.tree.DropFunction; import com.facebook.presto.sql.tree.DropRole; import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; @@ -102,8 +103,6 @@ import com.facebook.presto.sql.tree.NullLiteral; import com.facebook.presto.sql.tree.OrderBy; import com.facebook.presto.sql.tree.Parameter; -import com.facebook.presto.sql.tree.PathElement; -import com.facebook.presto.sql.tree.PathSpecification; import com.facebook.presto.sql.tree.Prepare; import com.facebook.presto.sql.tree.PrincipalSpecification; import com.facebook.presto.sql.tree.Property; @@ -121,12 +120,13 @@ import com.facebook.presto.sql.tree.RevokeRoles; import com.facebook.presto.sql.tree.Rollback; import com.facebook.presto.sql.tree.Rollup; +import com.facebook.presto.sql.tree.RoutineCharacteristics; +import com.facebook.presto.sql.tree.RoutineCharacteristics.Language; import com.facebook.presto.sql.tree.Row; import com.facebook.presto.sql.tree.SampledRelation; import com.facebook.presto.sql.tree.SearchedCaseExpression; import com.facebook.presto.sql.tree.Select; import com.facebook.presto.sql.tree.SelectItem; -import com.facebook.presto.sql.tree.SetPath; import com.facebook.presto.sql.tree.SetRole; import com.facebook.presto.sql.tree.SetSession; import com.facebook.presto.sql.tree.ShowCatalogs; @@ -144,6 +144,7 @@ import com.facebook.presto.sql.tree.SimpleGroupBy; import com.facebook.presto.sql.tree.SingleColumn; import com.facebook.presto.sql.tree.SortItem; +import com.facebook.presto.sql.tree.SqlParameterDeclaration; import com.facebook.presto.sql.tree.StartTransaction; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.sql.tree.StringLiteral; @@ -179,6 +180,13 @@ import java.util.Optional; import java.util.stream.Collectors; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.Determinism; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.Determinism.DETERMINISTIC; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.Determinism.NOT_DETERMINISTIC; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.NullCallClause; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.NullCallClause.RETURNS_NULL_ON_NULL_INPUT; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getOnlyElement; import static java.lang.String.format; @@ -208,12 +216,6 @@ public Node visitStandaloneExpression(SqlBaseParser.StandaloneExpressionContext return visit(context.expression()); } - @Override - public Node visitStandalonePathSpecification(SqlBaseParser.StandalonePathSpecificationContext context) - { - return visit(context.pathSpecification()); - } - // ******************* statements ********************** @Override @@ -400,6 +402,39 @@ public Node visitCreateView(SqlBaseParser.CreateViewContext context) context.REPLACE() != null); } + @Override + public Node visitCreateFunction(SqlBaseParser.CreateFunctionContext context) + { + Optional comment = Optional.empty(); + if (context.COMMENT() != null) { + comment = Optional.of(((StringLiteral) visit(context.string())).getValue()); + } + + return new CreateFunction( + getQualifiedName(context.functionName), + context.REPLACE() != null, + context.sqlParameterDeclaration().stream() + .map(this::getParameterDeclarations) + .collect(toImmutableList()), + getType(context.returnType), + comment, + getRoutineCharacteristics(context.routineCharacteristics()), + (Expression) visit(context.routineBody())); + } + + @Override + public Node visitDropFunction(SqlBaseParser.DropFunctionContext context) + { + Optional> parameterTypes = context.types() == null ? Optional.empty() : Optional.of(getTypes(context.types())); + return new DropFunction(getLocation(context), getQualifiedName(context.qualifiedName()), parameterTypes, context.EXISTS() != null); + } + + @Override + public Node visitRoutineBody(SqlBaseParser.RoutineBodyContext context) + { + return visit(context.expression()); + } + @Override public Node visitStartTransaction(SqlBaseParser.StartTransactionContext context) { @@ -955,12 +990,6 @@ public Node visitShowRoleGrants(SqlBaseParser.ShowRoleGrantsContext context) getIdentifierIfPresent(context.identifier())); } - @Override - public Node visitSetPath(SqlBaseParser.SetPathContext context) - { - return new SetPath(getLocation(context), (PathSpecification) visit(context.pathSpecification())); - } - // ***************** boolean expressions ****************** @Override @@ -1322,12 +1351,6 @@ public Node visitCurrentUser(SqlBaseParser.CurrentUserContext context) return new CurrentUser(getLocation(context.CURRENT_USER())); } - @Override - public Node visitCurrentPath(SqlBaseParser.CurrentPathContext context) - { - return new CurrentPath(getLocation(context.CURRENT_PATH())); - } - @Override public Node visitExtract(SqlBaseParser.ExtractContext context) { @@ -1430,6 +1453,8 @@ public Node visitFunctionCall(SqlBaseParser.FunctionCallContext context) boolean distinct = isDistinct(context.setQuantifier()); + boolean ignoreNulls = context.nullTreatment() != null && context.nullTreatment().IGNORE() != null; + if (name.toString().equalsIgnoreCase("if")) { check(context.expression().size() == 2 || context.expression().size() == 3, "Invalid number of arguments for 'if' function", context); check(!window.isPresent(), "OVER clause not valid for 'if' function", context); @@ -1503,6 +1528,7 @@ public Node visitFunctionCall(SqlBaseParser.FunctionCallContext context) filter, orderBy, distinct, + ignoreNulls, visit(context.expression(), Expression.class)); } @@ -1764,24 +1790,6 @@ public Node visitNamedArgument(SqlBaseParser.NamedArgumentContext context) return new CallArgument(getLocation(context), context.identifier().getText(), (Expression) visit(context.expression())); } - @Override - public Node visitQualifiedArgument(SqlBaseParser.QualifiedArgumentContext context) - { - return new PathElement(getLocation(context), (Identifier) visit(context.identifier(0)), (Identifier) visit(context.identifier(1))); - } - - @Override - public Node visitUnqualifiedArgument(SqlBaseParser.UnqualifiedArgumentContext context) - { - return new PathElement(getLocation(context), (Identifier) visit(context.identifier())); - } - - @Override - public Node visitPathSpecification(SqlBaseParser.PathSpecificationContext context) - { - return new PathSpecification(getLocation(context), visit(context.pathElement(), PathElement.class)); - } - // ***************** helpers ***************** @Override @@ -2152,6 +2160,13 @@ private static QuantifiedComparisonExpression.Quantifier getComparisonQuantifier throw new IllegalArgumentException("Unsupported quantifier: " + symbol.getText()); } + private List getTypes(SqlBaseParser.TypesContext types) + { + return types.type().stream() + .map(this::getType) + .collect(toImmutableList()); + } + private String getType(SqlBaseParser.TypeContext type) { if (type.baseType() != null) { @@ -2201,6 +2216,48 @@ private String getType(SqlBaseParser.TypeContext type) throw new IllegalArgumentException("Unsupported type specification: " + type.getText()); } + private SqlParameterDeclaration getParameterDeclarations(SqlBaseParser.SqlParameterDeclarationContext context) + { + return new SqlParameterDeclaration((Identifier) visit(context.identifier()), getType(context.type())); + } + + private RoutineCharacteristics getRoutineCharacteristics(SqlBaseParser.RoutineCharacteristicsContext context) + { + Language language = null; + Determinism determinism = null; + NullCallClause nullCallClause = null; + + for (SqlBaseParser.RoutineCharacteristicContext characteristic : context.routineCharacteristic()) { + if (characteristic.language() != null) { + checkArgument(characteristic.language().SQL() != null, "Unsupported language: %s", characteristic.language().getText()); + if (language != null) { + throw new ParsingException(format("Duplicate language clause: %s", characteristic.language().getText()), getLocation(characteristic.language())); + } + language = Language.SQL; + } + else if (characteristic.determinism() != null) { + if (determinism != null) { + throw new ParsingException(format("Duplicate determinism characteristics: %s", characteristic.determinism().getText()), getLocation(characteristic.determinism())); + } + determinism = characteristic.determinism().NOT() == null ? DETERMINISTIC : NOT_DETERMINISTIC; + } + else if (characteristic.nullCallClause() != null) { + if (nullCallClause != null) { + throw new ParsingException(format("Duplicate null-call clause: %s", characteristic.nullCallClause().getText()), getLocation(characteristic.nullCallClause())); + } + nullCallClause = characteristic.nullCallClause().CALLED() != null ? CALLED_ON_NULL_INPUT : RETURNS_NULL_ON_NULL_INPUT; + } + else { + throw new IllegalArgumentException(format("Unsupported RoutineCharacteristic: %s", characteristic.getText())); + } + } + + return new RoutineCharacteristics( + Optional.ofNullable(language), + Optional.ofNullable(determinism), + Optional.ofNullable(nullCallClause)); + } + private String typeParameterToString(SqlBaseParser.TypeParameterContext typeParameter) { if (typeParameter.INTEGER_VALUE() != null) { diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/parser/ErrorHandler.java b/presto-parser/src/main/java/com/facebook/presto/sql/parser/ErrorHandler.java index 7ea711acc4847..73a8ab4872faa 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/parser/ErrorHandler.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/parser/ErrorHandler.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.sql.parser; +import com.facebook.airlift.log.Logger; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import io.airlift.log.Logger; import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.NoViableAltException; import org.antlr.v4.runtime.Parser; diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/parser/SqlParser.java b/presto-parser/src/main/java/com/facebook/presto/sql/parser/SqlParser.java index 323671ecdf019..b84838068d15c 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/parser/SqlParser.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/parser/SqlParser.java @@ -15,7 +15,6 @@ import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.Node; -import com.facebook.presto.sql.tree.PathSpecification; import com.facebook.presto.sql.tree.Statement; import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.CharStreams; @@ -41,6 +40,7 @@ import java.util.function.Consumer; import java.util.function.Function; +import static com.facebook.presto.sql.parser.SqlParserOptions.RESERVED_WORDS_WARNING; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -112,11 +112,6 @@ public Expression createExpression(String expression, ParsingOptions parsingOpti return (Expression) invokeParser("expression", expression, SqlBaseParser::standaloneExpression, parsingOptions); } - public PathSpecification createPathSpecification(String expression) - { - return (PathSpecification) invokeParser("path specification", expression, SqlBaseParser::standalonePathSpecification, new ParsingOptions()); - } - private Node invokeParser(String name, String sql, Function parseFunction, ParsingOptions parsingOptions) { try { @@ -237,9 +232,13 @@ public void exitNonReserved(SqlBaseParser.NonReservedContext context) context.getParent().removeLastChild(); Token token = (Token) context.getChild(0).getPayload(); - if (token.getText().equalsIgnoreCase("CURRENT_ROLE")) { - warningConsumer.accept(new ParsingWarning(format("Reserved word used: %s", token.getText()), token.getLine(), token.getCharPositionInLine())); + if (RESERVED_WORDS_WARNING.contains(token.getText().toUpperCase())) { + warningConsumer.accept(new ParsingWarning( + format("%s should be a reserved word, please use double quote (\"%s\"). This will be made a reserved word in future release.", token.getText(), token.getText()), + token.getLine(), + token.getCharPositionInLine())); } + context.getParent().addChild(new CommonToken( new Pair<>(token.getTokenSource(), token.getInputStream()), SqlBaseLexer.IDENTIFIER, diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/parser/SqlParserOptions.java b/presto-parser/src/main/java/com/facebook/presto/sql/parser/SqlParserOptions.java index 3fe1ac03f7e23..a6d939863f073 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/parser/SqlParserOptions.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/parser/SqlParserOptions.java @@ -13,14 +13,19 @@ */ package com.facebook.presto.sql.parser; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import java.util.EnumSet; +import java.util.Set; import static java.util.Objects.requireNonNull; public class SqlParserOptions { + protected static final Set RESERVED_WORDS_WARNING = ImmutableSet.of( + "CALLED", "CURRENT_ROLE", "DETERMINISTIC", "FUNCTION", "LANGUAGE", "RETURN", "RETURNS", "SQL"); + private final EnumSet allowedIdentifierSymbols = EnumSet.noneOf(IdentifierSymbol.class); private boolean enhancedErrorHandlerEnabled = true; diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/ArrayConstructor.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/ArrayConstructor.java index 5465b15dc6186..448c02e8dae31 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/ArrayConstructor.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/ArrayConstructor.java @@ -24,7 +24,7 @@ public class ArrayConstructor extends Expression { - public static final String ARRAY_CONSTRUCTOR = "ARRAY_CONSTRUCTOR"; + public static final String ARRAY_CONSTRUCTOR = "array_constructor"; private final List values; public ArrayConstructor(List values) diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java index 7501c81760600..7451f88426fca 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java @@ -582,6 +582,21 @@ protected R visitDropView(DropView node, C context) return visitStatement(node, context); } + protected R visitCreateFunction(CreateFunction node, C context) + { + return visitStatement(node, context); + } + + protected R visitDropFunction(CreateFunction node, C context) + { + return visitStatement(node, context); + } + + protected R visitDropFunction(DropFunction node, C context) + { + return visitStatement(node, context); + } + protected R visitInsert(Insert node, C context) { return visitStatement(node, context); @@ -652,21 +667,6 @@ protected R visitShowRoleGrants(ShowRoleGrants node, C context) return visitStatement(node, context); } - protected R visitSetPath(SetPath node, C context) - { - return visitStatement(node, context); - } - - protected R visitPathSpecification(PathSpecification node, C context) - { - return visitNode(node, context); - } - - protected R visitPathElement(PathElement node, C context) - { - return visitNode(node, context); - } - protected R visitTransactionMode(TransactionMode node, C context) { return visitNode(node, context); @@ -756,9 +756,4 @@ protected R visitCurrentUser(CurrentUser node, C context) { return visitExpression(node, context); } - - protected R visitCurrentPath(CurrentPath node, C context) - { - return visitExpression(node, context); - } } diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/CreateFunction.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/CreateFunction.java new file mode 100644 index 0000000000000..2031da7b45db7 --- /dev/null +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/CreateFunction.java @@ -0,0 +1,143 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.tree; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class CreateFunction + extends Statement +{ + private final QualifiedName functionName; + private final boolean replace; + private final List parameters; + private final String returnType; + private final Optional comment; + private final RoutineCharacteristics characteristics; + private final Expression body; + + public CreateFunction(QualifiedName functionName, boolean replace, List parameters, String returnType, Optional comment, RoutineCharacteristics characteristics, Expression body) + { + this(Optional.empty(), replace, functionName, parameters, returnType, comment, characteristics, body); + } + + public CreateFunction(NodeLocation location, boolean replace, QualifiedName functionName, List parameters, String returnType, Optional comment, RoutineCharacteristics characteristics, Expression body) + { + this(Optional.of(location), replace, functionName, parameters, returnType, comment, characteristics, body); + } + + private CreateFunction(Optional location, boolean replace, QualifiedName functionName, List parameters, String returnType, Optional comment, RoutineCharacteristics characteristics, Expression body) + { + super(location); + this.functionName = requireNonNull(functionName, "functionName is null"); + this.replace = replace; + this.parameters = ImmutableList.copyOf(requireNonNull(parameters, "parameters is null")); + this.returnType = requireNonNull(returnType, "returnType is null"); + this.comment = requireNonNull(comment, "comment is null"); + this.characteristics = requireNonNull(characteristics, "routineCharacteristics is null"); + this.body = requireNonNull(body, "body is null"); + } + + public QualifiedName getFunctionName() + { + return functionName; + } + + public boolean isReplace() + { + return replace; + } + + public List getParameters() + { + return parameters; + } + + public String getReturnType() + { + return returnType; + } + + public Optional getComment() + { + return comment; + } + + public RoutineCharacteristics getCharacteristics() + { + return characteristics; + } + + public Expression getBody() + { + return body; + } + + @Override + public R accept(AstVisitor visitor, C context) + { + return visitor.visitCreateFunction(this, context); + } + + @Override + public List getChildren() + { + return ImmutableList.builder() + .add(body) + .build(); + } + + @Override + public int hashCode() + { + return Objects.hash(functionName, parameters, returnType, comment, characteristics, body); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + CreateFunction o = (CreateFunction) obj; + return Objects.equals(functionName, o.functionName) && + Objects.equals(parameters, o.parameters) && + Objects.equals(returnType, o.returnType) && + Objects.equals(comment, o.comment) && + Objects.equals(characteristics, o.characteristics) && + Objects.equals(body, o.body); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("functionName", functionName) + .add("parameters", parameters) + .add("returnType", returnType) + .add("comment", comment) + .add("characteristics", characteristics) + .add("body", body) + .toString(); + } +} diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/CurrentPath.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/CurrentPath.java deleted file mode 100644 index 1d7c03e7b09ad..0000000000000 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/CurrentPath.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.facebook.presto.sql.tree; - -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -public class CurrentPath - extends Expression -{ - public CurrentPath(NodeLocation location) - { - this(Optional.of(location)); - } - - private CurrentPath(Optional location) - { - super(location); - } - - @Override - public List getChildren() - { - return ImmutableList.of(); - } - - @Override - public R accept(AstVisitor visitor, C context) - { - return visitor.visitCurrentPath(this, context); - } - - @Override - public int hashCode() - { - return Objects.hash(); - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - return true; - } -} diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/DropFunction.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/DropFunction.java new file mode 100644 index 0000000000000..5522c29d2d99b --- /dev/null +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/DropFunction.java @@ -0,0 +1,107 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.tree; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class DropFunction + extends Statement +{ + private final QualifiedName functionName; + private final Optional> parameterTypes; + private final boolean exists; + + public DropFunction(QualifiedName functionName, Optional> parameterTypes, boolean exists) + { + this(Optional.empty(), functionName, parameterTypes, exists); + } + + public DropFunction(NodeLocation location, QualifiedName functionName, Optional> parameterTypes, boolean exists) + { + this(Optional.of(location), functionName, parameterTypes, exists); + } + + private DropFunction(Optional location, QualifiedName functionName, Optional> parameterTypes, boolean exists) + { + super(location); + this.functionName = requireNonNull(functionName, "functionName is null"); + this.parameterTypes = requireNonNull(parameterTypes, "parameterTypes is null").map(ImmutableList::copyOf); + this.exists = exists; + } + + public QualifiedName getFunctionName() + { + return functionName; + } + + public Optional> getParameterTypes() + { + return parameterTypes; + } + + public boolean isExists() + { + return exists; + } + + @Override + public R accept(AstVisitor visitor, C context) + { + return visitor.visitDropFunction(this, context); + } + + @Override + public List getChildren() + { + return ImmutableList.of(); + } + + @Override + public int hashCode() + { + return Objects.hash(functionName, parameterTypes, exists); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + DropFunction o = (DropFunction) obj; + return Objects.equals(functionName, o.functionName) + && Objects.equals(parameterTypes, o.parameterTypes) + && (exists == o.exists); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("functionName", functionName) + .add("parameterTypes", parameterTypes) + .add("exists", exists) + .toString(); + } +} diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/ExpressionRewriter.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/ExpressionRewriter.java index 8b54f9a0939ea..a266897ced761 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/ExpressionRewriter.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/ExpressionRewriter.java @@ -190,11 +190,6 @@ public Expression rewriteCurrentUser(CurrentUser node, C context, ExpressionTree return rewriteExpression(node, context, treeRewriter); } - public Expression rewriteCurrentPath(CurrentPath node, C context, ExpressionTreeRewriter treeRewriter) - { - return rewriteExpression(node, context, treeRewriter); - } - public Expression rewriteFieldReference(FieldReference node, C context, ExpressionTreeRewriter treeRewriter) { return rewriteExpression(node, context, treeRewriter); diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/ExpressionTreeRewriter.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/ExpressionTreeRewriter.java index 365eb573183e0..f0cc7cbd414c7 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/ExpressionTreeRewriter.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/ExpressionTreeRewriter.java @@ -539,7 +539,7 @@ public Expression visitFunctionCall(FunctionCall node, Context context) if (!sameElements(node.getArguments(), arguments) || !sameElements(rewrittenWindow, node.getWindow()) || !sameElements(filter, node.getFilter())) { - return new FunctionCall(node.getName(), rewrittenWindow, filter, node.getOrderBy().map(orderBy -> rewriteOrderBy(orderBy, context)), node.isDistinct(), arguments); + return new FunctionCall(node.getName(), rewrittenWindow, filter, node.getOrderBy().map(orderBy -> rewriteOrderBy(orderBy, context)), node.isDistinct(), node.isIgnoreNulls(), arguments); } return node; } @@ -883,19 +883,6 @@ protected Expression visitCurrentUser(CurrentUser node, Context context) return node; } - - @Override - protected Expression visitCurrentPath(CurrentPath node, Context context) - { - if (!context.isDefaultRewrite()) { - Expression result = rewriter.rewriteCurrentPath(node, context.get(), ExpressionTreeRewriter.this); - if (result != null) { - return result; - } - } - - return node; - } } public static class Context diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/FunctionCall.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/FunctionCall.java index b692cdb306fc5..2166ef545a3e2 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/FunctionCall.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/FunctionCall.java @@ -29,44 +29,55 @@ public class FunctionCall private final Optional filter; private final Optional orderBy; private final boolean distinct; + private final boolean ignoreNulls; private final List arguments; public FunctionCall(QualifiedName name, List arguments) { - this(Optional.empty(), name, Optional.empty(), Optional.empty(), Optional.empty(), false, arguments); + this(Optional.empty(), name, Optional.empty(), Optional.empty(), Optional.empty(), false, false, arguments); } public FunctionCall(NodeLocation location, QualifiedName name, List arguments) { - this(Optional.of(location), name, Optional.empty(), Optional.empty(), Optional.empty(), false, arguments); + this(Optional.of(location), name, Optional.empty(), Optional.empty(), Optional.empty(), false, false, arguments); } public FunctionCall(QualifiedName name, boolean distinct, List arguments) { - this(Optional.empty(), name, Optional.empty(), Optional.empty(), Optional.empty(), distinct, arguments); + this(Optional.empty(), name, Optional.empty(), Optional.empty(), Optional.empty(), distinct, false, arguments); } public FunctionCall(QualifiedName name, boolean distinct, List arguments, Optional filter) { - this(Optional.empty(), name, Optional.empty(), filter, Optional.empty(), distinct, arguments); + this(Optional.empty(), name, Optional.empty(), filter, Optional.empty(), distinct, false, arguments); } - public FunctionCall(QualifiedName name, Optional window, boolean distinct, List arguments) + public FunctionCall(QualifiedName name, Optional window, boolean distinct, boolean ignoreNulls, List arguments) { - this(Optional.empty(), name, window, Optional.empty(), Optional.empty(), distinct, arguments); + this(Optional.empty(), name, window, Optional.empty(), Optional.empty(), distinct, ignoreNulls, arguments); } public FunctionCall(QualifiedName name, Optional window, Optional filter, Optional orderBy, boolean distinct, List arguments) { - this(Optional.empty(), name, window, filter, orderBy, distinct, arguments); + this(Optional.empty(), name, window, filter, orderBy, distinct, false, arguments); + } + + public FunctionCall(QualifiedName name, Optional window, Optional filter, Optional orderBy, boolean distinct, boolean ignoreNulls, List arguments) + { + this(Optional.empty(), name, window, filter, orderBy, distinct, ignoreNulls, arguments); } public FunctionCall(NodeLocation location, QualifiedName name, Optional window, Optional filter, Optional orderBy, boolean distinct, List arguments) { - this(Optional.of(location), name, window, filter, orderBy, distinct, arguments); + this(Optional.of(location), name, window, filter, orderBy, distinct, false, arguments); } - private FunctionCall(Optional location, QualifiedName name, Optional window, Optional filter, Optional orderBy, boolean distinct, List arguments) + public FunctionCall(NodeLocation location, QualifiedName name, Optional window, Optional filter, Optional orderBy, boolean distinct, boolean ignoreNulls, List arguments) + { + this(Optional.of(location), name, window, filter, orderBy, distinct, ignoreNulls, arguments); + } + + private FunctionCall(Optional location, QualifiedName name, Optional window, Optional filter, Optional orderBy, boolean distinct, boolean ignoreNulls, List arguments) { super(location); requireNonNull(name, "name is null"); @@ -80,6 +91,7 @@ private FunctionCall(Optional location, QualifiedName name, Option this.filter = filter; this.orderBy = orderBy; this.distinct = distinct; + this.ignoreNulls = ignoreNulls; this.arguments = arguments; } @@ -103,6 +115,11 @@ public boolean isDistinct() return distinct; } + public boolean isIgnoreNulls() + { + return ignoreNulls; + } + public List getArguments() { return arguments; @@ -145,12 +162,13 @@ public boolean equals(Object obj) Objects.equals(filter, o.filter) && Objects.equals(orderBy, o.orderBy) && Objects.equals(distinct, o.distinct) && + Objects.equals(ignoreNulls, o.ignoreNulls) && Objects.equals(arguments, o.arguments); } @Override public int hashCode() { - return Objects.hash(name, distinct, window, filter, orderBy, arguments); + return Objects.hash(name, distinct, ignoreNulls, window, filter, orderBy, arguments); } } diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/PathElement.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/PathElement.java deleted file mode 100644 index d8363bb4b0bd1..0000000000000 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/PathElement.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sql.tree; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -public final class PathElement - extends Node -{ - private final Optional catalog; - private final Identifier schema; - - public PathElement(NodeLocation location, Identifier schema) - { - this(Optional.of(location), Optional.empty(), schema); - } - - @VisibleForTesting - public PathElement(Optional catalog, Identifier schema) - { - this(Optional.empty(), catalog, schema); - } - - public PathElement(NodeLocation location, Identifier catalog, Identifier schema) - { - this(Optional.of(location), Optional.of(catalog), schema); - } - - private PathElement(Optional location, Optional catalog, Identifier schema) - { - super(location); - this.catalog = requireNonNull(catalog, "catalog is null"); - this.schema = requireNonNull(schema, "schema is null"); - } - - public Optional getCatalog() - { - return catalog; - } - - public Identifier getSchema() - { - return schema; - } - - @Override - public R accept(AstVisitor visitor, C context) - { - return visitor.visitPathElement(this, context); - } - - @Override - public List getChildren() - { - return ImmutableList.of(); - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - PathElement o = (PathElement) obj; - return Objects.equals(schema, o.schema) && - Objects.equals(catalog, o.catalog); - } - - @Override - public int hashCode() - { - return Objects.hash(catalog, schema); - } - - @Override - public String toString() - { - if (catalog.isPresent()) { - return format("%s.%s", catalog.get().toString(), schema.toString()); - } - return schema.toString(); - } -} diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/PathSpecification.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/PathSpecification.java deleted file mode 100644 index b3b264bd7bfc8..0000000000000 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/PathSpecification.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sql.tree; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -public final class PathSpecification - extends Node -{ - private List path; - - public PathSpecification(NodeLocation location, List path) - { - this(Optional.of(location), path); - } - - @VisibleForTesting - public PathSpecification(Optional location, List path) - { - super(location); - this.path = ImmutableList.copyOf(requireNonNull(path, "path is null")); - } - - public List getPath() - { - return path; - } - - @Override - public R accept(AstVisitor visitor, C context) - { - return visitor.visitPathSpecification(this, context); - } - - @Override - public List getChildren() - { - return path; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - PathSpecification o = (PathSpecification) obj; - return Objects.equals(path, o.path); - } - - @Override - public int hashCode() - { - return Objects.hash(path); - } - - @Override - public String toString() - { - return Joiner.on(", ").join(path); - } -} diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/RoutineCharacteristics.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/RoutineCharacteristics.java new file mode 100644 index 0000000000000..1c573118f8519 --- /dev/null +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/RoutineCharacteristics.java @@ -0,0 +1,124 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.tree; + +import java.util.Objects; +import java.util.Optional; + +import static com.facebook.presto.sql.tree.RoutineCharacteristics.Determinism.DETERMINISTIC; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.Determinism.NOT_DETERMINISTIC; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.Language.SQL; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT; +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class RoutineCharacteristics +{ + public enum Language + { + SQL; + } + + public enum Determinism + { + DETERMINISTIC, + NOT_DETERMINISTIC; + } + + public enum NullCallClause + { + RETURNS_NULL_ON_NULL_INPUT, + CALLED_ON_NULL_INPUT; + } + + private final Language language; + private final Determinism determinism; + private final NullCallClause nullCallClause; + + public RoutineCharacteristics( + Optional language, + Optional determinism, + Optional nullCallClause) + { + this(language.orElse(SQL), + determinism.orElse(NOT_DETERMINISTIC), + nullCallClause.orElse(CALLED_ON_NULL_INPUT)); + } + + public RoutineCharacteristics( + Language language, + Determinism determinism, + NullCallClause nullCallClause) + { + this.language = requireNonNull(language, "language is null"); + this.determinism = requireNonNull(determinism, "determinism is null"); + this.nullCallClause = requireNonNull(nullCallClause, "nullCallClause is null"); + } + + public Language getLanguage() + { + return language; + } + + public Determinism getDeterminism() + { + return determinism; + } + + public NullCallClause getNullCallClause() + { + return nullCallClause; + } + + public boolean isDeterministic() + { + return determinism == DETERMINISTIC; + } + + public boolean isCalledOnNullInput() + { + return nullCallClause == CALLED_ON_NULL_INPUT; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RoutineCharacteristics that = (RoutineCharacteristics) o; + return language == that.language + && determinism == that.determinism + && nullCallClause == that.nullCallClause; + } + + @Override + public int hashCode() + { + return Objects.hash(language, determinism, nullCallClause); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("language", language) + .add("determinism", determinism) + .add("nullCallClause", nullCallClause) + .toString(); + } +} diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/SetPath.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/SetPath.java deleted file mode 100644 index 221d913c27159..0000000000000 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/SetPath.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sql.tree; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class SetPath - extends Statement -{ - private final PathSpecification pathSpecification; - - public SetPath(PathSpecification pathSpecification) - { - this(Optional.empty(), pathSpecification); - } - - public SetPath(NodeLocation location, PathSpecification pathSpecification) - { - this(Optional.of(location), pathSpecification); - } - - private SetPath(Optional location, PathSpecification pathSpecification) - { - super(location); - this.pathSpecification = requireNonNull(pathSpecification, "path is null"); - } - - public PathSpecification getPathSpecification() - { - return pathSpecification; - } - - @Override - public R accept(AstVisitor visitor, C context) - { - return visitor.visitSetPath(this, context); - } - - @Override - public List getChildren() - { - return ImmutableList.of(pathSpecification); - } - - @Override - public int hashCode() - { - return Objects.hash(pathSpecification); - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - SetPath o = (SetPath) obj; - return Objects.equals(pathSpecification, o.pathSpecification); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("pathSpecification", pathSpecification) - .toString(); - } -} diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/SqlParameterDeclaration.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/SqlParameterDeclaration.java new file mode 100644 index 0000000000000..484a443625d2b --- /dev/null +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/SqlParameterDeclaration.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.tree; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public final class SqlParameterDeclaration +{ + private final Identifier name; + private final String type; + + public SqlParameterDeclaration(Identifier name, String type) + { + this.name = requireNonNull(name, "name is null"); + this.type = requireNonNull(type, "type is null"); + } + + public Identifier getName() + { + return name; + } + + public String getType() + { + return type; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SqlParameterDeclaration that = (SqlParameterDeclaration) o; + return Objects.equals(name, that.name) + && Objects.equals(type, that.type); + } + + @Override + public int hashCode() + { + return Objects.hash(name, type); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("name", name) + .add("type", type) + .toString(); + } +} diff --git a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java index ebbee02ebf0a8..1e9c777db18e6 100644 --- a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java +++ b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java @@ -31,6 +31,7 @@ import com.facebook.presto.sql.tree.ColumnDefinition; import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.ComparisonExpression; +import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateRole; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; @@ -46,6 +47,7 @@ import com.facebook.presto.sql.tree.DescribeOutput; import com.facebook.presto.sql.tree.DoubleLiteral; import com.facebook.presto.sql.tree.DropColumn; +import com.facebook.presto.sql.tree.DropFunction; import com.facebook.presto.sql.tree.DropRole; import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; @@ -87,8 +89,6 @@ import com.facebook.presto.sql.tree.NullLiteral; import com.facebook.presto.sql.tree.OrderBy; import com.facebook.presto.sql.tree.Parameter; -import com.facebook.presto.sql.tree.PathElement; -import com.facebook.presto.sql.tree.PathSpecification; import com.facebook.presto.sql.tree.Prepare; import com.facebook.presto.sql.tree.PrincipalSpecification; import com.facebook.presto.sql.tree.Property; @@ -104,10 +104,10 @@ import com.facebook.presto.sql.tree.RevokeRoles; import com.facebook.presto.sql.tree.Rollback; import com.facebook.presto.sql.tree.Rollup; +import com.facebook.presto.sql.tree.RoutineCharacteristics; import com.facebook.presto.sql.tree.Row; import com.facebook.presto.sql.tree.Select; import com.facebook.presto.sql.tree.SelectItem; -import com.facebook.presto.sql.tree.SetPath; import com.facebook.presto.sql.tree.SetRole; import com.facebook.presto.sql.tree.SetSession; import com.facebook.presto.sql.tree.ShowCatalogs; @@ -122,6 +122,7 @@ import com.facebook.presto.sql.tree.SimpleGroupBy; import com.facebook.presto.sql.tree.SingleColumn; import com.facebook.presto.sql.tree.SortItem; +import com.facebook.presto.sql.tree.SqlParameterDeclaration; import com.facebook.presto.sql.tree.StartTransaction; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.sql.tree.StringLiteral; @@ -135,6 +136,7 @@ import com.facebook.presto.sql.tree.Union; import com.facebook.presto.sql.tree.Unnest; import com.facebook.presto.sql.tree.Values; +import com.facebook.presto.sql.tree.Window; import com.facebook.presto.sql.tree.With; import com.facebook.presto.sql.tree.WithQuery; import com.google.common.collect.ImmutableList; @@ -159,10 +161,16 @@ import static com.facebook.presto.sql.parser.IdentifierSymbol.AT_SIGN; import static com.facebook.presto.sql.parser.IdentifierSymbol.COLON; import static com.facebook.presto.sql.testing.TreeAssertions.assertFormattedSql; +import static com.facebook.presto.sql.tree.ArithmeticBinaryExpression.Operator.DIVIDE; import static com.facebook.presto.sql.tree.ArithmeticUnaryExpression.negative; import static com.facebook.presto.sql.tree.ArithmeticUnaryExpression.positive; import static com.facebook.presto.sql.tree.ComparisonExpression.Operator.GREATER_THAN; import static com.facebook.presto.sql.tree.ComparisonExpression.Operator.LESS_THAN; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.Determinism.DETERMINISTIC; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.Determinism.NOT_DETERMINISTIC; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.Language.SQL; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT; +import static com.facebook.presto.sql.tree.RoutineCharacteristics.NullCallClause.RETURNS_NULL_ON_NULL_INPUT; import static com.facebook.presto.sql.tree.SortItem.NullOrdering.UNDEFINED; import static com.facebook.presto.sql.tree.SortItem.Ordering.ASCENDING; import static com.facebook.presto.sql.tree.SortItem.Ordering.DESCENDING; @@ -578,8 +586,8 @@ public void testPrecedenceAndAssociativity() new LongLiteral("2")), new LongLiteral("3"))); - assertExpression("1 / 2 / 3", new ArithmeticBinaryExpression(ArithmeticBinaryExpression.Operator.DIVIDE, - new ArithmeticBinaryExpression(ArithmeticBinaryExpression.Operator.DIVIDE, + assertExpression("1 / 2 / 3", new ArithmeticBinaryExpression(DIVIDE, + new ArithmeticBinaryExpression(DIVIDE, new LongLiteral("1"), new LongLiteral("2")), new LongLiteral("3"))); @@ -1331,6 +1339,22 @@ public void testDropView() assertStatement("DROP VIEW IF EXISTS a.b.c", new DropView(QualifiedName.of("a", "b", "c"), true)); } + @Test + public void testDropFunction() + { + assertStatement("DROP FUNCTION a", new DropFunction(QualifiedName.of("a"), Optional.empty(), false)); + assertStatement("DROP FUNCTION a.b", new DropFunction(QualifiedName.of("a", "b"), Optional.empty(), false)); + assertStatement("DROP FUNCTION a.b.c", new DropFunction(QualifiedName.of("a", "b", "c"), Optional.empty(), false)); + + assertStatement("DROP FUNCTION a()", new DropFunction(QualifiedName.of("a"), Optional.of(ImmutableList.of()), false)); + assertStatement("DROP FUNCTION a.b()", new DropFunction(QualifiedName.of("a", "b"), Optional.of(ImmutableList.of()), false)); + assertStatement("DROP FUNCTION a.b.c()", new DropFunction(QualifiedName.of("a", "b", "c"), Optional.of(ImmutableList.of()), false)); + + assertStatement("DROP FUNCTION IF EXISTS a.b.c(int)", new DropFunction(QualifiedName.of("a", "b", "c"), Optional.of(ImmutableList.of("int")), true)); + assertStatement("DROP FUNCTION IF EXISTS a.b.c(bigint, double)", new DropFunction(QualifiedName.of("a", "b", "c"), Optional.of(ImmutableList.of("bigint", "double")), true)); + assertStatement("DROP FUNCTION IF EXISTS a.b.c(ARRAY(string), MAP(int,double))", new DropFunction(QualifiedName.of("a", "b", "c"), Optional.of(ImmutableList.of("ARRAY(string)", "MAP(int,double)")), true)); + } + @Test public void testInsertInto() { @@ -1416,6 +1440,62 @@ public void testCreateView() assertStatement("CREATE VIEW \"awesome schema\".\"awesome view\" AS SELECT * FROM t", new CreateView(QualifiedName.of("awesome schema", "awesome view"), query, false)); } + @Test + public void testCreateFunction() + { + assertStatement( + "CREATE FUNCTION tan (x double)\n" + + "RETURNS double\n" + + "COMMENT 'tangent trigonometric function'\n" + + "LANGUAGE SQL\n" + + "DETERMINISTIC\n" + + "RETURNS NULL ON NULL INPUT\n" + + "RETURN sin(x) / cos(x)", + new CreateFunction( + QualifiedName.of("tan"), + false, + ImmutableList.of(new SqlParameterDeclaration(identifier("x"), "double")), + "double", + Optional.of("tangent trigonometric function"), + new RoutineCharacteristics(SQL, DETERMINISTIC, RETURNS_NULL_ON_NULL_INPUT), + new ArithmeticBinaryExpression( + DIVIDE, + new FunctionCall(QualifiedName.of("sin"), ImmutableList.of(identifier("x"))), + new FunctionCall(QualifiedName.of("cos"), ImmutableList.of(identifier("x")))))); + + CreateFunction createFunctionRand = new CreateFunction( + QualifiedName.of("dev", "testing", "rand"), + true, + ImmutableList.of(), + "double", + Optional.empty(), + new RoutineCharacteristics(SQL, NOT_DETERMINISTIC, CALLED_ON_NULL_INPUT), + new FunctionCall(QualifiedName.of("rand"), ImmutableList.of())); + assertStatement( + "CREATE OR REPLACE FUNCTION dev.testing.rand ()\n" + + "RETURNS double\n" + + "LANGUAGE SQL\n" + + "NOT DETERMINISTIC\n" + + "CALLED ON NULL INPUT\n" + + "RETURN rand()", + createFunctionRand); + assertStatement( + "CREATE OR REPLACE FUNCTION dev.testing.rand ()\n" + + "RETURNS double\n" + + "RETURN rand()", + createFunctionRand); + + assertInvalidStatement( + "CREATE FUNCTION dev.testing.rand () RETURNS double LANGUAGE SQL LANGUAGE SQL RETURN rand()", + "Duplicate language clause: SQL"); + assertInvalidStatement( + "CREATE FUNCTION dev.testing.rand () RETURNS double DETERMINISTIC DETERMINISTIC RETURN rand()", + "Duplicate determinism characteristics: DETERMINISTIC"); + assertInvalidStatement( + "CREATE FUNCTION dev.testing.rand () RETURNS double CALLED ON NULL INPUT CALLED ON NULL INPUT RETURN rand()", + "Duplicate null-call clause: CALLEDONNULLINPUT"); + } + @Test public void testGrant() { @@ -1520,42 +1600,6 @@ public void testShowRoleGrants() new ShowRoleGrants(Optional.of(new Identifier("catalog")))); } - @Test - public void testSetPath() - { - assertStatement("SET PATH iLikeToEat.apples, andBananas", - new SetPath(new PathSpecification(Optional.empty(), ImmutableList.of( - new PathElement(Optional.of(new Identifier("iLikeToEat")), new Identifier("apples")), - new PathElement(Optional.empty(), new Identifier("andBananas")))))); - - assertStatement("SET PATH \"schemas,with\".\"grammar.in\", \"their!names\"", - new SetPath(new PathSpecification(Optional.empty(), ImmutableList.of( - new PathElement(Optional.of(new Identifier("schemas,with")), new Identifier("grammar.in")), - new PathElement(Optional.empty(), new Identifier("their!names")))))); - - assertStatement("SET PATH \"\"", - new SetPath(new PathSpecification(Optional.empty(), ImmutableList.of( - new PathElement(Optional.empty(), new Identifier("")))))); - - try { - assertStatement("SET PATH one.too.many, qualifiers", - new SetPath(new PathSpecification(Optional.empty(), ImmutableList.of( - new PathElement(Optional.empty(), new Identifier("dummyValue")))))); - fail(); - } - catch (RuntimeException e) { - //expected - schema can only be qualified by catalog - } - - try { - SQL_PARSER.createStatement("SET PATH ", new ParsingOptions()); - fail(); - } - catch (RuntimeException e) { - //expected - some form of parameter is required - } - } - @Test public void testWith() { @@ -2049,6 +2093,7 @@ public void testAggregationFilter() new LongLiteral("4"))), Optional.empty(), false, + false, ImmutableList.of(new Identifier("x")))), Optional.empty(), Optional.empty(), @@ -2305,6 +2350,29 @@ public void testSetRole() assertStatement("SET ROLE \"role\"", new SetRole(SetRole.Type.ROLE, Optional.of(new Identifier("role")))); } + @Test + public void testNullTreatment() + { + assertExpression("lead(x, 1) ignore nulls over()", + new FunctionCall( + QualifiedName.of("lead"), + Optional.of(new Window(ImmutableList.of(), Optional.empty(), Optional.empty())), + Optional.empty(), + Optional.empty(), + false, + true, + ImmutableList.of(new Identifier("x"), new LongLiteral("1")))); + assertExpression("lead(x, 1) respect nulls over()", + new FunctionCall( + QualifiedName.of("lead"), + Optional.of(new Window(ImmutableList.of(), Optional.empty(), Optional.empty())), + Optional.empty(), + Optional.empty(), + false, + false, + ImmutableList.of(new Identifier("x"), new LongLiteral("1")))); + } + private static void assertCast(String type) { assertCast(type, type); @@ -2336,6 +2404,19 @@ private static void assertParsed(String input, Node expected, Node parsed) } } + private static void assertInvalidStatement(String expression, String expectedErrorMessageRegex) + { + try { + Statement result = SQL_PARSER.createStatement(expression, ParsingOptions.builder().build()); + fail("Expected to throw ParsingException for input:[" + expression + "], but got: " + result); + } + catch (ParsingException e) { + if (!e.getErrorMessage().matches(expectedErrorMessageRegex)) { + fail(format("Expected error message to match '%s', but was: '%s'", expectedErrorMessageRegex, e.getErrorMessage())); + } + } + } + private static void assertInvalidExpression(String expression, String expectedErrorMessageRegex) { try { diff --git a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParserErrorHandling.java b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParserErrorHandling.java index 3773156a3e2b7..b17f32217d401 100644 --- a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParserErrorHandling.java +++ b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParserErrorHandling.java @@ -82,9 +82,9 @@ public Object[][] getStatements() {"select foo(DISTINCT ,1)", "line 1:21: mismatched input ','. Expecting: "}, {"CREATE TABLE foo () AS (VALUES 1)", - "line 1:19: mismatched input ')'. Expecting: 'OR', 'ROLE', 'SCHEMA', 'TABLE', 'VIEW'"}, + "line 1:19: mismatched input ')'. Expecting: 'FUNCTION', 'OR', 'ROLE', 'SCHEMA', 'TABLE', 'VIEW'"}, {"CREATE TABLE foo (*) AS (VALUES 1)", - "line 1:19: mismatched input '*'. Expecting: 'OR', 'ROLE', 'SCHEMA', 'TABLE', 'VIEW'"}, + "line 1:19: mismatched input '*'. Expecting: 'FUNCTION', 'OR', 'ROLE', 'SCHEMA', 'TABLE', 'VIEW'"}, {"SELECT grouping(a+2) FROM (VALUES (1)) AS t (a) GROUP BY a+2", "line 1:18: mismatched input '+'. Expecting: ')', ','"}, {"SELECT x() over (ROWS select) FROM t", diff --git a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestStatementBuilder.java b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestStatementBuilder.java index d39236876c578..b5c1fea0f4532 100644 --- a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestStatementBuilder.java +++ b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestStatementBuilder.java @@ -94,6 +94,8 @@ public void testStatementBuilder() ", sum(salary) over (partition by depname order by salary rows between current row and 3 following)\n" + ", sum(salary) over (partition by depname range unbounded preceding)\n" + ", sum(salary) over (rows between 2 preceding and unbounded following)\n" + + ", lag(salary, 1) ignore nulls over (partition by depname)\n" + + ", lag(salary, 1) respect nulls over (partition by depname)\n" + "from emp"); printStatement("" + diff --git a/presto-password-authenticators/pom.xml b/presto-password-authenticators/pom.xml index 9b9f073e5a9d6..6bcda1e3c1ab7 100644 --- a/presto-password-authenticators/pom.xml +++ b/presto-password-authenticators/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-password-authenticators @@ -18,17 +18,17 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift configuration - io.airlift + com.facebook.airlift log @@ -85,7 +85,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-password-authenticators/src/main/java/com/facebook/presto/password/LdapAuthenticator.java b/presto-password-authenticators/src/main/java/com/facebook/presto/password/LdapAuthenticator.java index 79b95716365f9..5ee61d68247e5 100644 --- a/presto-password-authenticators/src/main/java/com/facebook/presto/password/LdapAuthenticator.java +++ b/presto-password-authenticators/src/main/java/com/facebook/presto/password/LdapAuthenticator.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.password; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.security.AccessDeniedException; import com.facebook.presto.spi.security.BasicPrincipal; import com.facebook.presto.spi.security.PasswordAuthenticator; @@ -22,7 +23,6 @@ import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.UncheckedExecutionException; -import io.airlift.log.Logger; import javax.inject.Inject; import javax.naming.AuthenticationException; diff --git a/presto-password-authenticators/src/main/java/com/facebook/presto/password/LdapAuthenticatorFactory.java b/presto-password-authenticators/src/main/java/com/facebook/presto/password/LdapAuthenticatorFactory.java index 678f6bbb83f90..c81348badf9c7 100644 --- a/presto-password-authenticators/src/main/java/com/facebook/presto/password/LdapAuthenticatorFactory.java +++ b/presto-password-authenticators/src/main/java/com/facebook/presto/password/LdapAuthenticatorFactory.java @@ -13,16 +13,16 @@ */ package com.facebook.presto.password; +import com.facebook.airlift.bootstrap.Bootstrap; import com.facebook.presto.spi.security.PasswordAuthenticator; import com.facebook.presto.spi.security.PasswordAuthenticatorFactory; import com.google.inject.Injector; import com.google.inject.Scopes; -import io.airlift.bootstrap.Bootstrap; import java.util.Map; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static com.google.common.base.Throwables.throwIfUnchecked; -import static io.airlift.configuration.ConfigBinder.configBinder; public class LdapAuthenticatorFactory implements PasswordAuthenticatorFactory diff --git a/presto-password-authenticators/src/main/java/com/facebook/presto/password/LdapConfig.java b/presto-password-authenticators/src/main/java/com/facebook/presto/password/LdapConfig.java index 426d299f154d9..379ad89442570 100644 --- a/presto-password-authenticators/src/main/java/com/facebook/presto/password/LdapConfig.java +++ b/presto-password-authenticators/src/main/java/com/facebook/presto/password/LdapConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.password; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import io.airlift.units.Duration; import javax.validation.constraints.NotNull; diff --git a/presto-password-authenticators/src/test/java/com/facebook/presto/password/TestLdapConfig.java b/presto-password-authenticators/src/test/java/com/facebook/presto/password/TestLdapConfig.java index eaebc806a36fa..2651f1d283281 100644 --- a/presto-password-authenticators/src/test/java/com/facebook/presto/password/TestLdapConfig.java +++ b/presto-password-authenticators/src/test/java/com/facebook/presto/password/TestLdapConfig.java @@ -24,11 +24,11 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static io.airlift.testing.ValidationAssertions.assertFailsValidation; -import static io.airlift.testing.ValidationAssertions.assertValidates; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.testing.ValidationAssertions.assertFailsValidation; +import static com.facebook.airlift.testing.ValidationAssertions.assertValidates; public class TestLdapConfig { diff --git a/presto-pinot-toolkit/pom.xml b/presto-pinot-toolkit/pom.xml new file mode 100644 index 0000000000000..47511efb5e81b --- /dev/null +++ b/presto-pinot-toolkit/pom.xml @@ -0,0 +1,188 @@ + + + 4.0.0 + + com.facebook.presto + presto-root + 0.231-SNAPSHOT + + + presto-pinot-toolkit + presto-pinot-toolkit + Presto - Pinot Toolkit library + jar + + + ${project.parent.basedir} + + + + + com.facebook.presto.pinot + pinot-driver + + + + com.google.code.findbugs + jsr305 + + + + com.facebook.airlift + stats + + + + org.weakref + jmxutils + + + + com.facebook.airlift + bootstrap + + + + com.facebook.airlift + json + + + + com.facebook.airlift + log + + + + com.facebook.airlift + configuration + + + + io.airlift + units + provided + + + + com.google.guava + guava + + + + com.google.inject + guice + + + + javax.validation + validation-api + + + + javax.inject + javax.inject + + + + + com.facebook.presto + presto-spi + provided + + + + org.openjdk.jol + jol-core + provided + + + + io.airlift + slice + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + com.fasterxml.jackson.core + jackson-databind + + + + joda-time + joda-time + + + + + com.facebook.presto + presto-main + test + + + + com.facebook.presto + presto-expressions + + + + com.facebook.presto + presto-main + test-jar + test + + + + org.testng + testng + test + + + + com.facebook.airlift + testing + test + + + + com.facebook.airlift + http-server + test + + + + com.facebook.airlift + node + test + + + + com.facebook.presto + presto-parser + test + + + + javax.servlet + javax.servlet-api + test + + + + com.facebook.airlift + http-client + + + + com.facebook.airlift + concurrent + + + diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/ForPinot.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/ForPinot.java new file mode 100644 index 0000000000000..e570b18583727 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/ForPinot.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import javax.inject.Qualifier; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@Qualifier +public @interface ForPinot +{ +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotBrokerPageSource.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotBrokerPageSource.java new file mode 100644 index 0000000000000..62f94d5fdba09 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotBrokerPageSource.java @@ -0,0 +1,420 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.http.client.Request; +import com.facebook.presto.pinot.query.PinotQueryGenerator.GeneratedPql; +import com.facebook.presto.spi.ConnectorPageSource; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.PageBuilder; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.BooleanType; +import com.facebook.presto.spi.type.DecimalType; +import com.facebook.presto.spi.type.DoubleType; +import com.facebook.presto.spi.type.FixedWidthType; +import com.facebook.presto.spi.type.IntegerType; +import com.facebook.presto.spi.type.SmallintType; +import com.facebook.presto.spi.type.TimestampType; +import com.facebook.presto.spi.type.TinyintType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.VarcharType; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; + +import java.io.IOException; +import java.net.URI; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_DECODE_ERROR; +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_EXCEPTION; +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_INSUFFICIENT_SERVER_RESPONSE; +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNEXPECTED_RESPONSE; +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNSUPPORTED_COLUMN_TYPE; +import static com.facebook.presto.pinot.PinotUtils.doWithRetries; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static java.lang.Boolean.parseBoolean; +import static java.lang.Long.parseLong; +import static java.util.Objects.requireNonNull; + +public class PinotBrokerPageSource + implements ConnectorPageSource +{ + private static final String REQUEST_PAYLOAD_TEMPLATE = "{\"pql\" : \"%s\" }"; + private static final String QUERY_URL_TEMPLATE = "http://%s/query"; + + private static final String PINOT_INFINITY = "∞"; + private static final String PINOT_POSITIVE_INFINITY = "+" + PINOT_INFINITY; + private static final String PINOT_NEGATIVE_INFINITY = "-" + PINOT_INFINITY; + + private static final Double PRESTO_INFINITY = Double.POSITIVE_INFINITY; + private static final Double PRESTO_NEGATIVE_INFINITY = Double.NEGATIVE_INFINITY; + + private final GeneratedPql brokerPql; + private final PinotConfig pinotConfig; + private final List columnHandles; + private final PinotClusterInfoFetcher clusterInfoFetcher; + private final ConnectorSession session; + private final ObjectMapper objectMapper; + + private boolean finished; + private long readTimeNanos; + private long completedBytes; + + public PinotBrokerPageSource( + PinotConfig pinotConfig, + ConnectorSession session, + GeneratedPql brokerPql, + List columnHandles, + PinotClusterInfoFetcher clusterInfoFetcher, + ObjectMapper objectMapper) + { + this.pinotConfig = requireNonNull(pinotConfig, "pinot config is null"); + this.brokerPql = requireNonNull(brokerPql, "broker is null"); + this.clusterInfoFetcher = requireNonNull(clusterInfoFetcher, "cluster info fetcher is null"); + this.columnHandles = ImmutableList.copyOf(columnHandles); + this.session = requireNonNull(session, "session is null"); + this.objectMapper = requireNonNull(objectMapper, "object mapper is null"); + } + + private static Double parseDouble(String value) + { + try { + return Double.valueOf(value); + } + catch (NumberFormatException ne) { + switch (value) { + case PINOT_INFINITY: + case PINOT_POSITIVE_INFINITY: + return PRESTO_INFINITY; + case PINOT_NEGATIVE_INFINITY: + return PRESTO_NEGATIVE_INFINITY; + } + throw new PinotException(PINOT_DECODE_ERROR, Optional.empty(), "Cannot decode double value from pinot " + value, ne); + } + } + + private void setValue(Type type, BlockBuilder blockBuilder, String value) + { + if (value == null) { + blockBuilder.appendNull(); + return; + } + if (!(type instanceof FixedWidthType) && !(type instanceof VarcharType)) { + throw new PinotException(PINOT_UNSUPPORTED_COLUMN_TYPE, Optional.empty(), "type '" + type + "' not supported"); + } + if (type instanceof FixedWidthType) { + completedBytes += ((FixedWidthType) type).getFixedSize(); + if (type instanceof BigintType) { + type.writeLong(blockBuilder, parseDouble(value).longValue()); + } + else if (type instanceof IntegerType) { + blockBuilder.writeInt(parseDouble(value).intValue()); + } + else if (type instanceof TinyintType) { + blockBuilder.writeByte(parseDouble(value).byteValue()); + } + else if (type instanceof SmallintType) { + blockBuilder.writeShort(parseDouble(value).shortValue()); + } + else if (type instanceof BooleanType) { + type.writeBoolean(blockBuilder, parseBoolean(value)); + } + else if (type instanceof DecimalType || type instanceof DoubleType) { + type.writeDouble(blockBuilder, parseDouble(value)); + } + else if (type instanceof TimestampType) { + type.writeLong(blockBuilder, parseLong(value)); + } + else { + throw new PinotException(PINOT_UNSUPPORTED_COLUMN_TYPE, Optional.empty(), "type '" + type + "' not supported"); + } + } + else { + Slice slice = Slices.utf8Slice(value); + blockBuilder.writeBytes(slice, 0, slice.length()).closeEntry(); + completedBytes += slice.length(); + } + } + + private void setValuesForGroupby( + List blockBuilders, + List types, + int numGroupByClause, + JsonNode group, + String[] values) + { + for (int i = 0; i < group.size(); i++) { + setValue(types.get(i), blockBuilders.get(i), group.get(i).asText()); + } + for (int i = 0; i < values.length; i++) { + int metricColumnIndex = i + numGroupByClause; + if (metricColumnIndex < blockBuilders.size()) { + setValue(types.get(metricColumnIndex), blockBuilders.get(metricColumnIndex), values[i]); + } + } + } + + @Override + public long getCompletedBytes() + { + return completedBytes; + } + + @Override + public long getCompletedPositions() + { + return 0; // not available + } + + @Override + public long getReadTimeNanos() + { + return readTimeNanos; + } + + @Override + public boolean isFinished() + { + return finished; + } + + @Override + public Page getNextPage() + { + if (finished) { + return null; + } + + long start = System.nanoTime(); + try { + List expectedTypes = columnHandles.stream() + .map(PinotColumnHandle::getDataType) + .collect(Collectors.toList()); + PageBuilder pageBuilder = new PageBuilder(expectedTypes); + ImmutableList.Builder columnBlockBuilders = ImmutableList.builder(); + ImmutableList.Builder columnTypes = ImmutableList.builder(); + for (int i : brokerPql.getExpectedColumnIndices()) { + if (i == -1) { + continue; + } + BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i); + columnBlockBuilders.add(blockBuilder); + columnTypes.add(expectedTypes.get(i)); + } + + int counter = issuePqlAndPopulate( + brokerPql.getTable(), + brokerPql.getPql(), + brokerPql.getGroupByClauses(), + columnBlockBuilders.build(), + columnTypes.build()); + pageBuilder.declarePositions(counter); + Page page = pageBuilder.build(); + + // TODO: Implement chunking if the result set is ginormous + finished = true; + + return page; + } + finally { + readTimeNanos += System.nanoTime() - start; + } + } + + private int issuePqlAndPopulate( + String table, + String pql, + int numGroupByClause, + List blockBuilders, + List types) + { + return doWithRetries(PinotSessionProperties.getPinotRetryCount(session), (retryNumber) -> { + String queryHost; + Optional rpcService; + if (pinotConfig.getRestProxyUrl() != null) { + queryHost = pinotConfig.getRestProxyUrl(); + rpcService = Optional.ofNullable(pinotConfig.getRestProxyServiceForQuery()); + } + else { + queryHost = clusterInfoFetcher.getBrokerHost(table); + rpcService = Optional.empty(); + } + Request.Builder builder = Request.Builder + .preparePost() + .setUri(URI.create(String.format(QUERY_URL_TEMPLATE, queryHost))); + String body = clusterInfoFetcher.doHttpActionWithHeaders(builder, Optional.of(String.format(REQUEST_PAYLOAD_TEMPLATE, pql)), rpcService); + + return populateFromPqlResults(pql, numGroupByClause, blockBuilders, types, body); + }); + } + + @VisibleForTesting + public int populateFromPqlResults( + String pql, + int numGroupByClause, + List blockBuilders, + List types, + String body) + { + JsonNode jsonBody; + + try { + jsonBody = objectMapper.readTree(body); + } + catch (IOException e) { + throw new PinotException(PINOT_UNEXPECTED_RESPONSE, Optional.of(pql), "Couldn't parse response", e); + } + + JsonNode numServersResponded = jsonBody.get("numServersResponded"); + JsonNode numServersQueried = jsonBody.get("numServersQueried"); + + if (numServersQueried == null || numServersResponded == null || numServersQueried.asInt() > numServersResponded.asInt()) { + throw new PinotException( + PINOT_INSUFFICIENT_SERVER_RESPONSE, + Optional.of(pql), + String.format("Only %s out of %s servers responded for query %s", numServersResponded.asInt(), numServersQueried.asInt(), pql)); + } + + JsonNode exceptions = jsonBody.get("exceptions"); + if (exceptions != null && exceptions.isArray() && exceptions.size() > 0) { + // Pinot is known to return exceptions with benign errorcodes like 200 + // so we treat any exception as an error + throw new PinotException( + PINOT_EXCEPTION, + Optional.of(pql), + String.format("Query %s encountered exception %s", pql, exceptions.get(0))); + } + + JsonNode aggregationResults = jsonBody.get("aggregationResults"); + JsonNode selectionResults = jsonBody.get("selectionResults"); + + int rowCount; + if (aggregationResults != null && aggregationResults.isArray()) { + // This is map is populated only when we have multiple aggregates with a group by + checkState(aggregationResults.size() >= 1, "Expected at least one metric to be present"); + Map groupToValue = aggregationResults.size() == 1 || numGroupByClause == 0 ? null : new HashMap<>(); + rowCount = 0; + String[] singleAggregation = new String[1]; + Boolean seenGroupByResult = null; + for (int aggregationIndex = 0; aggregationIndex < aggregationResults.size(); aggregationIndex++) { + JsonNode result = aggregationResults.get(aggregationIndex); + + JsonNode metricValuesForEachGroup = result.get("groupByResult"); + + if (metricValuesForEachGroup != null) { + checkState(seenGroupByResult == null || seenGroupByResult); + seenGroupByResult = true; + checkState(numGroupByClause > 0, "Expected having non zero group by clauses"); + JsonNode groupByColumns = checkNotNull(result.get("groupByColumns"), "groupByColumns missing in %s", pql); + if (groupByColumns.size() != numGroupByClause) { + throw new PinotException( + PINOT_UNEXPECTED_RESPONSE, + Optional.of(pql), + String.format("Expected %d gby columns but got %s instead from pinot", numGroupByClause, groupByColumns)); + } + + // group by aggregation + for (int groupByIndex = 0; groupByIndex < metricValuesForEachGroup.size(); groupByIndex++) { + JsonNode row = metricValuesForEachGroup.get(groupByIndex); + JsonNode group = row.get("group"); + if (group == null || !group.isArray() || group.size() != numGroupByClause) { + throw new PinotException( + PINOT_UNEXPECTED_RESPONSE, + Optional.of(pql), + String.format("Expected %d group by columns but got only a group of size %d (%s)", numGroupByClause, group.size(), group)); + } + if (groupToValue == null) { + singleAggregation[0] = row.get("value").asText(); + setValuesForGroupby(blockBuilders, types, numGroupByClause, group, singleAggregation); + rowCount++; + } + else { + groupToValue.computeIfAbsent(group, (ignored) -> new String[aggregationResults.size()])[aggregationIndex] = row.get("value").asText(); + } + } + } + else { + checkState(seenGroupByResult == null || !seenGroupByResult); + seenGroupByResult = false; + // simple aggregation + // TODO: Validate that this is expected semantically + checkState(numGroupByClause == 0, "Expected no group by columns in pinot"); + setValue(types.get(aggregationIndex), blockBuilders.get(aggregationIndex), result.get("value").asText()); + rowCount = 1; + } + } + + if (groupToValue != null) { + checkState(rowCount == 0, "Row count shouldn't have changed from zero"); + groupToValue.forEach((group, values) -> setValuesForGroupby(blockBuilders, types, numGroupByClause, group, values)); + rowCount = groupToValue.size(); + } + } + else if (selectionResults != null) { + JsonNode columns = selectionResults.get("columns"); + JsonNode results = selectionResults.get("results"); + if (columns == null || results == null || !columns.isArray() || !results.isArray() || columns.size() != blockBuilders.size()) { + throw new PinotException( + PINOT_UNEXPECTED_RESPONSE, + Optional.of(pql), + String.format("Columns and results expected for %s, expected %d columns but got %d", pql, blockBuilders.size(), columns == null ? 0 : columns.size())); + } + for (int rowNumber = 0; rowNumber < results.size(); ++rowNumber) { + JsonNode result = results.get(rowNumber); + if (result == null || result.size() != blockBuilders.size()) { + throw new PinotException( + PINOT_UNEXPECTED_RESPONSE, + Optional.of(pql), + String.format("Expected row of %d columns", blockBuilders.size())); + } + for (int columnNumber = 0; columnNumber < blockBuilders.size(); columnNumber++) { + setValue(types.get(columnNumber), blockBuilders.get(columnNumber), result.get(columnNumber).asText()); + } + } + rowCount = results.size(); + } + else { + throw new PinotException( + PINOT_UNEXPECTED_RESPONSE, + Optional.of(pql), + "Expected one of aggregationResults or selectionResults to be present"); + } + + checkState(rowCount >= 0, "Expected row count to be initialized"); + return rowCount; + } + + @Override + public long getSystemMemoryUsage() + { + return 0; + } + + @Override + public void close() + { + finished = true; + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotClusterInfoFetcher.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotClusterInfoFetcher.java new file mode 100644 index 0000000000000..8189029dd5997 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotClusterInfoFetcher.java @@ -0,0 +1,430 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.StaticBodyGenerator; +import com.facebook.airlift.http.client.StringResponseHandler; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecBinder; +import com.facebook.airlift.log.Logger; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ticker; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.net.HttpHeaders; +import org.apache.pinot.common.data.Schema; + +import javax.inject.Inject; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static com.facebook.airlift.http.client.StringResponseHandler.createStringResponseHandler; +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_INVALID_CONFIGURATION; +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNABLE_TO_FIND_BROKER; +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNEXPECTED_RESPONSE; +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static org.apache.pinot.common.config.TableNameBuilder.extractRawTableName; + +public class PinotClusterInfoFetcher +{ + private static final Logger log = Logger.get(PinotClusterInfoFetcher.class); + private static final String APPLICATION_JSON = "application/json"; + private static final Pattern BROKER_PATTERN = Pattern.compile("Broker_(.*)_(\\d+)"); + + private static final String GET_ALL_TABLES_API_TEMPLATE = "tables"; + private static final String TABLE_INSTANCES_API_TEMPLATE = "tables/%s/instances"; + private static final String TABLE_SCHEMA_API_TEMPLATE = "tables/%s/schema"; + private static final String ROUTING_TABLE_API_TEMPLATE = "debug/routingTable/%s"; + private static final String TIME_BOUNDARY_API_TEMPLATE = "debug/timeBoundary/%s"; + + private final PinotConfig pinotConfig; + private final PinotMetrics pinotMetrics; + private final HttpClient httpClient; + + private final Ticker ticker = Ticker.systemTicker(); + + private final LoadingCache> brokersForTableCache; + + private final JsonCodec tablesJsonCodec; + private final JsonCodec brokersForTableJsonCodec; + private final JsonCodec routingTablesJsonCodec; + private final JsonCodec timeBoundaryJsonCodec; + + @Inject + public PinotClusterInfoFetcher( + PinotConfig pinotConfig, + PinotMetrics pinotMetrics, + @ForPinot HttpClient httpClient, + JsonCodec tablesJsonCodec, + JsonCodec brokersForTableJsonCodec, + JsonCodec routingTablesJsonCodec, + JsonCodec timeBoundaryJsonCodec) + { + this.brokersForTableJsonCodec = requireNonNull(brokersForTableJsonCodec, "brokers for table json codec is null"); + this.routingTablesJsonCodec = requireNonNull(routingTablesJsonCodec, "routing tables json codec is null"); + this.timeBoundaryJsonCodec = requireNonNull(timeBoundaryJsonCodec, "time boundary json codec is null"); + final long cacheExpiryMs = pinotConfig.getMetadataCacheExpiry().roundTo(TimeUnit.MILLISECONDS); + this.tablesJsonCodec = requireNonNull(tablesJsonCodec, "json codec is null"); + + this.pinotConfig = requireNonNull(pinotConfig, "pinotConfig is null"); + this.pinotMetrics = requireNonNull(pinotMetrics, "pinotMetrics is null"); + this.httpClient = requireNonNull(httpClient, "httpClient is null"); + this.brokersForTableCache = CacheBuilder.newBuilder() + .expireAfterWrite(cacheExpiryMs, TimeUnit.MILLISECONDS) + .build((CacheLoader.from(this::getAllBrokersForTable))); + } + + public static JsonCodecBinder addJsonBinders(JsonCodecBinder jsonCodecBinder) + { + jsonCodecBinder.bindJsonCodec(GetTables.class); + jsonCodecBinder.bindJsonCodec(BrokersForTable.InstancesInBroker.class); + jsonCodecBinder.bindJsonCodec(BrokersForTable.class); + jsonCodecBinder.bindJsonCodec(RoutingTables.class); + jsonCodecBinder.bindJsonCodec(RoutingTables.RoutingTableSnapshot.class); + jsonCodecBinder.bindJsonCodec(TimeBoundary.class); + return jsonCodecBinder; + } + + public String doHttpActionWithHeaders( + Request.Builder requestBuilder, + Optional requestBody, + Optional rpcService) + { + requestBuilder = requestBuilder + .setHeader(HttpHeaders.CONTENT_TYPE, APPLICATION_JSON) + .setHeader(HttpHeaders.ACCEPT, APPLICATION_JSON); + if (rpcService.isPresent()) { + requestBuilder + .setHeader(pinotConfig.getCallerHeaderParam(), pinotConfig.getCallerHeaderValue()) + .setHeader(pinotConfig.getServiceHeaderParam(), rpcService.get()); + } + if (requestBody.isPresent()) { + requestBuilder.setBodyGenerator(StaticBodyGenerator.createStaticBodyGenerator(requestBody.get(), StandardCharsets.UTF_8)); + } + pinotConfig.getExtraHttpHeaders().forEach(requestBuilder::setHeader); + Request request = requestBuilder.build(); + + long startTime = ticker.read(); + long duration; + StringResponseHandler.StringResponse response; + try { + response = httpClient.execute(request, createStringResponseHandler()); + } + finally { + duration = ticker.read() - startTime; + } + pinotMetrics.monitorRequest(request, response, duration, TimeUnit.NANOSECONDS); + String responseBody = response.getBody(); + if (PinotUtils.isValidPinotHttpResponseCode(response.getStatusCode())) { + return responseBody; + } + else { + throw new PinotException( + PinotErrorCode.PINOT_HTTP_ERROR, + Optional.empty(), + String.format( + "Unexpected response status: %d for request %s to url %s, with headers %s, full response %s", + response.getStatusCode(), + requestBody.orElse(""), + request.getUri(), + request.getHeaders(), + responseBody)); + } + } + + private String sendHttpGetToController(String path) + { + return doHttpActionWithHeaders( + Request.builder().prepareGet().setUri(URI.create(String.format("http://%s/%s", getControllerUrl(), path))), + Optional.empty(), + Optional.ofNullable(pinotConfig.getControllerRestService())); + } + + private String sendHttpGetToBroker(String table, String path) + { + return doHttpActionWithHeaders( + Request.builder().prepareGet().setUri(URI.create(String.format("http://%s/%s", getBrokerHost(table), path))), + Optional.empty(), + Optional.empty()); + } + + private String getControllerUrl() + { + List controllerUrls = pinotConfig.getControllerUrls(); + if (controllerUrls.isEmpty()) { + throw new PinotException(PINOT_INVALID_CONFIGURATION, Optional.empty(), "No pinot controllers specified"); + } + return controllerUrls.get(ThreadLocalRandom.current().nextInt(controllerUrls.size())); + } + + public static class GetTables + { + private final List tables; + + @JsonCreator + public GetTables(@JsonProperty("tables") List tables) + { + this.tables = tables; + } + + public List getTables() + { + return tables; + } + } + + public List getAllTables() + { + return tablesJsonCodec.fromJson(sendHttpGetToController(GET_ALL_TABLES_API_TEMPLATE)).getTables(); + } + + public Schema getTableSchema(String table) + throws Exception + { + String responseBody = sendHttpGetToController(String.format(TABLE_SCHEMA_API_TEMPLATE, table)); + return Schema.fromString(responseBody); + } + + public static class BrokersForTable + { + public static class InstancesInBroker + { + private final List instances; + + @JsonCreator + public InstancesInBroker(@JsonProperty("instances") List instances) + { + this.instances = instances; + } + + @JsonProperty("instances") + public List getInstances() + { + return instances; + } + } + + private final List brokers; + + @JsonCreator + public BrokersForTable(@JsonProperty("brokers") List brokers) + { + this.brokers = brokers; + } + + @JsonProperty("brokers") + public List getBrokers() + { + return brokers; + } + } + + @VisibleForTesting + List getAllBrokersForTable(String table) + { + String responseBody = sendHttpGetToController(String.format(TABLE_INSTANCES_API_TEMPLATE, table)); + ArrayList brokers = brokersForTableJsonCodec + .fromJson(responseBody) + .getBrokers() + .stream() + .flatMap(broker -> broker.getInstances().stream()) + .distinct() + .map(brokerToParse -> { + Matcher matcher = BROKER_PATTERN.matcher(brokerToParse); + if (matcher.matches() && matcher.groupCount() == 2) { + return matcher.group(1) + ":" + matcher.group(2); + } + else { + throw new PinotException( + PINOT_UNABLE_TO_FIND_BROKER, + Optional.empty(), + String.format("Cannot parse %s in the broker instance", brokerToParse)); + } + }) + .collect(Collectors.toCollection(() -> new ArrayList<>())); + Collections.shuffle(brokers); + return ImmutableList.copyOf(brokers); + } + + public String getBrokerHost(String table) + { + try { + List brokers = brokersForTableCache.get(table); + if (brokers.isEmpty()) { + throw new PinotException(PINOT_UNABLE_TO_FIND_BROKER, Optional.empty(), "No valid brokers found for " + table); + } + return brokers.get(ThreadLocalRandom.current().nextInt(brokers.size())); + } + catch (ExecutionException e) { + Throwable throwable = e.getCause(); + if (throwable instanceof PinotException) { + throw (PinotException) throwable; + } + else { + throw new PinotException(PINOT_UNABLE_TO_FIND_BROKER, Optional.empty(), "Error when getting brokers for table " + table, throwable); + } + } + } + + public static class RoutingTables + { + public static class RoutingTableSnapshot + { + private final String tableName; + private final List>> routingTableEntries; + + @JsonCreator + public RoutingTableSnapshot( + @JsonProperty("tableName") String tableName, + @JsonProperty("routingTableEntries") List>> routingTableEntries) + { + this.tableName = requireNonNull(tableName, "table name is null"); + this.routingTableEntries = requireNonNull(routingTableEntries, "routing table entries is null"); + } + + @JsonProperty("tableName") + public String getTableName() + { + return tableName; + } + + @JsonProperty("routingTableEntries") + public List>> getRoutingTableEntries() + { + return routingTableEntries; + } + } + + private final List routingTableSnapshot; + + @JsonCreator + public RoutingTables(@JsonProperty("routingTableSnapshot") List routingTableSnapshot) + { + this.routingTableSnapshot = routingTableSnapshot; + } + + public List getRoutingTableSnapshot() + { + return routingTableSnapshot; + } + } + + public Map>> getRoutingTableForTable(String tableName) + { + ImmutableMap.Builder>> routingTableMap = ImmutableMap.builder(); + log.debug("Trying to get routingTable for %s from broker", tableName); + String responseBody = sendHttpGetToBroker(tableName, String.format(ROUTING_TABLE_API_TEMPLATE, tableName)); + routingTablesJsonCodec.fromJson(responseBody).getRoutingTableSnapshot().forEach(snapshot -> { + String tableNameWithType = snapshot.getTableName(); + // Response could contain info for tableName that matches the original table by prefix. + // e.g. when table name is "table1", response could contain routingTable for "table1_staging" + if (!tableName.equals(extractRawTableName(tableNameWithType))) { + log.debug("Ignoring routingTable for %s", tableNameWithType); + } + else { + List>> routingTableEntriesList = snapshot.getRoutingTableEntries(); + if (routingTableEntriesList.isEmpty()) { + throw new PinotException( + PINOT_UNEXPECTED_RESPONSE, + Optional.empty(), + String.format("Empty routingTableEntries for %s. RoutingTable: %s", tableName, responseBody)); + } + + // We are given multiple routing tables for a table, each with different segment to host assignments + // We pick one randomly, so that a retry may hit a different server + Map> routingTableEntries = routingTableEntriesList.get(new Random().nextInt(routingTableEntriesList.size())); + ImmutableMap.Builder> routingTableBuilder = ImmutableMap.builder(); + routingTableEntries.forEach((host, segments) -> { + List segmentsCopied = new ArrayList<>(segments); + Collections.shuffle(segmentsCopied); + routingTableBuilder.put(host, ImmutableList.copyOf(segmentsCopied)); + }); + routingTableMap.put(tableNameWithType, routingTableBuilder.build()); + } + }); + return routingTableMap.build(); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("pinotConfig", pinotConfig) + .toString(); + } + + public static class TimeBoundary + { + private final Optional onlineTimePredicate; + private final Optional offlineTimePredicate; + + public TimeBoundary() + { + this(null, null); + } + + @JsonCreator + public TimeBoundary( + @JsonProperty String timeColumnName, + @JsonProperty String timeColumnValue) + { + if (timeColumnName != null && timeColumnValue != null) { + offlineTimePredicate = Optional.of(format("%s < %s", timeColumnName, timeColumnValue)); + onlineTimePredicate = Optional.of(format("%s >= %s", timeColumnName, timeColumnValue)); + } + else { + onlineTimePredicate = Optional.empty(); + offlineTimePredicate = Optional.empty(); + } + } + + public Optional getOnlineTimePredicate() + { + return onlineTimePredicate; + } + + public Optional getOfflineTimePredicate() + { + return offlineTimePredicate; + } + } + + public TimeBoundary getTimeBoundaryForTable(String table) + { + String responseBody = sendHttpGetToBroker(table, String.format(TIME_BOUNDARY_API_TEMPLATE, table)); + return timeBoundaryJsonCodec.fromJson(responseBody); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/SqlPathElement.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumn.java similarity index 50% rename from presto-main/src/main/java/com/facebook/presto/sql/SqlPathElement.java rename to presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumn.java index 2c59bd533b5ae..8c4f0a1af45f6 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/SqlPathElement.java +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumn.java @@ -11,42 +11,49 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql; +package com.facebook.presto.pinot; -import com.facebook.presto.sql.tree.Identifier; +import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Objects; -import java.util.Optional; -import static java.lang.String.format; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; import static java.util.Objects.requireNonNull; -public final class SqlPathElement +public final class PinotColumn { - private final Optional catalog; - private final Identifier schema; + private final String name; + private final Type type; @JsonCreator - public SqlPathElement( - @JsonProperty("catalog") Optional catalog, - @JsonProperty("schema") Identifier schema) + public PinotColumn( + @JsonProperty("name") String name, + @JsonProperty("type") Type type) { - this.catalog = requireNonNull(catalog, "catalog is null"); - this.schema = requireNonNull(schema, "schema is null"); + checkArgument(!isNullOrEmpty(name), "name is null or is empty"); + this.name = name; + this.type = requireNonNull(type, "type is null"); } @JsonProperty - public Optional getCatalog() + public String getName() { - return catalog; + return name; } @JsonProperty - public Identifier getSchema() + public Type getType() { - return schema; + return type; + } + + @Override + public int hashCode() + { + return Objects.hash(name, type); } @Override @@ -55,26 +62,17 @@ public boolean equals(Object obj) if (this == obj) { return true; } - if ((obj == null) || (getClass() != obj.getClass())) { + if (obj == null || getClass() != obj.getClass()) { return false; } - SqlPathElement that = (SqlPathElement) obj; - return Objects.equals(catalog, that.catalog) && - Objects.equals(schema, that.schema); - } - @Override - public int hashCode() - { - return Objects.hash(catalog, schema); + PinotColumn other = (PinotColumn) obj; + return Objects.equals(this.name, other.name) && Objects.equals(this.type, other.type); } @Override public String toString() { - if (catalog.isPresent()) { - return format("%s.%s", catalog.get().toString(), schema.toString()); - } - return schema.toString(); + return name + ":" + type; } } diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumnHandle.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumnHandle.java new file mode 100644 index 0000000000000..c655a704ff848 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumnHandle.java @@ -0,0 +1,112 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.Type; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public final class PinotColumnHandle + implements ColumnHandle +{ + private final String columnName; + private final Type dataType; + private final PinotColumnType type; + + public PinotColumnHandle( + VariableReferenceExpression variable, + PinotColumnType type) + { + this(variable.getName(), variable.getType(), type); + } + + @JsonCreator + public PinotColumnHandle( + @JsonProperty("columnName") String columnName, + @JsonProperty("dataType") Type dataType, + @JsonProperty("type") PinotColumnType type) + { + this.columnName = requireNonNull(columnName, "column name is null"); + this.dataType = requireNonNull(dataType, "data type name is null"); + this.type = requireNonNull(type, "type is null"); + } + + @JsonProperty("columnName") + public String getColumnName() + { + return columnName; + } + + @JsonProperty("dataType") + public Type getDataType() + { + return dataType; + } + + @JsonProperty + public PinotColumnType getType() + { + return type; + } + + public ColumnMetadata getColumnMetadata() + { + return new ColumnMetadata(getColumnName(), getDataType()); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + PinotColumnHandle that = (PinotColumnHandle) o; + return Objects.equals(getColumnName(), that.getColumnName()); + } + + @Override + public int hashCode() + { + return Objects.hash(columnName); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("columnName", columnName) + .add("dataType", dataType) + .add("type", type) + .toString(); + } + + public enum PinotColumnType + { + REGULAR, // refers to the column in table + DERIVED, // refers to a derived column that is created after a pushdown expression + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumnMetadata.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumnMetadata.java new file mode 100644 index 0000000000000..b19d08094c59e --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumnMetadata.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.type.Type; + +import java.util.Objects; + +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public class PinotColumnMetadata + extends ColumnMetadata +{ + // We need to preserve the case sensitivity of the column, store it here as the super class stores the value after lower-casing it + private final String name; + + public PinotColumnMetadata(String name, Type type) + { + super(requireNonNull(name, "name is null"), requireNonNull(type, "type is null")); + this.name = name; + } + + @Override + public String getName() + { + return name.toLowerCase(ENGLISH); + } + + public String getPinotName() + { + return name; + } + + @Override + public int hashCode() + { + return Objects.hash(name, getType(), getComment(), isHidden()); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PinotColumnMetadata other = (PinotColumnMetadata) obj; + return Objects.equals(this.name, other.name) && + Objects.equals(this.getType(), other.getType()) && + Objects.equals(this.getComment(), other.getComment()) && + Objects.equals(this.isHidden(), other.isHidden()); + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumnUtils.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumnUtils.java new file mode 100644 index 0000000000000..a1f896564bdd3 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotColumnUtils.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.BooleanType; +import com.facebook.presto.spi.type.DoubleType; +import com.facebook.presto.spi.type.IntegerType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.VarcharType; +import org.apache.pinot.common.data.FieldSpec; +import org.apache.pinot.common.data.FieldSpec.DataType; +import org.apache.pinot.common.data.Schema; + +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNSUPPORTED_COLUMN_TYPE; +import static com.google.common.collect.ImmutableList.toImmutableList; + +public class PinotColumnUtils +{ + private PinotColumnUtils() + { + } + + public static List getPinotColumnsForPinotSchema(Schema pinotTableSchema) + { + return pinotTableSchema.getColumnNames().stream() + .filter(columnName -> !columnName.startsWith("$")) // Hidden columns starts with "$", ignore them as we can't use them in PQL + .map(columnName -> new PinotColumn(columnName, getPrestoTypeFromPinotType(pinotTableSchema.getFieldSpecFor(columnName)))) + .collect(toImmutableList()); + } + + public static Type getPrestoTypeFromPinotType(FieldSpec field) + { + if (field.isSingleValueField()) { + return getPrestoTypeFromPinotType(field.getDataType()); + } + return VarcharType.VARCHAR; + } + + public static Type getPrestoTypeFromPinotType(DataType dataType) + { + switch (dataType) { + case BOOLEAN: + return BooleanType.BOOLEAN; + case DOUBLE: + case FLOAT: + return DoubleType.DOUBLE; + case INT: + return IntegerType.INTEGER; + case LONG: + return BigintType.BIGINT; + case STRING: + return VarcharType.VARCHAR; + default: + break; + } + throw new PinotException(PINOT_UNSUPPORTED_COLUMN_TYPE, Optional.empty(), "Not support type conversion for pinot data type: " + dataType); + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConfig.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConfig.java new file mode 100644 index 0000000000000..77697ee59ede8 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConfig.java @@ -0,0 +1,419 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.configuration.Config; +import com.google.common.base.Splitter; +import com.google.common.base.Splitter.MapSplitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; + +import javax.annotation.Nullable; +import javax.validation.constraints.NotNull; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static com.google.common.base.Preconditions.checkArgument; + +public class PinotConfig +{ + public static final int DEFAULT_LIMIT_LARGE_FOR_SEGMENT = Integer.MAX_VALUE; + public static final int DEFAULT_MAX_BACKLOG_PER_SERVER = 30; + public static final int DEFAULT_MAX_CONNECTIONS_PER_SERVER = 30; + public static final int DEFAULT_MIN_CONNECTIONS_PER_SERVER = 10; + public static final int DEFAULT_THREAD_POOL_SIZE = 30; + public static final int DEFAULT_NON_AGGREGATE_LIMIT_FOR_BROKER_QUERIES = 25_000; + + // There is a perf penalty of having a large topN since the structures are allocated to this size + // So size this judiciously + public static final int DEFAULT_TOPN_LARGE = 10_000; + + private static final Duration DEFAULT_IDLE_TIMEOUT = new Duration(5, TimeUnit.MINUTES); + private static final Duration DEFAULT_CONNECTION_TIMEOUT = new Duration(1, TimeUnit.MINUTES); + private static final int DEFAULT_ESTIMATED_SIZE_IN_BYTES_FOR_NON_NUMERIC_COLUMN = 20; + + private static final Splitter LIST_SPLITTER = Splitter.on(",").trimResults().omitEmptyStrings(); + private static final MapSplitter MAP_SPLITTER = Splitter.on(",").trimResults().omitEmptyStrings().withKeyValueSeparator(":"); + + private int maxConnectionsPerServer = DEFAULT_MAX_CONNECTIONS_PER_SERVER; + private String controllerRestService; + private String serviceHeaderParam = "RPC-Service"; + private String callerHeaderValue = "presto"; + private String callerHeaderParam = "RPC-Caller"; + + private List controllerUrls = ImmutableList.of(); + private String restProxyUrl; + private String restProxyServiceForQuery; + + private int limitLargeForSegment = DEFAULT_LIMIT_LARGE_FOR_SEGMENT; + private int topNLarge = DEFAULT_TOPN_LARGE; + + private Duration idleTimeout = DEFAULT_IDLE_TIMEOUT; + private Duration connectionTimeout = DEFAULT_CONNECTION_TIMEOUT; + + private int threadPoolSize = DEFAULT_THREAD_POOL_SIZE; + private int minConnectionsPerServer = DEFAULT_MIN_CONNECTIONS_PER_SERVER; + private int maxBacklogPerServer = DEFAULT_MAX_BACKLOG_PER_SERVER; + private int estimatedSizeInBytesForNonNumericColumn = DEFAULT_ESTIMATED_SIZE_IN_BYTES_FOR_NON_NUMERIC_COLUMN; + private Map extraHttpHeaders = ImmutableMap.of(); + private Duration metadataCacheExpiry = new Duration(2, TimeUnit.MINUTES); + + private boolean allowMultipleAggregations; + private boolean preferBrokerQueries = true; + private boolean forbidSegmentQueries; + private int numSegmentsPerSplit = 1; + private boolean ignoreEmptyResponses; + private int fetchRetryCount = 2; + private boolean useDateTrunc; + private int nonAggregateLimitForBrokerQueries = DEFAULT_NON_AGGREGATE_LIMIT_FOR_BROKER_QUERIES; + + @NotNull + public Map getExtraHttpHeaders() + { + return extraHttpHeaders; + } + + @Config("pinot.extra-http-headers") + public PinotConfig setExtraHttpHeaders(String headers) + { + extraHttpHeaders = ImmutableMap.copyOf(MAP_SPLITTER.split(headers)); + return this; + } + + @NotNull + public List getControllerUrls() + { + return controllerUrls; + } + + @Config("pinot.controller-urls") + public PinotConfig setControllerUrls(String controllerUrl) + { + this.controllerUrls = LIST_SPLITTER.splitToList(controllerUrl); + return this; + } + + @Nullable + public String getRestProxyUrl() + { + return restProxyUrl; + } + + @Config("pinot.rest-proxy-url") + public PinotConfig setRestProxyUrl(String restProxyUrl) + { + this.restProxyUrl = restProxyUrl; + return this; + } + + @Nullable + public String getControllerRestService() + { + return controllerRestService; + } + + @Config("pinot.controller-rest-service") + public PinotConfig setControllerRestService(String controllerRestService) + { + this.controllerRestService = controllerRestService; + return this; + } + + @NotNull + public boolean isAllowMultipleAggregations() + { + return allowMultipleAggregations; + } + + @Config("pinot.allow-multiple-aggregations") + public PinotConfig setAllowMultipleAggregations(boolean allowMultipleAggregations) + { + this.allowMultipleAggregations = allowMultipleAggregations; + return this; + } + + @NotNull + public int getLimitLargeForSegment() + { + return limitLargeForSegment; + } + + @Config("pinot.limit-large-for-segment") + public PinotConfig setLimitLargeForSegment(int limitLargeForSegment) + { + this.limitLargeForSegment = limitLargeForSegment; + return this; + } + + @NotNull + public int getTopNLarge() + { + return topNLarge; + } + + @Config("pinot.topn-large") + public PinotConfig setTopNLarge(int topNLarge) + { + this.topNLarge = topNLarge; + return this; + } + + @NotNull + public int getThreadPoolSize() + { + return threadPoolSize; + } + + @Config("pinot.thread-pool-size") + public PinotConfig setThreadPoolSize(int threadPoolSize) + { + this.threadPoolSize = threadPoolSize; + return this; + } + + @NotNull + public int getMinConnectionsPerServer() + { + return minConnectionsPerServer; + } + + @Config("pinot.min-connections-per-server") + public PinotConfig setMinConnectionsPerServer(int minConnectionsPerServer) + { + this.minConnectionsPerServer = minConnectionsPerServer; + return this; + } + + @NotNull + public int getMaxConnectionsPerServer() + { + return maxConnectionsPerServer; + } + + @Config("pinot.max-connections-per-server") + public PinotConfig setMaxConnectionsPerServer(int maxConnectionsPerServer) + { + this.maxConnectionsPerServer = maxConnectionsPerServer; + return this; + } + + @NotNull + public int getMaxBacklogPerServer() + { + return maxBacklogPerServer; + } + + @Config("pinot.max-backlog-per-server") + public PinotConfig setMaxBacklogPerServer(int maxBacklogPerServer) + { + this.maxBacklogPerServer = maxBacklogPerServer; + return this; + } + + @MinDuration("15s") + @NotNull + public Duration getIdleTimeout() + { + return idleTimeout; + } + + @Config("pinot.idle-timeout") + public PinotConfig setIdleTimeout(Duration idleTimeout) + { + this.idleTimeout = idleTimeout; + return this; + } + + @MinDuration("15s") + @NotNull + public Duration getConnectionTimeout() + { + return connectionTimeout; + } + + @Config("pinot.connection-timeout") + public PinotConfig setConnectionTimeout(Duration connectionTimeout) + { + this.connectionTimeout = connectionTimeout; + return this; + } + + @MinDuration("0s") + @NotNull + public Duration getMetadataCacheExpiry() + { + return metadataCacheExpiry; + } + + @Config("pinot.metadata-expiry") + public PinotConfig setMetadataCacheExpiry(Duration metadataCacheExpiry) + { + this.metadataCacheExpiry = metadataCacheExpiry; + return this; + } + + @NotNull + public int getEstimatedSizeInBytesForNonNumericColumn() + { + return estimatedSizeInBytesForNonNumericColumn; + } + + @Config("pinot.estimated-size-in-bytes-for-non-numeric-column") + public PinotConfig setEstimatedSizeInBytesForNonNumericColumn(int estimatedSizeInBytesForNonNumericColumn) + { + this.estimatedSizeInBytesForNonNumericColumn = estimatedSizeInBytesForNonNumericColumn; + return this; + } + + @NotNull + public String getServiceHeaderParam() + { + return serviceHeaderParam; + } + + @Config("pinot.service-header-param") + public PinotConfig setServiceHeaderParam(String serviceHeaderParam) + { + this.serviceHeaderParam = serviceHeaderParam; + return this; + } + + @NotNull + public String getCallerHeaderValue() + { + return callerHeaderValue; + } + + @Config("pinot.caller-header-value") + public PinotConfig setCallerHeaderValue(String callerHeaderValue) + { + this.callerHeaderValue = callerHeaderValue; + return this; + } + + @NotNull + public String getCallerHeaderParam() + { + return callerHeaderParam; + } + + @Config("pinot.caller-header-param") + public PinotConfig setCallerHeaderParam(String callerHeaderParam) + { + this.callerHeaderParam = callerHeaderParam; + return this; + } + + public boolean isPreferBrokerQueries() + { + return preferBrokerQueries; + } + + @Config("pinot.prefer-broker-queries") + public PinotConfig setPreferBrokerQueries(boolean preferBrokerQueries) + { + this.preferBrokerQueries = preferBrokerQueries; + return this; + } + + public boolean isForbidSegmentQueries() + { + return forbidSegmentQueries; + } + + @Config("pinot.forbid-segment-queries") + public PinotConfig setForbidSegmentQueries(boolean forbidSegmentQueries) + { + this.forbidSegmentQueries = forbidSegmentQueries; + return this; + } + + @Nullable + public String getRestProxyServiceForQuery() + { + return restProxyServiceForQuery; + } + + @Config("pinot.rest-proxy-service-for-query") + public PinotConfig setRestProxyServiceForQuery(String restProxyServiceForQuery) + { + this.restProxyServiceForQuery = restProxyServiceForQuery; + return this; + } + + public boolean isUseDateTrunc() + { + return useDateTrunc; + } + + @Config("pinot.use-date-trunc") + public PinotConfig setUseDateTrunc(boolean useDateTrunc) + { + this.useDateTrunc = useDateTrunc; + return this; + } + + public int getNumSegmentsPerSplit() + { + return this.numSegmentsPerSplit; + } + + @Config("pinot.num-segments-per-split") + public PinotConfig setNumSegmentsPerSplit(int numSegmentsPerSplit) + { + checkArgument(numSegmentsPerSplit > 0, "Number of segments per split must be more than zero"); + this.numSegmentsPerSplit = numSegmentsPerSplit; + return this; + } + + public boolean isIgnoreEmptyResponses() + { + return ignoreEmptyResponses; + } + + @Config("pinot.ignore-empty-responses") + public PinotConfig setIgnoreEmptyResponses(boolean ignoreEmptyResponses) + { + this.ignoreEmptyResponses = ignoreEmptyResponses; + return this; + } + + public int getFetchRetryCount() + { + return fetchRetryCount; + } + + @Config("pinot.fetch-retry-count") + public PinotConfig setFetchRetryCount(int fetchRetryCount) + { + this.fetchRetryCount = fetchRetryCount; + return this; + } + + public int getNonAggregateLimitForBrokerQueries() + { + return nonAggregateLimitForBrokerQueries; + } + + @Config("pinot.non-aggregate-limit-for-broker-queries") + public PinotConfig setNonAggregateLimitForBrokerQueries(int nonAggregateLimitForBrokerQueries) + { + this.nonAggregateLimitForBrokerQueries = nonAggregateLimitForBrokerQueries; + return this; + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnection.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnection.java new file mode 100644 index 0000000000000..970f6171d9ead --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnection.java @@ -0,0 +1,112 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.pinot.PinotClusterInfoFetcher.TimeBoundary; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import org.apache.pinot.common.data.Schema; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +import static com.google.common.cache.CacheLoader.asyncReloading; +import static java.util.Objects.requireNonNull; + +public class PinotConnection +{ + private static final Object ALL_TABLES_CACHE_KEY = new Object(); + + private final LoadingCache> pinotTableColumnCache; + private final LoadingCache> allTablesCache; + private final PinotConfig pinotConfig; + private final PinotClusterInfoFetcher pinotClusterInfoFetcher; + + @Inject + public PinotConnection( + PinotClusterInfoFetcher pinotClusterInfoFetcher, + PinotConfig pinotConfig, + @ForPinot Executor executor) + { + this.pinotConfig = requireNonNull(pinotConfig, "pinot config"); + final long metadataCacheExpiryMillis = this.pinotConfig.getMetadataCacheExpiry().roundTo(TimeUnit.MILLISECONDS); + this.pinotClusterInfoFetcher = requireNonNull(pinotClusterInfoFetcher, "cluster info fetcher is null"); + this.allTablesCache = CacheBuilder.newBuilder() + .refreshAfterWrite(metadataCacheExpiryMillis, TimeUnit.MILLISECONDS) + .build(asyncReloading(CacheLoader.from(pinotClusterInfoFetcher::getAllTables), executor)); + + this.pinotTableColumnCache = + CacheBuilder.newBuilder() + .refreshAfterWrite(metadataCacheExpiryMillis, TimeUnit.MILLISECONDS) + .build(asyncReloading(new CacheLoader>() + { + @Override + public List load(String tableName) + throws Exception + { + Schema tablePinotSchema = pinotClusterInfoFetcher.getTableSchema(tableName); + return PinotColumnUtils.getPinotColumnsForPinotSchema(tablePinotSchema); + } + }, executor)); + + executor.execute(() -> this.allTablesCache.refresh(ALL_TABLES_CACHE_KEY)); + } + + private static V getFromCache(LoadingCache cache, K key) + { + V value = cache.getIfPresent(key); + if (value != null) { + return value; + } + try { + return cache.get(key); + } + catch (ExecutionException e) { + throw new PinotException(PinotErrorCode.PINOT_UNCLASSIFIED_ERROR, Optional.empty(), "Cannot fetch from cache " + key, e.getCause()); + } + } + + public List getTableNames() + { + return getFromCache(allTablesCache, ALL_TABLES_CACHE_KEY); + } + + public PinotTable getTable(String tableName) + { + List columns = getPinotColumnsForTable(tableName); + return new PinotTable(tableName, columns); + } + + private List getPinotColumnsForTable(String tableName) + { + return getFromCache(pinotTableColumnCache, tableName); + } + + public Map>> getRoutingTable(String tableName) + { + return pinotClusterInfoFetcher.getRoutingTableForTable(tableName); + } + + public TimeBoundary getTimeBoundary(String tableName) + { + return pinotClusterInfoFetcher.getTimeBoundaryForTable(tableName); + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnector.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnector.java new file mode 100644 index 0000000000000..73724f6c0ad0f --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnector.java @@ -0,0 +1,134 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; +import com.facebook.presto.spi.ConnectorPlanOptimizer; +import com.facebook.presto.spi.connector.Connector; +import com.facebook.presto.spi.connector.ConnectorMetadata; +import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider; +import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; +import com.facebook.presto.spi.connector.ConnectorPlanOptimizerProvider; +import com.facebook.presto.spi.connector.ConnectorSplitManager; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.session.PropertyMetadata; +import com.facebook.presto.spi.transaction.IsolationLevel; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Set; + +import static com.facebook.presto.pinot.PinotTransactionHandle.INSTANCE; +import static java.util.Objects.requireNonNull; + +public class PinotConnector + implements Connector +{ + private static final Logger log = Logger.get(PinotConnector.class); + private final LifeCycleManager lifeCycleManager; + private final PinotMetadata metadata; + private final PinotSplitManager splitManager; + private final PinotPageSourceProvider pageSourceProvider; + private final PinotNodePartitioningProvider partitioningProvider; + private final List> sessionProperties; + private final ConnectorPlanOptimizer planOptimizer; + + @Inject + public PinotConnector(LifeCycleManager lifeCycleManager, + PinotMetadata metadata, + PinotSplitManager splitManager, + PinotPageSourceProvider pageSourceProvider, + PinotNodePartitioningProvider partitioningProvider, + PinotSessionProperties pinotSessionProperties, + PinotConnectorPlanOptimizer planOptimizer) + { + this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); + this.metadata = requireNonNull(metadata, "metadata is null"); + this.splitManager = requireNonNull(splitManager, "splitManager is null"); + this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); + this.partitioningProvider = requireNonNull(partitioningProvider, "partitioningProvider is null"); + this.sessionProperties = ImmutableList.copyOf(requireNonNull(pinotSessionProperties, "sessionProperties is null").getSessionProperties()); + this.planOptimizer = requireNonNull(planOptimizer, "plan optimizer is null"); + } + + @Override + public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) + { + return INSTANCE; + } + + @Override + public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) + { + return metadata; + } + + @Override + public ConnectorSplitManager getSplitManager() + { + return splitManager; + } + + @Override + public ConnectorPageSourceProvider getPageSourceProvider() + { + return pageSourceProvider; + } + + @Override + public ConnectorNodePartitioningProvider getNodePartitioningProvider() + { + return partitioningProvider; + } + + @Override + public List> getSessionProperties() + { + return sessionProperties; + } + + @Override + public ConnectorPlanOptimizerProvider getConnectorPlanOptimizerProvider() + { + return new ConnectorPlanOptimizerProvider() + { + @Override + public Set getLogicalPlanOptimizers() + { + return ImmutableSet.of(planOptimizer); + } + + @Override + public Set getPhysicalPlanOptimizers() + { + return ImmutableSet.of(); + } + }; + } + + @Override + public final void shutdown() + { + try { + lifeCycleManager.stop(); + } + catch (Exception e) { + log.error(e, "Error shutting down connector"); + } + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnectorFactory.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnectorFactory.java new file mode 100644 index 0000000000000..08348f268c016 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnectorFactory.java @@ -0,0 +1,97 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; +import com.facebook.presto.expressions.LogicalRowExpressions; +import com.facebook.presto.spi.ConnectorHandleResolver; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.connector.Connector; +import com.facebook.presto.spi.connector.ConnectorContext; +import com.facebook.presto.spi.connector.ConnectorFactory; +import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.RowExpressionService; +import com.facebook.presto.spi.type.TypeManager; +import com.google.inject.Injector; +import com.google.inject.Scopes; +import org.weakref.jmx.guice.MBeanModule; + +import javax.management.MBeanServer; + +import java.util.Map; + +import static java.lang.management.ManagementFactory.getPlatformMBeanServer; +import static java.util.Objects.requireNonNull; +import static org.weakref.jmx.ObjectNames.generatedNameOf; +import static org.weakref.jmx.guice.ExportBinder.newExporter; + +public class PinotConnectorFactory + implements ConnectorFactory +{ + public PinotConnectorFactory() + { + } + + @Override + public String getName() + { + return "pinot"; + } + + @Override + public ConnectorHandleResolver getHandleResolver() + { + return new PinotHandleResolver(); + } + + @Override + public Connector create(final String connectorId, Map config, ConnectorContext context) + { + requireNonNull(connectorId, "connectorId is null"); + requireNonNull(config, "config is null"); + + try { + Bootstrap app = new Bootstrap( + new JsonModule(), + new MBeanModule(), + new PinotModule(connectorId), binder -> { + binder.bind(MBeanServer.class).toInstance(new RebindSafeMBeanServer(getPlatformMBeanServer())); + binder.bind(ConnectorId.class).toInstance(new ConnectorId(connectorId)); + binder.bind(TypeManager.class).toInstance(context.getTypeManager()); + binder.bind(FunctionMetadataManager.class).toInstance(context.getFunctionMetadataManager()); + binder.bind(NodeManager.class).toInstance(context.getNodeManager()); + binder.bind(RowExpressionService.class).toInstance(context.getRowExpressionService()); + binder.bind(LogicalRowExpressions.class).toInstance(new LogicalRowExpressions(context.getRowExpressionService().getDeterminismEvaluator(), context.getStandardFunctionResolution(), context.getFunctionMetadataManager())); + binder.bind(StandardFunctionResolution.class).toInstance(context.getStandardFunctionResolution()); + binder.bind(PinotMetrics.class).in(Scopes.SINGLETON); + newExporter(binder).export(PinotMetrics.class).as(generatedNameOf(PinotMetrics.class, connectorId)); + binder.bind(ConnectorNodePartitioningProvider.class).to(PinotNodePartitioningProvider.class).in(Scopes.SINGLETON); + }); + + Injector injector = app.strictConfig() + .doNotInitializeLogging() + .setRequiredConfigurationProperties(config) + .initialize(); + + return injector.getInstance(PinotConnector.class); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnectorPlanOptimizer.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnectorPlanOptimizer.java new file mode 100644 index 0000000000000..47c0211181c5d --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnectorPlanOptimizer.java @@ -0,0 +1,234 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.expressions.LogicalRowExpressions; +import com.facebook.presto.pinot.query.PinotFilterExpressionConverter; +import com.facebook.presto.pinot.query.PinotQueryGenerator; +import com.facebook.presto.pinot.query.PinotQueryGeneratorContext; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorPlanOptimizer; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.VariableAllocator; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.PlanVisitor; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.TypeManager; +import com.google.common.collect.ImmutableList; + +import javax.inject.Inject; + +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNCLASSIFIED_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Objects.requireNonNull; + +public class PinotConnectorPlanOptimizer + implements ConnectorPlanOptimizer +{ + private final PinotQueryGenerator pinotQueryGenerator; + private final TypeManager typeManager; + private final FunctionMetadataManager functionMetadataManager; + private final LogicalRowExpressions logicalRowExpressions; + private final StandardFunctionResolution standardFunctionResolution; + + @Inject + public PinotConnectorPlanOptimizer( + PinotQueryGenerator pinotQueryGenerator, + TypeManager typeManager, + FunctionMetadataManager functionMetadataManager, + LogicalRowExpressions logicalRowExpressions, + StandardFunctionResolution standardFunctionResolution) + { + this.pinotQueryGenerator = requireNonNull(pinotQueryGenerator, "pinot query generator is null"); + this.typeManager = requireNonNull(typeManager, "type manager is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "function manager is null"); + this.logicalRowExpressions = requireNonNull(logicalRowExpressions, "logical row expressions is null"); + this.standardFunctionResolution = requireNonNull(standardFunctionResolution, "standard function resolution is null"); + } + + @Override + public PlanNode optimize(PlanNode maxSubplan, + ConnectorSession session, + VariableAllocator variableAllocator, + PlanNodeIdAllocator idAllocator) + { + Map scanNodes = maxSubplan.accept(new TableFindingVisitor(), null); + TableScanNode pinotTableScanNode = getOnlyPinotTable(scanNodes) + .orElseThrow(() -> new PrestoException(GENERIC_INTERNAL_ERROR, + "Expected to find the pinot table handle for the scan node")); + return maxSubplan.accept(new Visitor(pinotTableScanNode, session, idAllocator), null); + } + + private static Optional getPinotTableHandle(TableScanNode tableScanNode) + { + TableHandle table = tableScanNode.getTable(); + if (table != null) { + ConnectorTableHandle connectorHandle = table.getConnectorHandle(); + if (connectorHandle instanceof PinotTableHandle) { + return Optional.of((PinotTableHandle) connectorHandle); + } + } + return Optional.empty(); + } + + private static Optional getOnlyPinotTable(Map scanNodes) + { + if (scanNodes.size() == 1) { + TableScanNode tableScanNode = scanNodes.keySet().iterator().next(); + if (getPinotTableHandle(tableScanNode).isPresent()) { + return Optional.of(tableScanNode); + } + } + return Optional.empty(); + } + + private static PlanNode replaceChildren(PlanNode node, List children) + { + for (int i = 0; i < node.getSources().size(); i++) { + if (children.get(i) != node.getSources().get(i)) { + return node.replaceChildren(children); + } + } + return node; + } + + private static class TableFindingVisitor + extends PlanVisitor, Void> + { + @Override + public Map visitPlan(PlanNode node, Void context) + { + Map ret = new IdentityHashMap<>(); + node.getSources().forEach(source -> ret.putAll(source.accept(this, context))); + return ret; + } + + @Override + public Map visitTableScan(TableScanNode node, Void context) + { + Map ret = new IdentityHashMap<>(); + ret.put(node, null); + return ret; + } + } + + // Single use visitor that needs the pinot table handle + private class Visitor + extends PlanVisitor + { + private final PlanNodeIdAllocator idAllocator; + private final ConnectorSession session; + private final TableScanNode tableScanNode; + private final IdentityHashMap filtersSplitUp = new IdentityHashMap<>(); + + public Visitor(TableScanNode tableScanNode, ConnectorSession session, PlanNodeIdAllocator idAllocator) + { + this.session = session; + this.idAllocator = idAllocator; + this.tableScanNode = tableScanNode; + // Just making sure that the table exists + getPinotTableHandle(this.tableScanNode).get().getTableName(); + } + + private Optional tryCreatingNewScanNode(PlanNode plan) + { + Optional pql = pinotQueryGenerator.generate(plan, session); + if (!pql.isPresent()) { + return Optional.empty(); + } + PinotTableHandle pinotTableHandle = getPinotTableHandle(tableScanNode).orElseThrow(() -> new PinotException(PINOT_UNCLASSIFIED_ERROR, Optional.empty(), "Expected to find a pinot table handle")); + PinotQueryGeneratorContext context = pql.get().getContext(); + TableHandle oldTableHandle = tableScanNode.getTable(); + LinkedHashMap assignments = context.getAssignments(); + boolean isQueryShort = pql.get().getGeneratedPql().isQueryShort(); + TableHandle newTableHandle = new TableHandle( + oldTableHandle.getConnectorId(), + new PinotTableHandle(pinotTableHandle.getConnectorId(), pinotTableHandle.getSchemaName(), pinotTableHandle.getTableName(), Optional.of(isQueryShort), Optional.of(pql.get().getGeneratedPql())), + oldTableHandle.getTransaction(), + oldTableHandle.getLayout()); + return Optional.of( + new TableScanNode( + idAllocator.getNextId(), + newTableHandle, + ImmutableList.copyOf(assignments.keySet()), + assignments.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, (e) -> (ColumnHandle) (e.getValue()))), + tableScanNode.getCurrentConstraint(), + tableScanNode.getEnforcedConstraint())); + } + + @Override + public PlanNode visitPlan(PlanNode node, Void context) + { + Optional pushedDownPlan = tryCreatingNewScanNode(node); + return pushedDownPlan.orElseGet(() -> replaceChildren( + node, + node.getSources().stream().map(source -> source.accept(this, null)).collect(toImmutableList()))); + } + + @Override + public PlanNode visitFilter(FilterNode node, Void context) + { + if (filtersSplitUp.containsKey(node)) { + return this.visitPlan(node, context); + } + filtersSplitUp.put(node, null); + FilterNode nodeToRecurseInto = node; + List pushable = new ArrayList<>(); + List nonPushable = new ArrayList<>(); + PinotFilterExpressionConverter pinotFilterExpressionConverter = new PinotFilterExpressionConverter(typeManager, functionMetadataManager, standardFunctionResolution); + for (RowExpression conjunct : LogicalRowExpressions.extractConjuncts(node.getPredicate())) { + try { + conjunct.accept(pinotFilterExpressionConverter, (var) -> new PinotQueryGeneratorContext.Selection(var.getName(), PinotQueryGeneratorContext.Origin.DERIVED)); + pushable.add(conjunct); + } + catch (PinotException pe) { + nonPushable.add(conjunct); + } + } + if (!pushable.isEmpty()) { + FilterNode pushableFilter = new FilterNode(idAllocator.getNextId(), node.getSource(), logicalRowExpressions.combineConjuncts(pushable)); + Optional nonPushableFilter = nonPushable.isEmpty() ? Optional.empty() : Optional.of(new FilterNode(idAllocator.getNextId(), pushableFilter, logicalRowExpressions.combineConjuncts(nonPushable))); + + filtersSplitUp.put(pushableFilter, null); + if (nonPushableFilter.isPresent()) { + FilterNode nonPushableFilterNode = nonPushableFilter.get(); + filtersSplitUp.put(nonPushableFilterNode, null); + nodeToRecurseInto = nonPushableFilterNode; + } + else { + nodeToRecurseInto = pushableFilter; + } + } + return this.visitFilter(nodeToRecurseInto, context); + } + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotErrorCode.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotErrorCode.java new file mode 100644 index 0000000000000..73e6748a4b7d7 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotErrorCode.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.ErrorCode; +import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.spi.ErrorType; + +import static com.facebook.presto.spi.ErrorType.EXTERNAL; +import static com.facebook.presto.spi.ErrorType.INTERNAL_ERROR; + +public enum PinotErrorCode + implements ErrorCodeSupplier +{ + PINOT_UNSUPPORTED_COLUMN_TYPE(0, EXTERNAL), // schema issues + PINOT_QUERY_GENERATOR_FAILURE(1, INTERNAL_ERROR), // Accepted a query whose pql we couldn't generate + PINOT_INSUFFICIENT_SERVER_RESPONSE(2, EXTERNAL, true), // numServersResponded < numServersQueried + PINOT_EXCEPTION(3, EXTERNAL), // Exception reported by pinot + PINOT_HTTP_ERROR(4, EXTERNAL), // Some non okay http error code + PINOT_UNEXPECTED_RESPONSE(5, EXTERNAL), // Invalid json response with okay http return code + PINOT_UNSUPPORTED_EXPRESSION(6, INTERNAL_ERROR), // Unsupported function + PINOT_UNABLE_TO_FIND_BROKER(7, EXTERNAL), + PINOT_DECODE_ERROR(8, EXTERNAL), + PINOT_INVALID_PQL_GENERATED(9, INTERNAL_ERROR), + PINOT_INVALID_CONFIGURATION(10, INTERNAL_ERROR), + PINOT_UNCLASSIFIED_ERROR(100, EXTERNAL); + + /** + * Connectors can use error codes starting at the range 0x0100_0000 + * See https://github.com/prestodb/presto/wiki/Error-Codes + * + * @see com.facebook.presto.spi.StandardErrorCode + */ + + private final ErrorCode errorCode; + private final boolean retriable; + + PinotErrorCode(int code, ErrorType type, boolean retriable) + { + errorCode = new ErrorCode(code + 0x0505_0000, name(), type); + this.retriable = retriable; + } + + PinotErrorCode(int code, ErrorType type) + { + this(code, type, false); + } + + public boolean isRetriable() + { + return retriable; + } + + @Override + public ErrorCode toErrorCode() + { + return errorCode; + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotException.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotException.java new file mode 100644 index 0000000000000..30c6ce81a4f1c --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotException.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.PrestoException; + +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class PinotException + extends PrestoException +{ + private final Optional pql; + private final PinotErrorCode pinotErrorCode; + + public PinotException(PinotErrorCode errorCode, Optional pql, String message) + { + this(errorCode, pql, message, null); + } + + public PinotException(PinotErrorCode pinotErrorCode, Optional pql, String message, Throwable throwable) + { + super(requireNonNull(pinotErrorCode, "error code is null"), requireNonNull(message, "message is null"), throwable); + this.pinotErrorCode = pinotErrorCode; + this.pql = requireNonNull(pql, "pql is null"); + } + + public PinotErrorCode getPinotErrorCode() + { + return pinotErrorCode; + } + + @Override + public String getMessage() + { + String message = super.getMessage(); + if (pql.isPresent()) { + message += " with pql \"" + pql.get() + "\""; + } + return message; + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotHandleResolver.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotHandleResolver.java new file mode 100644 index 0000000000000..d4e462b77d01b --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotHandleResolver.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorHandleResolver; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; + +public class PinotHandleResolver + implements ConnectorHandleResolver +{ + @Override + public Class getTableLayoutHandleClass() + { + return PinotTableLayoutHandle.class; + } + + @Override + public Class getTableHandleClass() + { + return PinotTableHandle.class; + } + + @Override + public Class getColumnHandleClass() + { + return PinotColumnHandle.class; + } + + @Override + public Class getSplitClass() + { + return PinotSplit.class; + } + + @Override + public Class getTransactionHandleClass() + { + return PinotTransactionHandle.class; + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotMetadata.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotMetadata.java new file mode 100644 index 0000000000000..7388818166532 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotMetadata.java @@ -0,0 +1,181 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.ConnectorTableLayout; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.facebook.presto.spi.ConnectorTableLayoutResult; +import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.Constraint; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.SchemaTablePrefix; +import com.facebook.presto.spi.TableNotFoundException; +import com.facebook.presto.spi.connector.ConnectorMetadata; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.pinot.PinotColumnHandle.PinotColumnType.REGULAR; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public class PinotMetadata + implements ConnectorMetadata +{ + private final String connectorId; + private final PinotConnection pinotPrestoConnection; + + @Inject + public PinotMetadata(ConnectorId connectorId, PinotConnection pinotPrestoConnection) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); + this.pinotPrestoConnection = requireNonNull(pinotPrestoConnection, "pinotPrestoConnection is null"); + } + + @Override + public List listSchemaNames(ConnectorSession session) + { + return ImmutableList.of("default"); + } + + private String getPinotTableNameFromPrestoTableName(String prestoTableName) + { + List allTables = pinotPrestoConnection.getTableNames(); + for (String pinotTableName : allTables) { + if (prestoTableName.equalsIgnoreCase(pinotTableName)) { + return pinotTableName; + } + } + throw new PinotException(PinotErrorCode.PINOT_UNCLASSIFIED_ERROR, Optional.empty(), "Unable to find the presto table " + prestoTableName + " in " + allTables); + } + + @Override + public PinotTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) + { + String pinotTableName = getPinotTableNameFromPrestoTableName(tableName.getTableName()); + return new PinotTableHandle(connectorId, tableName.getSchemaName(), pinotTableName); + } + + @Override + public List getTableLayouts( + ConnectorSession session, + ConnectorTableHandle table, + Constraint constraint, + Optional> desiredColumns) + { + // Constraint's don't need to be pushed down since they are already taken care off by the pushdown logic + PinotTableHandle pinotTableHandle = (PinotTableHandle) table; + ConnectorTableLayout layout = new ConnectorTableLayout(new PinotTableLayoutHandle(pinotTableHandle)); + return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary())); + } + + @Override + public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle handle) + { + return new ConnectorTableLayout(handle); + } + + @Override + public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table) + { + PinotTableHandle pinotTableHandle = (PinotTableHandle) table; + checkArgument(pinotTableHandle.getConnectorId().equals(connectorId), "tableHandle is not for this connector"); + SchemaTableName tableName = new SchemaTableName(pinotTableHandle.getSchemaName(), pinotTableHandle.getTableName()); + + return getTableMetadata(tableName); + } + + @Override + public List listTables(ConnectorSession session, String schemaNameOrNull) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (String table : pinotPrestoConnection.getTableNames()) { + builder.add(new SchemaTableName("default", table)); + } + return builder.build(); + } + + @Override + public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) + { + PinotTableHandle pinotTableHandle = (PinotTableHandle) tableHandle; + checkArgument(pinotTableHandle.getConnectorId().equals(connectorId), "tableHandle is not for this connector"); + + String pinotTableName = getPinotTableNameFromPrestoTableName(pinotTableHandle.getTableName()); + PinotTable table = pinotPrestoConnection.getTable(pinotTableName); + if (table == null) { + throw new TableNotFoundException(pinotTableHandle.toSchemaTableName()); + } + ImmutableMap.Builder columnHandles = ImmutableMap.builder(); + for (ColumnMetadata column : table.getColumnsMetadata()) { + columnHandles.put(column.getName().toLowerCase(ENGLISH), + new PinotColumnHandle(((PinotColumnMetadata) column).getPinotName(), column.getType(), REGULAR)); + } + return columnHandles.build(); + } + + @Override + public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) + { + requireNonNull(prefix, "prefix is null"); + ImmutableMap.Builder> columns = ImmutableMap.builder(); + for (SchemaTableName tableName : listTables(session, prefix)) { + ConnectorTableMetadata tableMetadata = getTableMetadata(tableName); + // table can disappear during listing operation + if (tableMetadata != null) { + columns.put(tableName, tableMetadata.getColumns()); + } + } + return columns.build(); + } + + private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName) + { + String pinotTableName = getPinotTableNameFromPrestoTableName(tableName.getTableName()); + PinotTable table = pinotPrestoConnection.getTable(pinotTableName); + if (table == null) { + return null; + } + return new ConnectorTableMetadata(tableName, table.getColumnsMetadata()); + } + + private List listTables(ConnectorSession session, SchemaTablePrefix prefix) + { + if (prefix.getSchemaName() == null || prefix.getTableName() == null) { + return listTables(session, prefix.getSchemaName()); + } + return ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName())); + } + + @Override + public ColumnMetadata getColumnMetadata( + ConnectorSession session, + ConnectorTableHandle tableHandle, + ColumnHandle columnHandle) + { + return ((PinotColumnHandle) columnHandle).getColumnMetadata(); + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotMetrics.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotMetrics.java new file mode 100644 index 0000000000000..54ded495fb423 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotMetrics.java @@ -0,0 +1,114 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.StringResponseHandler.StringResponse; +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.concurrent.TimeUnit; + +import static java.util.Locale.ENGLISH; + +@ThreadSafe +public class PinotMetrics +{ + private final PinotMetricsStats getStats = new PinotMetricsStats(false); + private final PinotMetricsStats queryStats = new PinotMetricsStats(true); + private final PinotMetricsStats tablesStats = new PinotMetricsStats(true); + private final PinotMetricsStats schemaStats = new PinotMetricsStats(true); + private final PinotMetricsStats brokerTimeBoundaryStats = new PinotMetricsStats(false); + private final PinotMetricsStats brokerRoutingTableStats = new PinotMetricsStats(true); + + @Managed + @Nested + public PinotMetricsStats getQueryStats() + { + return queryStats; + } + + @Managed + @Nested + public PinotMetricsStats getGetStats() + { + return getStats; + } + + @Managed + @Nested + public PinotMetricsStats getTablesStats() + { + return tablesStats; + } + + @Managed + @Nested + public PinotMetricsStats getSchemaStats() + { + return schemaStats; + } + + @Managed + @Nested + public PinotMetricsStats getBrokerTimeBoundaryStats() + { + return brokerTimeBoundaryStats; + } + + @Managed + @Nested + public PinotMetricsStats getBrokerRoutingTableStats() + { + return brokerRoutingTableStats; + } + + public void monitorRequest( + Request request, + StringResponse response, + long duration, + TimeUnit timeUnit) + { + String[] split = request.getUri().getPath().split("/"); + String secondLast = split.length >= 2 ? split[split.length - 2].toLowerCase(ENGLISH) : null; + String last = split[split.length - 1].toLowerCase(ENGLISH); + if ("post".equalsIgnoreCase(request.getMethod()) && "query".equalsIgnoreCase(last)) { + queryStats.record(response, duration, timeUnit); + } + else if ("get".equalsIgnoreCase(request.getMethod())) { + switch (last) { + case "tables": + tablesStats.record(response, duration, timeUnit); + break; + case "schema": + schemaStats.record(response, duration, timeUnit); + break; + case "debug": + if (secondLast != null) { + switch (secondLast) { + case "routingtable": + brokerRoutingTableStats.record(response, duration, timeUnit); + break; + case "timeboundary": + brokerTimeBoundaryStats.record(response, duration, timeUnit); + break; + } + } + } + getStats.record(response, duration, timeUnit); + } + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotMetricsStats.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotMetricsStats.java new file mode 100644 index 0000000000000..842dc11ee0b1f --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotMetricsStats.java @@ -0,0 +1,85 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.http.client.StringResponseHandler.StringResponse; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.DistributionStat; +import com.facebook.airlift.stats.TimeStat; +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; + +import javax.annotation.concurrent.ThreadSafe; + +import java.util.concurrent.TimeUnit; + +import static com.facebook.presto.pinot.PinotUtils.isValidPinotHttpResponseCode; + +@ThreadSafe +public class PinotMetricsStats +{ + private final TimeStat time = new TimeStat(TimeUnit.MILLISECONDS); + private final CounterStat requests = new CounterStat(); + private final CounterStat errorRequests = new CounterStat(); + private DistributionStat responseSize; + + public PinotMetricsStats(boolean withResponse) + { + if (withResponse) { + responseSize = new DistributionStat(); + } + } + + public void record(StringResponse response, long duration, TimeUnit timeUnit) + { + time.add(duration, timeUnit); + requests.update(1); + if (isValidPinotHttpResponseCode(response.getStatusCode())) { + if (responseSize != null) { + responseSize.add(response.getBody().length()); + } + } + else { + errorRequests.update(1); + } + } + + @Managed + @Nested + public TimeStat getTime() + { + return time; + } + + @Managed + @Nested + public CounterStat getRequests() + { + return requests; + } + + @Managed + @Nested + public CounterStat getErrorRequests() + { + return errorRequests; + } + + @Managed + @Nested + public DistributionStat getResponseSize() + { + return responseSize; + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotModule.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotModule.java new file mode 100644 index 0000000000000..c0e09da286302 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotModule.java @@ -0,0 +1,108 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.pinot.query.PinotQueryGenerator; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; +import io.airlift.units.DataSize; +import io.airlift.units.Duration; + +import javax.inject.Inject; + +import java.util.concurrent.Executor; + +import static com.facebook.airlift.concurrent.Threads.threadsNamed; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.google.common.base.Preconditions.checkArgument; +import static io.airlift.units.DataSize.Unit.MEGABYTE; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.Executors.newSingleThreadExecutor; +import static java.util.concurrent.TimeUnit.SECONDS; + +/** + * Guice module for the Pinot connector. + */ +public class PinotModule + implements Module +{ + private final String catalogName; + + public PinotModule(String catalogName) + { + this.catalogName = catalogName; + } + + @Override + public void configure(Binder binder) + { + configBinder(binder).bindConfig(PinotConfig.class); + binder.bind(PinotConnector.class).in(Scopes.SINGLETON); + binder.bind(PinotMetadata.class).in(Scopes.SINGLETON); + binder.bind(PinotConnectorPlanOptimizer.class).in(Scopes.SINGLETON); + binder.bind(PinotSplitManager.class).in(Scopes.SINGLETON); + binder.bind(PinotPageSourceProvider.class).in(Scopes.SINGLETON); + binder.bind(PinotClusterInfoFetcher.class).in(Scopes.SINGLETON); + binder.bind(Executor.class).annotatedWith(ForPinot.class) + .toInstance(newSingleThreadExecutor(threadsNamed("pinot-metadata-fetcher-" + catalogName))); + + binder.bind(PinotConnection.class).in(Scopes.SINGLETON); + binder.bind(PinotSessionProperties.class).in(Scopes.SINGLETON); + binder.bind(PinotNodePartitioningProvider.class).in(Scopes.SINGLETON); + binder.bind(PinotQueryGenerator.class).in(Scopes.SINGLETON); + httpClientBinder(binder).bindHttpClient("pinot", ForPinot.class) + .withConfigDefaults(cfg -> { + cfg.setIdleTimeout(new Duration(300, SECONDS)); + cfg.setRequestTimeout(new Duration(300, SECONDS)); + cfg.setMaxConnectionsPerServer(250); + cfg.setMaxContentLength(new DataSize(32, MEGABYTE)); + }); + + jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); + jsonCodecBinder(binder).bindMapJsonCodec(String.class, listJsonCodec(PinotTable.class)); + PinotClusterInfoFetcher.addJsonBinders(jsonCodecBinder(binder)); + } + + @SuppressWarnings("serial") + public static final class TypeDeserializer + extends FromStringDeserializer + { + private final TypeManager typeManager; + + @Inject + public TypeDeserializer(TypeManager typeManager) + { + super(Type.class); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + } + + @Override + protected Type _deserialize(String value, DeserializationContext context) + { + Type type = typeManager.getType(parseTypeSignature(value)); + checkArgument(type != null, "Unknown type %s", value); + return type; + } + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotNodePartitioningProvider.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotNodePartitioningProvider.java new file mode 100644 index 0000000000000..c3995e8ead8f5 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotNodePartitioningProvider.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.BucketFunction; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.connector.ConnectorBucketNodeMap; +import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider; +import com.facebook.presto.spi.connector.ConnectorPartitioningHandle; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.type.Type; + +import java.util.List; +import java.util.function.ToIntFunction; + +public class PinotNodePartitioningProvider + implements ConnectorNodePartitioningProvider +{ + @Override + public ConnectorBucketNodeMap getBucketNodeMap( + ConnectorTransactionHandle transactionHandle, + ConnectorSession session, + ConnectorPartitioningHandle partitioningHandle) + { + return ConnectorBucketNodeMap.createBucketNodeMap(1); + } + + @Override + public ToIntFunction getSplitBucketFunction( + ConnectorTransactionHandle transactionHandle, + ConnectorSession session, + ConnectorPartitioningHandle partitioningHandle) + { + return value -> 0; + } + + @Override + public BucketFunction getBucketFunction( + ConnectorTransactionHandle transactionHandle, + ConnectorSession session, + ConnectorPartitioningHandle partitioningHandle, + List partitionChannelTypes, + int bucketCount) + { + return null; + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotPageSourceProvider.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotPageSourceProvider.java new file mode 100644 index 0000000000000..1cb063fc7fdb1 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotPageSourceProvider.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorPageSource; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.fasterxml.jackson.databind.ObjectMapper; + +import javax.inject.Inject; + +import java.util.ArrayList; +import java.util.List; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class PinotPageSourceProvider + implements ConnectorPageSourceProvider +{ + private final String connectorId; + private final PinotConfig pinotConfig; + private final PinotScatterGatherQueryClient pinotQueryClient; + private final PinotClusterInfoFetcher clusterInfoFetcher; + private final ObjectMapper objectMapper; + + @Inject + public PinotPageSourceProvider( + ConnectorId connectorId, + PinotConfig pinotConfig, + PinotClusterInfoFetcher clusterInfoFetcher, + ObjectMapper objectMapper) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); + this.pinotConfig = requireNonNull(pinotConfig, "pinotConfig is null"); + this.pinotQueryClient = new PinotScatterGatherQueryClient(new PinotScatterGatherQueryClient.Config( + pinotConfig.getIdleTimeout().toMillis(), + pinotConfig.getThreadPoolSize(), + pinotConfig.getMinConnectionsPerServer(), + pinotConfig.getMaxBacklogPerServer(), + pinotConfig.getMaxConnectionsPerServer())); + this.clusterInfoFetcher = requireNonNull(clusterInfoFetcher, "cluster info fetcher is null"); + this.objectMapper = requireNonNull(objectMapper, "object mapper is null"); + } + + @Override + public ConnectorPageSource createPageSource( + ConnectorTransactionHandle transactionHandle, + ConnectorSession session, + ConnectorSplit split, + ConnectorTableLayoutHandle tableLayoutHandle, + List columns) + { + requireNonNull(split, "split is null"); + + PinotSplit pinotSplit = (PinotSplit) split; + checkArgument(pinotSplit.getConnectorId().equals(connectorId), "split is not for this connector"); + + List handles = new ArrayList<>(); + for (ColumnHandle handle : columns) { + handles.add((PinotColumnHandle) handle); + } + + switch (pinotSplit.getSplitType()) { + case SEGMENT: + return new PinotSegmentPageSource( + session, + this.pinotConfig, + this.pinotQueryClient, + pinotSplit, + handles); + case BROKER: + return new PinotBrokerPageSource( + this.pinotConfig, + session, + pinotSplit.getBrokerPql().get(), + handles, + clusterInfoFetcher, + objectMapper); + default: + throw new UnsupportedOperationException("Unknown Pinot split type: " + pinotSplit.getSplitType()); + } + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotPushdownUtils.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotPushdownUtils.java new file mode 100644 index 0000000000000..0cfb3e51470fd --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotPushdownUtils.java @@ -0,0 +1,215 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.BooleanType; +import com.facebook.presto.spi.type.CharType; +import com.facebook.presto.spi.type.DecimalType; +import com.facebook.presto.spi.type.DoubleType; +import com.facebook.presto.spi.type.IntegerType; +import com.facebook.presto.spi.type.RealType; +import com.facebook.presto.spi.type.SmallintType; +import com.facebook.presto.spi.type.TinyintType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.VarcharType; +import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slice; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNSUPPORTED_EXPRESSION; +import static com.facebook.presto.spi.type.Decimals.decodeUnscaledValue; +import static com.google.common.base.Preconditions.checkState; +import static java.lang.Float.intBitsToFloat; +import static java.lang.String.format; + +public class PinotPushdownUtils +{ + private PinotPushdownUtils() {} + + public enum ExpressionType + { + GROUP_BY, + AGGREGATE, + } + + /** + * Group by field description + */ + public static class GroupByColumnNode + extends AggregationColumnNode + { + private final VariableReferenceExpression inputColumn; + + public GroupByColumnNode(VariableReferenceExpression inputColumn, VariableReferenceExpression output) + { + super(ExpressionType.GROUP_BY, output); + this.inputColumn = inputColumn; + } + + public VariableReferenceExpression getInputColumn() + { + return inputColumn; + } + + @Override + public String toString() + { + return inputColumn.toString(); + } + } + + /** + * Agg function description. + */ + public static class AggregationFunctionColumnNode + extends AggregationColumnNode + { + private final CallExpression callExpression; + + public AggregationFunctionColumnNode(VariableReferenceExpression output, CallExpression callExpression) + { + super(ExpressionType.AGGREGATE, output); + this.callExpression = callExpression; + } + + public CallExpression getCallExpression() + { + return callExpression; + } + + @Override + public String toString() + { + return callExpression.toString(); + } + } + + public static void checkSupported(boolean condition, String errorMessage, Object... errorMessageArgs) + { + if (!condition) { + throw new PinotException(PinotErrorCode.PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), String.format(errorMessage, errorMessageArgs)); + } + } + + public abstract static class AggregationColumnNode + { + private final ExpressionType expressionType; + private final VariableReferenceExpression outputColumn; + + public AggregationColumnNode(ExpressionType expressionType, VariableReferenceExpression outputColumn) + { + this.expressionType = expressionType; + this.outputColumn = outputColumn; + } + + public VariableReferenceExpression getOutputColumn() + { + return outputColumn; + } + + public ExpressionType getExpressionType() + { + return expressionType; + } + } + + public static List computeAggregationNodes(AggregationNode aggregationNode) + { + int groupByKeyIndex = 0; + ImmutableList.Builder nodeBuilder = ImmutableList.builder(); + for (VariableReferenceExpression outputColumn : aggregationNode.getOutputVariables()) { + AggregationNode.Aggregation agg = aggregationNode.getAggregations().get(outputColumn); + + if (agg != null) { + if (agg.getFilter().isPresent() + || agg.isDistinct() + || agg.getOrderBy().isPresent() + || agg.getMask().isPresent()) { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Unsupported aggregation node " + aggregationNode); + } + nodeBuilder.add(new AggregationFunctionColumnNode(outputColumn, agg.getCall())); + } + else { + // group by output + VariableReferenceExpression inputColumn = aggregationNode.getGroupingKeys().get(groupByKeyIndex); + nodeBuilder.add(new GroupByColumnNode(inputColumn, outputColumn)); + groupByKeyIndex++; + } + } + return nodeBuilder.build(); + } + + public static LinkedHashMap getOrderingScheme(TopNNode topNNode) + { + LinkedHashMap orderingScheme = new LinkedHashMap<>(); + topNNode.getOrderingScheme().getOrderByVariables().forEach(value -> orderingScheme.put(value, topNNode.getOrderingScheme().getOrdering(value))); + return orderingScheme; + } + + private static Number decodeDecimal(BigInteger unscaledValue, DecimalType type) + { + return new BigDecimal(unscaledValue, type.getScale(), new MathContext(type.getPrecision())); + } + + // Copied from com.facebook.presto.sql.planner.LiteralInterpreter.evaluate + public static String getLiteralAsString(ConstantExpression node) + { + Type type = node.getType(); + + if (node.getValue() == null) { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), String.format("Null constant expression %s with value of type %s", node, type)); + } + if (type instanceof BooleanType) { + return String.valueOf(((Boolean) node.getValue()).booleanValue()); + } + if (type instanceof BigintType || type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType) { + Number number = (Number) node.getValue(); + return format("%d", number.longValue()); + } + if (type instanceof DoubleType) { + return node.getValue().toString(); + } + if (type instanceof RealType) { + Long number = (Long) node.getValue(); + return format("%f", intBitsToFloat(number.intValue())); + } + if (type instanceof DecimalType) { + DecimalType decimalType = (DecimalType) type; + if (decimalType.isShort()) { + checkState(node.getValue() instanceof Long); + return decodeDecimal(BigInteger.valueOf((long) node.getValue()), decimalType).toString(); + } + checkState(node.getValue() instanceof Slice); + Slice value = (Slice) node.getValue(); + return decodeDecimal(decodeUnscaledValue(value), decimalType).toString(); + } + if (type instanceof VarcharType || type instanceof CharType) { + return "'" + ((Slice) node.getValue()).toStringUtf8() + "'"; + } + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), String.format("Cannot handle the constant expression %s with value of type %s", node, type)); + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSegmentPageSource.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSegmentPageSource.java new file mode 100644 index 0000000000000..bc26d263ecb64 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSegmentPageSource.java @@ -0,0 +1,434 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.pinot.PinotScatterGatherQueryClient.ErrorCode; +import com.facebook.presto.spi.ConnectorPageSource; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.PageBuilder; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableMap; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import org.apache.pinot.common.data.FieldSpec.DataType; +import org.apache.pinot.common.response.ServerInstance; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.DataTable; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNSUPPORTED_COLUMN_TYPE; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.google.common.base.Preconditions.checkArgument; +import static io.airlift.slice.Slices.utf8Slice; +import static java.util.Objects.requireNonNull; + +/** + * This class retrieves Pinot data from a Pinot client, and re-constructs the data into Presto Pages. + */ + +public class PinotSegmentPageSource + implements ConnectorPageSource +{ + private static final Map PINOT_ERROR_CODE_MAP = ImmutableMap.of( + ErrorCode.PINOT_UNCLASSIFIED_ERROR, PinotErrorCode.PINOT_UNCLASSIFIED_ERROR, + ErrorCode.PINOT_INSUFFICIENT_SERVER_RESPONSE, PinotErrorCode.PINOT_INSUFFICIENT_SERVER_RESPONSE, + ErrorCode.PINOT_INVALID_PQL_GENERATED, PinotErrorCode.PINOT_INVALID_PQL_GENERATED); + + private final List columnHandles; + private final PinotConfig pinotConfig; + private final PinotSplit split; + private final PinotScatterGatherQueryClient pinotQueryClient; + private final ConnectorSession session; + + private List columnTypes; + // dataTableList stores the dataTable returned from each server. Each dataTable is constructed to a Page, and then destroyed to save memory. + private LinkedList dataTableList = new LinkedList<>(); + private long completedBytes; + private long readTimeNanos; + private long estimatedMemoryUsageInBytes; + private PinotDataTableWithSize currentDataTable; + private boolean closed; + private boolean isPinotDataFetched; + + public PinotSegmentPageSource( + ConnectorSession session, + PinotConfig pinotConfig, + PinotScatterGatherQueryClient pinotQueryClient, + PinotSplit split, + List columnHandles) + { + this.pinotConfig = requireNonNull(pinotConfig, "pinotConfig is null"); + this.split = requireNonNull(split, "split is null"); + this.pinotQueryClient = requireNonNull(pinotQueryClient, "pinotQueryClient is null"); + this.columnHandles = requireNonNull(columnHandles, "columnHandles is null"); + this.session = requireNonNull(session, "session is null"); + } + + private static void checkExceptions(DataTable dataTable, PinotSplit split) + { + Map metadata = dataTable.getMetadata(); + List exceptions = new ArrayList<>(); + metadata.forEach((k, v) -> { + if (k.startsWith(DataTable.EXCEPTION_METADATA_KEY)) { + exceptions.add(v); + } + }); + if (!exceptions.isEmpty()) { + throw new PinotException(PinotErrorCode.PINOT_EXCEPTION, split.getSegmentPql(), String.format("Encountered %d pinot exceptions for split %s: %s", exceptions.size(), split, exceptions)); + } + } + + @Override + public long getCompletedBytes() + { + return completedBytes; + } + + @Override + public long getReadTimeNanos() + { + return readTimeNanos; + } + + @Override + public long getSystemMemoryUsage() + { + return estimatedMemoryUsageInBytes; + } + + /** + * @return true if is closed or all Pinot data have been processed. + */ + @Override + public boolean isFinished() + { + return closed || (isPinotDataFetched && dataTableList.isEmpty()); + } + + /** + * @return constructed page for pinot data. + */ + @Override + public Page getNextPage() + { + if (isFinished()) { + close(); + return null; + } + if (!isPinotDataFetched) { + fetchPinotData(); + } + // To reduce memory usage, remove dataTable from dataTableList once it's processed. + if (currentDataTable != null) { + estimatedMemoryUsageInBytes -= currentDataTable.getEstimatedSizeInBytes(); + } + if (dataTableList.size() == 0) { + close(); + return null; + } + currentDataTable = dataTableList.pop(); + + PageBuilder pageBuilder = new PageBuilder(columnTypes); + // Note that declared positions in the Page should be the same with number of rows in each Block + pageBuilder.declarePositions(currentDataTable.getDataTable().getNumberOfRows()); + for (int columnHandleIdx = 0; columnHandleIdx < columnHandles.size(); columnHandleIdx++) { + BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(columnHandleIdx); + Type columnType = columnTypes.get(columnHandleIdx); + // Write a block for each column in the original order. + writeBlock(blockBuilder, columnType, columnHandleIdx); + } + + return pageBuilder.build(); + } + + /** + * Fetch data from Pinot for the current split and store the data returned from each Pinot server. + */ + private void fetchPinotData() + { + long startTimeNanos = System.nanoTime(); + try { + Map dataTableMap = queryPinot(session, split); + dataTableMap.values().stream() + // ignore empty tables and tables with 0 rows + .filter(table -> table != null && table.getNumberOfRows() > 0) + .forEach(dataTable -> + { + checkExceptions(dataTable, split); + // Store each dataTable which will later be constructed into Pages. + // Also update estimatedMemoryUsage, mostly represented by the size of all dataTables, using numberOfRows and fieldTypes combined as an estimate + int estimatedTableSizeInBytes = IntStream.rangeClosed(0, dataTable.getDataSchema().size() - 1) + .map(i -> getEstimatedColumnSizeInBytes(dataTable.getDataSchema().getColumnDataType(i)) * dataTable.getNumberOfRows()) + .reduce(0, Integer::sum); + dataTableList.add(new PinotDataTableWithSize(dataTable, estimatedTableSizeInBytes)); + estimatedMemoryUsageInBytes += estimatedTableSizeInBytes; + }); + + this.columnTypes = columnHandles + .stream() + .map(columnHandle -> getTypeForBlock(columnHandle)) + .collect(Collectors.toList()); + isPinotDataFetched = true; + } + finally { + readTimeNanos += System.nanoTime() - startTimeNanos; + } + } + + private Map queryPinot(ConnectorSession session, PinotSplit split) + { + String pql = split.getSegmentPql().orElseThrow(() -> new PinotException(PinotErrorCode.PINOT_INVALID_PQL_GENERATED, Optional.empty(), "Expected the segment split to contain the pql")); + String host = split.getSegmentHost().orElseThrow(() -> new PinotException(PinotErrorCode.PINOT_INVALID_PQL_GENERATED, Optional.empty(), "Expected the segment split to contain the host")); + try { + return ImmutableMap.copyOf( + pinotQueryClient.queryPinotServerForDataTable( + pql, + host, + split.getSegments(), + PinotSessionProperties.getConnectionTimeout(session).toMillis(), + PinotSessionProperties.isIgnoreEmptyResponses(session), + PinotSessionProperties.getPinotRetryCount(session))); + } + catch (PinotScatterGatherQueryClient.PinotException pe) { + throw new PinotException(PINOT_ERROR_CODE_MAP.getOrDefault(pe.getErrorCode(), PinotErrorCode.PINOT_UNCLASSIFIED_ERROR), Optional.of(pql), String.format("Error when hitting host %s", host), pe); + } + } + + @Override + public void close() + { + if (closed) { + return; + } + closed = true; + } + + /** + * Generates the {@link com.facebook.presto.spi.block.Block} for the specific column from the {@link #currentDataTable}. + * + *

    Based on the original Pinot column types, write as Presto-supported values to {@link com.facebook.presto.spi.block.BlockBuilder}, e.g. + * FLOAT -> Double, INT -> Long, String -> Slice. + * + * @param blockBuilder blockBuilder for the current column + * @param columnType type of the column + * @param columnIdx column index + */ + + private void writeBlock(BlockBuilder blockBuilder, Type columnType, int columnIdx) + { + Class javaType = columnType.getJavaType(); + DataSchema.ColumnDataType pinotColumnType = currentDataTable.getDataTable().getDataSchema().getColumnDataType(columnIdx); + if (javaType.equals(boolean.class)) { + writeBooleanBlock(blockBuilder, columnType, columnIdx); + } + else if (javaType.equals(long.class)) { + writeLongBlock(blockBuilder, columnType, columnIdx); + } + else if (javaType.equals(double.class)) { + writeDoubleBlock(blockBuilder, columnType, columnIdx); + } + else if (javaType.equals(Slice.class)) { + writeSliceBlock(blockBuilder, columnType, columnIdx); + } + else { + throw new PrestoException( + PINOT_UNSUPPORTED_COLUMN_TYPE, + String.format( + "Failed to write column %s. pinotColumnType %s, javaType %s", + columnHandles.get(columnIdx).getColumnName(), pinotColumnType, javaType)); + } + } + + private void writeBooleanBlock(BlockBuilder blockBuilder, Type columnType, int columnIndex) + { + for (int i = 0; i < currentDataTable.getDataTable().getNumberOfRows(); i++) { + columnType.writeBoolean(blockBuilder, getBoolean(i, columnIndex)); + completedBytes++; + } + } + + private void writeLongBlock(BlockBuilder blockBuilder, Type columnType, int columnIndex) + { + for (int i = 0; i < currentDataTable.getDataTable().getNumberOfRows(); i++) { + columnType.writeLong(blockBuilder, getLong(i, columnIndex)); + completedBytes += Long.BYTES; + } + } + + private void writeDoubleBlock(BlockBuilder blockBuilder, Type columnType, int columnIndex) + { + for (int i = 0; i < currentDataTable.getDataTable().getNumberOfRows(); i++) { + columnType.writeDouble(blockBuilder, getDouble(i, columnIndex)); + completedBytes += Double.BYTES; + } + } + + private void writeSliceBlock(BlockBuilder blockBuilder, Type columnType, int columnIndex) + { + for (int i = 0; i < currentDataTable.getDataTable().getNumberOfRows(); i++) { + Slice slice = getSlice(i, columnIndex); + columnType.writeSlice(blockBuilder, slice, 0, slice.length()); + completedBytes += slice.getBytes().length; + } + } + + Type getType(int columnIndex) + { + checkArgument(columnIndex < columnHandles.size(), "Invalid field index"); + return columnHandles.get(columnIndex).getDataType(); + } + + boolean getBoolean(int rowIdx, int columnIndex) + { + return Boolean.getBoolean(currentDataTable.getDataTable().getString(rowIdx, columnIndex)); + } + + long getLong(int rowIndex, int columnIndex) + { + DataSchema.ColumnDataType dataType = currentDataTable.getDataTable().getDataSchema().getColumnDataType(columnIndex); + // Note columnType in the dataTable could be different from the original columnType in the columnHandle. + // e.g. when original column type is int/long and aggregation value is requested, the returned dataType from Pinot would be double. + // So need to cast it back to the original columnType. + if (dataType.equals(DataType.DOUBLE)) { + return (long) currentDataTable.getDataTable().getDouble(rowIndex, columnIndex); + } + if (dataType.equals(DataType.INT)) { + return (long) currentDataTable.getDataTable().getInt(rowIndex, columnIndex); + } + else { + return currentDataTable.getDataTable().getLong(rowIndex, columnIndex); + } + } + + double getDouble(int rowIndex, int columnIndex) + { + DataSchema.ColumnDataType dataType = currentDataTable.getDataTable().getDataSchema().getColumnDataType(columnIndex); + if (dataType.equals(DataType.FLOAT)) { + return currentDataTable.getDataTable().getFloat(rowIndex, columnIndex); + } + else { + return currentDataTable.getDataTable().getDouble(rowIndex, columnIndex); + } + } + + Slice getSlice(int rowIndex, int columnIndex) + { + checkColumnType(columnIndex, VARCHAR); + DataSchema.ColumnDataType columnType = currentDataTable.getDataTable().getDataSchema().getColumnDataType(columnIndex); + switch (columnType) { + case INT_ARRAY: + int[] intArray = currentDataTable.getDataTable().getIntArray(rowIndex, columnIndex); + return utf8Slice(Arrays.toString(intArray)); + case LONG_ARRAY: + long[] longArray = currentDataTable.getDataTable().getLongArray(rowIndex, columnIndex); + return utf8Slice(Arrays.toString(longArray)); + case FLOAT_ARRAY: + float[] floatArray = currentDataTable.getDataTable().getFloatArray(rowIndex, columnIndex); + return utf8Slice(Arrays.toString(floatArray)); + case DOUBLE_ARRAY: + double[] doubleArray = currentDataTable.getDataTable().getDoubleArray(rowIndex, columnIndex); + return utf8Slice(Arrays.toString(doubleArray)); + case STRING_ARRAY: + String[] stringArray = currentDataTable.getDataTable().getStringArray(rowIndex, columnIndex); + return utf8Slice(Arrays.toString(stringArray)); + case STRING: + String field = currentDataTable.getDataTable().getString(rowIndex, columnIndex); + if (field == null || field.isEmpty()) { + return Slices.EMPTY_SLICE; + } + return Slices.utf8Slice(field); + } + return Slices.EMPTY_SLICE; + } + + /** + * Get estimated size in bytes for the Pinot column. + * Deterministic for numeric fields; use estimate for other types to save calculation. + * + * @param dataType FieldSpec.dataType for Pinot column. + * @return estimated size in bytes. + */ + private int getEstimatedColumnSizeInBytes(DataSchema.ColumnDataType dataType) + { + if (dataType.isNumber()) { + switch (dataType) { + case LONG: + return Long.BYTES; + case FLOAT: + return Float.BYTES; + case DOUBLE: + return Double.BYTES; + case INT: + default: + return Integer.BYTES; + } + } + return pinotConfig.getEstimatedSizeInBytesForNonNumericColumn(); + } + + void checkColumnType(int columnIndex, Type expected) + { + Type actual = getType(columnIndex); + checkArgument(actual.equals(expected), "Expected column %s to be type %s but is %s", columnIndex, expected, actual); + } + + Type getTypeForBlock(PinotColumnHandle pinotColumnHandle) + { + if (pinotColumnHandle.getDataType().equals(INTEGER)) { + return BIGINT; + } + return pinotColumnHandle.getDataType(); + } + + @Override + public long getCompletedPositions() + { + return 0; + } + + private static class PinotDataTableWithSize + { + DataTable dataTable; + int estimatedSizeInBytes; + + PinotDataTableWithSize(DataTable dataTable, int estimatedSizeInBytes) + { + this.dataTable = dataTable; + this.estimatedSizeInBytes = estimatedSizeInBytes; + } + + DataTable getDataTable() + { + return dataTable; + } + + int getEstimatedSizeInBytes() + { + return estimatedSizeInBytes; + } + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSessionProperties.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSessionProperties.java new file mode 100644 index 0000000000000..d6bfaff0d49ce --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSessionProperties.java @@ -0,0 +1,153 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.session.PropertyMetadata; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import io.airlift.units.Duration; + +import javax.inject.Inject; + +import java.util.List; + +import static com.facebook.presto.spi.session.PropertyMetadata.booleanProperty; +import static com.facebook.presto.spi.session.PropertyMetadata.integerProperty; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType; +import static com.google.common.base.Preconditions.checkArgument; + +public class PinotSessionProperties +{ + private static final String CONNECTION_TIMEOUT = "connection_timeout"; + private static final String PREFER_BROKER_QUERIES = "prefer_broker_queries"; + private static final String IGNORE_EMPTY_RESPONSES = "ignore_empty_responses"; + private static final String RETRY_COUNT = "retry_count"; + private static final String USE_DATE_TRUNC = "use_date_trunc"; + private static final String NON_AGGREGATE_LIMIT_FOR_BROKER_QUERIES = "non_aggregate_limit_for_broker_queries"; + + @VisibleForTesting + public static final String FORBID_SEGMENT_QUERIES = "forbid_segment_queries"; + + @VisibleForTesting + public static final String NUM_SEGMENTS_PER_SPLIT = "num_segments_per_split"; + + private final List> sessionProperties; + + public static int getNumSegmentsPerSplit(ConnectorSession session) + { + int segmentsPerSplit = session.getProperty(NUM_SEGMENTS_PER_SPLIT, Integer.class); + return segmentsPerSplit <= 0 ? Integer.MAX_VALUE : segmentsPerSplit; + } + + public static boolean isPreferBrokerQueries(ConnectorSession session) + { + return session.getProperty(PREFER_BROKER_QUERIES, Boolean.class); + } + + public static boolean isForbidSegmentQueries(ConnectorSession session) + { + return session.getProperty(FORBID_SEGMENT_QUERIES, Boolean.class); + } + + public static Duration getConnectionTimeout(ConnectorSession session) + { + return session.getProperty(CONNECTION_TIMEOUT, Duration.class); + } + + public static boolean isIgnoreEmptyResponses(ConnectorSession session) + { + return session.getProperty(IGNORE_EMPTY_RESPONSES, Boolean.class); + } + + public static int getPinotRetryCount(ConnectorSession session) + { + return session.getProperty(RETRY_COUNT, Integer.class); + } + + public static boolean isUseDateTruncation(ConnectorSession session) + { + return session.getProperty(USE_DATE_TRUNC, Boolean.class); + } + + public static int getNonAggregateLimitForBrokerQueries(ConnectorSession session) + { + return session.getProperty(NON_AGGREGATE_LIMIT_FOR_BROKER_QUERIES, Integer.class); + } + + @Inject + public PinotSessionProperties(PinotConfig pinotConfig) + { + sessionProperties = ImmutableList.of( + booleanProperty( + PREFER_BROKER_QUERIES, + "Prefer queries to broker even when parallel scan is enabled for aggregation queries", + pinotConfig.isPreferBrokerQueries(), + false), + booleanProperty( + FORBID_SEGMENT_QUERIES, + "Forbid segment queries", + pinotConfig.isForbidSegmentQueries(), + false), + booleanProperty( + IGNORE_EMPTY_RESPONSES, + "Ignore empty or missing pinot server responses", + pinotConfig.isIgnoreEmptyResponses(), + false), + integerProperty( + RETRY_COUNT, + "Retry count for retriable pinot data fetch calls", + pinotConfig.getFetchRetryCount(), + false), + integerProperty( + NON_AGGREGATE_LIMIT_FOR_BROKER_QUERIES, + "Max limit for non aggregate queries to the pinot broker", + pinotConfig.getNonAggregateLimitForBrokerQueries(), + false), + booleanProperty( + USE_DATE_TRUNC, + "Use the new UDF dateTrunc in pinot that is more presto compatible", + pinotConfig.isUseDateTrunc(), + false), + new PropertyMetadata<>( + CONNECTION_TIMEOUT, + "Connection Timeout to talk to Pinot servers", + createUnboundedVarcharType(), + Duration.class, + pinotConfig.getConnectionTimeout(), + false, + value -> Duration.valueOf((String) value), + Duration::toString), + new PropertyMetadata<>( + NUM_SEGMENTS_PER_SPLIT, + "Number of segments of the same host per split", + INTEGER, + Integer.class, + pinotConfig.getNumSegmentsPerSplit(), + false, + value -> { + int ret = ((Number) value).intValue(); + checkArgument(ret > 0, "Number of segments per split must be more than zero"); + return ret; + }, + object -> object)); + } + + public List> getSessionProperties() + { + return sessionProperties; + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSplit.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSplit.java new file mode 100644 index 0000000000000..6f02c311cb1e4 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSplit.java @@ -0,0 +1,165 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.pinot.query.PinotQueryGenerator; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.HostAddress; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class PinotSplit + implements ConnectorSplit +{ + private final String connectorId; + private final SplitType splitType; + + // Properties needed for broker split type + private final Optional brokerPql; + + // Properties needed for segment split type + private final Optional segmentPql; + private final List segments; + private final Optional segmentHost; + + @JsonCreator + public PinotSplit( + @JsonProperty("connectorId") String connectorId, + @JsonProperty("splitType") SplitType splitType, + @JsonProperty("brokerPql") Optional brokerPql, + @JsonProperty("segmentPql") Optional segmentPql, + @JsonProperty("segments") List segments, + @JsonProperty("segmentHost") Optional segmentHost) + { + this.connectorId = requireNonNull(connectorId, "connector id is null"); + this.splitType = requireNonNull(splitType, "splitType id is null"); + this.brokerPql = requireNonNull(brokerPql, "brokerPql is null"); + this.segmentPql = requireNonNull(segmentPql, "table name is null"); + this.segments = ImmutableList.copyOf(requireNonNull(segments, "segment is null")); + this.segmentHost = requireNonNull(segmentHost, "host is null"); + + // make sure the segment properties are present when the split type is segment + if (splitType == SplitType.SEGMENT) { + checkArgument(segmentPql.isPresent(), "Table name is missing from the split"); + checkArgument(!segments.isEmpty(), "Segments are missing from the split"); + checkArgument(segmentHost.isPresent(), "Segment host address is missing from the split"); + } + else { + checkArgument(brokerPql.isPresent(), "brokerPql is missing from the split"); + } + } + + public static PinotSplit createBrokerSplit(String connectorId, PinotQueryGenerator.GeneratedPql brokerPql) + { + return new PinotSplit( + requireNonNull(connectorId, "connector id is null"), + SplitType.BROKER, + Optional.of(requireNonNull(brokerPql, "brokerPql is null")), + Optional.empty(), + ImmutableList.of(), + Optional.empty()); + } + + public static PinotSplit createSegmentSplit(String connectorId, String pql, List segments, String segmentHost) + { + return new PinotSplit( + requireNonNull(connectorId, "connector id is null"), + SplitType.SEGMENT, + Optional.empty(), + Optional.of(requireNonNull(pql, "pql is null")), + requireNonNull(segments, "segments are null"), + Optional.of(requireNonNull(segmentHost, "segmentHost is null"))); + } + + @JsonProperty + public String getConnectorId() + { + return connectorId; + } + + @JsonProperty + public SplitType getSplitType() + { + return splitType; + } + + @JsonProperty + public Optional getBrokerPql() + { + return brokerPql; + } + + @JsonProperty + public Optional getSegmentPql() + { + return segmentPql; + } + + @JsonProperty + public Optional getSegmentHost() + { + return segmentHost; + } + + @JsonProperty + public List getSegments() + { + return segments; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("connectorId", connectorId) + .add("splitType", splitType) + .add("segmentPql", segmentPql) + .add("brokerPql", brokerPql) + .add("segments", segments) + .add("segmentHost", segmentHost) + .toString(); + } + + @Override + public boolean isRemotelyAccessible() + { + return true; + } + + @Override + public List getAddresses() + { + return null; + } + + @Override + public Object getInfo() + { + return this; + } + + public enum SplitType + { + SEGMENT, + BROKER, + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSplitManager.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSplitManager.java new file mode 100644 index 0000000000000..34bb63bc83ac2 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSplitManager.java @@ -0,0 +1,192 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.pinot.query.PinotQueryGenerator.GeneratedPql; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorSplitSource; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.facebook.presto.spi.ErrorCode; +import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.spi.ErrorType; +import com.facebook.presto.spi.FixedSplitSource; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.connector.ConnectorSplitManager; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.google.common.collect.Iterables; + +import javax.inject.Inject; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +import static com.facebook.presto.pinot.PinotSplit.createBrokerSplit; +import static com.facebook.presto.pinot.PinotSplit.createSegmentSplit; +import static com.facebook.presto.pinot.query.PinotQueryGeneratorContext.TABLE_NAME_SUFFIX_TEMPLATE; +import static com.facebook.presto.pinot.query.PinotQueryGeneratorContext.TIME_BOUNDARY_FILTER_TEMPLATE; +import static com.facebook.presto.spi.ErrorType.USER_ERROR; +import static java.util.Collections.singletonList; +import static java.util.Objects.requireNonNull; + +public class PinotSplitManager + implements ConnectorSplitManager +{ + private final String connectorId; + private final PinotConnection pinotPrestoConnection; + + @Inject + public PinotSplitManager(ConnectorId connectorId, PinotConnection pinotPrestoConnection) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); + this.pinotPrestoConnection = requireNonNull(pinotPrestoConnection, "pinotPrestoConnection is null"); + } + + protected ConnectorSplitSource generateSplitForBrokerBasedScan(GeneratedPql brokerPql) + { + return new FixedSplitSource(singletonList(createBrokerSplit(connectorId, brokerPql))); + } + + protected ConnectorSplitSource generateSplitsForSegmentBasedScan( + PinotTableLayoutHandle pinotLayoutHandle, + ConnectorSession session) + { + PinotTableHandle tableHandle = pinotLayoutHandle.getTable(); + String tableName = tableHandle.getTableName(); + Map>> routingTable; + + routingTable = pinotPrestoConnection.getRoutingTable(tableName); + + List splits = new ArrayList<>(); + if (!routingTable.isEmpty()) { + GeneratedPql segmentPql = tableHandle.getPql().orElseThrow(() -> new PinotException(PinotErrorCode.PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Expected to find realtime and offline pql in " + tableHandle)); + PinotClusterInfoFetcher.TimeBoundary timeBoundary = pinotPrestoConnection.getTimeBoundary(tableName); + String realtime = getSegmentPql(segmentPql, "_REALTIME", timeBoundary.getOnlineTimePredicate()); + String offline = getSegmentPql(segmentPql, "_OFFLINE", timeBoundary.getOfflineTimePredicate()); + generateSegmentSplits(splits, routingTable, tableName, "_REALTIME", session, realtime); + generateSegmentSplits(splits, routingTable, tableName, "_OFFLINE", session, offline); + } + + Collections.shuffle(splits); + return new FixedSplitSource(splits); + } + + private String getSegmentPql(GeneratedPql basePql, String suffix, Optional timePredicate) + { + String pql = basePql.getPql().replace(TABLE_NAME_SUFFIX_TEMPLATE, suffix); + if (timePredicate.isPresent()) { + String tp = timePredicate.get(); + pql = pql.replace(TIME_BOUNDARY_FILTER_TEMPLATE, basePql.isHaveFilter() ? tp : " WHERE " + tp); + } + else { + pql = pql.replace(TIME_BOUNDARY_FILTER_TEMPLATE, ""); + } + return pql; + } + + protected void generateSegmentSplits( + List splits, + Map>> routingTable, + String tableName, + String tableNameSuffix, + ConnectorSession session, + String pql) + { + final String finalTableName = tableName + tableNameSuffix; + int segmentsPerSplitConfigured = PinotSessionProperties.getNumSegmentsPerSplit(session); + for (String routingTableName : routingTable.keySet()) { + if (!routingTableName.equalsIgnoreCase(finalTableName)) { + continue; + } + + Map> hostToSegmentsMap = routingTable.get(routingTableName); + hostToSegmentsMap.forEach((host, segments) -> { + int numSegmentsInThisSplit = Math.min(segments.size(), segmentsPerSplitConfigured); + // segments is already shuffled + Iterables.partition(segments, numSegmentsInThisSplit).forEach( + segmentsForThisSplit -> splits.add( + createSegmentSplit(connectorId, pql, segmentsForThisSplit, host))); + }); + } + } + + public enum QueryNotAdequatelyPushedDownErrorCode + implements ErrorCodeSupplier + { + PQL_NOT_PRESENT(1, USER_ERROR, "Query uses unsupported expressions that cannot be pushed into the storage engine. Please see https://XXX for more details"); + + private final ErrorCode errorCode; + + QueryNotAdequatelyPushedDownErrorCode(int code, ErrorType type, String guidance) + { + errorCode = new ErrorCode(code + 0x0625_0000, name() + ": " + guidance, type); + } + + @Override + public ErrorCode toErrorCode() + { + return errorCode; + } + } + + public static class QueryNotAdequatelyPushedDownException + extends PrestoException + { + private final String connectorId; + private final ConnectorTableHandle connectorTableHandle; + + public QueryNotAdequatelyPushedDownException( + QueryNotAdequatelyPushedDownErrorCode errorCode, + ConnectorTableHandle connectorTableHandle, + String connectorId) + { + super(requireNonNull(errorCode, "error code is null"), (String) null); + this.connectorId = requireNonNull(connectorId, "connector id is null"); + this.connectorTableHandle = requireNonNull(connectorTableHandle, "connector table handle is null"); + } + + @Override + public String getMessage() + { + return super.getMessage() + String.format(" table: %s:%s", connectorId, connectorTableHandle); + } + } + + @Override + public ConnectorSplitSource getSplits( + ConnectorTransactionHandle transactionHandle, + ConnectorSession session, + ConnectorTableLayoutHandle layout, + SplitSchedulingContext splitSchedulingContext) + { + PinotTableLayoutHandle pinotLayoutHandle = (PinotTableLayoutHandle) layout; + PinotTableHandle pinotTableHandle = pinotLayoutHandle.getTable(); + Supplier errorSupplier = () -> new QueryNotAdequatelyPushedDownException(QueryNotAdequatelyPushedDownErrorCode.PQL_NOT_PRESENT, pinotTableHandle, connectorId); + if (!pinotTableHandle.getIsQueryShort().orElseThrow(errorSupplier)) { + if (PinotSessionProperties.isForbidSegmentQueries(session)) { + throw errorSupplier.get(); + } + return generateSplitsForSegmentBasedScan(pinotLayoutHandle, session); + } + else { + return generateSplitForBrokerBasedScan(pinotTableHandle.getPql().orElseThrow(errorSupplier)); + } + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTable.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTable.java new file mode 100644 index 0000000000000..79fcdddd80b47 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTable.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.ColumnMetadata; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.stream.Collectors; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.util.Objects.requireNonNull; + +public class PinotTable +{ + private final String name; + private final List columns; + private final List columnsMetadata; + + @JsonCreator + public PinotTable( + @JsonProperty("name") String name, + @JsonProperty("columns") List columns) + { + checkArgument(!isNullOrEmpty(name), "name is null or is empty"); + this.name = requireNonNull(name, "name is null"); + this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); + + this.columnsMetadata = columns.stream().map(c -> new PinotColumnMetadata(c.getName(), c.getType())).collect(Collectors.toList()); + } + + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public List getColumns() + { + return columns; + } + + public List getColumnsMetadata() + { + return columnsMetadata; + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTableHandle.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTableHandle.java new file mode 100644 index 0000000000000..0e7f134608339 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTableHandle.java @@ -0,0 +1,129 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.pinot.query.PinotQueryGenerator; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.SchemaTableName; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public final class PinotTableHandle + implements ConnectorTableHandle +{ + private final String connectorId; + private final String schemaName; + private final String tableName; + private final Optional isQueryShort; + private final Optional pql; + + public PinotTableHandle( + String connectorId, + String schemaName, + String tableName) + { + this(connectorId, schemaName, tableName, Optional.empty(), Optional.empty()); + } + + @JsonCreator + public PinotTableHandle( + @JsonProperty("connectorId") String connectorId, + @JsonProperty("schemaName") String schemaName, + @JsonProperty("tableName") String tableName, + @JsonProperty("isQueryShort") Optional isQueryShort, + @JsonProperty("pql") Optional pql) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null"); + this.schemaName = requireNonNull(schemaName, "schemaName is null"); + this.tableName = requireNonNull(tableName, "tableName is null"); + this.isQueryShort = requireNonNull(isQueryShort, "safe to execute is null"); + this.pql = requireNonNull(pql, "broker pql is null"); + } + + @JsonProperty + public Optional getPql() + { + return pql; + } + + @JsonProperty + public String getConnectorId() + { + return connectorId; + } + + @JsonProperty + public String getSchemaName() + { + return schemaName; + } + + @JsonProperty + public String getTableName() + { + return tableName; + } + + @JsonProperty + public Optional getIsQueryShort() + { + return isQueryShort; + } + + public SchemaTableName toSchemaTableName() + { + return new SchemaTableName(schemaName, tableName); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PinotTableHandle that = (PinotTableHandle) o; + return Objects.equals(connectorId, that.connectorId) && + Objects.equals(schemaName, that.schemaName) && + Objects.equals(tableName, that.tableName) && + Objects.equals(isQueryShort, that.isQueryShort) && + Objects.equals(pql, that.pql); + } + + @Override + public int hashCode() + { + return Objects.hash(connectorId, schemaName, tableName, isQueryShort, pql); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("connectorId", connectorId) + .add("schemaName", schemaName) + .add("tableName", tableName) + .add("isQueryShort", isQueryShort) + .add("pql", pql) + .toString(); + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTableLayoutHandle.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTableLayoutHandle.java new file mode 100644 index 0000000000000..dd234c6b7c185 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTableLayoutHandle.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class PinotTableLayoutHandle + implements ConnectorTableLayoutHandle +{ + private final PinotTableHandle table; + + @JsonCreator + public PinotTableLayoutHandle( + @JsonProperty("table") PinotTableHandle table) + { + this.table = requireNonNull(table, "table is null"); + } + + @JsonProperty + public PinotTableHandle getTable() + { + return table; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PinotTableLayoutHandle that = (PinotTableLayoutHandle) o; + return Objects.equals(table, that.table); + } + + @Override + public int hashCode() + { + return Objects.hash(table); + } + + @Override + public String toString() + { + return table.toString(); + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTransactionHandle.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTransactionHandle.java new file mode 100644 index 0000000000000..5e0fe3f7b31e2 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotTransactionHandle.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; + +public enum PinotTransactionHandle + implements ConnectorTransactionHandle +{ + INSTANCE +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotUtils.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotUtils.java new file mode 100644 index 0000000000000..1f13c7e97b478 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotUtils.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.google.common.base.Preconditions; + +import java.util.function.Function; + +import static java.net.HttpURLConnection.HTTP_MULT_CHOICE; +import static java.net.HttpURLConnection.HTTP_OK; + +public class PinotUtils +{ + private PinotUtils() + { + } + + static boolean isValidPinotHttpResponseCode(int status) + { + return status >= HTTP_OK && status < HTTP_MULT_CHOICE; + } + + public static T doWithRetries(int retries, Function caller) + { + PinotException firstError = null; + Preconditions.checkState(retries > 0, "Invalid num of retries %d", retries); + for (int i = 0; i < retries; ++i) { + try { + return caller.apply(i); + } + catch (PinotException e) { + if (firstError == null) { + firstError = e; + } + if (!e.getPinotErrorCode().isRetriable()) { + throw e; + } + } + } + throw firstError; + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/RebindSafeMBeanServer.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/RebindSafeMBeanServer.java new file mode 100644 index 0000000000000..1b7f8b8312660 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/RebindSafeMBeanServer.java @@ -0,0 +1,344 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import javax.annotation.concurrent.ThreadSafe; +import javax.management.Attribute; +import javax.management.AttributeList; +import javax.management.AttributeNotFoundException; +import javax.management.InstanceAlreadyExistsException; +import javax.management.InstanceNotFoundException; +import javax.management.IntrospectionException; +import javax.management.InvalidAttributeValueException; +import javax.management.ListenerNotFoundException; +import javax.management.MBeanException; +import javax.management.MBeanInfo; +import javax.management.MBeanRegistrationException; +import javax.management.MBeanServer; +import javax.management.NotCompliantMBeanException; +import javax.management.NotificationFilter; +import javax.management.NotificationListener; +import javax.management.ObjectInstance; +import javax.management.ObjectName; +import javax.management.OperationsException; +import javax.management.QueryExp; +import javax.management.ReflectionException; +import javax.management.loading.ClassLoaderRepository; + +import java.io.ObjectInputStream; +import java.util.Set; + +/** + * MBeanServer wrapper that a ignores calls to registerMBean when there is already + * a MBean registered with the specified object name. + *

    + * This originally existed in hive, raptor and cassandra and I am promoting it to SPI + */ +@ThreadSafe +public class RebindSafeMBeanServer + implements MBeanServer +{ + private final MBeanServer mbeanServer; + + public RebindSafeMBeanServer(MBeanServer mbeanServer) + { + this.mbeanServer = mbeanServer; + } + + /** + * Delegates to the wrapped mbean server, but if a mbean is already registered + * with the specified name, the existing instance is returned. + */ + @Override + public ObjectInstance registerMBean(Object object, ObjectName name) + throws MBeanRegistrationException, NotCompliantMBeanException + { + while (true) { + try { + // try to register the mbean + return mbeanServer.registerMBean(object, name); + } + catch (InstanceAlreadyExistsException ignored) { + } + + try { + // a mbean is already installed, try to return the already registered instance + ObjectInstance objectInstance = mbeanServer.getObjectInstance(name); + return objectInstance; + } + catch (InstanceNotFoundException ignored) { + // the mbean was removed before we could get the reference + // start the whole process over again + } + } + } + + @Override + public void unregisterMBean(ObjectName name) + throws InstanceNotFoundException, MBeanRegistrationException + { + mbeanServer.unregisterMBean(name); + } + + @Override + public ObjectInstance getObjectInstance(ObjectName name) + throws InstanceNotFoundException + { + return mbeanServer.getObjectInstance(name); + } + + @Override + public Set queryMBeans(ObjectName name, QueryExp query) + { + return mbeanServer.queryMBeans(name, query); + } + + @Override + public Set queryNames(ObjectName name, QueryExp query) + { + return mbeanServer.queryNames(name, query); + } + + @Override + public boolean isRegistered(ObjectName name) + { + return mbeanServer.isRegistered(name); + } + + @Override + public Integer getMBeanCount() + { + return mbeanServer.getMBeanCount(); + } + + @Override + public Object getAttribute(ObjectName name, String attribute) + throws MBeanException, AttributeNotFoundException, InstanceNotFoundException, ReflectionException + { + return mbeanServer.getAttribute(name, attribute); + } + + @Override + public AttributeList getAttributes(ObjectName name, String[] attributes) + throws InstanceNotFoundException, ReflectionException + { + return mbeanServer.getAttributes(name, attributes); + } + + @Override + public void setAttribute(ObjectName name, Attribute attribute) + throws InstanceNotFoundException, AttributeNotFoundException, InvalidAttributeValueException, MBeanException, ReflectionException + { + mbeanServer.setAttribute(name, attribute); + } + + @Override + public AttributeList setAttributes(ObjectName name, AttributeList attributes) + throws InstanceNotFoundException, ReflectionException + { + return mbeanServer.setAttributes(name, attributes); + } + + @Override + public Object invoke(ObjectName name, String operationName, Object[] params, String[] signature) + throws InstanceNotFoundException, MBeanException, ReflectionException + { + return mbeanServer.invoke(name, operationName, params, signature); + } + + @Override + public String getDefaultDomain() + { + return mbeanServer.getDefaultDomain(); + } + + @Override + public String[] getDomains() + { + return mbeanServer.getDomains(); + } + + @Override + public void addNotificationListener(ObjectName name, NotificationListener listener, NotificationFilter filter, Object context) + throws InstanceNotFoundException + { + mbeanServer.addNotificationListener(name, listener, filter, context); + } + + @Override + public void addNotificationListener(ObjectName name, ObjectName listener, NotificationFilter filter, Object context) + throws InstanceNotFoundException + { + mbeanServer.addNotificationListener(name, listener, filter, context); + } + + @Override + public void removeNotificationListener(ObjectName name, ObjectName listener) + throws InstanceNotFoundException, ListenerNotFoundException + { + mbeanServer.removeNotificationListener(name, listener); + } + + @Override + public void removeNotificationListener(ObjectName name, ObjectName listener, NotificationFilter filter, Object context) + throws InstanceNotFoundException, ListenerNotFoundException + { + mbeanServer.removeNotificationListener(name, listener, filter, context); + } + + @Override + public void removeNotificationListener(ObjectName name, NotificationListener listener) + throws InstanceNotFoundException, ListenerNotFoundException + { + mbeanServer.removeNotificationListener(name, listener); + } + + @Override + public void removeNotificationListener(ObjectName name, NotificationListener listener, NotificationFilter filter, Object context) + throws InstanceNotFoundException, ListenerNotFoundException + { + mbeanServer.removeNotificationListener(name, listener, filter, context); + } + + @Override + public MBeanInfo getMBeanInfo(ObjectName name) + throws InstanceNotFoundException, IntrospectionException, ReflectionException + { + return mbeanServer.getMBeanInfo(name); + } + + @Override + public boolean isInstanceOf(ObjectName name, String className) + throws InstanceNotFoundException + { + return mbeanServer.isInstanceOf(name, className); + } + + @Override + public Object instantiate(String className) + throws ReflectionException, MBeanException + { + return mbeanServer.instantiate(className); + } + + @Override + public Object instantiate(String className, ObjectName loaderName) + throws ReflectionException, MBeanException, InstanceNotFoundException + { + return mbeanServer.instantiate(className, loaderName); + } + + @Override + public Object instantiate(String className, Object[] params, String[] signature) + throws ReflectionException, MBeanException + { + return mbeanServer.instantiate(className, params, signature); + } + + @Override + public Object instantiate(String className, ObjectName loaderName, Object[] params, String[] signature) + throws ReflectionException, MBeanException, InstanceNotFoundException + { + return mbeanServer.instantiate(className, loaderName, params, signature); + } + + @Override + @Deprecated + @SuppressWarnings("deprecation") + public ObjectInputStream deserialize(ObjectName name, byte[] data) + throws OperationsException + { + return mbeanServer.deserialize(name, data); + } + + @Override + @Deprecated + @SuppressWarnings("deprecation") + public ObjectInputStream deserialize(String className, byte[] data) + throws OperationsException, ReflectionException + { + return mbeanServer.deserialize(className, data); + } + + @Override + @Deprecated + @SuppressWarnings("deprecation") + public ObjectInputStream deserialize(String className, ObjectName loaderName, byte[] data) + throws OperationsException, ReflectionException + { + return mbeanServer.deserialize(className, loaderName, data); + } + + @Override + public ClassLoader getClassLoaderFor(ObjectName mbeanName) + throws InstanceNotFoundException + { + return mbeanServer.getClassLoaderFor(mbeanName); + } + + @Override + public ClassLoader getClassLoader(ObjectName loaderName) + throws InstanceNotFoundException + { + return mbeanServer.getClassLoader(loaderName); + } + + @Override + public ClassLoaderRepository getClassLoaderRepository() + { + return mbeanServer.getClassLoaderRepository(); + } + + @Override + public ObjectInstance createMBean(String className, ObjectName name) + throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException + { + return mbeanServer.createMBean(className, name); + } + + @Override + public ObjectInstance createMBean(String className, ObjectName name, ObjectName loaderName) + throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException, InstanceNotFoundException + { + return mbeanServer.createMBean(className, name, loaderName); + } + + @Override + public ObjectInstance createMBean(String className, ObjectName name, Object[] params, String[] signature) + throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException + { + return mbeanServer.createMBean(className, name, params, signature); + } + + @Override + public ObjectInstance createMBean(String className, ObjectName name, ObjectName loaderName, Object[] params, String[] signature) + throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException, InstanceNotFoundException + { + return mbeanServer.createMBean(className, name, loaderName, params, signature); + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotAggregationProjectConverter.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotAggregationProjectConverter.java new file mode 100644 index 0000000000000..308b028ba3553 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotAggregationProjectConverter.java @@ -0,0 +1,262 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot.query; + +import com.facebook.presto.pinot.PinotException; +import com.facebook.presto.pinot.PinotSessionProperties; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.TypeManager; +import com.google.common.collect.ImmutableMap; +import io.airlift.slice.Slice; +import org.joda.time.DateTimeZone; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNSUPPORTED_EXPRESSION; +import static com.facebook.presto.pinot.PinotPushdownUtils.getLiteralAsString; +import static com.facebook.presto.pinot.query.PinotExpression.derived; +import static java.lang.String.format; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public class PinotAggregationProjectConverter + extends PinotProjectExpressionConverter +{ + // Pinot does not support modulus yet + private static final Map PRESTO_TO_PINOT_OPERATORS = ImmutableMap.of( + "-", "SUB", + "+", "ADD", + "*", "MULT", + "/", "DIV"); + private static final String FROM_UNIXTIME = "from_unixtime"; + + private final FunctionMetadataManager functionMetadataManager; + private final ConnectorSession session; + + public PinotAggregationProjectConverter(TypeManager typeManager, FunctionMetadataManager functionMetadataManager, StandardFunctionResolution standardFunctionResolution, ConnectorSession session) + { + super(typeManager, standardFunctionResolution); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + this.session = requireNonNull(session, "session is null"); + } + + @Override + public PinotExpression visitCall( + CallExpression call, + Map context) + { + Optional basicCallHandlingResult = basicCallHandling(call, context); + if (basicCallHandlingResult.isPresent()) { + return basicCallHandlingResult.get(); + } + FunctionMetadata functionMetadata = functionMetadataManager.getFunctionMetadata(call.getFunctionHandle()); + Optional operatorTypeOptional = functionMetadata.getOperatorType(); + if (operatorTypeOptional.isPresent()) { + OperatorType operatorType = operatorTypeOptional.get(); + if (operatorType.isArithmeticOperator()) { + return handleArithmeticExpression(call, operatorType, context); + } + if (operatorType.isComparisonOperator()) { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Comparison operator not supported: " + call); + } + } + return handleFunction(call, context); + } + + @Override + public PinotExpression visitConstant( + ConstantExpression literal, + Map context) + { + return new PinotExpression(getLiteralAsString(literal), PinotQueryGeneratorContext.Origin.LITERAL); + } + + private PinotExpression handleDateTruncationViaDateTimeConvert( + CallExpression function, + Map context) + { + // Convert SQL standard function `DATE_TRUNC(INTERVAL, DATE/TIMESTAMP COLUMN)` to + // Pinot's equivalent function `dateTimeConvert(columnName, inputFormat, outputFormat, outputGranularity)` + // Pinot doesn't have a DATE/TIMESTAMP type. That means the input column (second argument) has been converted from numeric type to DATE/TIMESTAMP using one of the + // conversion functions in SQL. First step is find the function and find its input column units (seconds, secondsSinceEpoch etc.) + RowExpression timeInputParameter = function.getArguments().get(1); + String inputColumn; + String inputFormat; + + CallExpression timeConversion = getExpressionAsFunction(timeInputParameter, timeInputParameter); + switch (timeConversion.getDisplayName().toLowerCase(ENGLISH)) { + case FROM_UNIXTIME: + inputColumn = timeConversion.getArguments().get(0).accept(this, context).getDefinition(); + inputFormat = "'1:SECONDS:EPOCH'"; + break; + default: + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "not supported: " + timeConversion.getDisplayName()); + } + + String outputFormat = "'1:MILLISECONDS:EPOCH'"; + String outputGranularity; + + RowExpression intervalParameter = function.getArguments().get(0); + if (!(intervalParameter instanceof ConstantExpression)) { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), + "interval unit in date_trunc is not supported: " + intervalParameter); + } + + String value = getStringFromConstant(intervalParameter); + switch (value) { + case "second": + outputGranularity = "'1:SECONDS'"; + break; + case "minute": + outputGranularity = "'1:MINUTES'"; + break; + case "hour": + outputGranularity = "'1:HOURS'"; + break; + case "day": + outputGranularity = "'1:DAYS'"; + break; + case "week": + outputGranularity = "'1:WEEKS'"; + break; + case "month": + outputGranularity = "'1:MONTHS'"; + break; + case "quarter": + outputGranularity = "'1:QUARTERS'"; + break; + case "year": + outputGranularity = "'1:YEARS'"; + break; + default: + throw new PinotException( + PINOT_UNSUPPORTED_EXPRESSION, + Optional.empty(), + "interval in date_trunc is not supported: " + value); + } + + return derived("dateTimeConvert(" + inputColumn + ", " + inputFormat + ", " + outputFormat + ", " + outputGranularity + ")"); + } + + private PinotExpression handleDateTruncationViaDateTruncation( + CallExpression function, + Map context) + { + RowExpression timeInputParameter = function.getArguments().get(1); + String inputColumn; + String inputTimeZone; + String inputFormat; + + CallExpression timeConversion = getExpressionAsFunction(timeInputParameter, timeInputParameter); + switch (timeConversion.getDisplayName().toLowerCase(ENGLISH)) { + case FROM_UNIXTIME: + inputColumn = timeConversion.getArguments().get(0).accept(this, context).getDefinition(); + inputTimeZone = timeConversion.getArguments().size() > 1 ? getStringFromConstant(timeConversion.getArguments().get(1)) : DateTimeZone.UTC.getID(); + inputFormat = "seconds"; + break; + default: + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "not supported: " + timeConversion.getDisplayName()); + } + + RowExpression intervalParameter = function.getArguments().get(0); + if (!(intervalParameter instanceof ConstantExpression)) { + throw new PinotException( + PINOT_UNSUPPORTED_EXPRESSION, + Optional.empty(), + "interval unit in date_trunc is not supported: " + intervalParameter); + } + + return derived("dateTrunc(" + inputColumn + "," + inputFormat + ", " + inputTimeZone + ", " + getStringFromConstant(intervalParameter) + ")"); + } + + private PinotExpression handleArithmeticExpression( + CallExpression expression, + OperatorType operatorType, + Map context) + { + List arguments = expression.getArguments(); + if (arguments.size() == 1) { + String prefix = operatorType == OperatorType.NEGATION ? "-" : ""; + return derived(prefix + arguments.get(0).accept(this, context).getDefinition()); + } + if (arguments.size() == 2) { + PinotExpression left = arguments.get(0).accept(this, context); + PinotExpression right = arguments.get(1).accept(this, context); + String prestoOperator = operatorType.getOperator(); + String pinotOperator = PRESTO_TO_PINOT_OPERATORS.get(prestoOperator); + if (pinotOperator == null) { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Unsupported binary expression " + prestoOperator); + } + return derived(format("%s(%s, %s)", pinotOperator, left.getDefinition(), right.getDefinition())); + } + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), format("Don't know how to interpret %s as an arithmetic expression", expression)); + } + + private PinotExpression handleFunction( + CallExpression function, + Map context) + { + switch (function.getDisplayName().toLowerCase(ENGLISH)) { + case "date_trunc": + boolean useDateTruncation = PinotSessionProperties.isUseDateTruncation(session); + return useDateTruncation ? + handleDateTruncationViaDateTruncation(function, context) : + handleDateTruncationViaDateTimeConvert(function, context); + default: + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), format("function %s not supported yet", function.getDisplayName())); + } + } + + private static String getStringFromConstant(RowExpression expression) + { + if (expression instanceof ConstantExpression) { + Object value = ((ConstantExpression) expression).getValue(); + if (value instanceof String) { + return (String) value; + } + if (value instanceof Slice) { + return ((Slice) value).toStringUtf8(); + } + } + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Expected string literal but found " + expression); + } + + private CallExpression getExpressionAsFunction( + RowExpression originalExpression, + RowExpression expression) + { + if (expression instanceof CallExpression) { + CallExpression call = (CallExpression) expression; + if (standardFunctionResolution.isCastFunction(call.getFunctionHandle())) { + if (isImplicitCast(call.getArguments().get(0).getType(), call.getType())) { + return getExpressionAsFunction(originalExpression, call.getArguments().get(0)); + } + } + else { + return call; + } + } + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Could not dig function out of expression: " + originalExpression + ", inside of " + expression); + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotExpression.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotExpression.java new file mode 100644 index 0000000000000..13deee4f76eb3 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotExpression.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot.query; + +import com.facebook.presto.pinot.query.PinotQueryGeneratorContext.Origin; + +import static java.util.Objects.requireNonNull; + +public class PinotExpression +{ + private final String definition; + private final Origin origin; + + PinotExpression(String definition, Origin origin) + { + this.definition = requireNonNull(definition, "definition is null"); + this.origin = requireNonNull(origin, "origin is null"); + } + + static PinotExpression derived(String definition) + { + return new PinotExpression(definition, Origin.DERIVED); + } + + public String getDefinition() + { + return definition; + } + + public Origin getOrigin() + { + return origin; + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotFilterExpressionConverter.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotFilterExpressionConverter.java new file mode 100644 index 0000000000000..736521dc30c87 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotFilterExpressionConverter.java @@ -0,0 +1,229 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot.query; + +import com.facebook.presto.pinot.PinotException; +import com.facebook.presto.pinot.query.PinotQueryGeneratorContext.Origin; +import com.facebook.presto.pinot.query.PinotQueryGeneratorContext.Selection; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.OperatorType; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.RowExpressionVisitor; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNSUPPORTED_EXPRESSION; +import static com.facebook.presto.pinot.PinotPushdownUtils.getLiteralAsString; +import static com.facebook.presto.pinot.query.PinotExpression.derived; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +/** + * Convert {@link RowExpression} in filter into Pinot complaint expression text + */ +public class PinotFilterExpressionConverter + implements RowExpressionVisitor> +{ + private static final Set LOGICAL_BINARY_OPS_FILTER = ImmutableSet.of("=", "<", "<=", ">", ">=", "<>"); + private final TypeManager typeManager; + private final FunctionMetadataManager functionMetadataManager; + private final StandardFunctionResolution standardFunctionResolution; + + public PinotFilterExpressionConverter( + TypeManager typeManager, + FunctionMetadataManager functionMetadataManager, + StandardFunctionResolution standardFunctionResolution) + { + this.typeManager = requireNonNull(typeManager, "type manager is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "function metadata manager is null"); + this.standardFunctionResolution = requireNonNull(standardFunctionResolution, "standardFunctionResolution is null"); + } + + private PinotExpression handleIn( + SpecialFormExpression specialForm, + boolean isWhitelist, + Function context) + { + return derived(format("(%s %s (%s))", + specialForm.getArguments().get(0).accept(this, context).getDefinition(), + isWhitelist ? "IN" : "NOT IN", + specialForm.getArguments().subList(1, specialForm.getArguments().size()).stream() + .map(argument -> argument.accept(this, context).getDefinition()) + .collect(Collectors.joining(", ")))); + } + + private PinotExpression handleLogicalBinary( + String operator, + CallExpression call, + Function context) + { + if (!LOGICAL_BINARY_OPS_FILTER.contains(operator)) { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), format("'%s' is not supported in filter", operator)); + } + List arguments = call.getArguments(); + if (arguments.size() == 2) { + return derived(format( + "(%s %s %s)", + arguments.get(0).accept(this, context).getDefinition(), + operator, + arguments.get(1).accept(this, context).getDefinition())); + } + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), format("Unknown logical binary: '%s'", call)); + } + + private PinotExpression handleBetween( + CallExpression between, + Function context) + { + if (between.getArguments().size() == 3) { + RowExpression value = between.getArguments().get(0); + RowExpression min = between.getArguments().get(1); + RowExpression max = between.getArguments().get(2); + + return derived(format( + "(%s BETWEEN %s AND %s)", + value.accept(this, context).getDefinition(), + min.accept(this, context).getDefinition(), + max.accept(this, context).getDefinition())); + } + + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), format("Between operator not supported: %s", between)); + } + + private PinotExpression handleNot(CallExpression not, Function context) + { + if (not.getArguments().size() == 1) { + RowExpression input = not.getArguments().get(0); + if (input instanceof SpecialFormExpression) { + SpecialFormExpression specialFormExpression = (SpecialFormExpression) input; + // NOT operator is only supported on top of the IN expression + if (specialFormExpression.getForm() == SpecialFormExpression.Form.IN) { + return handleIn(specialFormExpression, false, context); + } + } + } + + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), format("NOT operator is supported only on top of IN operator. Received: %s", not)); + } + + private PinotExpression handleCast(CallExpression cast, Function context) + { + if (cast.getArguments().size() == 1) { + RowExpression input = cast.getArguments().get(0); + Type expectedType = cast.getType(); + if (typeManager.canCoerce(input.getType(), expectedType)) { + return input.accept(this, context); + } + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Non implicit casts not supported: " + cast); + } + + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), format("This type of CAST operator not supported. Received: %s", cast)); + } + + @Override + public PinotExpression visitCall(CallExpression call, Function context) + { + FunctionHandle functionHandle = call.getFunctionHandle(); + if (standardFunctionResolution.isNotFunction(functionHandle)) { + return handleNot(call, context); + } + if (standardFunctionResolution.isCastFunction(functionHandle)) { + return handleCast(call, context); + } + if (standardFunctionResolution.isBetweenFunction(functionHandle)) { + return handleBetween(call, context); + } + FunctionMetadata functionMetadata = functionMetadataManager.getFunctionMetadata(call.getFunctionHandle()); + Optional operatorTypeOptional = functionMetadata.getOperatorType(); + if (operatorTypeOptional.isPresent()) { + OperatorType operatorType = operatorTypeOptional.get(); + if (operatorType.isArithmeticOperator()) { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Arithmetic expressions are not supported in filter: " + call); + } + if (operatorType.isComparisonOperator()) { + return handleLogicalBinary(operatorType.getOperator(), call, context); + } + } + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), format("function %s not supported in filter", call)); + } + + @Override + public PinotExpression visitInputReference(InputReferenceExpression reference, Function context) + { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Pinot does not support struct dereferencing " + reference); + } + + @Override + public PinotExpression visitConstant(ConstantExpression literal, Function context) + { + return new PinotExpression(getLiteralAsString(literal), Origin.LITERAL); + } + + @Override + public PinotExpression visitLambda(LambdaDefinitionExpression lambda, Function context) + { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Pinot does not support lambda " + lambda); + } + + @Override + public PinotExpression visitVariableReference(VariableReferenceExpression reference, Function context) + { + Selection input = requireNonNull(context.apply(reference), format("Input column %s does not exist in the input: %s", reference, context)); + return new PinotExpression(input.getDefinition(), input.getOrigin()); + } + + @Override + public PinotExpression visitSpecialForm(SpecialFormExpression specialForm, Function context) + { + switch (specialForm.getForm()) { + case IF: + case NULL_IF: + case SWITCH: + case WHEN: + case IS_NULL: + case COALESCE: + case DEREFERENCE: + case ROW_CONSTRUCTOR: + case BIND: + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Pinot does not support the special form " + specialForm); + case IN: + return handleIn(specialForm, true, context); + case AND: + case OR: + return derived(format( + "(%s %s %s)", + specialForm.getArguments().get(0).accept(this, context).getDefinition(), + specialForm.getForm().toString(), + specialForm.getArguments().get(1).accept(this, context).getDefinition())); + default: + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Unexpected special form: " + specialForm); + } + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotProjectExpressionConverter.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotProjectExpressionConverter.java new file mode 100644 index 0000000000000..51e0ba8f20679 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotProjectExpressionConverter.java @@ -0,0 +1,137 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot.query; + +import com.facebook.presto.pinot.PinotException; +import com.facebook.presto.pinot.query.PinotQueryGeneratorContext.Selection; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.RowExpressionVisitor; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.google.common.collect.ImmutableSet; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNSUPPORTED_EXPRESSION; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +class PinotProjectExpressionConverter + implements RowExpressionVisitor> +{ + private static final Set TIME_EQUIVALENT_TYPES = ImmutableSet.of(StandardTypes.BIGINT, StandardTypes.INTEGER, StandardTypes.TINYINT, StandardTypes.SMALLINT); + + protected final TypeManager typeManager; + protected final StandardFunctionResolution standardFunctionResolution; + + public PinotProjectExpressionConverter( + TypeManager typeManager, + StandardFunctionResolution standardFunctionResolution) + { + this.typeManager = requireNonNull(typeManager, "type manager"); + this.standardFunctionResolution = requireNonNull(standardFunctionResolution, "standardFunctionResolution is null"); + } + + @Override + public PinotExpression visitVariableReference( + VariableReferenceExpression reference, + Map context) + { + Selection input = requireNonNull(context.get(reference), format("Input column %s does not exist in the input", reference)); + return new PinotExpression(input.getDefinition(), input.getOrigin()); + } + + @Override + public PinotExpression visitLambda( + LambdaDefinitionExpression lambda, + Map context) + { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Pinot does not support lambda " + lambda); + } + + protected boolean isImplicitCast(Type inputType, Type resultType) + { + if (typeManager.canCoerce(inputType, resultType)) { + return true; + } + return resultType.getTypeSignature().getBase().equals(StandardTypes.TIMESTAMP) && TIME_EQUIVALENT_TYPES.contains(inputType.getTypeSignature().getBase()); + } + + private PinotExpression handleCast( + CallExpression cast, + Map context) + { + if (cast.getArguments().size() == 1) { + RowExpression input = cast.getArguments().get(0); + Type expectedType = cast.getType(); + if (isImplicitCast(input.getType(), expectedType)) { + return input.accept(this, context); + } + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Non implicit casts not supported: " + cast); + } + + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), format("This type of CAST operator not supported. Received: %s", cast)); + } + + protected Optional basicCallHandling(CallExpression call, Map context) + { + FunctionHandle functionHandle = call.getFunctionHandle(); + if (standardFunctionResolution.isNotFunction(functionHandle) || standardFunctionResolution.isBetweenFunction(functionHandle)) { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Unsupported function in pinot aggregation: " + functionHandle); + } + if (standardFunctionResolution.isCastFunction(functionHandle)) { + return Optional.of(handleCast(call, context)); + } + return Optional.empty(); + } + + @Override + public PinotExpression visitInputReference( + InputReferenceExpression reference, + Map context) + { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Input reference not supported: " + reference); + } + + @Override + public PinotExpression visitSpecialForm( + SpecialFormExpression specialForm, + Map context) + { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Special form not supported: " + specialForm); + } + + @Override + public PinotExpression visitCall(CallExpression call, Map context) + { + return basicCallHandling(call, context).orElseThrow(() -> new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Call not supported: " + call)); + } + + @Override + public PinotExpression visitConstant(ConstantExpression literal, Map context) + { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Constant not supported: " + literal); + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotQueryGenerator.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotQueryGenerator.java new file mode 100644 index 0000000000000..c98e409c323e2 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotQueryGenerator.java @@ -0,0 +1,467 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot.query; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.pinot.PinotColumnHandle; +import com.facebook.presto.pinot.PinotConfig; +import com.facebook.presto.pinot.PinotException; +import com.facebook.presto.pinot.PinotPushdownUtils.AggregationColumnNode; +import com.facebook.presto.pinot.PinotPushdownUtils.AggregationFunctionColumnNode; +import com.facebook.presto.pinot.PinotPushdownUtils.GroupByColumnNode; +import com.facebook.presto.pinot.PinotSessionProperties; +import com.facebook.presto.pinot.PinotTableHandle; +import com.facebook.presto.pinot.query.PinotQueryGeneratorContext.Selection; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanVisitor; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.TypeManager; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; + +import javax.inject.Inject; + +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNSUPPORTED_EXPRESSION; +import static com.facebook.presto.pinot.PinotPushdownUtils.checkSupported; +import static com.facebook.presto.pinot.PinotPushdownUtils.computeAggregationNodes; +import static com.facebook.presto.pinot.PinotPushdownUtils.getLiteralAsString; +import static com.facebook.presto.pinot.PinotPushdownUtils.getOrderingScheme; +import static com.facebook.presto.pinot.query.PinotQueryGeneratorContext.Origin.DERIVED; +import static com.facebook.presto.pinot.query.PinotQueryGeneratorContext.Origin.LITERAL; +import static com.facebook.presto.pinot.query.PinotQueryGeneratorContext.Origin.TABLE_COLUMN; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.lang.String.format; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public class PinotQueryGenerator +{ + private static final Logger log = Logger.get(PinotQueryGenerator.class); + private static final Map UNARY_AGGREGATION_MAP = ImmutableMap.of( + "min", "min", + "max", "max", + "avg", "avg", + "sum", "sum", + "approx_distinct", "DISTINCTCOUNTHLL"); + + private final PinotConfig pinotConfig; + private final TypeManager typeManager; + private final FunctionMetadataManager functionMetadataManager; + private final StandardFunctionResolution standardFunctionResolution; + private final PinotFilterExpressionConverter pinotFilterExpressionConverter; + private final PinotProjectExpressionConverter pinotProjectExpressionConverter; + + @Inject + public PinotQueryGenerator( + PinotConfig pinotConfig, + TypeManager typeManager, + FunctionMetadataManager functionMetadataManager, + StandardFunctionResolution standardFunctionResolution) + { + this.pinotConfig = requireNonNull(pinotConfig, "pinot config is null"); + this.typeManager = requireNonNull(typeManager, "type manager is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "function metadata manager is null"); + this.standardFunctionResolution = requireNonNull(standardFunctionResolution, "standardFunctionResolution is null"); + this.pinotFilterExpressionConverter = new PinotFilterExpressionConverter(this.typeManager, this.functionMetadataManager, standardFunctionResolution); + this.pinotProjectExpressionConverter = new PinotProjectExpressionConverter(typeManager, standardFunctionResolution); + } + + public static class PinotQueryGeneratorResult + { + private final GeneratedPql generatedPql; + private final PinotQueryGeneratorContext context; + + public PinotQueryGeneratorResult( + GeneratedPql generatedPql, + PinotQueryGeneratorContext context) + { + this.generatedPql = requireNonNull(generatedPql, "generatedPql is null"); + this.context = requireNonNull(context, "context is null"); + } + + public GeneratedPql getGeneratedPql() + { + return generatedPql; + } + + public PinotQueryGeneratorContext getContext() + { + return context; + } + } + + public Optional generate(PlanNode plan, ConnectorSession session) + { + try { + boolean preferBrokerQueries = PinotSessionProperties.isPreferBrokerQueries(session); + PinotQueryGeneratorContext context = requireNonNull(plan.accept(new PinotQueryPlanVisitor(session, preferBrokerQueries), new PinotQueryGeneratorContext()), "Resulting context is null"); + boolean isQueryShort = context.isQueryShort(PinotSessionProperties.getNonAggregateLimitForBrokerQueries(session)); + return Optional.of(new PinotQueryGeneratorResult(context.toQuery(pinotConfig, preferBrokerQueries, isQueryShort), context)); + } + catch (PinotException e) { + log.debug(e, "Possibly benign error when pushing plan into scan node %s", plan); + return Optional.empty(); + } + } + + public static class GeneratedPql + { + final String table; + final String pql; + final List expectedColumnIndices; + final int groupByClauses; + final boolean haveFilter; + final boolean isQueryShort; + + @JsonCreator + public GeneratedPql( + @JsonProperty("table") String table, + @JsonProperty("pql") String pql, + @JsonProperty("expectedColumnIndices") List expectedColumnIndices, + @JsonProperty("groupByClauses") int groupByClauses, + @JsonProperty("haveFilter") boolean haveFilter, + @JsonProperty("isQueryShort") boolean isQueryShort) + { + this.table = table; + this.pql = pql; + this.expectedColumnIndices = expectedColumnIndices; + this.groupByClauses = groupByClauses; + this.haveFilter = haveFilter; + this.isQueryShort = isQueryShort; + } + + @JsonProperty("pql") + public String getPql() + { + return pql; + } + + @JsonProperty("expectedColumnIndices") + public List getExpectedColumnIndices() + { + return expectedColumnIndices; + } + + @JsonProperty("groupByClauses") + public int getGroupByClauses() + { + return groupByClauses; + } + + @JsonProperty("table") + public String getTable() + { + return table; + } + + @JsonProperty("haveFilter") + public boolean isHaveFilter() + { + return haveFilter; + } + + @JsonProperty("isQueryShort") + public boolean isQueryShort() + { + return isQueryShort; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("pql", pql) + .add("table", table) + .add("expectedColumnIndices", expectedColumnIndices) + .add("groupByClauses", groupByClauses) + .add("haveFilter", haveFilter) + .add("isQueryShort", isQueryShort) + .toString(); + } + } + + class PinotQueryPlanVisitor + extends PlanVisitor + { + private final ConnectorSession session; + private final boolean preferBrokerQueries; + + protected PinotQueryPlanVisitor(ConnectorSession session, boolean preferBrokerQueries) + { + this.session = session; + this.preferBrokerQueries = preferBrokerQueries; + } + + @Override + public PinotQueryGeneratorContext visitPlan(PlanNode node, PinotQueryGeneratorContext context) + { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Don't know how to handle plan node of type " + node); + } + + protected VariableReferenceExpression getVariableReference(RowExpression expression) + { + if (expression instanceof VariableReferenceExpression) { + return ((VariableReferenceExpression) expression); + } + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Expected a variable reference but got " + expression); + } + + @Override + public PinotQueryGeneratorContext visitFilter(FilterNode node, PinotQueryGeneratorContext context) + { + context = node.getSource().accept(this, context); + requireNonNull(context, "context is null"); + LinkedHashMap selections = context.getSelections(); + String filter = node.getPredicate().accept(pinotFilterExpressionConverter, selections::get).getDefinition(); + return context.withFilter(filter).withOutputColumns(node.getOutputVariables()); + } + + @Override + public PinotQueryGeneratorContext visitProject(ProjectNode node, PinotQueryGeneratorContext contextIn) + { + PinotQueryGeneratorContext context = node.getSource().accept(this, contextIn); + requireNonNull(context, "context is null"); + LinkedHashMap newSelections = new LinkedHashMap<>(); + + node.getOutputVariables().forEach(variable -> { + RowExpression expression = node.getAssignments().get(variable); + PinotExpression pinotExpression = expression.accept( + contextIn.getVariablesInAggregation().contains(variable) ? + new PinotAggregationProjectConverter(typeManager, functionMetadataManager, standardFunctionResolution, session) : pinotProjectExpressionConverter, + context.getSelections()); + newSelections.put( + variable, + new Selection(pinotExpression.getDefinition(), pinotExpression.getOrigin())); + }); + return context.withProject(newSelections); + } + + @Override + public PinotQueryGeneratorContext visitTableScan(TableScanNode node, PinotQueryGeneratorContext contextIn) + { + PinotTableHandle tableHandle = (PinotTableHandle) node.getTable().getConnectorHandle(); + checkSupported(!tableHandle.getPql().isPresent(), "Expect to see no existing pql"); + checkSupported(!tableHandle.getIsQueryShort().isPresent(), "Expect to see no existing pql"); + LinkedHashMap selections = new LinkedHashMap<>(); + node.getOutputVariables().forEach(outputColumn -> { + PinotColumnHandle pinotColumn = (PinotColumnHandle) (node.getAssignments().get(outputColumn)); + checkSupported(pinotColumn.getType().equals(PinotColumnHandle.PinotColumnType.REGULAR), "Unexpected pinot column handle that is not regular: %s", pinotColumn); + selections.put(outputColumn, new Selection(pinotColumn.getColumnName(), TABLE_COLUMN)); + }); + return new PinotQueryGeneratorContext(selections, tableHandle.getTableName()); + } + + private String handleAggregationFunction(CallExpression aggregation, Map inputSelections) + { + String prestoAggregation = aggregation.getDisplayName().toLowerCase(ENGLISH); + List parameters = aggregation.getArguments(); + switch (prestoAggregation) { + case "count": + if (parameters.size() <= 1) { + return format("count(%s)", parameters.isEmpty() ? "*" : inputSelections.get(getVariableReference(parameters.get(0)))); + } + break; + case "approx_percentile": + return handleApproxPercentile(aggregation, inputSelections); + default: + if (UNARY_AGGREGATION_MAP.containsKey(prestoAggregation) && aggregation.getArguments().size() == 1) { + return format("%s(%s)", UNARY_AGGREGATION_MAP.get(prestoAggregation), inputSelections.get(getVariableReference(parameters.get(0)))); + } + } + + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), format("aggregation function '%s' not supported yet", aggregation)); + } + + private String handleApproxPercentile(CallExpression aggregation, Map inputSelections) + { + List inputs = aggregation.getArguments(); + if (inputs.size() != 2) { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Cannot handle approx_percentile function " + aggregation); + } + + String fractionString; + RowExpression fractionInput = inputs.get(1); + + if (fractionInput instanceof ConstantExpression) { + fractionString = getLiteralAsString((ConstantExpression) fractionInput); + } + else if (fractionInput instanceof VariableReferenceExpression) { + Selection fraction = inputSelections.get(fractionInput); + if (fraction.getOrigin() != LITERAL) { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), + "Cannot handle approx_percentile percentage argument be a non literal " + aggregation); + } + fractionString = fraction.getDefinition(); + } + else { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "Expected the fraction to be a constant or a variable " + fractionInput); + } + + int percentile = getValidPercentile(fractionString); + if (percentile < 0) { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), + format("Cannot handle approx_percentile parsed as %d from input %s (function %s)", percentile, fractionString, aggregation)); + } + return format("PERCENTILEEST%d(%s)", percentile, inputSelections.get(getVariableReference(inputs.get(0)))); + } + + private int getValidPercentile(String fraction) + { + try { + double percent = Double.parseDouble(fraction); + if (percent < 0 || percent > 1) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Percentile must be between 0 and 1"); + } + percent = percent * 100.0; + if (percent == Math.floor(percent)) { + return (int) percent; + } + } + catch (NumberFormatException ne) { + // Skip + } + return -1; + } + + @Override + public PinotQueryGeneratorContext visitAggregation(AggregationNode node, PinotQueryGeneratorContext contextIn) + { + List aggregationColumnNodes = computeAggregationNodes(node); + + // Make two passes over the aggregatinColumnNodes: In the first pass identify all the variables that will be used + // Then pass that context to the source + // And finally, in the second pass actually generate the PQL + + // 1st pass + Set variablesInAggregation = new HashSet<>(); + for (AggregationColumnNode expression : aggregationColumnNodes) { + switch (expression.getExpressionType()) { + case GROUP_BY: { + GroupByColumnNode groupByColumn = (GroupByColumnNode) expression; + VariableReferenceExpression groupByInputColumn = getVariableReference(groupByColumn.getInputColumn()); + variablesInAggregation.add(groupByInputColumn); + break; + } + case AGGREGATE: { + AggregationFunctionColumnNode aggregationNode = (AggregationFunctionColumnNode) expression; + variablesInAggregation.addAll( + aggregationNode + .getCallExpression() + .getArguments() + .stream() + .filter(argument -> argument instanceof VariableReferenceExpression) + .map(argument -> (VariableReferenceExpression) argument) + .collect(Collectors.toList())); + break; + } + default: + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "unknown aggregation expression: " + expression.getExpressionType()); + } + } + + // now visit the child project node + PinotQueryGeneratorContext context = node.getSource().accept(this, contextIn.withVariablesInAggregation(variablesInAggregation)); + requireNonNull(context, "context is null"); + checkSupported(!node.getStep().isOutputPartial(), "partial aggregations are not supported in Pinot pushdown framework"); + checkSupported(preferBrokerQueries, "Cannot push aggregation in segment mode"); + + // 2nd pass + LinkedHashMap newSelections = new LinkedHashMap<>(); + LinkedHashSet groupByColumns = new LinkedHashSet<>(); + Set hiddenColumnSet = new HashSet<>(context.getHiddenColumnSet()); + int aggregations = 0; + boolean groupByExists = false; + + for (AggregationColumnNode expression : aggregationColumnNodes) { + switch (expression.getExpressionType()) { + case GROUP_BY: { + GroupByColumnNode groupByColumn = (GroupByColumnNode) expression; + VariableReferenceExpression groupByInputColumn = getVariableReference(groupByColumn.getInputColumn()); + VariableReferenceExpression outputColumn = getVariableReference(groupByColumn.getOutputColumn()); + Selection pinotColumn = requireNonNull(context.getSelections().get(groupByInputColumn), "Group By column " + groupByInputColumn + " doesn't exist in input " + context.getSelections()); + + newSelections.put(outputColumn, new Selection(pinotColumn.getDefinition(), pinotColumn.getOrigin())); + groupByColumns.add(outputColumn); + groupByExists = true; + break; + } + case AGGREGATE: { + AggregationFunctionColumnNode aggregationNode = (AggregationFunctionColumnNode) expression; + String pinotAggFunction = handleAggregationFunction(aggregationNode.getCallExpression(), context.getSelections()); + newSelections.put(getVariableReference(aggregationNode.getOutputColumn()), new Selection(pinotAggFunction, DERIVED)); + aggregations++; + break; + } + default: + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.empty(), "unknown aggregation expression: " + expression.getExpressionType()); + } + } + + // Handling non-aggregated group by + if (groupByExists && aggregations == 0) { + VariableReferenceExpression hidden = new VariableReferenceExpression(UUID.randomUUID().toString(), BigintType.BIGINT); + newSelections.put(hidden, new Selection("count(*)", DERIVED)); + hiddenColumnSet.add(hidden); + aggregations++; + } + return context.withAggregation(newSelections, groupByColumns, aggregations, hiddenColumnSet); + } + + @Override + public PinotQueryGeneratorContext visitLimit(LimitNode node, PinotQueryGeneratorContext context) + { + checkSupported(!node.isPartial(), String.format("pinot query generator cannot handle partial limit")); + checkSupported(preferBrokerQueries, "Cannot push limit in segment mode"); + context = node.getSource().accept(this, context); + requireNonNull(context, "context is null"); + return context.withLimit(node.getCount()).withOutputColumns(node.getOutputVariables()); + } + + @Override + public PinotQueryGeneratorContext visitTopN(TopNNode node, PinotQueryGeneratorContext context) + { + context = node.getSource().accept(this, context); + requireNonNull(context, "context is null"); + checkSupported(preferBrokerQueries, "Cannot push topn in segment mode"); + checkSupported(node.getStep().equals(TopNNode.Step.SINGLE), "Can only push single logical topn in"); + return context.withTopN(getOrderingScheme(node), node.getCount()).withOutputColumns(node.getOutputVariables()); + } + } +} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotQueryGeneratorContext.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotQueryGeneratorContext.java new file mode 100644 index 0000000000000..e7388f3a3aea8 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotQueryGeneratorContext.java @@ -0,0 +1,468 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot.query; + +import com.facebook.presto.pinot.PinotColumnHandle; +import com.facebook.presto.pinot.PinotConfig; +import com.facebook.presto.pinot.PinotException; +import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_QUERY_GENERATOR_FAILURE; +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNSUPPORTED_EXPRESSION; +import static com.facebook.presto.pinot.PinotPushdownUtils.checkSupported; +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.lang.StrictMath.toIntExact; +import static java.lang.String.format; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +/** + * Encapsulates the components needed to construct a PQL query and provides methods to update the current context with new operations. + */ +public class PinotQueryGeneratorContext +{ + public static final String TIME_BOUNDARY_FILTER_TEMPLATE = "__TIME_BOUNDARY_FILTER_TEMPLATE__"; + public static final String TABLE_NAME_SUFFIX_TEMPLATE = "__TABLE_NAME_SUFFIX_TEMPLATE__"; + // Fields defining the query + // order map that maps the column definition in terms of input relation column(s) + private final LinkedHashMap selections; + private final LinkedHashSet groupByColumns; + private final LinkedHashMap topNColumnOrderingMap; + private final Set hiddenColumnSet; + private final Set variablesInAggregation; + private final Optional from; + private final Optional filter; + private final OptionalInt limit; + private final int aggregations; + + public boolean isQueryShort(int nonAggregateRowLimit) + { + return hasAggregation() || limit.orElse(Integer.MAX_VALUE) < nonAggregateRowLimit; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("selections", selections) + .add("groupByColumns", groupByColumns) + .add("hiddenColumnSet", hiddenColumnSet) + .add("from", from) + .add("filter", filter) + .add("limit", limit) + .add("aggregations", aggregations) + .toString(); + } + + PinotQueryGeneratorContext() + { + this(new LinkedHashMap<>(), null); + } + + PinotQueryGeneratorContext( + LinkedHashMap selections, + String from) + { + this( + selections, + Optional.ofNullable(from), + Optional.empty(), + 0, + new LinkedHashSet<>(), + new LinkedHashMap<>(), + OptionalInt.empty(), + new HashSet<>(), + new HashSet<>()); + } + + private PinotQueryGeneratorContext( + LinkedHashMap selections, + Optional from, + Optional filter, + int aggregations, + LinkedHashSet groupByColumns, + LinkedHashMap topNColumnOrderingMap, + OptionalInt limit, + Set variablesInAggregation, + Set hiddenColumnSet) + { + this.selections = new LinkedHashMap<>(requireNonNull(selections, "selections can't be null")); + this.from = requireNonNull(from, "source can't be null"); + this.aggregations = aggregations; + this.groupByColumns = new LinkedHashSet<>(requireNonNull(groupByColumns, "groupByColumns can't be null. It could be empty if not available")); + this.topNColumnOrderingMap = new LinkedHashMap<>(requireNonNull(topNColumnOrderingMap, "topNColumnOrderingMap can't be null. It could be empty if not available")); + this.filter = requireNonNull(filter, "filter is null"); + this.limit = requireNonNull(limit, "limit is null"); + this.hiddenColumnSet = requireNonNull(hiddenColumnSet, "hidden column set is null"); + this.variablesInAggregation = requireNonNull(variablesInAggregation, "variables in aggregation is null"); + } + + /** + * Apply the given filter to current context and return the updated context. Throws error for invalid operations. + */ + public PinotQueryGeneratorContext withFilter(String filter) + { + checkSupported(!hasFilter(), "There already exists a filter. Pinot doesn't support filters at multiple levels"); + checkSupported(!hasAggregation(), "Pinot doesn't support filtering the results of aggregation"); + checkSupported(!hasLimit(), "Pinot doesn't support filtering on top of the limit"); + return new PinotQueryGeneratorContext( + selections, + from, + Optional.of(filter), + aggregations, + groupByColumns, + topNColumnOrderingMap, + limit, + variablesInAggregation, + hiddenColumnSet); + } + + /** + * Apply the aggregation to current context and return the updated context. Throws error for invalid operations. + */ + public PinotQueryGeneratorContext withAggregation( + LinkedHashMap newSelections, + LinkedHashSet groupByColumns, + int aggregations, + Set hiddenColumnSet) + { + // there is only one aggregation supported. + checkSupported(!hasAggregation(), "Pinot doesn't support aggregation on top of the aggregated data"); + checkSupported(!hasLimit(), "Pinot doesn't support aggregation on top of the limit"); + checkSupported(aggregations > 0, "Invalid number of aggregations"); + return new PinotQueryGeneratorContext(newSelections, from, filter, aggregations, groupByColumns, topNColumnOrderingMap, limit, variablesInAggregation, hiddenColumnSet); + } + + /** + * Apply new selections/project to current context and return the updated context. Throws error for invalid operations. + */ + public PinotQueryGeneratorContext withProject(LinkedHashMap newSelections) + { + checkSupported(groupByColumns.isEmpty(), "Pinot doesn't yet support new selections on top of the grouped by data"); + return new PinotQueryGeneratorContext( + newSelections, + from, + filter, + aggregations, + groupByColumns, + topNColumnOrderingMap, + limit, + variablesInAggregation, + hiddenColumnSet); + } + + private static int checkForValidLimit(long limit) + { + if (limit <= 0 || limit > Integer.MAX_VALUE) { + throw new PinotException(PINOT_QUERY_GENERATOR_FAILURE, Optional.empty(), "Limit " + limit + " not supported: Limit is not being pushed down"); + } + return toIntExact(limit); + } + + /** + * Apply limit to current context and return the updated context. Throws error for invalid operations. + */ + public PinotQueryGeneratorContext withLimit(long limit) + { + int intLimit = checkForValidLimit(limit); + checkSupported(!hasLimit(), "Limit already exists. Pinot doesn't support limit on top of another limit"); + return new PinotQueryGeneratorContext( + selections, + from, + filter, + aggregations, + groupByColumns, + topNColumnOrderingMap, + OptionalInt.of(intLimit), + variablesInAggregation, + hiddenColumnSet); + } + + /** + * Apply order by to current context and return the updated context. Throws error for invalid operations. + */ + public PinotQueryGeneratorContext withTopN(LinkedHashMap orderByColumnOrderingMap, long limit) + { + checkSupported(!hasLimit(), "Limit already exists. Pinot doesn't support order by limit on top of another limit"); + checkSupported(!hasAggregation(), "Pinot doesn't support ordering on top of the aggregated data"); + int intLimit = checkForValidLimit(limit); + return new PinotQueryGeneratorContext( + selections, + from, + filter, + aggregations, + groupByColumns, + orderByColumnOrderingMap, + OptionalInt.of(intLimit), + variablesInAggregation, + hiddenColumnSet); + } + + private boolean hasFilter() + { + return filter.isPresent(); + } + + private boolean hasLimit() + { + return limit.isPresent(); + } + + private boolean hasAggregation() + { + return aggregations > 0; + } + + private boolean hasOrderBy() + { + return !topNColumnOrderingMap.isEmpty(); + } + + public LinkedHashMap getSelections() + { + return selections; + } + + public Set getHiddenColumnSet() + { + return hiddenColumnSet; + } + + Set getVariablesInAggregation() + { + return variablesInAggregation; + } + + /** + * Convert the current context to a PQL + */ + public PinotQueryGenerator.GeneratedPql toQuery(PinotConfig pinotConfig, boolean preferBrokerQueries, boolean isQueryShort) + { + boolean forBroker = preferBrokerQueries && isQueryShort; + if (!pinotConfig.isAllowMultipleAggregations() && aggregations > 1 && !groupByColumns.isEmpty()) { + throw new PinotException(PINOT_QUERY_GENERATOR_FAILURE, Optional.empty(), "Multiple aggregates in the presence of group by is forbidden"); + } + + if (hasLimit() && aggregations > 1 && !groupByColumns.isEmpty()) { + throw new PinotException(PINOT_QUERY_GENERATOR_FAILURE, Optional.empty(), "Multiple aggregates in the presence of group by and limit is forbidden"); + } + + String expressions = selections.entrySet().stream() + .filter(s -> !groupByColumns.contains(s.getKey())) // remove the group by columns from the query as Pinot barfs if the group by column is an expression + .map(s -> s.getValue().getDefinition()) + .collect(Collectors.joining(", ")); + + String tableName = from.orElseThrow(() -> new PinotException(PINOT_QUERY_GENERATOR_FAILURE, Optional.empty(), "Table name not encountered yet")); + String query = "SELECT " + expressions + " FROM " + tableName + (forBroker ? "" : TABLE_NAME_SUFFIX_TEMPLATE); + if (filter.isPresent()) { + String filterString = filter.get(); + // this is hack!!!. Ideally we want to clone the scan pipeline and create/update the filter in the scan pipeline to contain this filter and + // at the same time add the time column to scan so that the query generator doesn't fail when it looks up the time column in scan output columns + query += format(" WHERE %s%s", filterString, forBroker ? "" : TIME_BOUNDARY_FILTER_TEMPLATE); + } + else if (!forBroker) { + query += TIME_BOUNDARY_FILTER_TEMPLATE; + } + + if (!groupByColumns.isEmpty()) { + String groupByExpr = groupByColumns.stream().map(x -> selections.get(x).getDefinition()).collect(Collectors.joining(", ")); + query = query + " GROUP BY " + groupByExpr; + } + + if (hasOrderBy()) { + String orderByExpressions = topNColumnOrderingMap.entrySet().stream().map(entry -> selections.get(entry.getKey()).getDefinition() + (entry.getValue().isAscending() ? "" : " DESC")).collect(Collectors.joining(", ")); + query = query + " ORDER BY " + orderByExpressions; + } + // Rules for limit: + // - If its a selection query: + // + given limit or configured limit + // - Else if has group by: + // + ensure that only one aggregation + // + default limit or configured top limit + // - Fail if limit is invalid + + String limitKeyWord = ""; + int queryLimit = -1; + + if (!hasAggregation()) { + if (!limit.isPresent() && forBroker) { + throw new PinotException(PINOT_QUERY_GENERATOR_FAILURE, Optional.empty(), "Broker non aggregate queries have to have a limit"); + } + else { + queryLimit = limit.orElseGet(pinotConfig::getLimitLargeForSegment); + } + limitKeyWord = "LIMIT"; + } + else if (!groupByColumns.isEmpty()) { + limitKeyWord = "TOP"; + if (limit.isPresent()) { + if (aggregations > 1) { + throw new PinotException(PINOT_QUERY_GENERATOR_FAILURE, Optional.of(query), + "Pinot has weird semantics with group by and multiple aggregation functions and limits"); + } + else { + queryLimit = limit.getAsInt(); + } + } + else { + queryLimit = pinotConfig.getTopNLarge(); + } + } + + if (!limitKeyWord.isEmpty()) { + query += " " + limitKeyWord + " " + queryLimit; + } + + List columnHandles = ImmutableList.copyOf(getAssignments().values()); + return new PinotQueryGenerator.GeneratedPql(tableName, query, getIndicesMappingFromPinotSchemaToPrestoSchema(query, columnHandles), groupByColumns.size(), filter.isPresent(), isQueryShort); + } + + private List getIndicesMappingFromPinotSchemaToPrestoSchema(String query, List handles) + { + LinkedHashMap expressionsInPinotOrder = new LinkedHashMap<>(); + for (VariableReferenceExpression groupByColumn : groupByColumns) { + Selection groupByColumnDefinition = selections.get(groupByColumn); + if (groupByColumnDefinition == null) { + throw new IllegalStateException(format( + "Group By column (%s) definition not found in input selections: %s", + groupByColumn, + Joiner.on(",").withKeyValueSeparator(":").join(selections))); + } + expressionsInPinotOrder.put(groupByColumn, groupByColumnDefinition); + } + expressionsInPinotOrder.putAll(selections); + + checkSupported( + handles.size() == expressionsInPinotOrder.keySet().stream().filter(key -> !hiddenColumnSet.contains(key)).count(), + "Expected returned expressions %s to match selections %s", + Joiner.on(",").withKeyValueSeparator(":").join(expressionsInPinotOrder), Joiner.on(",").join(handles)); + + Map nameToIndex = new HashMap<>(); + for (int i = 0; i < handles.size(); i++) { + PinotColumnHandle columnHandle = handles.get(i); + VariableReferenceExpression columnName = new VariableReferenceExpression(columnHandle.getColumnName().toLowerCase(ENGLISH), columnHandle.getDataType()); + Integer previous = nameToIndex.put(columnName, i); + if (previous != null) { + throw new PinotException(PINOT_UNSUPPORTED_EXPRESSION, Optional.of(query), format("Expected Pinot column handle %s to occur only once, but we have: %s", columnName, Joiner.on(",").join(handles))); + } + } + + ImmutableList.Builder outputIndices = ImmutableList.builder(); + for (Map.Entry expression : expressionsInPinotOrder.entrySet()) { + Integer index = nameToIndex.get(expression.getKey()); + if (hiddenColumnSet.contains(expression.getKey())) { + index = -1; // negative output index means to skip this value returned by pinot at query time + } + if (index == null) { + throw new PinotException( + PINOT_UNSUPPORTED_EXPRESSION, Optional.of(query), + format( + "Expected to find a Pinot column handle for the expression %s, but we have %s", + expression, + Joiner.on(",").withKeyValueSeparator(":").join(nameToIndex))); + } + outputIndices.add(index); + } + return outputIndices.build(); + } + + public LinkedHashMap getAssignments() + { + LinkedHashMap result = new LinkedHashMap<>(); + selections.entrySet().stream().filter(e -> !hiddenColumnSet.contains(e.getKey())).forEach(entry -> { + VariableReferenceExpression variable = entry.getKey(); + Selection selection = entry.getValue(); + PinotColumnHandle handle = selection.getOrigin() == Origin.TABLE_COLUMN ? new PinotColumnHandle(selection.getDefinition(), variable.getType(), PinotColumnHandle.PinotColumnType.REGULAR) : new PinotColumnHandle(variable, PinotColumnHandle.PinotColumnType.DERIVED); + result.put(variable, handle); + }); + return result; + } + + public PinotQueryGeneratorContext withOutputColumns(List outputColumns) + { + LinkedHashMap newSelections = new LinkedHashMap<>(); + outputColumns.forEach(o -> newSelections.put(o, requireNonNull(selections.get(o), String.format("Cannot find the selection %s in the original context %s", o, this)))); + + // Hidden columns flow as is from the previous + selections.entrySet().stream().filter(e -> hiddenColumnSet.contains(e.getKey())).forEach(e -> newSelections.put(e.getKey(), e.getValue())); + return new PinotQueryGeneratorContext(newSelections, from, filter, aggregations, groupByColumns, topNColumnOrderingMap, limit, variablesInAggregation, hiddenColumnSet); + } + + public PinotQueryGeneratorContext withVariablesInAggregation(Set newVariablesInAggregation) + { + return new PinotQueryGeneratorContext( + selections, + from, + filter, + aggregations, + groupByColumns, + topNColumnOrderingMap, + limit, + newVariablesInAggregation, + hiddenColumnSet); + } + + /** + * Where is the selection/projection originated from + */ + public enum Origin + { + TABLE_COLUMN, // refers to direct column in table + DERIVED, // expression is derived from one or more input columns or a combination of input columns and literals + LITERAL, // derived from literal + } + + // Projected/selected column definition in query + public static class Selection + { + private final String definition; + private final Origin origin; + + public Selection(String definition, Origin origin) + { + this.definition = definition; + this.origin = origin; + } + + public String getDefinition() + { + return definition; + } + + public Origin getOrigin() + { + return origin; + } + + @Override + public String toString() + { + return definition; + } + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/MetadataUtil.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/MetadataUtil.java new file mode 100644 index 0000000000000..0a4eebde65806 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/MetadataUtil.java @@ -0,0 +1,87 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; +import com.google.common.collect.ImmutableMap; + +import java.util.List; +import java.util.Map; + +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public final class MetadataUtil +{ + public static final JsonCodec>> CATALOG_CODEC; + public static final JsonCodec TABLE_CODEC; + public static final JsonCodec COLUMN_CODEC; + public static final JsonCodec TABLES_JSON_CODEC; + public static final JsonCodec BROKERS_FOR_TABLE_JSON_CODEC; + public static final JsonCodec ROUTING_TABLES_JSON_CODEC; + public static final JsonCodec TIME_BOUNDARY_JSON_CODEC; + + private MetadataUtil() + { + } + + public static final class TestingTypeDeserializer + extends FromStringDeserializer + { + private final Map types = ImmutableMap.of( + StandardTypes.BOOLEAN, BOOLEAN, + StandardTypes.BIGINT, BIGINT, + StandardTypes.INTEGER, INTEGER, + StandardTypes.DOUBLE, DOUBLE, + StandardTypes.VARCHAR, VARCHAR); + + public TestingTypeDeserializer() + { + super(Type.class); + } + + @Override + protected Type _deserialize(String value, DeserializationContext context) + { + Type type = types.get(value.toLowerCase(ENGLISH)); + return requireNonNull(type, "Unknown type " + value); + } + } + + static { + ObjectMapperProvider objectMapperProvider = new ObjectMapperProvider(); + objectMapperProvider.setJsonDeserializers(ImmutableMap., JsonDeserializer>of(Type.class, new TestingTypeDeserializer())); + JsonCodecFactory codecFactory = new JsonCodecFactory(objectMapperProvider); + CATALOG_CODEC = codecFactory.mapJsonCodec(String.class, listJsonCodec(PinotTable.class)); + TABLE_CODEC = codecFactory.jsonCodec(PinotTable.class); + COLUMN_CODEC = codecFactory.jsonCodec(PinotColumnHandle.class); + TABLES_JSON_CODEC = codecFactory.jsonCodec(PinotClusterInfoFetcher.GetTables.class); + BROKERS_FOR_TABLE_JSON_CODEC = codecFactory.jsonCodec(PinotClusterInfoFetcher.BrokersForTable.class); + ROUTING_TABLES_JSON_CODEC = codecFactory.jsonCodec(PinotClusterInfoFetcher.RoutingTables.class); + TIME_BOUNDARY_JSON_CODEC = codecFactory.jsonCodec(PinotClusterInfoFetcher.TimeBoundary.class); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/MockPinotClusterInfoFetcher.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/MockPinotClusterInfoFetcher.java new file mode 100644 index 0000000000000..8d77f917d1e45 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/MockPinotClusterInfoFetcher.java @@ -0,0 +1,420 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.http.client.testing.TestingHttpClient; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.pinot.common.data.Schema; + +import java.util.List; +import java.util.Map; + +import static com.facebook.presto.pinot.MetadataUtil.BROKERS_FOR_TABLE_JSON_CODEC; +import static com.facebook.presto.pinot.MetadataUtil.ROUTING_TABLES_JSON_CODEC; +import static com.facebook.presto.pinot.MetadataUtil.TABLES_JSON_CODEC; +import static com.facebook.presto.pinot.MetadataUtil.TIME_BOUNDARY_JSON_CODEC; + +public class MockPinotClusterInfoFetcher + extends PinotClusterInfoFetcher +{ + public MockPinotClusterInfoFetcher(PinotConfig pinotConfig) + { + super( + pinotConfig, + new PinotMetrics(), + new TestingHttpClient(request -> null), + TABLES_JSON_CODEC, + BROKERS_FOR_TABLE_JSON_CODEC, + ROUTING_TABLES_JSON_CODEC, + TIME_BOUNDARY_JSON_CODEC); + } + + @Override + public List getAllTables() + { + return ImmutableList.of(TestPinotSplitManager.realtimeOnlyTable.getTableName(), TestPinotSplitManager.hybridTable.getTableName()); + } + + @Override + public Map>> getRoutingTableForTable(String tableName) + { + ImmutableMap.Builder>> routingTable = ImmutableMap.builder(); + + if (TestPinotSplitManager.realtimeOnlyTable.getTableName().equalsIgnoreCase(tableName) || TestPinotSplitManager.hybridTable.getTableName().equalsIgnoreCase(tableName)) { + routingTable.put(tableName + "_REALTIME", ImmutableMap.of( + "server1", ImmutableList.of("segment11", "segment12"), + "server2", ImmutableList.of("segment21", "segment22"))); + } + + if (TestPinotSplitManager.hybridTable.getTableName().equalsIgnoreCase(tableName)) { + routingTable.put(tableName + "_OFFLINE", ImmutableMap.of( + "server3", ImmutableList.of("segment31", "segment32"), + "server4", ImmutableList.of("segment41", "segment42"))); + } + + return routingTable.build(); + } + + @Override + public Schema getTableSchema(String table) + throws Exception + { + // From the test pinot table airlineStats + return Schema.fromString("{\n" + + " \"schemaName\": \"airlineStats\",\n" + + " \"dimensionFieldSpecs\": [\n" + + " {\n" + + " \"name\": \"ActualElapsedTime\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"AirTime\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"AirlineID\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"ArrDel15\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"ArrDelay\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"ArrDelayMinutes\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"ArrTime\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"ArrTimeBlk\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"ArrivalDelayGroups\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"CRSArrTime\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"CRSDepTime\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"CRSElapsedTime\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"CancellationCode\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"Cancelled\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"Carrier\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"CarrierDelay\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DayOfWeek\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DayofMonth\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DepDel15\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DepDelay\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DepDelayMinutes\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DepTime\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DepTimeBlk\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DepartureDelayGroups\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"Dest\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DestAirportID\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DestAirportSeqID\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DestCityMarketID\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DestCityName\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DestState\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DestStateFips\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DestStateName\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DestWac\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"Distance\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DistanceGroup\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DivActualElapsedTime\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DivAirportIDs\",\n" + + " \"dataType\": \"INT\",\n" + + " \"singleValueField\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"DivAirportLandings\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DivAirportSeqIDs\",\n" + + " \"dataType\": \"INT\",\n" + + " \"singleValueField\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"DivAirports\",\n" + + " \"dataType\": \"STRING\",\n" + + " \"singleValueField\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"DivArrDelay\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DivDistance\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DivLongestGTimes\",\n" + + " \"dataType\": \"INT\",\n" + + " \"singleValueField\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"DivReachedDest\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"DivTailNums\",\n" + + " \"dataType\": \"STRING\",\n" + + " \"singleValueField\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"DivTotalGTimes\",\n" + + " \"dataType\": \"INT\",\n" + + " \"singleValueField\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"DivWheelsOffs\",\n" + + " \"dataType\": \"INT\",\n" + + " \"singleValueField\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"DivWheelsOns\",\n" + + " \"dataType\": \"INT\",\n" + + " \"singleValueField\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"Diverted\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"FirstDepTime\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"FlightDate\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"FlightNum\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"Flights\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"LateAircraftDelay\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"LongestAddGTime\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"Month\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"NASDelay\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"Origin\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"OriginAirportID\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"OriginAirportSeqID\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"OriginCityMarketID\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"OriginCityName\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"OriginState\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"OriginStateFips\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"OriginStateName\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"OriginWac\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"Quarter\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"RandomAirports\",\n" + + " \"dataType\": \"STRING\",\n" + + " \"singleValueField\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"SecurityDelay\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"TailNum\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"TaxiIn\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"TaxiOut\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"Year\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"WheelsOn\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"WheelsOff\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"WeatherDelay\",\n" + + " \"dataType\": \"INT\"\n" + + " },\n" + + " {\n" + + " \"name\": \"UniqueCarrier\",\n" + + " \"dataType\": \"STRING\"\n" + + " },\n" + + " {\n" + + " \"name\": \"TotalAddGTime\",\n" + + " \"dataType\": \"INT\"\n" + + " }\n" + + " ],\n" + + " \"timeFieldSpec\": {\n" + + " \"incomingGranularitySpec\": {\n" + + " \"name\": \"DaysSinceEpoch\",\n" + + " \"dataType\": \"INT\",\n" + + " \"timeType\": \"DAYS\"\n" + + " }\n" + + " },\n" + + " \"updateSemantic\": null\n" + + "}"); + } + + @Override + public TimeBoundary getTimeBoundaryForTable(String table) + { + if (TestPinotSplitManager.hybridTable.getTableName().equalsIgnoreCase(table)) { + return new TimeBoundary("secondsSinceEpoch", "4562345"); + } + + return new TimeBoundary(); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotBrokerPageSource.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotBrokerPageSource.java new file mode 100644 index 0000000000000..56034a7ad9f71 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotBrokerPageSource.java @@ -0,0 +1,205 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.json.ObjectMapperProvider; +import com.facebook.presto.pinot.query.PinotQueryGenerator; +import com.facebook.presto.spi.PageBuilder; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.testing.TestingConnectorSession; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Streams; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestPinotBrokerPageSource + extends TestPinotQueryBase +{ + private static PinotTableHandle pinotTable = new PinotTableHandle("connId", "schema", "tbl"); + private final ObjectMapper objectMapper = new ObjectMapperProvider().get(); + + private static class PqlParsedInfo + { + final int groupByColumns; + final int columns; + final int rows; + + private PqlParsedInfo(int groupByColumns, int columns, int rows) + { + this.groupByColumns = groupByColumns; + this.columns = columns; + this.rows = rows; + } + + public static PqlParsedInfo forSelection(int columns, int rows) + { + return new PqlParsedInfo(0, columns, rows); + } + + public static PqlParsedInfo forAggregation(int groups, int aggregates, int rows) + { + return new PqlParsedInfo(groups, groups + aggregates, rows); + } + } + + PqlParsedInfo getBasicInfoFromPql(String pqlResponse) + throws IOException + { + JsonNode pqlJson = objectMapper.readTree(pqlResponse); + JsonNode selectionResults = pqlJson.get("selectionResults"); + if (selectionResults != null) { + return PqlParsedInfo.forSelection(selectionResults.get("columns").size(), selectionResults.get("results").size()); + } + + JsonNode aggregationResults = pqlJson.get("aggregationResults"); + int aggregates = aggregationResults.size(); + Set> groups = new HashSet<>(); + int groupByColumns = 0; + int pureAggregates = 0; + for (int i = 0; i < aggregates; i++) { + JsonNode groupByResult = aggregationResults.get(i).get("groupByResult"); + if (groupByResult != null) { + for (int j = 0; j < groupByResult.size(); ++j) { + JsonNode groupJson = groupByResult.get(j).get("group"); + List group = Streams.stream(groupJson.iterator()).map(JsonNode::asText).collect(toImmutableList()); + groups.add(group); + if (groupByColumns == 0) { + groupByColumns = group.size(); + } + } + } + else { + pureAggregates++; + } + } + assertTrue(pureAggregates == 0 || pureAggregates == aggregates, String.format("In pql response %s, got mixed aggregates %d of %d", pqlResponse, pureAggregates, aggregates)); + if (pureAggregates == 0) { + return PqlParsedInfo.forAggregation(groupByColumns, aggregates, groups.size()); + } + return PqlParsedInfo.forAggregation(0, pureAggregates, 1); + } + + @DataProvider(name = "pqlResponses") + public static Object[][] pqlResponsesProvider() + { + return new Object[][] { + {"SELECT count(*), sum(regionId) FROM eats_job_state GROUP BY jobState TOP 1000000", + "{\"aggregationResults\":[{\"groupByResult\":[{\"value\":\"10646777\",\"group\":[\"CREATED\"]},{\"value\":\"9441201\",\"group\":[\"ASSIGNED\"]},{\"value\":\"5329962\",\"group\":[\"SUBMITTED_TO_BILLING\"]},{\"value\":\"5281666\",\"group\":[\"PICKUP_COMPLETED\"]},{\"value\":\"5225839\",\"group\":[\"OFFERED\"]},{\"value\":\"5088568\",\"group\":[\"READY\"]},{\"value\":\"5027369\",\"group\":[\"COMPLETED\"]},{\"value\":\"3677267\",\"group\":[\"SUBMITTED_TO_MANIFEST\"]},{\"value\":\"1559953\",\"group\":[\"SCHEDULED\"]},{\"value\":\"1532913\",\"group\":[\"ACCEPTED\"]},{\"value\":\"1532891\",\"group\":[\"RELEASED\"]},{\"value\":\"531719\",\"group\":[\"UNASSIGNED\"]},{\"value\":\"252977\",\"group\":[\"PREP_TIME_UPDATED\"]},{\"value\":\"243463\",\"group\":[\"CANCELED\"]},{\"value\":\"211553\",\"group\":[\"PAYMENT_PENDING\"]},{\"value\":\"148548\",\"group\":[\"PAYMENT_CONFIRMED\"]},{\"value\":\"108057\",\"group\":[\"UNFULFILLED_WARNED\"]},{\"value\":\"47043\",\"group\":[\"DELIVERY_FAILED\"]},{\"value\":\"30832\",\"group\":[\"UNFULFILLED\"]},{\"value\":\"18009\",\"group\":[\"SCHEDULE_ORDER_CREATED\"]},{\"value\":\"16459\",\"group\":[\"SCHEDULE_ORDER_ACCEPTED\"]},{\"value\":\"11086\",\"group\":[\"FAILED\"]},{\"value\":\"9976\",\"group\":[\"SCHEDULE_ORDER_OFFERED\"]},{\"value\":\"3094\",\"group\":[\"PAYMENT_FAILED\"]}],\"function\":\"count_star\",\"groupByColumns\":[\"jobState\"]},{\"groupByResult\":[{\"value\":\"3274799599.00000\",\"group\":[\"CREATED\"]},{\"value\":\"2926585674.00000\",\"group\":[\"ASSIGNED\"]},{\"value\":\"1645707788.00000\",\"group\":[\"SUBMITTED_TO_BILLING\"]},{\"value\":\"1614715326.00000\",\"group\":[\"OFFERED\"]},{\"value\":\"1608041994.00000\",\"group\":[\"PICKUP_COMPLETED\"]},{\"value\":\"1568036720.00000\",\"group\":[\"READY\"]},{\"value\":\"1541977381.00000\",\"group\":[\"COMPLETED\"]},{\"value\":\"1190457213.00000\",\"group\":[\"SUBMITTED_TO_MANIFEST\"]},{\"value\":\"430246171.00000\",\"group\":[\"SCHEDULED\"]},{\"value\":\"422020881.00000\",\"group\":[\"RELEASED\"]},{\"value\":\"421937782.00000\",\"group\":[\"ACCEPTED\"]},{\"value\":\"147557783.00000\",\"group\":[\"UNASSIGNED\"]},{\"value\":\"94882088.00000\",\"group\":[\"PREP_TIME_UPDATED\"]},{\"value\":\"86447788.00000\",\"group\":[\"CANCELED\"]},{\"value\":\"77505566.00000\",\"group\":[\"PAYMENT_PENDING\"]},{\"value\":\"53955037.00000\",\"group\":[\"PAYMENT_CONFIRMED\"]},{\"value\":\"36026660.00000\",\"group\":[\"UNFULFILLED_WARNED\"]},{\"value\":\"15306755.00000\",\"group\":[\"DELIVERY_FAILED\"]},{\"value\":\"8811788.00000\",\"group\":[\"UNFULFILLED\"]},{\"value\":\"5301567.00000\",\"group\":[\"SCHEDULE_ORDER_CREATED\"]},{\"value\":\"4855342.00000\",\"group\":[\"SCHEDULE_ORDER_ACCEPTED\"]},{\"value\":\"3113490.00000\",\"group\":[\"FAILED\"]},{\"value\":\"2811789.00000\",\"group\":[\"SCHEDULE_ORDER_OFFERED\"]},{\"value\":\"1053944.00000\",\"group\":[\"PAYMENT_FAILED\"]}],\"function\":\"sum_regionId\",\"groupByColumns\":[\"jobState\"]}],\"exceptions\":[],\"numServersQueried\":7,\"numServersResponded\":7,\"numDocsScanned\":55977222,\"numEntriesScannedInFilter\":0,\"numEntriesScannedPostFilter\":111954444,\"totalDocs\":55977222,\"numGroupsLimitReached\":false,\"timeUsedMs\":775,\"segmentStatistics\":[],\"traceInfo\":{}}", + ImmutableList.of(VARCHAR, BIGINT, BIGINT), Optional.empty()}, + {"SELECT count(*) FROM eats_job_state GROUP BY jobState TOP 1000000", + "{\"traceInfo\":{},\"numEntriesScannedPostFilter\":55979949,\"numDocsScanned\":55979949,\"numServersResponded\":7,\"numGroupsLimitReached\":false,\"aggregationResults\":[{\"groupByResult\":[{\"value\":\"10647363\",\"group\":[\"CREATED\"]},{\"value\":\"9441638\",\"group\":[\"ASSIGNED\"]},{\"value\":\"5330203\",\"group\":[\"SUBMITTED_TO_BILLING\"]},{\"value\":\"5281905\",\"group\":[\"PICKUP_COMPLETED\"]},{\"value\":\"5226090\",\"group\":[\"OFFERED\"]},{\"value\":\"5088813\",\"group\":[\"READY\"]},{\"value\":\"5027589\",\"group\":[\"COMPLETED\"]},{\"value\":\"3677424\",\"group\":[\"SUBMITTED_TO_MANIFEST\"]},{\"value\":\"1560029\",\"group\":[\"SCHEDULED\"]},{\"value\":\"1533006\",\"group\":[\"ACCEPTED\"]},{\"value\":\"1532980\",\"group\":[\"RELEASED\"]},{\"value\":\"531745\",\"group\":[\"UNASSIGNED\"]},{\"value\":\"252989\",\"group\":[\"PREP_TIME_UPDATED\"]},{\"value\":\"243477\",\"group\":[\"CANCELED\"]},{\"value\":\"211571\",\"group\":[\"PAYMENT_PENDING\"]},{\"value\":\"148557\",\"group\":[\"PAYMENT_CONFIRMED\"]},{\"value\":\"108062\",\"group\":[\"UNFULFILLED_WARNED\"]},{\"value\":\"47048\",\"group\":[\"DELIVERY_FAILED\"]},{\"value\":\"30832\",\"group\":[\"UNFULFILLED\"]},{\"value\":\"18009\",\"group\":[\"SCHEDULE_ORDER_CREATED\"]},{\"value\":\"16461\",\"group\":[\"SCHEDULE_ORDER_ACCEPTED\"]},{\"value\":\"11086\",\"group\":[\"FAILED\"]},{\"value\":\"9978\",\"group\":[\"SCHEDULE_ORDER_OFFERED\"]},{\"value\":\"3094\",\"group\":[\"PAYMENT_FAILED\"]}],\"function\":\"count_star\",\"groupByColumns\":[\"jobState\"]}],\"exceptions\":[],\"numEntriesScannedInFilter\":0,\"timeUsedMs\":402,\"segmentStatistics\":[],\"numServersQueried\":7,\"totalDocs\":55979949}", + ImmutableList.of(VARCHAR, BIGINT), Optional.empty()}, + {"SELECT count(*) FROM eats_job_state", + "{\"traceInfo\":{},\"numEntriesScannedPostFilter\":0,\"numDocsScanned\":55981101,\"numServersResponded\":7,\"numGroupsLimitReached\":false,\"aggregationResults\":[{\"function\":\"count_star\",\"value\":\"55981101\"}],\"exceptions\":[],\"numEntriesScannedInFilter\":0,\"timeUsedMs\":7,\"segmentStatistics\":[],\"numServersQueried\":7,\"totalDocs\":55981101}", + ImmutableList.of(BIGINT), Optional.empty()}, + {"SELECT sum(regionId), count(*) FROM eats_job_state", + "{\"traceInfo\":{},\"numEntriesScannedPostFilter\":55981641,\"numDocsScanned\":55981641,\"numServersResponded\":7,\"numGroupsLimitReached\":false,\"aggregationResults\":[{\"function\":\"sum_regionId\",\"value\":\"17183585871.00000\"},{\"function\":\"count_star\",\"value\":\"55981641\"}],\"exceptions\":[],\"numEntriesScannedInFilter\":0,\"timeUsedMs\":549,\"segmentStatistics\":[],\"numServersQueried\":7,\"totalDocs\":55981641}", + ImmutableList.of(BIGINT, BIGINT), Optional.empty()}, + {"SELECT jobState, regionId FROM eats_job_state LIMIT 10", + "{\"selectionResults\":{\"columns\":[\"jobState\",\"regionId\"],\"results\":[[\"CREATED\",\"197\"],[\"SUBMITTED_TO_BILLING\",\"227\"],[\"ASSIGNED\",\"188\"],[\"SCHEDULED\",\"1479\"],[\"CANCELED\",\"1708\"],[\"CREATED\",\"134\"],[\"CREATED\",\"12\"],[\"OFFERED\",\"30\"],[\"COMPLETED\",\"215\"],[\"CREATED\",\"7\"]]},\"exceptions\":[],\"numServersQueried\":7,\"numServersResponded\":7,\"numDocsScanned\":380,\"numEntriesScannedInFilter\":0,\"numEntriesScannedPostFilter\":760,\"totalDocs\":55988817,\"numGroupsLimitReached\":false,\"timeUsedMs\":2,\"segmentStatistics\":[],\"traceInfo\":{}}", + ImmutableList.of(VARCHAR, BIGINT), Optional.empty()}, + {"SELECT shoppingCartUUID, $validUntil, $validFrom, jobState, tenancy, accountUUID, vehicleViewId, $partition, clientUUID, orderJobUUID, productTypeUUID, demandJobUUID, regionId, workflowUUID, jobType, kafkaOffset, productUUID, timestamp, flowType, ts FROM eats_job_state LIMIT 10", + "{\"selectionResults\":{\"columns\":[\"shoppingCartUUID\",\"$validUntil\",\"$validFrom\",\"jobState\",\"tenancy\",\"accountUUID\",\"vehicleViewId\",\"$partition\",\"clientUUID\",\"orderJobUUID\",\"productTypeUUID\",\"demandJobUUID\",\"regionId\",\"workflowUUID\",\"jobType\",\"kafkaOffset\",\"productUUID\",\"timestamp\",\"flowType\",\"ts\"],\"results\":[]},\"traceInfo\":{},\"numEntriesScannedPostFilter\":0,\"numDocsScanned\":0,\"numServersResponded\":7,\"numGroupsLimitReached\":false,\"exceptions\":[{\"errorCode\":200,\"message\":\"QueryExecutionError:\\njava.lang.NullPointerException\\n\\tat java.lang.Class.forName0(Native Method\\n\\tat\"}],\"numEntriesScannedInFilter\":0,\"timeUsedMs\":3,\"segmentStatistics\":[],\"numServersQueried\":7,\"totalDocs\":0}", + ImmutableList.of(), Optional.of(PinotException.class)}, + {"SELECT * from eats_utilization_summarized", + "{\n" + + " \"selectionResults\": {\n" + + " \"columns\": [\"activeTrips\", \"numDrivers\", \"region\", \"rowtime\", \"secondsSinceEpoch\", \"utilization\", \"utilizedDrivers\", \"vehicleViewId\", \"windowEnd\", \"windowStart\"],\n" + + " \"results\": [\n" + + " [\"0\", \"0\", \"foobar\", \"null\", \"4588780800\", \"-∞\", \"0\", \"20017545\", \"4588780740000\", \"4588780725000\"],\n" + + " [\"8699\", \"11452\", \"doobar\", \"null\", \"4588780800\", \"0.730701685\", \"8368\", \"0\", \"4588780740000\", \"4588780725000\"],\n" + + " [\"0\", \"14\", \"zoobar\", \"null\", \"4588780800\", \"0.5\", \"7\", \"20014789\", \"4588780740000\", \"4588780725000\"],\n" + + " [\"0\", \"23\", \"moobar\", \"null\", \"4588780800\", \"0.4336180091\", \"10\", \"20009983\", \"4588780740000\", \"4588780725000\"],\n" + + " [\"0\", \"840\", \"koobar\", \"null\", \"4588780800\", \"0.6597985029\", \"554\", \"20006875\", \"4588780740000\", \"4588780725000\"],\n" + + " [\"0\", \"0\", \"loobar\", \"null\", \"4588780800\", \"-∞\", \"0\", \"20006291\", \"4588780740000\", \"4588780725000\"],\n" + + " [\"15\", \"1832\", \"monkeybar\", \"null\", \"4588780800\", \"0.8792306185\", \"1610\", \"20004007\", \"4588780740000\", \"4588780725000\"],\n" + + " [\"0\", \"0\", \"donkeybar\", \"null\", \"4588780800\", \"-∞\", \"0\", \"0\", \"4588780740000\", \"4588780725000\"],\n" + + " [\"1\", \"7\", \"horseybar\", \"null\", \"4588780800\", \"0.2857142985\", \"2\", \"20016753\", \"4588780740000\", \"4588780725000\"],\n" + + " [\"0\", \"130\", \"ginbar\", \"null\", \"4588780800\", \"0.8052611947\", \"105\", \"10000942\", \"4588780740000\", \"4588780725000\"]\n" + + " ]\n" + + " },\n" + + " \"exceptions\": [],\n" + + " \"numServersQueried\": 4,\n" + + " \"numServersResponded\": 4,\n" + + " \"numSegmentsQueried\": 24,\n" + + " \"numSegmentsProcessed\": 24,\n" + + " \"numSegmentsMatched\": 24,\n" + + " \"numDocsScanned\": 240,\n" + + " \"numEntriesScannedInFilter\": 0,\n" + + " \"numEntriesScannedPostFilter\": 240,\n" + + " \"numGroupsLimitReached\": false,\n" + + " \"totalDocs\": 1000,\n" + + " \"timeUsedMs\": 6,\n" + + " \"segmentStatistics\": [],\n" + + " \"traceInfo\": {}\n" + + "}", + ImmutableList.of(BIGINT, BIGINT, VARCHAR, VARCHAR, BIGINT, BIGINT, BIGINT, BIGINT, BIGINT, BIGINT), Optional.empty()} + }; + } + + @Test(dataProvider = "pqlResponses") + public void testPopulateFromPql(String pql, String pqlResponse, List types, Optional> expectedError) + throws IOException + { + PqlParsedInfo pqlParsedInfo = getBasicInfoFromPql(pqlResponse); + ImmutableList.Builder blockBuilders = ImmutableList.builder(); + PageBuilder pageBuilder = new PageBuilder(types); + PinotBrokerPageSource pageSource = getPinotBrokerPageSource(); + for (int i = 0; i < types.size(); i++) { + blockBuilders.add(pageBuilder.getBlockBuilder(i)); + } + + Optional thrown = Optional.empty(); + int rows = -1; + try { + rows = pageSource.populateFromPqlResults(pql, pqlParsedInfo.groupByColumns, blockBuilders.build(), types, pqlResponse); + } + catch (PrestoException e) { + thrown = Optional.of(e); + } + + Optional> thrownType = thrown.map(e -> e.getClass()); + Optional errorString = thrown.map(e -> Throwables.getStackTraceAsString(e)); + assertEquals(thrownType, expectedError, String.format("Expected error %s, but got error of type %s: %s", expectedError, thrownType, errorString)); + if (!expectedError.isPresent()) { + assertEquals(types.size(), pqlParsedInfo.columns); + assertEquals(rows, pqlParsedInfo.rows); + } + } + + private PinotBrokerPageSource getPinotBrokerPageSource() + { + List pinotColumnHandles = ImmutableList.of(regionId, fare, city, fare, secondsSinceEpoch); + PinotConfig pinotConfig = new PinotConfig(); + PinotQueryGenerator.GeneratedPql generatedPql = new PinotQueryGenerator.GeneratedPql(pinotTable.getTableName(), String.format("SELECT %s, %s FROM %s LIMIT %d", city.getColumnName(), regionId.getColumnName(), pinotTable.getTableName(), pinotConfig.getLimitLargeForSegment()), ImmutableList.of(0, 1), 0, false, true); + return new PinotBrokerPageSource(pinotConfig, new TestingConnectorSession(ImmutableList.of()), generatedPql, pinotColumnHandles, new MockPinotClusterInfoFetcher(pinotConfig), objectMapper); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotClusterInfoFetcher.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotClusterInfoFetcher.java new file mode 100644 index 0000000000000..e4493b3831c63 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotClusterInfoFetcher.java @@ -0,0 +1,80 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.pinot; + +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpStatus; +import com.facebook.airlift.http.client.testing.TestingHttpClient; +import com.facebook.airlift.http.client.testing.TestingResponse; +import com.facebook.presto.testing.assertions.Assert; +import com.google.common.collect.ImmutableSet; +import com.google.common.net.MediaType; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.concurrent.TimeUnit; + +public class TestPinotClusterInfoFetcher +{ + @Test + public void testBrokersParsed() + { + HttpClient httpClient = new TestingHttpClient((request) -> TestingResponse.mockResponse(HttpStatus.OK, MediaType.JSON_UTF_8, "{\n" + + " \"tableName\": \"dummy\",\n" + + " \"brokers\": [\n" + + " {\n" + + " \"tableType\": \"offline\",\n" + + " \"instances\": [\n" + + " \"Broker_dummy-broker-host1-datacenter1_6513\",\n" + + " \"Broker_dummy-broker-host2-datacenter1_6513\",\n" + + " \"Broker_dummy-broker-host4-datacenter1_6513\"\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"tableType\": \"realtime\",\n" + + " \"instances\": [\n" + + " \"Broker_dummy-broker-host1-datacenter1_6513\",\n" + + " \"Broker_dummy-broker-host2-datacenter1_6513\",\n" + + " \"Broker_dummy-broker-host3-datacenter1_6513\"\n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"server\": [\n" + + " {\n" + + " \"tableType\": \"offline\",\n" + + " \"instances\": [\n" + + " \"Server_dummy-server-host8-datacenter1_7090\",\n" + + " \"Server_dummy-server-host9-datacenter1_7090\"\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"tableType\": \"realtime\",\n" + + " \"instances\": [\n" + + " \"Server_dummy-server-host7-datacenter1_7090\",\n" + + " \"Server_dummy-server-host4-datacenter1_7090\",\n" + + " \"Server_dummy-server-host5-datacenter1_7090\",\n" + + " \"Server_dummy-server-host6-datacenter1_7090\"\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}")); + PinotConfig pinotConfig = new PinotConfig() + .setMetadataCacheExpiry(new Duration(0, TimeUnit.MILLISECONDS)) + .setControllerUrls("localhost:7900"); + PinotClusterInfoFetcher pinotClusterInfoFetcher = new PinotClusterInfoFetcher(pinotConfig, new PinotMetrics(), httpClient, MetadataUtil.TABLES_JSON_CODEC, MetadataUtil.BROKERS_FOR_TABLE_JSON_CODEC, MetadataUtil.ROUTING_TABLES_JSON_CODEC, MetadataUtil.TIME_BOUNDARY_JSON_CODEC); + ImmutableSet brokers = ImmutableSet.copyOf(pinotClusterInfoFetcher.getAllBrokersForTable("dummy")); + Assert.assertEquals(ImmutableSet.of("dummy-broker-host1-datacenter1:6513", "dummy-broker-host2-datacenter1:6513", "dummy-broker-host3-datacenter1:6513", "dummy-broker-host4-datacenter1:6513"), brokers); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotColumnHandle.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotColumnHandle.java new file mode 100644 index 0000000000000..b6b50bc3f6e12 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotColumnHandle.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.testing.EquivalenceTester; +import org.testng.annotations.Test; + +import static com.facebook.presto.pinot.MetadataUtil.COLUMN_CODEC; +import static com.facebook.presto.pinot.PinotColumnHandle.PinotColumnType.DERIVED; +import static com.facebook.presto.pinot.PinotColumnHandle.PinotColumnType.REGULAR; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static org.testng.Assert.assertEquals; + +public class TestPinotColumnHandle +{ + private final PinotColumnHandle columnHandle = new PinotColumnHandle("columnName", VARCHAR, REGULAR); + + @Test + public void testJsonRoundTrip() + { + String json = COLUMN_CODEC.toJson(columnHandle); + PinotColumnHandle copy = COLUMN_CODEC.fromJson(json); + assertEquals(copy, columnHandle); + } + + @Test + public void testEquivalence() + { + EquivalenceTester + .equivalenceTester() + .addEquivalentGroup( + new PinotColumnHandle("columnName", VARCHAR, REGULAR), + new PinotColumnHandle("columnName", VARCHAR, DERIVED), + new PinotColumnHandle("columnName", BIGINT, REGULAR), + new PinotColumnHandle("columnName", BIGINT, DERIVED)) + .addEquivalentGroup( + new PinotColumnHandle("columnNameX", VARCHAR, REGULAR), + new PinotColumnHandle("columnNameX", VARCHAR, DERIVED), + new PinotColumnHandle("columnNameX", BIGINT, REGULAR), + new PinotColumnHandle("columnNameX", BIGINT, DERIVED)) + .check(); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotConfig.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotConfig.java new file mode 100644 index 0000000000000..fe3bae05965c7 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotConfig.java @@ -0,0 +1,123 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.configuration.testing.ConfigAssertions; +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class TestPinotConfig +{ + @Test + public void testDefaults() + { + ConfigAssertions.assertRecordedDefaults( + ConfigAssertions.recordDefaults(PinotConfig.class) + .setExtraHttpHeaders("") + .setControllerUrls("") + .setIdleTimeout(new Duration(5, TimeUnit.MINUTES)) + .setLimitLargeForSegment(PinotConfig.DEFAULT_LIMIT_LARGE_FOR_SEGMENT) + .setTopNLarge(PinotConfig.DEFAULT_TOPN_LARGE) + .setMaxBacklogPerServer(PinotConfig.DEFAULT_MAX_BACKLOG_PER_SERVER) + .setMaxConnectionsPerServer(PinotConfig.DEFAULT_MAX_CONNECTIONS_PER_SERVER) + .setMinConnectionsPerServer(PinotConfig.DEFAULT_MIN_CONNECTIONS_PER_SERVER) + .setThreadPoolSize(PinotConfig.DEFAULT_THREAD_POOL_SIZE) + .setEstimatedSizeInBytesForNonNumericColumn(20) + .setConnectionTimeout(new Duration(1, TimeUnit.MINUTES)) + .setControllerRestService(null) + .setServiceHeaderParam("RPC-Service") + .setCallerHeaderValue("presto") + .setCallerHeaderParam("RPC-Caller") + .setMetadataCacheExpiry(new Duration(2, TimeUnit.MINUTES)) + .setAllowMultipleAggregations(false) + .setPreferBrokerQueries(true) + .setRestProxyServiceForQuery(null) + .setRestProxyUrl(null) + .setNumSegmentsPerSplit(1) + .setFetchRetryCount(2) + .setIgnoreEmptyResponses(false) + .setUseDateTrunc(false) + .setForbidSegmentQueries(false) + .setNonAggregateLimitForBrokerQueries(PinotConfig.DEFAULT_NON_AGGREGATE_LIMIT_FOR_BROKER_QUERIES) + .setUseDateTrunc(false)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("pinot.extra-http-headers", "k:v") + .put("pinot.controller-rest-service", "pinot-controller-service") + .put("pinot.controller-urls", "host1:1111,host2:1111") + .put("pinot.idle-timeout", "1h") + .put("pinot.topn-large", "1000") + .put("pinot.max-backlog-per-server", "15") + .put("pinot.max-connections-per-server", "10") + .put("pinot.min-connections-per-server", "1") + .put("pinot.thread-pool-size", "100") + .put("pinot.estimated-size-in-bytes-for-non-numeric-column", "30") + .put("pinot.connection-timeout", "8m") + .put("pinot.metadata-expiry", "1m") + .put("pinot.caller-header-value", "myCaller") + .put("pinot.caller-header-param", "myParam") + .put("pinot.service-header-param", "myServiceHeader") + .put("pinot.allow-multiple-aggregations", "true") + .put("pinot.prefer-broker-queries", "false") + .put("pinot.rest-proxy-url", "localhost:1111") + .put("pinot.rest-proxy-service-for-query", "pinot-rest-proxy-service") + .put("pinot.num-segments-per-split", "2") + .put("pinot.ignore-empty-responses", "true") + .put("pinot.fetch-retry-count", "3") + .put("pinot.non-aggregate-limit-for-broker-queries", "10") + .put("pinot.use-date-trunc", "true") + .put("pinot.limit-large-for-segment", "100") + .put("pinot.forbid-segment-queries", "true") + .build(); + + PinotConfig expected = new PinotConfig() + .setExtraHttpHeaders("k:v") + .setControllerRestService("pinot-controller-service") + .setControllerUrls("host1:1111,host2:1111") + .setRestProxyUrl("localhost:1111") + .setIdleTimeout(new Duration(1, TimeUnit.HOURS)) + .setLimitLargeForSegment(100000) + .setTopNLarge(1000) + .setMaxBacklogPerServer(15) + .setMaxConnectionsPerServer(10) + .setMinConnectionsPerServer(1) + .setThreadPoolSize(100) + .setEstimatedSizeInBytesForNonNumericColumn(30) + .setConnectionTimeout(new Duration(8, TimeUnit.MINUTES)) + .setServiceHeaderParam("myServiceHeader") + .setCallerHeaderValue("myCaller") + .setCallerHeaderParam("myParam") + .setMetadataCacheExpiry(new Duration(1, TimeUnit.MINUTES)) + .setAllowMultipleAggregations(true) + .setPreferBrokerQueries(false) + .setRestProxyServiceForQuery("pinot-rest-proxy-service") + .setNumSegmentsPerSplit(2) + .setIgnoreEmptyResponses(true) + .setFetchRetryCount(3) + .setNonAggregateLimitForBrokerQueries(10) + .setLimitLargeForSegment(100) + .setForbidSegmentQueries(true) + .setUseDateTrunc(true); + + ConfigAssertions.assertFullMapping(properties, expected); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotMetadata.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotMetadata.java new file mode 100644 index 0000000000000..9ec5185124f67 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotMetadata.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.SchemaTableName; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.concurrent.Executors; + +import static org.testng.Assert.assertEquals; + +public class TestPinotMetadata +{ + private final PinotConfig pinotConfig = new PinotConfig(); + private final PinotConnection pinotConnection = new PinotConnection(new MockPinotClusterInfoFetcher(pinotConfig), pinotConfig, Executors.newSingleThreadExecutor()); + private final PinotMetadata metadata = new PinotMetadata(TestPinotSplitManager.pinotConnectorId, pinotConnection); + + @Test + public void testTables() + { + ConnectorSession session = TestPinotSplitManager.createSessionWithNumSplits(1, false, pinotConfig); + List schemaTableNames = metadata.listTables(session, (String) null); + assertEquals(ImmutableSet.copyOf(schemaTableNames), ImmutableSet.of(new SchemaTableName("default", TestPinotSplitManager.realtimeOnlyTable.getTableName()), new SchemaTableName("default", TestPinotSplitManager.hybridTable.getTableName()))); + List schemas = metadata.listSchemaNames(session); + assertEquals(ImmutableList.copyOf(schemas), ImmutableList.of("default")); + PinotTableHandle withWeirdSchema = metadata.getTableHandle(session, new SchemaTableName("foo", TestPinotSplitManager.realtimeOnlyTable.getTableName())); + assertEquals(withWeirdSchema.getTableName(), TestPinotSplitManager.realtimeOnlyTable.getTableName()); + PinotTableHandle withAnotherSchema = metadata.getTableHandle(session, new SchemaTableName(TestPinotSplitManager.realtimeOnlyTable.getTableName(), TestPinotSplitManager.realtimeOnlyTable.getTableName())); + assertEquals(withAnotherSchema.getTableName(), TestPinotSplitManager.realtimeOnlyTable.getTableName()); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotQueryBase.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotQueryBase.java new file mode 100644 index 0000000000000..ea01e0c9c1ba5 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotQueryBase.java @@ -0,0 +1,225 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.Session; +import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.execution.warnings.WarningCollector; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.metadata.SessionPropertyManager; +import com.facebook.presto.pinot.query.PinotQueryGeneratorContext; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.sql.ExpressionUtils; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.parser.ParsingOptions; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.NodeRef; +import com.facebook.presto.testing.TestingConnectorSession; +import com.facebook.presto.testing.TestingSession; +import com.facebook.presto.testing.TestingTransactionHandle; +import com.facebook.presto.type.TypeRegistry; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.IntStream; + +import static com.facebook.presto.pinot.PinotColumnHandle.PinotColumnType.REGULAR; +import static com.facebook.presto.pinot.query.PinotQueryGeneratorContext.Origin.DERIVED; +import static com.facebook.presto.pinot.query.PinotQueryGeneratorContext.Origin.TABLE_COLUMN; +import static com.facebook.presto.spi.plan.LimitNode.Step.FINAL; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypes; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; + +public class TestPinotQueryBase +{ + protected static final TypeManager typeManager = new TypeRegistry(); + protected static final FunctionManager functionMetadataManager = new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + protected static final StandardFunctionResolution standardFunctionResolution = new FunctionResolution(functionMetadataManager); + + protected static ConnectorId pinotConnectorId = new ConnectorId("id"); + protected static PinotTableHandle realtimeOnlyTable = new PinotTableHandle(pinotConnectorId.getCatalogName(), "schema", "realtimeOnly"); + protected static PinotTableHandle hybridTable = new PinotTableHandle(pinotConnectorId.getCatalogName(), "schema", "hybrid"); + protected static PinotColumnHandle regionId = new PinotColumnHandle("regionId", BIGINT, REGULAR); + protected static PinotColumnHandle city = new PinotColumnHandle("city", VARCHAR, REGULAR); + protected static final PinotColumnHandle fare = new PinotColumnHandle("fare", DOUBLE, REGULAR); + protected static final PinotColumnHandle secondsSinceEpoch = new PinotColumnHandle("secondsSinceEpoch", BIGINT, REGULAR); + + protected static final Metadata metadata = MetadataManager.createTestMetadataManager(); + + protected final PinotConfig pinotConfig = new PinotConfig(); + + protected static final Map testInput = ImmutableMap.of( + new VariableReferenceExpression("regionid", BIGINT), new PinotQueryGeneratorContext.Selection("regionId", TABLE_COLUMN), // direct column reference + new VariableReferenceExpression("city", VARCHAR), new PinotQueryGeneratorContext.Selection("city", TABLE_COLUMN), // direct column reference + new VariableReferenceExpression("fare", DOUBLE), new PinotQueryGeneratorContext.Selection("fare", TABLE_COLUMN), // direct column reference + new VariableReferenceExpression("totalfare", DOUBLE), new PinotQueryGeneratorContext.Selection("(fare + trip)", DERIVED), // derived column + new VariableReferenceExpression("secondssinceepoch", BIGINT), new PinotQueryGeneratorContext.Selection("secondsSinceEpoch", TABLE_COLUMN)); // column for datetime functions + + protected final TypeProvider typeProvider = TypeProvider.fromVariables(testInput.keySet()); + + protected static class SessionHolder + { + private final ConnectorSession connectorSession; + private final Session session; + + public SessionHolder(PinotConfig pinotConfig) + { + connectorSession = new TestingConnectorSession(new PinotSessionProperties(pinotConfig).getSessionProperties()); + session = TestingSession.testSessionBuilder(new SessionPropertyManager(new SystemSessionProperties().getSessionProperties())).build(); + } + + public SessionHolder(boolean useDateTrunc) + { + this(new PinotConfig().setUseDateTrunc(useDateTrunc)); + } + + public ConnectorSession getConnectorSession() + { + return connectorSession; + } + + public Session getSession() + { + return session; + } + } + + protected VariableReferenceExpression v(String name) + { + return testInput.keySet().stream().filter(v -> v.getName().equals(name)).findFirst().orElseThrow(() -> new IllegalArgumentException("Cannot find variable " + name)); + } + + protected TableScanNode tableScan(PlanBuilder planBuilder, PinotTableHandle connectorTableHandle, PinotColumnHandle... columnHandles) + { + List variables = Arrays.stream(columnHandles).map(ch -> new VariableReferenceExpression(ch.getColumnName().toLowerCase(ENGLISH), ch.getDataType())).collect(toImmutableList()); + ImmutableMap.Builder assignments = ImmutableMap.builder(); + for (int i = 0; i < variables.size(); ++i) { + assignments.put(variables.get(i), columnHandles[i]); + } + TableHandle tableHandle = new TableHandle( + pinotConnectorId, + connectorTableHandle, + TestingTransactionHandle.create(), + Optional.empty()); + return planBuilder.tableScan( + tableHandle, + variables, + assignments.build()); + } + + protected FilterNode filter(PlanBuilder planBuilder, PlanNode source, RowExpression predicate) + { + return planBuilder.filter(predicate, source); + } + + protected ProjectNode project(PlanBuilder planBuilder, PlanNode source, List columnNames) + { + Map incomingColumns = source.getOutputVariables().stream().collect(toMap(VariableReferenceExpression::getName, identity())); + Assignments.Builder assignmentsBuilder = Assignments.builder(); + columnNames.forEach(columnName -> { + VariableReferenceExpression variable = requireNonNull(incomingColumns.get(columnName), "Couldn't find the incoming column " + columnName); + assignmentsBuilder.put(variable, variable); + }); + return planBuilder.project(assignmentsBuilder.build(), source); + } + + protected ProjectNode project(PlanBuilder planBuilder, PlanNode source, LinkedHashMap toProject, SessionHolder sessionHolder) + { + Assignments.Builder assignmentsBuilder = Assignments.builder(); + toProject.forEach((columnName, expression) -> { + RowExpression rowExpression = getRowExpression(expression, sessionHolder); + VariableReferenceExpression variable = new VariableReferenceExpression(columnName, rowExpression.getType()); + assignmentsBuilder.put(variable, rowExpression); + }); + return planBuilder.project(assignmentsBuilder.build(), source); + } + + public static Expression expression(String sql) + { + return ExpressionUtils.rewriteIdentifiersToSymbolReferences(new SqlParser().createExpression(sql, new ParsingOptions(ParsingOptions.DecimalLiteralTreatment.AS_DECIMAL))); + } + + protected RowExpression toRowExpression(Expression expression, Session session) + { + Map, Type> expressionTypes = getExpressionTypes( + session, + metadata, + new SqlParser(), + typeProvider, + expression, + ImmutableList.of(), + WarningCollector.NOOP); + return SqlToRowExpressionTranslator.translate(expression, expressionTypes, ImmutableMap.of(), functionMetadataManager, typeManager, session); + } + + protected LimitNode limit(PlanBuilder pb, long count, PlanNode source) + { + return new LimitNode(pb.getIdAllocator().getNextId(), source, count, FINAL); + } + + protected TopNNode topN(PlanBuilder pb, long count, List orderingColumns, List ascending, PlanNode source) + { + ImmutableList ordering = IntStream.range(0, orderingColumns.size()).boxed().map(i -> new Ordering(v(orderingColumns.get(i)), ascending.get(i) ? SortOrder.ASC_NULLS_FIRST : SortOrder.DESC_NULLS_FIRST)).collect(toImmutableList()); + return new TopNNode(pb.getIdAllocator().getNextId(), source, count, new OrderingScheme(ordering), TopNNode.Step.SINGLE); + } + + protected RowExpression getRowExpression(String sqlExpression, SessionHolder sessionHolder) + { + return toRowExpression(expression(sqlExpression), sessionHolder.getSession()); + } + + protected PlanBuilder createPlanBuilder(SessionHolder sessionHolder) + { + return new PlanBuilder(sessionHolder.getSession(), new PlanNodeIdAllocator(), metadata); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSessionProperties.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSessionProperties.java new file mode 100644 index 0000000000000..7de528cbb67bd --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSessionProperties.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.testing.TestingConnectorSession; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.concurrent.TimeUnit; + +import static com.facebook.presto.testing.assertions.Assert.assertEquals; + +public class TestPinotSessionProperties +{ + @Test(expectedExceptions = IllegalArgumentException.class) + public void testInvalidNumSegmentSplits() + { + new PinotConfig().setNumSegmentsPerSplit(-3); + } + + @Test + public void testConnectionTimeoutParsedProperly() + { + PinotConfig pinotConfig = new PinotConfig().setConnectionTimeout(new Duration(15, TimeUnit.SECONDS)); + PinotSessionProperties pinotSessionProperties = new PinotSessionProperties(pinotConfig); + ConnectorSession session = new TestingConnectorSession(pinotSessionProperties.getSessionProperties()); + assertEquals(PinotSessionProperties.getConnectionTimeout(session), new Duration(0.25, TimeUnit.MINUTES)); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSplitManager.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSplitManager.java new file mode 100644 index 0000000000000..f52aac2aff924 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSplitManager.java @@ -0,0 +1,165 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.pinot.query.PinotQueryGenerator; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorSplitSource; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; +import com.facebook.presto.testing.TestingConnectorSession; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executors; + +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.presto.pinot.PinotSplit.SplitType.BROKER; +import static com.facebook.presto.pinot.PinotSplit.SplitType.SEGMENT; +import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; +import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY; +import static java.util.Locale.ENGLISH; +import static java.util.stream.Collectors.toList; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestPinotSplitManager + extends TestPinotQueryBase +{ + // Test table and related info + private final PinotConfig pinotConfig = new PinotConfig(); + private final PinotConnection pinotConnection = new PinotConnection(new MockPinotClusterInfoFetcher(pinotConfig), pinotConfig, Executors.newSingleThreadExecutor()); + private final PinotSplitManager pinotSplitManager = new PinotSplitManager(pinotConnectorId, pinotConnection); + + @Test + public void testRealtimeSegmentSplitsOneSegmentPerServer() + { + testSegmentSplitsHelperNoFilter(realtimeOnlyTable, 1, 4, false); // 2 servers with 2 segments each + } + + private void testSegmentSplitsHelperNoFilter(PinotTableHandle table, int segmentsPerSplit, int expectedNumSplits, boolean expectFilter) + { + PinotConfig pinotConfig = new PinotConfig().setPreferBrokerQueries(false); + SessionHolder sessionHolder = new SessionHolder(pinotConfig); + PlanBuilder planBuilder = createPlanBuilder(sessionHolder); + PlanNode plan = tableScan(planBuilder, table, regionId, city, fare, secondsSinceEpoch); + PinotQueryGenerator.GeneratedPql generatedPql = new PinotQueryGenerator(pinotConfig, typeManager, functionMetadataManager, standardFunctionResolution).generate(plan, sessionHolder.getConnectorSession()).get().getGeneratedPql(); + PinotTableHandle pinotTableHandle = new PinotTableHandle(table.getConnectorId(), table.getSchemaName(), table.getTableName(), Optional.of(false), Optional.of(generatedPql)); + List splits = getSplitsHelper(pinotTableHandle, segmentsPerSplit, false); + assertSplits(splits, expectedNumSplits, SEGMENT); + splits.forEach(s -> assertSegmentSplitWellFormed(s, expectFilter)); + } + + private void testSegmentSplitsHelperWithFilter(PinotTableHandle table, int segmentsPerSplit, int expectedNumSplits) + { + PinotConfig pinotConfig = new PinotConfig().setPreferBrokerQueries(false); + SessionHolder sessionHolder = new SessionHolder(pinotConfig); + PlanBuilder planBuilder = createPlanBuilder(sessionHolder); + PlanNode plan = filter(planBuilder, tableScan(planBuilder, table, regionId, city, fare, secondsSinceEpoch), getRowExpression("city = 'Boston'", sessionHolder)); + PinotQueryGenerator.GeneratedPql generatedPql = new PinotQueryGenerator(pinotConfig, typeManager, functionMetadataManager, standardFunctionResolution).generate(plan, sessionHolder.getConnectorSession()).get().getGeneratedPql(); + PinotTableHandle pinotTableHandle = new PinotTableHandle(table.getConnectorId(), table.getSchemaName(), table.getTableName(), Optional.of(false), Optional.of(generatedPql)); + List splits = getSplitsHelper(pinotTableHandle, segmentsPerSplit, false); + assertSplits(splits, expectedNumSplits, SEGMENT); + splits.forEach(s -> assertSegmentSplitWellFormed(s, true)); + } + + @Test + public void testSplitsBroker() + { + PinotQueryGenerator.GeneratedPql generatedPql = new PinotQueryGenerator.GeneratedPql(realtimeOnlyTable.getTableName(), String.format("SELECT %s, COUNT(1) FROM %s GROUP BY %s TOP %d", city.getColumnName(), realtimeOnlyTable.getTableName(), city.getColumnName(), pinotConfig.getTopNLarge()), ImmutableList.of(0, 1), 1, false, true); + PinotTableHandle pinotTableHandle = new PinotTableHandle(realtimeOnlyTable.getConnectorId(), realtimeOnlyTable.getSchemaName(), realtimeOnlyTable.getTableName(), Optional.of(true), Optional.of(generatedPql)); + List splits = getSplitsHelper(pinotTableHandle, 1, false); + assertSplits(splits, 1, BROKER); + } + + @Test(expectedExceptions = PinotSplitManager.QueryNotAdequatelyPushedDownException.class) + public void testBrokerNonShortQuery() + { + PinotQueryGenerator.GeneratedPql generatedPql = new PinotQueryGenerator.GeneratedPql(realtimeOnlyTable.getTableName(), String.format("SELECT %s FROM %s", city.getColumnName(), realtimeOnlyTable.getTableName()), ImmutableList.of(0), 0, false, false); + PinotTableHandle pinotTableHandle = new PinotTableHandle(realtimeOnlyTable.getConnectorId(), realtimeOnlyTable.getSchemaName(), realtimeOnlyTable.getTableName(), Optional.of(false), Optional.of(generatedPql)); + List splits = getSplitsHelper(pinotTableHandle, 1, true); + assertSplits(splits, 1, BROKER); + } + + @Test + public void testRealtimeSegmentSplitsManySegmentPerServer() + { + testSegmentSplitsHelperNoFilter(realtimeOnlyTable, Integer.MAX_VALUE, 2, false); + } + + @Test + public void testHybridSegmentSplitsOneSegmentPerServer() + { + testSegmentSplitsHelperNoFilter(hybridTable, 1, 8, true); + testSegmentSplitsHelperWithFilter(hybridTable, 1, 8); + } + + private void assertSplits(List splits, int numSplitsExpected, PinotSplit.SplitType splitType) + { + assertEquals(splits.size(), numSplitsExpected); + splits.forEach(s -> assertEquals(s.getSplitType(), splitType)); + } + + private void assertSegmentSplitWellFormed(PinotSplit split, boolean expectFilter) + { + assertEquals(split.getSplitType(), SEGMENT); + assertTrue(split.getSegmentPql().isPresent()); + assertTrue(split.getSegmentHost().isPresent()); + assertFalse(split.getSegments().isEmpty()); + String pql = split.getSegmentPql().get(); + assertFalse(pql.contains("__")); // templates should be fully resolved + List splitOnWhere = Splitter.on(" WHERE ").splitToList(pql); + // There should be exactly one WHERE clause and it should partition the pql into two + assertEquals(splitOnWhere.size(), expectFilter ? 2 : 1, "Expected to find only one WHERE clause in " + pql); + } + + public static ConnectorSession createSessionWithNumSplits(int numSegmentsPerSplit, boolean forbidSegmentQueries, PinotConfig pinotConfig) + { + return new TestingConnectorSession( + "user", + Optional.of("test"), + Optional.empty(), + UTC_KEY, + ENGLISH, + System.currentTimeMillis(), + new PinotSessionProperties(pinotConfig).getSessionProperties(), + ImmutableMap.of( + PinotSessionProperties.NUM_SEGMENTS_PER_SPLIT, + numSegmentsPerSplit, + PinotSessionProperties.FORBID_SEGMENT_QUERIES, + forbidSegmentQueries), + new FeaturesConfig().isLegacyTimestamp(), + Optional.empty()); + } + + private List getSplitsHelper(PinotTableHandle pinotTable, int numSegmentsPerSplit, boolean forbidSegmentQueries) + { + PinotTableLayoutHandle pinotTableLayout = new PinotTableLayoutHandle(pinotTable); + ConnectorSession session = createSessionWithNumSplits(numSegmentsPerSplit, forbidSegmentQueries, pinotConfig); + ConnectorSplitSource splitSource = pinotSplitManager.getSplits(null, session, pinotTableLayout, null); + List splits = new ArrayList<>(); + while (!splitSource.isFinished()) { + splits.addAll(getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, 1000)).getSplits().stream().map(s -> (PinotSplit) s).collect(toList())); + } + + return splits; + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotTableHandle.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotTableHandle.java new file mode 100644 index 0000000000000..054097694c0c8 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotTableHandle.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.testing.EquivalenceTester; +import org.testng.annotations.Test; + +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static org.testng.Assert.assertEquals; + +public class TestPinotTableHandle +{ + private final PinotTableHandle tableHandle = new PinotTableHandle("connectorId", "schemaName", "tableName"); + + @Test + public void testJsonRoundTrip() + { + JsonCodec codec = jsonCodec(PinotTableHandle.class); + String json = codec.toJson(tableHandle); + PinotTableHandle copy = codec.fromJson(json); + assertEquals(copy, tableHandle); + } + + @Test + public void testEquivalence() + { + EquivalenceTester.equivalenceTester() + .addEquivalentGroup( + new PinotTableHandle("connector", "schema", "table"), + new PinotTableHandle("connector", "schema", "table")) + .addEquivalentGroup( + new PinotTableHandle("connectorX", "schema", "table"), + new PinotTableHandle("connectorX", "schema", "table")) + .addEquivalentGroup( + new PinotTableHandle("connector", "schemaX", "table"), + new PinotTableHandle("connector", "schemaX", "table")) + .addEquivalentGroup( + new PinotTableHandle("connector", "schema", "tableX"), + new PinotTableHandle("connector", "schema", "tableX")) + .check(); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/query/TestPinotConnectorPlanOptimizer.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/query/TestPinotConnectorPlanOptimizer.java new file mode 100644 index 0000000000000..70ef43fb39bb8 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/query/TestPinotConnectorPlanOptimizer.java @@ -0,0 +1,233 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot.query; + +import com.facebook.presto.Session; +import com.facebook.presto.cost.PlanNodeStatsEstimate; +import com.facebook.presto.cost.StatsAndCosts; +import com.facebook.presto.cost.StatsProvider; +import com.facebook.presto.expressions.LogicalRowExpressions; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.pinot.PinotConfig; +import com.facebook.presto.pinot.PinotConnectorPlanOptimizer; +import com.facebook.presto.pinot.PinotTableHandle; +import com.facebook.presto.pinot.TestPinotQueryBase; +import com.facebook.presto.pinot.TestPinotSplitManager; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.Plan; +import com.facebook.presto.sql.planner.PlanVariableAllocator; +import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.planner.assertions.ExpectedValueProvider; +import com.facebook.presto.sql.planner.assertions.MatchResult; +import com.facebook.presto.sql.planner.assertions.Matcher; +import com.facebook.presto.sql.planner.assertions.PlanAssert; +import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; +import com.facebook.presto.sql.planner.assertions.SymbolAliases; +import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; +import com.facebook.presto.sql.tree.FunctionCall; +import com.facebook.presto.sql.tree.SymbolReference; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Pattern; + +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkState; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; + +public class TestPinotConnectorPlanOptimizer + extends TestPinotQueryBase +{ + private static final SessionHolder defaultSessionHolder = new SessionHolder(false); + private final LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions( + new RowExpressionDeterminismEvaluator(functionMetadataManager), + new FunctionResolution(functionMetadataManager), + functionMetadataManager); + private final PinotTableHandle pinotTable = TestPinotSplitManager.hybridTable; + + private static void assertPlanMatch(PlanNode actual, PlanMatchPattern expected, TypeProvider typeProvider) + { + PlanAssert.assertPlan( + defaultSessionHolder.getSession(), + metadata, + (node, sourceStats, lookup, session, types) -> PlanNodeStatsEstimate.unknown(), + new Plan(actual, typeProvider, StatsAndCosts.empty()), + expected); + } + + private static final class PinotTableScanMatcher + implements Matcher + { + private final ConnectorId connectorId; + private final String tableName; + private final Optional pqlRegex; + private final Optional scanParallelismExpected; + private final String[] columns; + + static PlanMatchPattern match( + String connectorName, + String tableName, + Optional pqlRegex, + Optional scanParallelismExpected, + String... columnNames) + { + return node(TableScanNode.class) + .with(new PinotTableScanMatcher( + new ConnectorId(connectorName), + tableName, + pqlRegex, scanParallelismExpected, columnNames)); + } + + static PlanMatchPattern match( + PinotTableHandle tableHandle, + Optional pqlRegex, + Optional scanParallelismExpected, + List variables) + { + return match(tableHandle.getConnectorId(), + tableHandle.getTableName(), + pqlRegex, + scanParallelismExpected, + variables.stream().map(VariableReferenceExpression::getName).toArray(String[]::new)); + } + + private PinotTableScanMatcher( + ConnectorId connectorId, + String tableName, + Optional pqlRegex, + Optional scanParallelismExpected, + String... columns) + { + this.connectorId = connectorId; + this.pqlRegex = pqlRegex; + this.scanParallelismExpected = scanParallelismExpected; + this.columns = columns; + this.tableName = tableName; + } + + @Override + public boolean shapeMatches(PlanNode node) + { + return node instanceof TableScanNode; + } + + private static boolean checkPqlMatches(Optional regex, Optional pql) + { + if (!pql.isPresent() && !regex.isPresent()) { + return true; + } + if (pql.isPresent() && regex.isPresent()) { + String toMatch = pql.get(); + Pattern compiled = Pattern.compile(regex.get(), Pattern.CASE_INSENSITIVE); + return compiled.matcher(toMatch).matches(); + } + return false; + } + + @Override + public MatchResult detailMatches( + PlanNode node, + StatsProvider stats, + Session session, + Metadata metadata, + SymbolAliases symbolAliases) + { + checkState(shapeMatches(node), "Plan testing framework error: shapeMatches returned false in detailMatches in %s", this.getClass().getName()); + + TableScanNode tableScanNode = (TableScanNode) node; + if (connectorId.equals(tableScanNode.getTable().getConnectorId())) { + PinotTableHandle pinotTableHandle = (PinotTableHandle) tableScanNode.getTable().getConnectorHandle(); + if (pinotTableHandle.getTableName().equals(tableName)) { + Optional pql = pinotTableHandle.getPql().map(PinotQueryGenerator.GeneratedPql::getPql); + if (checkPqlMatches(pqlRegex, pql)) { + return MatchResult.match(SymbolAliases.builder().putAll(Arrays.stream(columns).collect(toMap(identity(), SymbolReference::new))).build()); + } + } + } + return MatchResult.NO_MATCH; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("connectorId", connectorId) + .add("tableName", tableName) + .add("pqlRegex", pqlRegex) + .add("scanParallelismExpected", scanParallelismExpected) + .add("columns", columns) + .toString(); + } + } + + @Test + public void testLimitPushdownWithStarSelection() + { + PlanBuilder pb = createPlanBuilder(defaultSessionHolder); + PlanNode originalPlan = limit(pb, 50L, tableScan(pb, pinotTable, regionId, city, fare, secondsSinceEpoch)); + PlanNode optimized = getOptimizedPlan(pb, originalPlan, true); + assertPlanMatch(optimized, PinotTableScanMatcher.match(pinotTable, Optional.of("SELECT regionId, city, fare, secondsSinceEpoch FROM hybrid LIMIT 50"), Optional.of(false), originalPlan.getOutputVariables()), typeProvider); + } + + @Test + public void testPartialPredicatePushdown() + { + PlanBuilder pb = createPlanBuilder(defaultSessionHolder); + TableScanNode tableScanNode = tableScan(pb, pinotTable, regionId, city, fare, secondsSinceEpoch); + FilterNode filter = filter(pb, tableScanNode, getRowExpression("lower(substr(city, 0, 3)) = 'del' AND fare > 100", defaultSessionHolder)); + PlanNode originalPlan = limit(pb, 50L, filter); + PlanNode optimized = getOptimizedPlan(pb, originalPlan, true); + PlanMatchPattern tableScanMatcher = PinotTableScanMatcher.match(pinotTable, Optional.of("SELECT regionId, city, fare, secondsSinceEpoch FROM hybrid__TABLE_NAME_SUFFIX_TEMPLATE__ WHERE \\(fare > 100\\).*"), Optional.of(true), filter.getOutputVariables()); + assertPlanMatch(optimized, PlanMatchPattern.limit(50L, PlanMatchPattern.filter("lower(substr(city, 0, 3)) = 'del'", tableScanMatcher)), typeProvider); + } + + @Test + public void testUnsupportedPredicatePushdown() + { + Map> aggregationsSecond = ImmutableMap.of( + "count", PlanMatchPattern.functionCall("count", false, ImmutableList.of())); + + PlanBuilder planBuilder = createPlanBuilder(defaultSessionHolder); + PlanNode limit = limit(planBuilder, 50L, tableScan(planBuilder, pinotTable, regionId, city, fare, secondsSinceEpoch)); + PlanNode originalPlan = planBuilder.aggregation(builder -> builder.source(limit).globalGrouping().addAggregation(new VariableReferenceExpression("count", BIGINT), getRowExpression("count(*)", defaultSessionHolder))); + + PlanNode optimized = getOptimizedPlan(planBuilder, originalPlan, true); + + PlanMatchPattern tableScanMatcher = PinotTableScanMatcher.match(pinotTable, Optional.of("SELECT regionId, city, fare, secondsSinceEpoch FROM hybrid LIMIT 50"), Optional.of(false), originalPlan.getOutputVariables()); + assertPlanMatch(optimized, aggregation(aggregationsSecond, tableScanMatcher), typeProvider); + } + + private PlanNode getOptimizedPlan(PlanBuilder planBuilder, PlanNode originalPlan, boolean scanParallelism) + { + PinotConfig pinotConfig = new PinotConfig().setPreferBrokerQueries(scanParallelism); + PinotQueryGenerator pinotQueryGenerator = new PinotQueryGenerator(pinotConfig, typeManager, functionMetadataManager, standardFunctionResolution); + PinotConnectorPlanOptimizer optimizer = new PinotConnectorPlanOptimizer(pinotQueryGenerator, typeManager, functionMetadataManager, logicalRowExpressions, standardFunctionResolution); + return optimizer.optimize(originalPlan, defaultSessionHolder.getConnectorSession(), new PlanVariableAllocator(), planBuilder.getIdAllocator()); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/query/TestPinotExpressionConverters.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/query/TestPinotExpressionConverters.java new file mode 100644 index 0000000000000..52d474b11ffc4 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/query/TestPinotExpressionConverters.java @@ -0,0 +1,171 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot.query; + +import com.facebook.presto.pinot.PinotException; +import com.facebook.presto.pinot.TestPinotQueryBase; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import org.testng.annotations.Test; + +import java.util.function.Function; + +import static com.facebook.presto.pinot.PinotErrorCode.PINOT_UNSUPPORTED_EXPRESSION; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; + +public class TestPinotExpressionConverters + extends TestPinotQueryBase +{ + private final Function testInputFunction = testInput::get; + + @Test + public void testProjectExpressionConverter() + { + SessionHolder sessionHolder = new SessionHolder(false); + testProject("secondssinceepoch", "secondsSinceEpoch", sessionHolder); + // functions + testAggregationProject("date_trunc('hour', from_unixtime(secondssinceepoch))", + "dateTimeConvert(secondsSinceEpoch, '1:SECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '1:HOURS')", sessionHolder); + + // arithmetic + testAggregationProject("regionid + 1", "ADD(regionId, 1)", sessionHolder); + testAggregationProject("regionid - 1", "SUB(regionId, 1)", sessionHolder); + testAggregationProject("1 * regionid", "MULT(1, regionId)", sessionHolder); + testAggregationProject("1 / regionid", "DIV(1, regionId)", sessionHolder); + + // TODO ... this one is failing + testAggregationProject("secondssinceepoch + 1559978258.674", "ADD(secondsSinceEpoch, 1559978258.674)", sessionHolder); + + testAggregationProject("secondssinceepoch + 1559978258", "ADD(secondsSinceEpoch, 1559978258)", sessionHolder); + + testAggregationProjectUnsupported("secondssinceepoch > 0", sessionHolder); + + testAggregationProject("date_trunc('hour', from_unixtime(secondssinceepoch + 2))", + "dateTimeConvert(ADD(secondsSinceEpoch, 2), '1:SECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '1:HOURS')", sessionHolder); + } + + private void testProject(String sqlExpression, String expectedPinotExpression, SessionHolder sessionHolder) + { + RowExpression pushDownExpression = getRowExpression(sqlExpression, sessionHolder); + String actualPinotExpression = pushDownExpression.accept(new PinotProjectExpressionConverter( + typeManager, + standardFunctionResolution), + testInput).getDefinition(); + assertEquals(actualPinotExpression, expectedPinotExpression); + } + + @Test + public void testAdhoc() + { + SessionHolder sessionHolder = new SessionHolder(false); + + testAggregationProject("secondssinceepoch + 1559978258.674", "ADD(secondsSinceEpoch, 1559978258.674)", sessionHolder); + } + + @Test + public void testDateTruncationConversion() + { + SessionHolder sessionHolder = new SessionHolder(true); + testAggregationProject("date_trunc('hour', from_unixtime(secondssinceepoch + 2))", + "dateTrunc(ADD(secondsSinceEpoch, 2),seconds, UTC, hour)", sessionHolder); + + testAggregationProject("date_trunc('hour', from_unixtime(secondssinceepoch + 2, 'America/New_York'))", + "dateTrunc(ADD(secondsSinceEpoch, 2),seconds, America/New_York, hour)", sessionHolder); + } + + @Test + public void testFilterExpressionConverter() + { + SessionHolder sessionHolder = new SessionHolder(false); + + // Simple comparisons + testFilter("regionid = 20", "(regionId = 20)", sessionHolder); + testFilter("regionid >= 20", "(regionId >= 20)", sessionHolder); + testFilter("city = 'Campbell'", "(city = 'Campbell')", sessionHolder); + + // between + testFilter("totalfare between 20 and 30", "((fare + trip) BETWEEN 20 AND 30)", sessionHolder); + + // in, not in + testFilter("regionid in (20, 30, 40)", "(regionId IN (20, 30, 40))", sessionHolder); + testFilter("regionid not in (20, 30, 40)", "(regionId NOT IN (20, 30, 40))", sessionHolder); + testFilter("city in ('San Jose', 'Campbell', 'Union City')", "(city IN ('San Jose', 'Campbell', 'Union City'))", sessionHolder); + testFilter("city not in ('San Jose', 'Campbell', 'Union City')", "(city NOT IN ('San Jose', 'Campbell', 'Union City'))", sessionHolder); + testFilterUnsupported("secondssinceepoch + 1 in (234, 24324)", sessionHolder); + testFilterUnsupported("NOT (secondssinceepoch = 2323)", sessionHolder); + + // combinations + testFilter("totalfare between 20 and 30 AND regionid > 20 OR city = 'Campbell'", + "((((fare + trip) BETWEEN 20 AND 30) AND (regionId > 20)) OR (city = 'Campbell'))", sessionHolder); + + testFilter("secondssinceepoch > 1559978258", "(secondsSinceEpoch > 1559978258)", sessionHolder); + } + + private void testAggregationProject(String sqlExpression, String expectedPinotExpression, SessionHolder sessionHolder) + { + RowExpression pushDownExpression = getRowExpression(sqlExpression, sessionHolder); + String actualPinotExpression = pushDownExpression.accept(new PinotAggregationProjectConverter( + typeManager, + functionMetadataManager, + standardFunctionResolution, + sessionHolder.getConnectorSession()), + testInput).getDefinition(); + assertEquals(actualPinotExpression, expectedPinotExpression); + } + + private void testAggregationProjectUnsupported(String sqlExpression, SessionHolder sessionHolder) + { + try { + RowExpression pushDownExpression = getRowExpression(sqlExpression, sessionHolder); + String actualPinotExpression = pushDownExpression.accept(new PinotAggregationProjectConverter( + typeManager, + functionMetadataManager, + standardFunctionResolution, + sessionHolder.getConnectorSession()), + testInput).getDefinition(); + fail("expected to not reach here: Generated " + actualPinotExpression); + } + catch (PinotException e) { + assertEquals(e.getErrorCode(), PINOT_UNSUPPORTED_EXPRESSION.toErrorCode()); + } + } + + private void testFilter(String sqlExpression, String expectedPinotExpression, SessionHolder sessionHolder) + { + RowExpression pushDownExpression = getRowExpression(sqlExpression, sessionHolder); + String actualPinotExpression = pushDownExpression.accept(new PinotFilterExpressionConverter( + typeManager, + functionMetadataManager, + standardFunctionResolution), + testInputFunction).getDefinition(); + assertEquals(actualPinotExpression, expectedPinotExpression); + } + + private void testFilterUnsupported(String sqlExpression, SessionHolder sessionHolder) + { + try { + RowExpression pushDownExpression = getRowExpression(sqlExpression, sessionHolder); + String actualPinotExpression = pushDownExpression.accept(new PinotFilterExpressionConverter( + typeManager, + functionMetadataManager, + standardFunctionResolution), + testInputFunction).getDefinition(); + fail("expected to not reach here: Generated " + actualPinotExpression); + } + catch (PinotException e) { + assertEquals(e.getErrorCode(), PINOT_UNSUPPORTED_EXPRESSION.toErrorCode()); + } + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/query/TestPinotQueryGenerator.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/query/TestPinotQueryGenerator.java new file mode 100644 index 0000000000000..f203fd9750244 --- /dev/null +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/query/TestPinotQueryGenerator.java @@ -0,0 +1,238 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot.query; + +import com.facebook.presto.pinot.PinotConfig; +import com.facebook.presto.pinot.PinotTableHandle; +import com.facebook.presto.pinot.TestPinotQueryBase; +import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static java.lang.String.format; +import static org.testng.Assert.assertEquals; + +public class TestPinotQueryGenerator + extends TestPinotQueryBase +{ + private static final SessionHolder defaultSessionHolder = new SessionHolder(false); + private static final PinotTableHandle pinotTable = realtimeOnlyTable; + + private void testPQL( + PinotConfig givenPinotConfig, + Function planBuilderConsumer, + String expectedPQL, SessionHolder sessionHolder, + Map outputVariables) + { + PlanNode planNode = planBuilderConsumer.apply(createPlanBuilder(sessionHolder)); + testPQL(givenPinotConfig, planNode, expectedPQL, sessionHolder, outputVariables); + } + + private void testPQL( + PinotConfig givenPinotConfig, + PlanNode planNode, + String expectedPQL, + SessionHolder sessionHolder, + Map outputVariables) + { + PinotQueryGenerator.PinotQueryGeneratorResult pinotQueryGeneratorResult = new PinotQueryGenerator(givenPinotConfig, typeManager, functionMetadataManager, standardFunctionResolution).generate(planNode, sessionHolder.getConnectorSession()).get(); + if (expectedPQL.contains("__expressions__")) { + String expressions = planNode.getOutputVariables().stream().map(v -> outputVariables.get(v.getName())).filter(v -> v != null).collect(Collectors.joining(", ")); + expectedPQL = expectedPQL.replace("__expressions__", expressions); + } + assertEquals(pinotQueryGeneratorResult.getGeneratedPql().getPql(), expectedPQL); + } + + private void testPQL(Function planBuilderConsumer, String expectedPQL, SessionHolder sessionHolder, Map outputVariables) + { + testPQL(pinotConfig, planBuilderConsumer, expectedPQL, sessionHolder, outputVariables); + } + + private void testPQL(Function planBuilderConsumer, String expectedPQL, SessionHolder sessionHolder) + { + testPQL(planBuilderConsumer, expectedPQL, sessionHolder, ImmutableMap.of()); + } + + private void testPQL(Function planBuilderConsumer, String expectedPQL) + { + testPQL(planBuilderConsumer, expectedPQL, defaultSessionHolder); + } + + private PlanNode buildPlan(Function consumer) + { + PlanBuilder planBuilder = createPlanBuilder(defaultSessionHolder); + return consumer.apply(planBuilder); + } + + private void testUnaryAggregationHelper(BiConsumer aggregationFunctionBuilder, String expectedAggOutput) + { + PlanNode justScan = buildPlan(planBuilder -> tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare)); + PlanNode filter = buildPlan(planBuilder -> filter(planBuilder, tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare), getRowExpression("fare > 3", defaultSessionHolder))); + PlanNode anotherFilter = buildPlan(planBuilder -> filter(planBuilder, tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare), getRowExpression("secondssinceepoch between 200 and 300 and regionid >= 40", defaultSessionHolder))); + testPQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(justScan).globalGrouping())), + format("SELECT %s FROM realtimeOnly", expectedAggOutput)); + testPQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(filter).globalGrouping())), + format("SELECT %s FROM realtimeOnly WHERE (fare > 3)", expectedAggOutput)); + testPQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(filter).singleGroupingSet(v("regionid")))), + format("SELECT %s FROM realtimeOnly WHERE (fare > 3) GROUP BY regionId TOP 10000", expectedAggOutput)); + testPQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(justScan).singleGroupingSet(v("regionid")))), + format("SELECT %s FROM realtimeOnly GROUP BY regionId TOP 10000", expectedAggOutput)); + testPQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggregationFunctionBuilder.accept(planBuilder, aggBuilder.source(anotherFilter).singleGroupingSet(v("regionid"), v("city")))), + format("SELECT %s FROM realtimeOnly WHERE ((secondsSinceEpoch BETWEEN 200 AND 300) AND (regionId >= 40)) GROUP BY regionId, city TOP 10000", expectedAggOutput)); + } + + @Test + public void testSimpleSelectStar() + { + testPQL(planBuilder -> limit(planBuilder, 50L, tableScan(planBuilder, pinotTable, regionId, city, fare, secondsSinceEpoch)), + "SELECT regionId, city, fare, secondsSinceEpoch FROM realtimeOnly LIMIT 50"); + testPQL(planBuilder -> limit(planBuilder, 50L, tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch)), + "SELECT regionId, secondsSinceEpoch FROM realtimeOnly LIMIT 50"); + } + + @Test + public void testSimpleSelectWithFilterLimit() + { + testPQL(planBuilder -> limit(planBuilder, 50L, project(planBuilder, filter(planBuilder, tableScan(planBuilder, pinotTable, regionId, city, fare, secondsSinceEpoch), getRowExpression("secondssinceepoch > 20", defaultSessionHolder)), ImmutableList.of("city", "secondssinceepoch"))), + "SELECT city, secondsSinceEpoch FROM realtimeOnly WHERE (secondsSinceEpoch > 20) LIMIT 50"); + } + + @Test + public void testCountStar() + { + testUnaryAggregationHelper((planBuilder, aggregationBuilder) -> aggregationBuilder.addAggregation(planBuilder.variable("agg"), getRowExpression("count(*)", defaultSessionHolder)), "count(*)"); + } + + @Test + public void testDistinctSelection() + { + PlanNode justScan = buildPlan(planBuilder -> tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare)); + testPQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggBuilder.source(justScan).singleGroupingSet(v("regionid"))), + "SELECT count(*) FROM realtimeOnly GROUP BY regionId TOP 10000"); + } + + @Test + public void testPercentileAggregation() + { + testUnaryAggregationHelper((planBuilder, aggregationBuilder) -> aggregationBuilder.addAggregation(planBuilder.variable("agg"), getRowExpression("approx_percentile(fare, 0.10)", defaultSessionHolder)), "PERCENTILEEST10(fare)"); + } + + @Test + public void testApproxDistinct() + { + testUnaryAggregationHelper((planBuilder, aggregationBuilder) -> aggregationBuilder.addAggregation(planBuilder.variable("agg"), getRowExpression("approx_distinct(fare)", defaultSessionHolder)), "DISTINCTCOUNTHLL(fare)"); + } + + @Test + public void testAggWithUDFInGroupBy() + { + LinkedHashMap aggProjection = new LinkedHashMap<>(); + aggProjection.put("date", "date_trunc('day', cast(from_unixtime(secondssinceepoch - 50) AS TIMESTAMP))"); + PlanNode justDate = buildPlan(planBuilder -> project(planBuilder, tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare), aggProjection, defaultSessionHolder)); + testPQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggBuilder.source(justDate).singleGroupingSet(new VariableReferenceExpression("date", TIMESTAMP)).addAggregation(planBuilder.variable("agg"), getRowExpression("count(*)", defaultSessionHolder))), + "SELECT count(*) FROM realtimeOnly GROUP BY dateTimeConvert(SUB(secondsSinceEpoch, 50), '1:SECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '1:DAYS') TOP 10000"); + aggProjection.put("city", "city"); + PlanNode newScanWithCity = buildPlan(planBuilder -> project(planBuilder, tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare), aggProjection, defaultSessionHolder)); + testPQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggBuilder.source(newScanWithCity).singleGroupingSet(new VariableReferenceExpression("date", TIMESTAMP), v("city")).addAggregation(planBuilder.variable("agg"), getRowExpression("count(*)", defaultSessionHolder))), + "SELECT count(*) FROM realtimeOnly GROUP BY dateTimeConvert(SUB(secondsSinceEpoch, 50), '1:SECONDS:EPOCH', '1:MILLISECONDS:EPOCH', '1:DAYS'), city TOP 10000"); + } + + @Test + public void testMultipleAggregatesWithOutGroupBy() + { + Map outputVariables = ImmutableMap.of("agg", "count(*)", "min", "min(fare)"); + PlanNode justScan = buildPlan(planBuilder -> tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare)); + testPQL(planBuilder -> planBuilder.aggregation(aggBuilder -> aggBuilder.source(justScan).globalGrouping().addAggregation(planBuilder.variable("agg"), getRowExpression("count(*)", defaultSessionHolder)).addAggregation(planBuilder.variable("min"), getRowExpression("min(fare)", defaultSessionHolder))), + "SELECT __expressions__ FROM realtimeOnly", defaultSessionHolder, outputVariables); + testPQL(planBuilder -> planBuilder.limit(50L, planBuilder.aggregation(aggBuilder -> aggBuilder.source(justScan).globalGrouping().addAggregation(planBuilder.variable("agg"), getRowExpression("count(*)", defaultSessionHolder)).addAggregation(planBuilder.variable("min"), getRowExpression("min(fare)", defaultSessionHolder)))), + "SELECT __expressions__ FROM realtimeOnly", defaultSessionHolder, outputVariables); + } + + @Test + public void testMultipleAggregatesWhenAllowed() + { + helperTestMultipleAggregatesWithGroupBy(new PinotConfig().setAllowMultipleAggregations(true)); + } + + @Test(expectedExceptions = NoSuchElementException.class) + public void testMultipleAggregatesNotAllowed() + { + helperTestMultipleAggregatesWithGroupBy(pinotConfig); + } + + private void helperTestMultipleAggregatesWithGroupBy(PinotConfig givenPinotConfig) + { + Map outputVariables = ImmutableMap.of("agg", "count(*)", "min", "min(fare)"); + PlanNode justScan = buildPlan(planBuilder -> tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare)); + testPQL(givenPinotConfig, planBuilder -> planBuilder.aggregation(aggBuilder -> aggBuilder.source(justScan).singleGroupingSet(v("city")).addAggregation(planBuilder.variable("agg"), getRowExpression("count(*)", defaultSessionHolder)).addAggregation(planBuilder.variable("min"), getRowExpression("min(fare)", defaultSessionHolder))), + "SELECT __expressions__ FROM realtimeOnly GROUP BY city TOP 10000", defaultSessionHolder, outputVariables); + } + + @Test(expectedExceptions = NoSuchElementException.class) + public void testMultipleAggregateGroupByWithLimitFails() + { + Map outputVariables = ImmutableMap.of("agg", "count(*)", "min", "min(fare)"); + PlanNode justScan = buildPlan(planBuilder -> tableScan(planBuilder, pinotTable, regionId, secondsSinceEpoch, city, fare)); + testPQL(planBuilder -> planBuilder.limit(50L, planBuilder.aggregation(aggBuilder -> aggBuilder.source(justScan).singleGroupingSet(v("city")).addAggregation(planBuilder.variable("agg"), getRowExpression("count(*)", defaultSessionHolder)).addAggregation(planBuilder.variable("min"), getRowExpression("min(fare)", defaultSessionHolder)))), + "SELECT __expressions__ FROM realtimeOnly GROUP BY city TOP 50", defaultSessionHolder, outputVariables); + } + + @Test(expectedExceptions = NoSuchElementException.class) + public void testForbiddenProjectionOutsideOfAggregation() + { + LinkedHashMap projections = new LinkedHashMap<>(ImmutableMap.of("hour", "date_trunc('hour', from_unixtime(secondssinceepoch))", "regionid", "regionid")); + PlanNode plan = buildPlan(planBuilder -> limit(planBuilder, 10, project(planBuilder, tableScan(planBuilder, pinotTable, secondsSinceEpoch, regionId), projections, defaultSessionHolder))); + testPQL(pinotConfig, plan, "Should fail", defaultSessionHolder, ImmutableMap.of()); + } + + @Test + public void testSimpleSelectWithTopN() + { + PlanBuilder planBuilder = createPlanBuilder(defaultSessionHolder); + TableScanNode tableScanNode = tableScan(planBuilder, pinotTable, regionId, city, fare); + TopNNode topNFare = topN(planBuilder, 50L, ImmutableList.of("fare"), ImmutableList.of(false), tableScanNode); + testPQL(pinotConfig, topNFare, + "SELECT regionId, city, fare FROM realtimeOnly ORDER BY fare DESC LIMIT 50", defaultSessionHolder, ImmutableMap.of()); + TopNNode topnFareAndCity = topN(planBuilder, 50L, ImmutableList.of("fare", "city"), ImmutableList.of(true, false), tableScanNode); + testPQL(pinotConfig, topnFareAndCity, + "SELECT regionId, city, fare FROM realtimeOnly ORDER BY fare, city DESC LIMIT 50", defaultSessionHolder, ImmutableMap.of()); + } + + @Test(expectedExceptions = NoSuchElementException.class) + public void testAggregationWithOrderByPushDownInTopN() + { + PlanBuilder planBuilder = createPlanBuilder(defaultSessionHolder); + TableScanNode tableScanNode = tableScan(planBuilder, pinotTable, city, fare); + AggregationNode agg = planBuilder.aggregation(aggBuilder -> aggBuilder.source(tableScanNode).singleGroupingSet(v("city")).addAggregation(planBuilder.variable("agg"), getRowExpression("sum(fare)", defaultSessionHolder))); + TopNNode topN = new TopNNode(planBuilder.getIdAllocator().getNextId(), agg, 50L, new OrderingScheme(ImmutableList.of(new Ordering(v("city"), SortOrder.DESC_NULLS_FIRST))), TopNNode.Step.FINAL); + testPQL(pinotConfig, topN, "", defaultSessionHolder, ImmutableMap.of()); + } +} diff --git a/presto-pinot/pom.xml b/presto-pinot/pom.xml new file mode 100644 index 0000000000000..ea0d5c28615fb --- /dev/null +++ b/presto-pinot/pom.xml @@ -0,0 +1,64 @@ + + + 4.0.0 + + com.facebook.presto + presto-root + 0.231-SNAPSHOT + + + presto-pinot + presto-pinot + Presto - Pinot Native Connector + presto-plugin + + + ${project.parent.basedir} + + + + + com.facebook.presto + presto-pinot-toolkit + ${project.version} + + + + com.google.guava + guava + + + + + com.facebook.presto + presto-spi + provided + + + + org.openjdk.jol + jol-core + provided + + + + io.airlift + slice + provided + + + + io.airlift + units + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + diff --git a/presto-pinot/src/main/java/com/facebook/presto/pinot/PinotPlugin.java b/presto-pinot/src/main/java/com/facebook/presto/pinot/PinotPlugin.java new file mode 100644 index 0000000000000..7f18c769a5398 --- /dev/null +++ b/presto-pinot/src/main/java/com/facebook/presto/pinot/PinotPlugin.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import com.facebook.presto.spi.Plugin; +import com.facebook.presto.spi.connector.ConnectorFactory; +import com.google.common.collect.ImmutableList; + +public class PinotPlugin + implements Plugin +{ + @Override + public Iterable getConnectorFactories() + { + return ImmutableList.of(new PinotConnectorFactory()); + } +} diff --git a/presto-pinot/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin b/presto-pinot/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin new file mode 100644 index 0000000000000..5d236ddb0c2a4 --- /dev/null +++ b/presto-pinot/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin @@ -0,0 +1,2 @@ +com.facebook.presto.pinot.PinotPlugin + diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index 0c0b8040b3a02..38a54bae6b5f5 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-plugin-toolkit @@ -23,12 +23,12 @@ - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift configuration @@ -53,7 +53,7 @@ - io.airlift + com.facebook.airlift log @@ -81,7 +81,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/JsonUtils.java b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/JsonUtils.java index 9ed100ac93bb8..974b67b6b3b83 100644 --- a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/JsonUtils.java +++ b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/JsonUtils.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.plugin.base; +import com.facebook.airlift.json.ObjectMapperProvider; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import io.airlift.json.ObjectMapperProvider; import java.io.IOException; import java.nio.file.Files; diff --git a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/FileBasedAccessControlConfig.java b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/FileBasedAccessControlConfig.java index b82923bd7f619..b65e4f1461b15 100644 --- a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/FileBasedAccessControlConfig.java +++ b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/FileBasedAccessControlConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.plugin.base.security; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import io.airlift.units.Duration; import io.airlift.units.MinDuration; diff --git a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/FileBasedAccessControlModule.java b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/FileBasedAccessControlModule.java index 2c771053de81c..fd2cfa930cbe9 100644 --- a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/FileBasedAccessControlModule.java +++ b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/FileBasedAccessControlModule.java @@ -13,15 +13,15 @@ */ package com.facebook.presto.plugin.base.security; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.connector.ConnectorAccessControl; import com.google.inject.Binder; import com.google.inject.Inject; import com.google.inject.Module; import com.google.inject.Provides; -import io.airlift.log.Logger; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static com.google.common.base.Suppliers.memoizeWithExpiration; -import static io.airlift.configuration.ConfigBinder.configBinder; import static java.util.concurrent.TimeUnit.MILLISECONDS; public class FileBasedAccessControlModule diff --git a/presto-plugin-toolkit/src/test/java/com/facebook/presto/plugin/base/security/TestFileBasedAccessControlConfig.java b/presto-plugin-toolkit/src/test/java/com/facebook/presto/plugin/base/security/TestFileBasedAccessControlConfig.java index 3ec7b97fcc7f6..63e956c89fa62 100644 --- a/presto-plugin-toolkit/src/test/java/com/facebook/presto/plugin/base/security/TestFileBasedAccessControlConfig.java +++ b/presto-plugin-toolkit/src/test/java/com/facebook/presto/plugin/base/security/TestFileBasedAccessControlConfig.java @@ -13,20 +13,20 @@ */ package com.facebook.presto.plugin.base.security; +import com.facebook.airlift.configuration.ConfigurationFactory; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; import com.google.inject.ConfigurationException; -import io.airlift.configuration.ConfigurationFactory; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.Duration; import org.testng.annotations.Test; import java.util.Map; import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static com.facebook.presto.plugin.base.security.FileBasedAccessControlConfig.SECURITY_CONFIG_FILE; import static com.facebook.presto.plugin.base.security.FileBasedAccessControlConfig.SECURITY_REFRESH_PERIOD; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestFileBasedAccessControlConfig diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 7231f6b704861..95459b79e47be 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-postgresql @@ -23,7 +23,7 @@ - io.airlift + com.facebook.airlift configuration @@ -86,13 +86,13 @@ - io.airlift + com.facebook.airlift testing test - io.airlift + com.facebook.airlift json test @@ -122,7 +122,7 @@ - io.airlift + com.facebook.airlift testing-postgresql-server test diff --git a/presto-postgresql/src/main/java/com/facebook/presto/plugin/postgresql/PostgreSqlClient.java b/presto-postgresql/src/main/java/com/facebook/presto/plugin/postgresql/PostgreSqlClient.java index 4bebdd79bf4c0..9fb3d7abd3360 100644 --- a/presto-postgresql/src/main/java/com/facebook/presto/plugin/postgresql/PostgreSqlClient.java +++ b/presto-postgresql/src/main/java/com/facebook/presto/plugin/postgresql/PostgreSqlClient.java @@ -17,6 +17,8 @@ import com.facebook.presto.plugin.jdbc.BaseJdbcConfig; import com.facebook.presto.plugin.jdbc.DriverConnectionFactory; import com.facebook.presto.plugin.jdbc.JdbcConnectorId; +import com.facebook.presto.plugin.jdbc.JdbcIdentity; +import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; @@ -30,6 +32,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Optional; import static com.facebook.presto.plugin.jdbc.JdbcErrorCode.JDBC_ERROR; import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS; @@ -58,15 +61,15 @@ public PreparedStatement getPreparedStatement(Connection connection, String sql) } @Override - protected ResultSet getTables(Connection connection, String schemaName, String tableName) + protected ResultSet getTables(Connection connection, Optional schemaName, Optional tableName) throws SQLException { DatabaseMetaData metadata = connection.getMetaData(); - String escape = metadata.getSearchStringEscape(); + Optional escape = Optional.ofNullable(metadata.getSearchStringEscape()); return metadata.getTables( connection.getCatalog(), - escapeNamePattern(schemaName, escape), - escapeNamePattern(tableName, escape), + escapeNamePattern(schemaName, escape).orElse(null), + escapeNamePattern(tableName, escape).orElse(null), new String[] {"TABLE", "VIEW", "MATERIALIZED VIEW", "FOREIGN TABLE"}); } @@ -81,10 +84,10 @@ protected String toSqlType(Type type) } @Override - public void createTable(ConnectorTableMetadata tableMetadata) + public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata) { try { - createTable(tableMetadata, null, tableMetadata.getTable().getTableName()); + createTable(tableMetadata, session, tableMetadata.getTable().getTableName()); } catch (SQLException e) { if (DUPLICATE_TABLE_SQLSTATE.equals(e.getSQLState())) { @@ -95,10 +98,10 @@ public void createTable(ConnectorTableMetadata tableMetadata) } @Override - protected void renameTable(String catalogName, SchemaTableName oldTable, SchemaTableName newTable) + protected void renameTable(JdbcIdentity identity, String catalogName, SchemaTableName oldTable, SchemaTableName newTable) { // PostgreSQL does not allow qualifying the target of a rename - try (Connection connection = connectionFactory.openConnection()) { + try (Connection connection = connectionFactory.openConnection(identity)) { String sql = format( "ALTER TABLE %s RENAME TO %s", quoted(catalogName, oldTable.getSchemaName(), oldTable.getTableName()), diff --git a/presto-postgresql/src/main/java/com/facebook/presto/plugin/postgresql/PostgreSqlClientModule.java b/presto-postgresql/src/main/java/com/facebook/presto/plugin/postgresql/PostgreSqlClientModule.java index 58dfb0e692730..8919ad74ff47c 100644 --- a/presto-postgresql/src/main/java/com/facebook/presto/plugin/postgresql/PostgreSqlClientModule.java +++ b/presto-postgresql/src/main/java/com/facebook/presto/plugin/postgresql/PostgreSqlClientModule.java @@ -19,7 +19,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; public class PostgreSqlClientModule implements Module diff --git a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/PostgreSqlQueryRunner.java b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/PostgreSqlQueryRunner.java index f9a8425e03004..efcb056efe005 100644 --- a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/PostgreSqlQueryRunner.java +++ b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/PostgreSqlQueryRunner.java @@ -13,25 +13,26 @@ */ package com.facebook.presto.plugin.postgresql; +import com.facebook.airlift.testing.postgresql.TestingPostgreSqlServer; import com.facebook.presto.Session; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.DistributedQueryRunner; import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.testing.postgresql.TestingPostgreSqlServer; import io.airlift.tpch.TpchTable; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.util.HashMap; import java.util.Map; +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tests.QueryAssertions.copyTpchTables; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static io.airlift.testing.Closeables.closeAllSuppress; public final class PostgreSqlQueryRunner { @@ -44,10 +45,10 @@ private PostgreSqlQueryRunner() public static QueryRunner createPostgreSqlQueryRunner(TestingPostgreSqlServer server, TpchTable... tables) throws Exception { - return createPostgreSqlQueryRunner(server, ImmutableList.copyOf(tables)); + return createPostgreSqlQueryRunner(server, ImmutableMap.of(), ImmutableList.copyOf(tables)); } - public static QueryRunner createPostgreSqlQueryRunner(TestingPostgreSqlServer server, Iterable> tables) + public static QueryRunner createPostgreSqlQueryRunner(TestingPostgreSqlServer server, Map connectorProperties, Iterable> tables) throws Exception { DistributedQueryRunner queryRunner = null; @@ -57,15 +58,14 @@ public static QueryRunner createPostgreSqlQueryRunner(TestingPostgreSqlServer se queryRunner.installPlugin(new TpchPlugin()); queryRunner.createCatalog("tpch", "tpch"); - Map properties = ImmutableMap.builder() - .put("connection-url", server.getJdbcUrl()) - .put("allow-drop-table", "true") - .build(); + connectorProperties = new HashMap<>(ImmutableMap.copyOf(connectorProperties)); + connectorProperties.putIfAbsent("connection-url", server.getJdbcUrl()); + connectorProperties.putIfAbsent("allow-drop-table", "true"); createSchema(server.getJdbcUrl(), "tpch"); queryRunner.installPlugin(new PostgreSqlPlugin()); - queryRunner.createCatalog("postgresql", "postgresql", properties); + queryRunner.createCatalog("postgresql", "postgresql", connectorProperties); copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), tables); diff --git a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlCaseInsensitiveMapping.java b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlCaseInsensitiveMapping.java new file mode 100644 index 0000000000000..ec606b3fd9976 --- /dev/null +++ b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlCaseInsensitiveMapping.java @@ -0,0 +1,190 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.postgresql; + +import com.facebook.airlift.testing.postgresql.TestingPostgreSqlServer; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.stream.Stream; + +import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static java.lang.String.format; +import static java.util.Locale.ENGLISH; +import static org.assertj.core.api.Assertions.assertThat; + +@Test(singleThreaded = true) +public class TestPostgreSqlCaseInsensitiveMapping + extends AbstractTestQueryFramework +{ + private final TestingPostgreSqlServer postgreSqlServer; + + public TestPostgreSqlCaseInsensitiveMapping() + throws Exception + { + this(new TestingPostgreSqlServer("testuser", "tpch")); + } + + public TestPostgreSqlCaseInsensitiveMapping(TestingPostgreSqlServer postgreSqlServer) + { + super(() -> PostgreSqlQueryRunner.createPostgreSqlQueryRunner( + postgreSqlServer, + ImmutableMap.of("case-insensitive-name-matching", "true"), + ImmutableSet.of())); + this.postgreSqlServer = postgreSqlServer; + } + + @AfterClass(alwaysRun = true) + public final void destroy() + throws IOException + { + postgreSqlServer.close(); + } + + @Test + public void testNonLowerCaseSchemaName() + throws Exception + { + try (AutoCloseable ignore1 = withSchema("\"NonLowerCaseSchema\""); + AutoCloseable ignore2 = withTable("\"NonLowerCaseSchema\".lower_case_name", "(c varchar(5))"); + AutoCloseable ignore3 = withTable("\"NonLowerCaseSchema\".\"Mixed_Case_Name\"", "(c varchar(5))"); + AutoCloseable ignore4 = withTable("\"NonLowerCaseSchema\".\"UPPER_CASE_NAME\"", "(c varchar(5))")) { + assertThat(computeActual("SHOW SCHEMAS").getOnlyColumn()).contains("nonlowercaseschema"); + assertQuery("SHOW SCHEMAS LIKE 'nonlowerc%'", "VALUES 'nonlowercaseschema'"); + assertQuery("SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE '%nonlowercaseschema'", "VALUES 'nonlowercaseschema'"); + assertQuery("SHOW TABLES FROM nonlowercaseschema", "VALUES 'lower_case_name', 'mixed_case_name', 'upper_case_name'"); + assertQuery("SELECT table_name FROM information_schema.tables WHERE table_schema = 'nonlowercaseschema'", "VALUES 'lower_case_name', 'mixed_case_name', 'upper_case_name'"); + assertQueryReturnsEmptyResult("SELECT * FROM nonlowercaseschema.lower_case_name"); + } + } + + @Test + public void testNonLowerCaseTableName() + throws Exception + { + try (AutoCloseable ignore1 = withSchema("\"SomeSchema\""); + AutoCloseable ignore2 = withTable( + "\"SomeSchema\".\"NonLowerCaseTable\"", "AS SELECT * FROM (VALUES ('a', 'b', 'c')) t(lower_case_name, \"Mixed_Case_Name\", \"UPPER_CASE_NAME\")")) { + assertQuery( + "SELECT column_name FROM information_schema.columns WHERE table_schema = 'someschema' AND table_name = 'nonlowercasetable'", + "VALUES 'lower_case_name', 'mixed_case_name', 'upper_case_name'"); + assertQuery( + "SELECT column_name FROM information_schema.columns WHERE table_name = 'nonlowercasetable'", + "VALUES 'lower_case_name', 'mixed_case_name', 'upper_case_name'"); + assertEquals( + computeActual("SHOW COLUMNS FROM someschema.nonlowercasetable").getMaterializedRows().stream() + .map(row -> row.getField(0)) + .collect(toImmutableSet()), + ImmutableSet.of("lower_case_name", "mixed_case_name", "upper_case_name")); + + // Note: until https://github.com/prestodb/presto/issues/2863 is resolved, this is *the* way to access the tables. + + assertQuery("SELECT lower_case_name FROM someschema.nonlowercasetable", "VALUES 'a'"); + assertQuery("SELECT mixed_case_name FROM someschema.nonlowercasetable", "VALUES 'b'"); + assertQuery("SELECT upper_case_name FROM someschema.nonlowercasetable", "VALUES 'c'"); + assertQuery("SELECT upper_case_name FROM SomeSchema.NonLowerCaseTable", "VALUES 'c'"); + assertQuery("SELECT upper_case_name FROM \"SomeSchema\".\"NonLowerCaseTable\"", "VALUES 'c'"); + + assertUpdate("INSERT INTO someschema.nonlowercasetable (lower_case_name) VALUES ('lower')", 1); + assertUpdate("INSERT INTO someschema.nonlowercasetable (mixed_case_name) VALUES ('mixed')", 1); + assertUpdate("INSERT INTO someschema.nonlowercasetable (upper_case_name) VALUES ('upper')", 1); + assertQuery( + "SELECT * FROM someschema.nonlowercasetable", + "VALUES ('a', 'b', 'c')," + + "('lower', NULL, NULL)," + + "(NULL, 'mixed', NULL)," + + "(NULL, NULL, 'upper')"); + } + } + + @Test + public void testSchemaNameClash() + throws Exception + { + String[] nameVariants = {"casesensitivename", "\"CaseSensitiveName\"", "\"CASESENSITIVENAME\""}; + assertThat(Stream.of(nameVariants) + .map(name -> name.replace("\"", "").toLowerCase(ENGLISH)) + .collect(toImmutableSet())) + .hasSize(1); + + for (int i = 0; i < nameVariants.length; i++) { + for (int j = i + 1; j < nameVariants.length; j++) { + String schemaName = nameVariants[i]; + String otherSchemaName = nameVariants[j]; + try (AutoCloseable ignore1 = withSchema(schemaName); + AutoCloseable ignore2 = withSchema(otherSchemaName); + AutoCloseable ignore3 = withTable(schemaName + ".some_table_name", "(c varchar(5))")) { + assertThat(computeActual("SHOW SCHEMAS").getOnlyColumn()).contains("casesensitivename"); + assertThat(computeActual("SHOW SCHEMAS").getOnlyColumn().filter("casesensitivename"::equals)).hasSize(1); // TODO change io.prestosql.plugin.jdbc.JdbcClient.getSchemaNames to return a List + assertQueryFails("SHOW TABLES FROM casesensitivename", "Failed to find remote schema name:.*Multiple entries with same key.*"); + assertQueryFails("SELECT * FROM casesensitivename.some_table_name", "Failed to find remote schema name:.*Multiple entries with same key.*"); + } + } + } + } + + @Test + public void testTableNameClash() + throws Exception + { + String[] nameVariants = {"casesensitivename", "\"CaseSensitiveName\"", "\"CASESENSITIVENAME\""}; + assertThat(Stream.of(nameVariants) + .map(name -> name.replace("\"", "").toLowerCase(ENGLISH)) + .collect(toImmutableSet())) + .hasSize(1); + + for (int i = 0; i < nameVariants.length; i++) { + for (int j = i + 1; j < nameVariants.length; j++) { + try (AutoCloseable ignore1 = withTable("tpch." + nameVariants[i], "(c varchar(5))"); + AutoCloseable ignore2 = withTable("tpch." + nameVariants[j], "(d varchar(5))")) { + assertThat(computeActual("SHOW TABLES").getOnlyColumn()).contains("casesensitivename"); + assertThat(computeActual("SHOW TABLES").getOnlyColumn().filter("casesensitivename"::equals)).hasSize(1); // TODO, should be 2 + assertQueryFails("SHOW COLUMNS FROM casesensitivename", "Failed to find remote table name:.*Multiple entries with same key.*"); + assertQueryFails("SELECT * FROM casesensitivename", "Failed to find remote table name:.*Multiple entries with same key.*"); + } + } + } + } + + private AutoCloseable withSchema(String schemaName) + { + execute("CREATE SCHEMA " + schemaName); + return () -> execute("DROP SCHEMA " + schemaName); + } + + private AutoCloseable withTable(String tableName, String tableDefinition) + { + execute(format("CREATE TABLE %s %s", tableName, tableDefinition)); + return () -> execute(format("DROP TABLE %s", tableName)); + } + + private void execute(String sql) + { + try (Connection connection = DriverManager.getConnection(postgreSqlServer.getJdbcUrl()); + Statement statement = connection.createStatement()) { + statement.execute(sql); + } + catch (Exception e) { + throw new RuntimeException("Failed to execute statement: " + sql, e); + } + } +} diff --git a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlDistributedQueries.java b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlDistributedQueries.java index 30a3175b4d803..65e013d15cd9c 100644 --- a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlDistributedQueries.java +++ b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlDistributedQueries.java @@ -13,8 +13,9 @@ */ package com.facebook.presto.plugin.postgresql; +import com.facebook.airlift.testing.postgresql.TestingPostgreSqlServer; import com.facebook.presto.tests.AbstractTestDistributedQueries; -import io.airlift.testing.postgresql.TestingPostgreSqlServer; +import com.google.common.collect.ImmutableMap; import io.airlift.tpch.TpchTable; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -37,7 +38,7 @@ public TestPostgreSqlDistributedQueries() public TestPostgreSqlDistributedQueries(TestingPostgreSqlServer postgreSqlServer) { - super(() -> createPostgreSqlQueryRunner(postgreSqlServer, TpchTable.getTables())); + super(() -> createPostgreSqlQueryRunner(postgreSqlServer, ImmutableMap.of(), TpchTable.getTables())); this.postgreSqlServer = postgreSqlServer; } diff --git a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlIntegrationSmokeTest.java b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlIntegrationSmokeTest.java index 504dc225b5887..91bf469dc68c5 100644 --- a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlIntegrationSmokeTest.java +++ b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlIntegrationSmokeTest.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.plugin.postgresql; +import com.facebook.airlift.testing.postgresql.TestingPostgreSqlServer; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestIntegrationSmokeTest; import com.facebook.presto.tests.DistributedQueryRunner; import com.google.common.collect.ImmutableMap; -import io.airlift.testing.postgresql.TestingPostgreSqlServer; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; diff --git a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlTypeMapping.java b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlTypeMapping.java index 5fe0f83743ae9..3b3ddee730658 100644 --- a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlTypeMapping.java +++ b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlTypeMapping.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.plugin.postgresql; +import com.facebook.airlift.testing.postgresql.TestingPostgreSqlServer; import com.facebook.presto.Session; import com.facebook.presto.spi.type.TimeZoneKey; import com.facebook.presto.tests.AbstractTestQueryFramework; @@ -24,7 +25,7 @@ import com.facebook.presto.tests.sql.JdbcSqlExecutor; import com.facebook.presto.tests.sql.PrestoSqlExecutor; import com.google.common.collect.ImmutableList; -import io.airlift.testing.postgresql.TestingPostgreSqlServer; +import com.google.common.collect.ImmutableMap; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -53,7 +54,6 @@ import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_16LE; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Collections.emptyList; @Test public class TestPostgreSqlTypeMapping @@ -69,7 +69,7 @@ public TestPostgreSqlTypeMapping() private TestPostgreSqlTypeMapping(TestingPostgreSqlServer postgreSqlServer) { - super(() -> createPostgreSqlQueryRunner(postgreSqlServer, emptyList())); + super(() -> createPostgreSqlQueryRunner(postgreSqlServer, ImmutableMap.of(), ImmutableList.of())); this.postgreSqlServer = postgreSqlServer; } diff --git a/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh b/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh index eebc9d0ac1b6d..b804cba484611 100755 --- a/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh +++ b/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh @@ -4,16 +4,16 @@ set -euo pipefail CONFIG="$1" +shift 1 + PRESTO_CONFIG_DIRECTORY="/docker/volumes/conf/presto/etc" CONFIG_PROPERTIES_LOCATION="${PRESTO_CONFIG_DIRECTORY}/${CONFIG}.properties" -if [[ ! -e ${CONFIG_PROPERTIES_LOCATION} ]]; then - echo "${CONFIG_PROPERTIES_LOCATION} does not exist" +if [[ ! -f "${CONFIG_PROPERTIES_LOCATION}" ]]; then + echo "${CONFIG_PROPERTIES_LOCATION} does not exist" >&2 exit 1 fi -shift 1 - /docker/volumes/presto-server/bin/launcher \ -Dnode.id="${HOSTNAME}" \ --etc-dir="${PRESTO_CONFIG_DIRECTORY}" \ diff --git a/presto-product-tests/conf/docker/multinode/docker-compose.yml b/presto-product-tests/conf/docker/multinode/docker-compose.yml index 30f497c74a6bf..172e36f75c1e4 100644 --- a/presto-product-tests/conf/docker/multinode/docker-compose.yml +++ b/presto-product-tests/conf/docker/multinode/docker-compose.yml @@ -3,6 +3,8 @@ services: presto-master: command: /docker/volumes/conf/docker/files/presto-launcher-wrapper.sh multinode-master run + volumes: + - ../../../conf/presto/etc/multinode-master-jvm.config:/docker/volumes/conf/presto/etc/jvm.config presto-worker: extends: @@ -13,3 +15,5 @@ services: - presto-master volumes_from: - presto-master + volumes: + - ../../../conf/presto/etc/multinode-worker-jvm.config:/docker/volumes/conf/presto/etc/jvm.config diff --git a/presto-product-tests/conf/presto/etc/jvm.config b/presto-product-tests/conf/presto/etc/jvm.config index 8b8a99a119e8f..9a1ebfe6669e2 100644 --- a/presto-product-tests/conf/presto/etc/jvm.config +++ b/presto-product-tests/conf/presto/etc/jvm.config @@ -13,6 +13,8 @@ -XX:+HeapDumpOnOutOfMemoryError -XX:+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=150M +# jdk.nio.maxCachedBufferSize controls what buffers can be allocated in per-thread "temporary buffer cache" (sun.nio.ch.Util). Value of 0 disables the cache. +-Djdk.nio.maxCachedBufferSize=0 -DHADOOP_USER_NAME=hive -Duser.timezone=Asia/Kathmandu #-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 diff --git a/presto-product-tests/conf/presto/etc/log.properties b/presto-product-tests/conf/presto/etc/log.properties index e2d6f6bc084cc..b4f37b590dbff 100644 --- a/presto-product-tests/conf/presto/etc/log.properties +++ b/presto-product-tests/conf/presto/etc/log.properties @@ -9,4 +9,4 @@ com.facebook.presto=INFO com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory=WARN com.ning.http.client=DEBUG com.facebook.presto.server.PluginManager=INFO -io.airlift.discovery.client=INFO +com.facebook.airlift.discovery.client=INFO diff --git a/presto-product-tests/conf/presto/etc/multinode-master-jvm.config b/presto-product-tests/conf/presto/etc/multinode-master-jvm.config new file mode 100644 index 0000000000000..c02ab0e4f0f77 --- /dev/null +++ b/presto-product-tests/conf/presto/etc/multinode-master-jvm.config @@ -0,0 +1,23 @@ +# +# WARNING +# ^^^^^^^ +# This configuration file is for development only and should NOT be used be +# used in production. For example configuration, see the Presto documentation. +# + +-server +# coordinator is not a worker +-Xmx1G +-XX:-UseBiasedLocking +# Disable G1 to enable UseGCOverheadLimit (JDK-8212084) and reduce JVM's memory footprint +-XX:+UseParallelGC +-XX:+ExitOnOutOfMemoryError +-XX:+UseGCOverheadLimit +-XX:+HeapDumpOnOutOfMemoryError +-XX:ReservedCodeCacheSize=150M +# jdk.nio.maxCachedBufferSize controls what buffers can be allocated in per-thread "temporary buffer cache" (sun.nio.ch.Util). Value of 0 disables the cache. +-Djdk.nio.maxCachedBufferSize=0 +-DHADOOP_USER_NAME=hive +-Duser.timezone=Asia/Kathmandu +#-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 +-XX:ErrorFile=/docker/volumes/logs/product-tests-presto-jvm-error-file.log diff --git a/presto-product-tests/conf/presto/etc/multinode-master.properties b/presto-product-tests/conf/presto/etc/multinode-master.properties index df0a4325c3d1f..051bccaddc9ce 100644 --- a/presto-product-tests/conf/presto/etc/multinode-master.properties +++ b/presto-product-tests/conf/presto/etc/multinode-master.properties @@ -12,7 +12,5 @@ coordinator=true node-scheduler.include-coordinator=false http-server.http.port=8080 query.max-memory=1GB -query.max-memory-per-node=512MB -query.max-total-memory-per-node=1GB discovery-server.enabled=true discovery.uri=http://presto-master:8080 diff --git a/presto-product-tests/conf/presto/etc/multinode-worker-jvm.config b/presto-product-tests/conf/presto/etc/multinode-worker-jvm.config new file mode 100644 index 0000000000000..72cef8c38638a --- /dev/null +++ b/presto-product-tests/conf/presto/etc/multinode-worker-jvm.config @@ -0,0 +1,23 @@ +# +# WARNING +# ^^^^^^^ +# This configuration file is for development only and should NOT be used +# in production. For example configuration, see the Presto documentation. +# + +-server +-Xmx2G +-XX:-UseBiasedLocking +# Disable G1 to enable UseGCOverheadLimit (JDK-8212084) and reduce JVM's memory footprint +-XX:-UseG1GC +-XX:+ExitOnOutOfMemoryError +-XX:+UseGCOverheadLimit +-XX:+HeapDumpOnOutOfMemoryError +-XX:ReservedCodeCacheSize=150M +-Djdk.attach.allowAttachSelf=true +# jdk.nio.maxCachedBufferSize controls what buffers can be allocated in per-thread "temporary buffer cache" (sun.nio.ch.Util). Value of 0 disables the cache. +-Djdk.nio.maxCachedBufferSize=0 +-DHADOOP_USER_NAME=hive +-Duser.timezone=Asia/Kathmandu +#-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 +-XX:ErrorFile=/docker/volumes/logs/product-tests-presto-jvm-error-file.log diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 1ad43d87cb227..6b050c50dfb17 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-product-tests @@ -81,7 +81,7 @@ testng - io.airlift + com.facebook.airlift testing @@ -89,7 +89,7 @@ javax.inject - io.airlift + com.facebook.airlift log @@ -97,7 +97,7 @@ units - io.airlift + com.facebook.airlift json @@ -105,7 +105,7 @@ jackson-databind - io.airlift + com.facebook.airlift http-client diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/AlterTableTests.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/AlterTableTests.java index b30bfafd5ab68..d14e4465258ed 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/AlterTableTests.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/AlterTableTests.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.tests; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import io.prestodb.tempto.AfterTestWithContext; import io.prestodb.tempto.BeforeTestWithContext; import io.prestodb.tempto.ProductTest; diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/SqlCancelTests.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/SqlCancelTests.java index e55eb33e164cc..6035bb814465c 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/SqlCancelTests.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/SqlCancelTests.java @@ -13,17 +13,17 @@ */ package com.facebook.presto.tests; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpClientConfig; +import com.facebook.airlift.http.client.HttpStatus; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.ResponseHandler; +import com.facebook.airlift.http.client.jetty.JettyHttpClient; import com.google.common.base.Stopwatch; import com.google.common.io.Closer; import com.google.inject.Inject; import com.google.inject.name.Named; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpClientConfig; -import io.airlift.http.client.HttpStatus; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.ResponseHandler; -import io.airlift.http.client.jetty.JettyHttpClient; import io.prestodb.tempto.AfterTestWithContext; import io.prestodb.tempto.BeforeTestWithContext; import io.prestodb.tempto.ProductTest; @@ -39,11 +39,11 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.Request.Builder.prepareDelete; +import static com.facebook.airlift.http.client.ResponseHandlerUtils.propagate; import static com.facebook.presto.tests.TestGroups.CANCEL_QUERY; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.Request.Builder.prepareDelete; -import static io.airlift.http.client.ResponseHandlerUtils.propagate; import static io.prestodb.tempto.assertions.QueryAssert.assertThat; import static io.prestodb.tempto.query.QueryExecutor.query; import static java.lang.String.format; diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/cli/PrestoCliTests.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/cli/PrestoCliTests.java index 7be4f4c9b303a..2d3514fc894b4 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/cli/PrestoCliTests.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/cli/PrestoCliTests.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.tests.cli; +import com.facebook.airlift.testing.TempFile; import com.facebook.presto.cli.Presto; import com.google.common.collect.ImmutableList; import com.google.common.io.Files; import com.google.inject.Inject; import com.google.inject.name.Named; -import io.airlift.testing.TempFile; import io.prestodb.tempto.AfterTestWithContext; import io.prestodb.tempto.Requirement; import io.prestodb.tempto.RequirementsProvider; diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestGrantRevoke.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestGrantRevoke.java index 839d84cd3a4f8..e88c987efd7e1 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestGrantRevoke.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestGrantRevoke.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.tests.hive; +import com.facebook.airlift.log.Logger; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; import io.prestodb.tempto.AfterTestWithContext; import io.prestodb.tempto.BeforeTestWithContext; import io.prestodb.tempto.ProductTest; diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveCoercion.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveCoercion.java index e595b242b23f4..ae7bb329fa4ba 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveCoercion.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveCoercion.java @@ -39,13 +39,13 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static com.facebook.presto.tests.TestGroups.HIVE_COERCION; import static com.facebook.presto.tests.TestGroups.JDBC; import static com.facebook.presto.tests.utils.JdbcDriverUtils.usingPrestoJdbcDriver; import static com.facebook.presto.tests.utils.JdbcDriverUtils.usingTeradataJdbcDriver; import static com.facebook.presto.tests.utils.QueryExecutors.onHive; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static io.prestodb.tempto.assertions.QueryAssert.Row.row; import static io.prestodb.tempto.assertions.QueryAssert.assertThat; import static io.prestodb.tempto.context.ThreadLocalTestContextHolder.testContext; diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/jdbc/JdbcTests.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/jdbc/JdbcTests.java index 6e2286bbc86cb..5e165c3ecef13 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/jdbc/JdbcTests.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/jdbc/JdbcTests.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.tests.jdbc; +import com.facebook.airlift.log.Logger; import com.facebook.presto.jdbc.PrestoConnection; import com.facebook.presto.sql.analyzer.FeaturesConfig; -import io.airlift.log.Logger; import io.prestodb.tempto.ProductTest; import io.prestodb.tempto.Requirement; import io.prestodb.tempto.RequirementsProvider; diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/jdbc/PreparedStatements.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/jdbc/PreparedStatements.java index 22e656b2a35c6..6bb51fc182229 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/jdbc/PreparedStatements.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/jdbc/PreparedStatements.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.tests.jdbc; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import io.prestodb.tempto.ProductTest; import io.prestodb.tempto.Requirement; import io.prestodb.tempto.RequirementsProvider; diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/mysql/CreateTableAsSelect.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/mysql/CreateTableAsSelect.java index 39b7ea3c5750b..3bd14598f8e50 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/mysql/CreateTableAsSelect.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/mysql/CreateTableAsSelect.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.tests.mysql; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import io.prestodb.tempto.AfterTestWithContext; import io.prestodb.tempto.BeforeTestWithContext; import io.prestodb.tempto.ProductTest; diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/querystats/HttpQueryStatsClient.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/querystats/HttpQueryStatsClient.java index 6bc471f245cee..e03c8fa42ae2f 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/querystats/HttpQueryStatsClient.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/querystats/HttpQueryStatsClient.java @@ -13,23 +13,23 @@ */ package com.facebook.presto.tests.querystats; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpStatus; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.ResponseHandler; import com.facebook.presto.execution.QueryStats; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpStatus; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.ResponseHandler; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; import java.util.Optional; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.http.client.ResponseHandlerUtils.propagate; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.airlift.http.client.ResponseHandlerUtils.propagate; /** * Implementation of {@link QueryStatsClient} using Presto's /v1/query HTTP API. diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/querystats/QueryStatsClientModuleProvider.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/querystats/QueryStatsClientModuleProvider.java index 2b1d2f0e6e04d..a6c01afec2187 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/querystats/QueryStatsClientModuleProvider.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/querystats/QueryStatsClientModuleProvider.java @@ -13,15 +13,15 @@ */ package com.facebook.presto.tests.querystats; +import com.facebook.airlift.http.client.HttpClientConfig; +import com.facebook.airlift.http.client.jetty.JettyHttpClient; +import com.facebook.airlift.json.ObjectMapperProvider; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Exposed; import com.google.inject.Inject; import com.google.inject.Module; import com.google.inject.PrivateModule; import com.google.inject.Provides; -import io.airlift.http.client.HttpClientConfig; -import io.airlift.http.client.jetty.JettyHttpClient; -import io.airlift.json.ObjectMapperProvider; import io.prestodb.tempto.configuration.Configuration; import io.prestodb.tempto.initialization.AutoModuleProvider; import io.prestodb.tempto.initialization.SuiteModuleProvider; diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/sqlserver/TestInsert.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/sqlserver/TestInsert.java index 302a51d9d4773..e6cc22f32dd99 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/sqlserver/TestInsert.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/sqlserver/TestInsert.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.tests.sqlserver; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import io.prestodb.tempto.AfterTestWithContext; import io.prestodb.tempto.BeforeTestWithContext; import io.prestodb.tempto.ProductTest; diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/sqlserver/TestSelect.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/sqlserver/TestSelect.java index b69a8ed01d1b7..9c6391402ee87 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/sqlserver/TestSelect.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/sqlserver/TestSelect.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.tests.sqlserver; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import io.prestodb.tempto.AfterTestWithContext; import io.prestodb.tempto.BeforeTestWithContext; import io.prestodb.tempto.ProductTest; diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/utils/JdbcDriverUtils.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/utils/JdbcDriverUtils.java index 722569659e433..5a23163d17f24 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/utils/JdbcDriverUtils.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/utils/JdbcDriverUtils.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.tests.utils; +import com.facebook.airlift.log.Logger; import com.facebook.presto.jdbc.PrestoConnection; -import io.airlift.log.Logger; import java.math.BigDecimal; import java.sql.Connection; diff --git a/presto-product-tests/src/main/resources/sql-tests/testcases/system/selectInformationSchemaColumns.result b/presto-product-tests/src/main/resources/sql-tests/testcases/system/selectInformationSchemaColumns.result index 576f7e11452bc..86ddcd16cb196 100644 --- a/presto-product-tests/src/main/resources/sql-tests/testcases/system/selectInformationSchemaColumns.result +++ b/presto-product-tests/src/main/resources/sql-tests/testcases/system/selectInformationSchemaColumns.result @@ -77,6 +77,7 @@ system| runtime| queries| last_heartbeat| timestamp| YES| null| null| system| runtime| queries| end| timestamp| YES| null| null| system| runtime| tasks| node_id| varchar| YES| null| null| system| runtime| tasks| task_id| varchar| YES| null| null| +system| runtime| tasks| stage_execution_id| varchar| YES| null| null| system| runtime| tasks| stage_id| varchar| YES| null| null| system| runtime| tasks| query_id| varchar| YES| null| null| system| runtime| tasks| state| varchar| YES| null| null| diff --git a/presto-product-tests/src/test/java/com/facebook/presto/tests/querystats/TestHttpQueryStatsClient.java b/presto-product-tests/src/test/java/com/facebook/presto/tests/querystats/TestHttpQueryStatsClient.java index 548b599b871de..2702ff31793a7 100644 --- a/presto-product-tests/src/test/java/com/facebook/presto/tests/querystats/TestHttpQueryStatsClient.java +++ b/presto-product-tests/src/test/java/com/facebook/presto/tests/querystats/TestHttpQueryStatsClient.java @@ -13,15 +13,15 @@ */ package com.facebook.presto.tests.querystats; +import com.facebook.airlift.http.client.HttpStatus; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.testing.TestingHttpClient; +import com.facebook.airlift.http.client.testing.TestingResponse; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.execution.QueryStats; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableListMultimap; import com.google.common.io.Resources; -import io.airlift.http.client.HttpStatus; -import io.airlift.http.client.Response; -import io.airlift.http.client.testing.TestingHttpClient; -import io.airlift.http.client.testing.TestingResponse; -import io.airlift.json.ObjectMapperProvider; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; diff --git a/presto-proxy/pom.xml b/presto-proxy/pom.xml index 2d7998475516a..35723178b5364 100644 --- a/presto-proxy/pom.xml +++ b/presto-proxy/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-proxy @@ -18,27 +18,27 @@ - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift node - io.airlift + com.facebook.airlift jmx - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift bootstrap @@ -48,47 +48,47 @@ - io.airlift + com.facebook.airlift configuration - io.airlift + com.facebook.airlift http-client - io.airlift + com.facebook.airlift http-server - io.airlift + com.facebook.airlift jaxrs - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift log-manager - io.airlift + com.facebook.airlift event - io.airlift + com.facebook.airlift security - io.airlift + com.facebook.airlift trace-token diff --git a/presto-proxy/src/main/java/com/facebook/presto/proxy/JsonWebTokenHandler.java b/presto-proxy/src/main/java/com/facebook/presto/proxy/JsonWebTokenHandler.java index d78058ddff046..dd1b7a4333cfe 100644 --- a/presto-proxy/src/main/java/com/facebook/presto/proxy/JsonWebTokenHandler.java +++ b/presto-proxy/src/main/java/com/facebook/presto/proxy/JsonWebTokenHandler.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.proxy; -import io.airlift.security.pem.PemReader; +import com.facebook.airlift.security.pem.PemReader; import io.jsonwebtoken.JwtBuilder; import io.jsonwebtoken.Jwts; import io.jsonwebtoken.SignatureAlgorithm; diff --git a/presto-proxy/src/main/java/com/facebook/presto/proxy/JwtHandlerConfig.java b/presto-proxy/src/main/java/com/facebook/presto/proxy/JwtHandlerConfig.java index af1f1cef3b9a4..807e9f993643c 100644 --- a/presto-proxy/src/main/java/com/facebook/presto/proxy/JwtHandlerConfig.java +++ b/presto-proxy/src/main/java/com/facebook/presto/proxy/JwtHandlerConfig.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.proxy; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.ConfigSecuritySensitive; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.ConfigSecuritySensitive; import java.io.File; diff --git a/presto-proxy/src/main/java/com/facebook/presto/proxy/PrestoProxy.java b/presto-proxy/src/main/java/com/facebook/presto/proxy/PrestoProxy.java index 223ecb5e8b8ce..95a542d7b75e2 100644 --- a/presto-proxy/src/main/java/com/facebook/presto/proxy/PrestoProxy.java +++ b/presto-proxy/src/main/java/com/facebook/presto/proxy/PrestoProxy.java @@ -13,18 +13,18 @@ */ package com.facebook.presto.proxy; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.event.client.EventModule; +import com.facebook.airlift.http.server.HttpServerModule; +import com.facebook.airlift.jaxrs.JaxrsModule; +import com.facebook.airlift.jmx.JmxModule; +import com.facebook.airlift.json.JsonModule; +import com.facebook.airlift.log.LogJmxModule; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.node.NodeModule; +import com.facebook.airlift.tracetoken.TraceTokenModule; import com.google.common.collect.ImmutableList; import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.event.client.EventModule; -import io.airlift.http.server.HttpServerModule; -import io.airlift.jaxrs.JaxrsModule; -import io.airlift.jmx.JmxModule; -import io.airlift.json.JsonModule; -import io.airlift.log.LogJmxModule; -import io.airlift.log.Logger; -import io.airlift.node.NodeModule; -import io.airlift.tracetoken.TraceTokenModule; import org.weakref.jmx.guice.MBeanModule; public final class PrestoProxy diff --git a/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyConfig.java b/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyConfig.java index 2cc67c8b9158f..1b1a48a0e6491 100644 --- a/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyConfig.java +++ b/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.proxy; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import javax.validation.constraints.NotNull; diff --git a/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyModule.java b/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyModule.java index 2b983c0278b2c..c8baa9f01a453 100644 --- a/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyModule.java +++ b/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyModule.java @@ -17,9 +17,9 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.http.client.HttpClientBinder.httpClientBinder; -import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; +import static com.facebook.airlift.jaxrs.JaxrsBinder.jaxrsBinder; public class ProxyModule implements Module diff --git a/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyResource.java b/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyResource.java index 5cc7d89a3e015..46f1eb1228091 100644 --- a/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyResource.java +++ b/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyResource.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.proxy; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.log.Logger; import com.facebook.presto.proxy.ProxyResponseHandler.ProxyResponse; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; @@ -22,9 +25,6 @@ import com.google.common.hash.HashFunction; import com.google.common.util.concurrent.FluentFuture; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.Request; -import io.airlift.log.Logger; import io.airlift.units.Duration; import javax.annotation.PreDestroy; @@ -54,6 +54,13 @@ import java.util.concurrent.ExecutorService; import java.util.function.Function; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.Request.Builder.prepareDelete; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.airlift.http.client.Request.Builder.preparePost; +import static com.facebook.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; +import static com.facebook.airlift.http.server.AsyncResponseHandler.bindAsyncResponse; import static com.fasterxml.jackson.core.JsonFactory.Feature.CANONICALIZE_FIELD_NAMES; import static com.fasterxml.jackson.core.JsonToken.END_OBJECT; import static com.fasterxml.jackson.core.JsonToken.FIELD_NAME; @@ -64,14 +71,8 @@ import static com.google.common.net.HttpHeaders.COOKIE; import static com.google.common.net.HttpHeaders.SET_COOKIE; import static com.google.common.net.HttpHeaders.USER_AGENT; +import static com.google.common.net.HttpHeaders.X_FORWARDED_FOR; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.Request.Builder.prepareDelete; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.http.client.Request.Builder.preparePost; -import static io.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; -import static io.airlift.http.server.AsyncResponseHandler.bindAsyncResponse; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.file.Files.readAllBytes; @@ -189,6 +190,7 @@ private void performRequest( Request.Builder requestBuilder, Function responseBuilder) { + setupXForwardedFor(servletRequest, requestBuilder); setupBearerToken(servletRequest, requestBuilder); for (String name : list(servletRequest.getHeaderNames())) { @@ -262,6 +264,16 @@ private void setupBearerToken(HttpServletRequest servletRequest, Request.Builder requestBuilder.addHeader(AUTHORIZATION, "Bearer " + accessToken); } + private void setupXForwardedFor(HttpServletRequest servletRequest, Request.Builder requestBuilder) + { + StringBuilder xForwardedFor = new StringBuilder(); + if (servletRequest.getHeader(X_FORWARDED_FOR) != null) { + xForwardedFor.append(servletRequest.getHeader(X_FORWARDED_FOR) + ","); + } + xForwardedFor.append(servletRequest.getRemoteAddr()); + requestBuilder.addHeader(X_FORWARDED_FOR, xForwardedFor.toString()); + } + private static T handleProxyException(Request request, ProxyException e) { log.warn(e, "Proxy request failed: %s %s", request.getMethod(), request.getUri()); diff --git a/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyResponseHandler.java b/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyResponseHandler.java index 60c2015d477ae..e7a9309e61167 100644 --- a/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyResponseHandler.java +++ b/presto-proxy/src/main/java/com/facebook/presto/proxy/ProxyResponseHandler.java @@ -13,20 +13,20 @@ */ package com.facebook.presto.proxy; +import com.facebook.airlift.http.client.HeaderName; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.ResponseHandler; import com.facebook.presto.proxy.ProxyResponseHandler.ProxyResponse; import com.google.common.collect.ListMultimap; import com.google.common.net.MediaType; -import io.airlift.http.client.HeaderName; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.ResponseHandler; import java.io.IOException; +import static com.facebook.airlift.http.client.HttpStatus.NO_CONTENT; +import static com.facebook.airlift.http.client.HttpStatus.OK; import static com.google.common.io.ByteStreams.toByteArray; import static com.google.common.net.HttpHeaders.CONTENT_TYPE; -import static io.airlift.http.client.HttpStatus.NO_CONTENT; -import static io.airlift.http.client.HttpStatus.OK; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.US_ASCII; import static java.util.Objects.requireNonNull; diff --git a/presto-proxy/src/test/java/com/facebook/presto/proxy/TestJwtHandlerConfig.java b/presto-proxy/src/test/java/com/facebook/presto/proxy/TestJwtHandlerConfig.java index f3c950fa6ef11..a66caba0e61a2 100644 --- a/presto-proxy/src/test/java/com/facebook/presto/proxy/TestJwtHandlerConfig.java +++ b/presto-proxy/src/test/java/com/facebook/presto/proxy/TestJwtHandlerConfig.java @@ -19,9 +19,9 @@ import java.io.File; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestJwtHandlerConfig { diff --git a/presto-proxy/src/test/java/com/facebook/presto/proxy/TestProxyConfig.java b/presto-proxy/src/test/java/com/facebook/presto/proxy/TestProxyConfig.java index f606895ad22e5..6aac2061211c0 100644 --- a/presto-proxy/src/test/java/com/facebook/presto/proxy/TestProxyConfig.java +++ b/presto-proxy/src/test/java/com/facebook/presto/proxy/TestProxyConfig.java @@ -20,9 +20,9 @@ import java.net.URI; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestProxyConfig { diff --git a/presto-proxy/src/test/java/com/facebook/presto/proxy/TestProxyServer.java b/presto-proxy/src/test/java/com/facebook/presto/proxy/TestProxyServer.java index c27b75ae7244a..f17bcaa43b7a6 100644 --- a/presto-proxy/src/test/java/com/facebook/presto/proxy/TestProxyServer.java +++ b/presto-proxy/src/test/java/com/facebook/presto/proxy/TestProxyServer.java @@ -13,6 +13,15 @@ */ package com.facebook.presto.proxy; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.http.server.HttpServerInfo; +import com.facebook.airlift.http.server.testing.TestingHttpServerModule; +import com.facebook.airlift.jaxrs.JaxrsModule; +import com.facebook.airlift.jmx.testing.TestingJmxModule; +import com.facebook.airlift.json.JsonModule; +import com.facebook.airlift.log.Logging; +import com.facebook.airlift.node.testing.TestingNodeModule; import com.facebook.presto.execution.QueryState; import com.facebook.presto.jdbc.PrestoResultSet; import com.facebook.presto.jdbc.PrestoStatement; @@ -20,15 +29,6 @@ import com.facebook.presto.server.testing.TestingPrestoServer; import com.facebook.presto.tpch.TpchPlugin; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.http.server.HttpServerInfo; -import io.airlift.http.server.testing.TestingHttpServerModule; -import io.airlift.jaxrs.JaxrsModule; -import io.airlift.jmx.testing.TestingJmxModule; -import io.airlift.json.JsonModule; -import io.airlift.log.Logging; -import io.airlift.node.testing.TestingNodeModule; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -46,8 +46,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicReference; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.execution.QueryState.FAILED; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.US_ASCII; import static java.util.concurrent.Executors.newCachedThreadPool; diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index 849283d0cf728..1fd0a49519d01 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-raptor @@ -18,15 +18,20 @@ - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift json + + com.facebook.presto + presto-cache + + com.facebook.presto presto-plugin-toolkit @@ -53,42 +58,42 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift configuration - io.airlift + com.facebook.airlift dbpool - io.airlift + com.facebook.airlift discovery - io.airlift + com.facebook.airlift http-client - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift node - io.airlift + com.facebook.airlift stats @@ -161,7 +166,7 @@ - io.airlift + com.facebook.airlift log-manager runtime @@ -205,19 +210,19 @@ - io.airlift + com.facebook.airlift testing test - io.airlift + com.facebook.airlift http-server test - io.airlift + com.facebook.airlift jaxrs test @@ -260,8 +265,14 @@ - io.airlift - testing-mysql-server + com.facebook.presto + testing-mysql-server-5 + test + + + + com.facebook.presto + testing-mysql-server-base test diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/PluginInfo.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/PluginInfo.java index 7294176905d68..6b6c7e97c5013 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/PluginInfo.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/PluginInfo.java @@ -31,6 +31,11 @@ public Module getMetadataModule() return new DatabaseMetadataModule(); } + public Map getFileSystemProviders() + { + return ImmutableMap.of(); + } + public Map getBackupProviders() { return ImmutableMap.of(); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorConnector.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorConnector.java index ca1d4c14de6bf..84e53e56395aa 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorConnector.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorConnector.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.raptor; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.raptor.metadata.ForMetadata; import com.facebook.presto.raptor.metadata.MetadataDao; import com.facebook.presto.spi.NodeManager; @@ -25,12 +27,11 @@ import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; import com.facebook.presto.spi.connector.ConnectorSplitManager; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.procedure.Procedure; import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spi.transaction.IsolationLevel; import com.google.common.collect.HashMultimap; import com.google.common.collect.SetMultimap; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import org.skife.jdbi.v2.IDBI; import javax.annotation.PostConstruct; @@ -44,12 +45,12 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.raptor.util.DatabaseUtil.onDemandDao; import static com.facebook.presto.spi.transaction.IsolationLevel.READ_COMMITTED; import static com.facebook.presto.spi.transaction.IsolationLevel.checkConnectorSupports; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Verify.verify; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; import static java.util.concurrent.TimeUnit.SECONDS; @@ -71,6 +72,7 @@ public class RaptorConnector private final MetadataDao dao; private final ConnectorAccessControl accessControl; private final boolean coordinator; + private final Set procedures; private final ConcurrentMap transactions = new ConcurrentHashMap<>(); @@ -92,7 +94,8 @@ public RaptorConnector( RaptorTableProperties tableProperties, Set systemTables, ConnectorAccessControl accessControl, - @ForMetadata IDBI dbi) + @ForMetadata IDBI dbi, + Set procedures) { this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); this.metadataFactory = requireNonNull(metadataFactory, "metadataFactory is null"); @@ -106,6 +109,7 @@ public RaptorConnector( this.accessControl = requireNonNull(accessControl, "accessControl is null"); this.dao = onDemandDao(dbi, MetadataDao.class); this.coordinator = nodeManager.getCurrentNode().isCoordinator(); + this.procedures = requireNonNull(procedures, "procedures is null"); } @PostConstruct @@ -203,6 +207,12 @@ public ConnectorAccessControl getAccessControl() return accessControl; } + @Override + public Set getProcedures() + { + return procedures; + } + @Override public final void shutdown() { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorConnectorFactory.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorConnectorFactory.java index 7544b20bbc248..8f69857351e03 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorConnectorFactory.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorConnectorFactory.java @@ -13,7 +13,10 @@ */ package com.facebook.presto.raptor; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.raptor.backup.BackupModule; +import com.facebook.presto.raptor.filesystem.FileSystemModule; import com.facebook.presto.raptor.security.RaptorSecurityModule; import com.facebook.presto.raptor.storage.StorageModule; import com.facebook.presto.raptor.util.RebindSafeMBeanServer; @@ -27,8 +30,6 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Injector; import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import org.weakref.jmx.guice.MBeanModule; import javax.management.MBeanServer; @@ -46,13 +47,15 @@ public class RaptorConnectorFactory { private final String name; private final Module metadataModule; + private final Map fileSystemProviders; private final Map backupProviders; - public RaptorConnectorFactory(String name, Module metadataModule, Map backupProviders) + public RaptorConnectorFactory(String name, Module metadataModule, Map fileSystemProviders, Map backupProviders) { checkArgument(!isNullOrEmpty(name), "name is null or empty"); this.name = name; this.metadataModule = requireNonNull(metadataModule, "metadataModule is null"); + this.fileSystemProviders = requireNonNull(fileSystemProviders, "fileSystemProviders is null"); this.backupProviders = ImmutableMap.copyOf(requireNonNull(backupProviders, "backupProviders is null")); } @@ -84,10 +87,12 @@ public Connector create(String catalogName, Map config, Connecto binder.bind(TypeManager.class).toInstance(context.getTypeManager()); }, metadataModule, + new FileSystemModule(fileSystemProviders), new BackupModule(backupProviders), new StorageModule(catalogName), new RaptorModule(catalogName), - new RaptorSecurityModule()); + new RaptorSecurityModule(), + new RaptorProcedureModule()); Injector injector = app .strictConfig() diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java index ecafb08057adb..b9a2304981ede 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorErrorCode.java @@ -38,7 +38,9 @@ public enum RaptorErrorCode RAPTOR_BACKUP_CORRUPTION(13, EXTERNAL), RAPTOR_NOT_ENOUGH_NODES(14, EXTERNAL), RAPTOR_WRITER_DATA_ERROR(15, EXTERNAL), - RAPTOR_UNSUPPORTED_COMPRESSION_KIND(16, EXTERNAL); + RAPTOR_UNSUPPORTED_COMPRESSION_KIND(16, EXTERNAL), + RAPTOR_FILE_SYSTEM_ERROR(17, EXTERNAL), + RAPTOR_TOO_MANY_FILES_CREATED(18, EXTERNAL); private final ErrorCode errorCode; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadata.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadata.java index 59a81e372b04a..1f66661d534cd 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadata.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadata.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.raptor; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.facebook.presto.raptor.metadata.ColumnInfo; import com.facebook.presto.raptor.metadata.Distribution; import com.facebook.presto.raptor.metadata.MetadataDao; @@ -22,6 +24,7 @@ import com.facebook.presto.raptor.metadata.Table; import com.facebook.presto.raptor.metadata.TableColumn; import com.facebook.presto.raptor.metadata.ViewResult; +import com.facebook.presto.raptor.storage.StorageTypeConverter; import com.facebook.presto.raptor.systemtables.ColumnRangesSystemTable; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; @@ -49,14 +52,13 @@ import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.statistics.ComputedStatistics; import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Multimaps; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import org.skife.jdbi.v2.IDBI; @@ -77,6 +79,7 @@ import java.util.function.LongConsumer; import java.util.stream.Collectors; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.raptor.RaptorBucketFunction.validateBucketType; import static com.facebook.presto.raptor.RaptorColumnHandle.BUCKET_NUMBER_COLUMN_NAME; import static com.facebook.presto.raptor.RaptorColumnHandle.SHARD_UUID_COLUMN_NAME; @@ -93,6 +96,7 @@ import static com.facebook.presto.raptor.RaptorTableProperties.DISTRIBUTION_NAME_PROPERTY; import static com.facebook.presto.raptor.RaptorTableProperties.ORDERING_PROPERTY; import static com.facebook.presto.raptor.RaptorTableProperties.ORGANIZED_PROPERTY; +import static com.facebook.presto.raptor.RaptorTableProperties.TABLE_SUPPORTS_DELTA_DELETE; import static com.facebook.presto.raptor.RaptorTableProperties.TEMPORAL_COLUMN_PROPERTY; import static com.facebook.presto.raptor.RaptorTableProperties.getBucketColumns; import static com.facebook.presto.raptor.RaptorTableProperties.getBucketCount; @@ -100,6 +104,7 @@ import static com.facebook.presto.raptor.RaptorTableProperties.getSortColumns; import static com.facebook.presto.raptor.RaptorTableProperties.getTemporalColumn; import static com.facebook.presto.raptor.RaptorTableProperties.isOrganized; +import static com.facebook.presto.raptor.RaptorTableProperties.isTableSupportsDeltaDelete; import static com.facebook.presto.raptor.systemtables.ColumnRangesSystemTable.getSourceTable; import static com.facebook.presto.raptor.util.DatabaseUtil.daoTransaction; import static com.facebook.presto.raptor.util.DatabaseUtil.onDemandDao; @@ -116,7 +121,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.json.JsonCodec.jsonCodec; import static java.lang.String.format; import static java.util.Collections.nCopies; import static java.util.Objects.requireNonNull; @@ -134,26 +138,29 @@ public class RaptorMetadata private final IDBI dbi; private final MetadataDao dao; private final ShardManager shardManager; + private final TypeManager typeManager; private final String connectorId; private final LongConsumer beginDeleteForTableId; private final AtomicReference currentTransactionId = new AtomicReference<>(); - public RaptorMetadata(String connectorId, IDBI dbi, ShardManager shardManager) + public RaptorMetadata(String connectorId, IDBI dbi, ShardManager shardManager, TypeManager typeManager) { - this(connectorId, dbi, shardManager, tableId -> {}); + this(connectorId, dbi, shardManager, typeManager, tableId -> {}); } public RaptorMetadata( String connectorId, IDBI dbi, ShardManager shardManager, + TypeManager typeManager, LongConsumer beginDeleteForTableId) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.dbi = requireNonNull(dbi, "dbi is null"); this.dao = onDemandDao(dbi, MetadataDao.class); this.shardManager = requireNonNull(shardManager, "shardManager is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.beginDeleteForTableId = requireNonNull(beginDeleteForTableId, "beginDeleteForTableId is null"); } @@ -190,7 +197,8 @@ private RaptorTableHandle getTableHandle(SchemaTableName tableName) table.isOrganized(), OptionalLong.empty(), Optional.empty(), - false); + false, + table.isTableSupportsDeltaDelete()); } @Override @@ -232,10 +240,13 @@ public ConnectorTableMetadata getTableMetadata(ConnectorSession session, Connect handle.getBucketCount().ifPresent(bucketCount -> properties.put(BUCKET_COUNT_PROPERTY, bucketCount)); handle.getDistributionName().ifPresent(distributionName -> properties.put(DISTRIBUTION_NAME_PROPERTY, distributionName)); - // Only display organization property if set + // Only display organization and table_supports_delta_delete property if set if (handle.isOrganized()) { properties.put(ORGANIZED_PROPERTY, true); } + if (handle.isTableSupportsDeltaDelete()) { + properties.put(TABLE_SUPPORTS_DELTA_DELETE, true); + } List columns = tableColumns.stream() .map(TableColumn::toColumnMetadata) @@ -345,7 +356,9 @@ public Optional getNewTableLayout(ConnectorSession sess { ImmutableMap.Builder map = ImmutableMap.builder(); long columnId = 1; + StorageTypeConverter storageTypeConverter = new StorageTypeConverter(typeManager); for (ColumnMetadata column : metadata.getColumns()) { + checkState(storageTypeConverter.toStorageType(column.getType()) != null, "storage type cannot be null"); map.put(column.getName(), new RaptorColumnHandle(connectorId, column.getName(), columnId, column.getType())); columnId++; } @@ -469,6 +482,8 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle long columnId = lastColumn.getColumnId() + 1; int ordinalPosition = lastColumn.getOrdinalPosition() + 1; + StorageTypeConverter storageTypeConverter = new StorageTypeConverter(typeManager); + checkState(storageTypeConverter.toStorageType(column.getType()) != null, "storage type cannot be null"); String type = column.getType().getTypeSignature().toString(); daoTransaction(dbi, MetadataDao.class, dao -> { dao.insertColumn(table.getTableId(), columnId, column.getName(), ordinalPosition, type, null, null); @@ -587,8 +602,9 @@ public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, Con temporalColumnHandle, distribution.map(info -> OptionalLong.of(info.getDistributionId())).orElse(OptionalLong.empty()), distribution.map(info -> OptionalInt.of(info.getBucketCount())).orElse(OptionalInt.empty()), + distribution.map(DistributionInfo::getBucketColumns).orElse(ImmutableList.of()), organized, - distribution.map(DistributionInfo::getBucketColumns).orElse(ImmutableList.of())); + isTableSupportsDeltaDelete(tableMetadata.getProperties())); } private DistributionInfo getDistributionInfo(long distributionId, Map columnHandleMap, Map properties) @@ -650,7 +666,7 @@ public Optional finishCreateTable(ConnectorSession sess Long distributionId = table.getDistributionId().isPresent() ? table.getDistributionId().getAsLong() : null; // TODO: update default value of organization_enabled to true - long tableId = dao.insertTable(table.getSchemaName(), table.getTableName(), true, table.isOrganized(), distributionId, updateTime); + long tableId = dao.insertTable(table.getSchemaName(), table.getTableName(), true, table.isOrganized(), distributionId, updateTime, table.isTableSupportsDeltaDelete()); List sortColumnHandles = table.getSortColumnHandles(); List bucketColumnHandles = table.getBucketColumnHandles(); @@ -680,7 +696,7 @@ public Optional finishCreateTable(ConnectorSession sess .orElse(OptionalLong.empty()); // TODO: refactor this to avoid creating an empty table on failure - shardManager.createTable(newTableId, columns, table.getBucketCount().isPresent(), temporalColumnId); + shardManager.createTable(newTableId, columns, table.getBucketCount().isPresent(), temporalColumnId, table.isTableSupportsDeltaDelete()); shardManager.commitShards(transactionId, newTableId, columns, parseFragments(fragments), Optional.empty(), updateTime); clearRollback(); @@ -790,7 +806,8 @@ public ConnectorTableHandle beginDelete(ConnectorSession session, ConnectorTable handle.isOrganized(), OptionalLong.of(transactionId), Optional.of(columnTypes), - true); + true, + handle.isTableSupportsDeltaDelete()); } @Override @@ -825,7 +842,7 @@ public void finishDelete(ConnectorSession session, ConnectorTableHandle tableHan } @Override - public boolean supportsMetadataDelete(ConnectorSession session, ConnectorTableHandle tableHandle) + public boolean supportsMetadataDelete(ConnectorSession session, ConnectorTableHandle tableHandle, Optional tableLayoutHandle) { return false; } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadataFactory.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadataFactory.java index eda7f7371d8dc..ef253f425e45d 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadataFactory.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorMetadataFactory.java @@ -15,6 +15,7 @@ import com.facebook.presto.raptor.metadata.ForMetadata; import com.facebook.presto.raptor.metadata.ShardManager; +import com.facebook.presto.spi.type.TypeManager; import org.skife.jdbi.v2.IDBI; import javax.inject.Inject; @@ -28,20 +29,23 @@ public class RaptorMetadataFactory private final String connectorId; private final IDBI dbi; private final ShardManager shardManager; + private final TypeManager typeManager; @Inject public RaptorMetadataFactory( RaptorConnectorId connectorId, @ForMetadata IDBI dbi, - ShardManager shardManager) + ShardManager shardManager, + TypeManager typeManager) { this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); this.dbi = requireNonNull(dbi, "dbi is null"); this.shardManager = requireNonNull(shardManager, "shardManager is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); } public RaptorMetadata create(LongConsumer beginDeleteForTableId) { - return new RaptorMetadata(connectorId, dbi, shardManager, beginDeleteForTableId); + return new RaptorMetadata(connectorId, dbi, shardManager, typeManager, beginDeleteForTableId); } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorOutputTableHandle.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorOutputTableHandle.java index 958454acc23f0..4ea6f48ba70cf 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorOutputTableHandle.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorOutputTableHandle.java @@ -45,6 +45,7 @@ public class RaptorOutputTableHandle private final OptionalInt bucketCount; private final List bucketColumnHandles; private final boolean organized; + private final boolean tableSupportsDeltaDelete; @JsonCreator public RaptorOutputTableHandle( @@ -59,8 +60,9 @@ public RaptorOutputTableHandle( @JsonProperty("temporalColumnHandle") Optional temporalColumnHandle, @JsonProperty("distributionId") OptionalLong distributionId, @JsonProperty("bucketCount") OptionalInt bucketCount, + @JsonProperty("bucketColumnHandles") List bucketColumnHandles, @JsonProperty("organized") boolean organized, - @JsonProperty("bucketColumnHandles") List bucketColumnHandles) + @JsonProperty("tableSupportsDeltaDelete") boolean tableSupportsDeltaDelete) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.transactionId = transactionId; @@ -75,6 +77,7 @@ public RaptorOutputTableHandle( this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); this.bucketColumnHandles = ImmutableList.copyOf(requireNonNull(bucketColumnHandles, "bucketColumnHandles is null")); this.organized = organized; + this.tableSupportsDeltaDelete = tableSupportsDeltaDelete; } @JsonProperty @@ -155,6 +158,12 @@ public boolean isOrganized() return organized; } + @JsonProperty + public boolean isTableSupportsDeltaDelete() + { + return tableSupportsDeltaDelete; + } + @Override public String toString() { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSink.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSink.java index a67a1ddc85611..f78e0656e9b13 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSink.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSink.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.raptor; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.raptor.filesystem.FileSystemContext; import com.facebook.presto.raptor.metadata.ShardInfo; import com.facebook.presto.raptor.storage.StorageManager; import com.facebook.presto.raptor.storage.organization.TemporalFunction; @@ -22,12 +24,12 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PageBuilder; import com.facebook.presto.spi.PageSorter; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.block.SortOrder; import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import io.airlift.units.DataSize; @@ -41,11 +43,13 @@ import java.util.OptionalInt; import java.util.concurrent.CompletableFuture; +import static com.facebook.airlift.concurrent.MoreFutures.allAsList; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_TOO_MANY_FILES_CREATED; import static com.facebook.presto.spi.type.DateType.DATE; import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.MoreFutures.allAsList; -import static io.airlift.json.JsonCodec.jsonCodec; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; @@ -67,10 +71,13 @@ public class RaptorPageSink private final OptionalInt temporalColumnIndex; private final Optional temporalColumnType; private final TemporalFunction temporalFunction; + private final int maxAllowedFilesPerWriter; + private final FileSystemContext context; private final PageWriter pageWriter; public RaptorPageSink( + FileSystemContext context, PageSorter pageSorter, StorageManager storageManager, TemporalFunction temporalFunction, @@ -82,7 +89,8 @@ public RaptorPageSink( OptionalInt bucketCount, List bucketColumnIds, Optional temporalColumnHandle, - DataSize maxBufferSize) + DataSize maxBufferSize, + int maxAllowedFilesPerWriter) { this.transactionId = transactionId; this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); @@ -91,12 +99,14 @@ public RaptorPageSink( this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); this.storageManager = requireNonNull(storageManager, "storageManager is null"); this.maxBufferBytes = requireNonNull(maxBufferSize, "maxBufferSize is null").toBytes(); + this.maxAllowedFilesPerWriter = maxAllowedFilesPerWriter; this.sortFields = ImmutableList.copyOf(sortColumnIds.stream().map(columnIds::indexOf).collect(toList())); this.sortOrders = ImmutableList.copyOf(requireNonNull(sortOrders, "sortOrders is null")); this.bucketCount = bucketCount; this.bucketFields = bucketColumnIds.stream().mapToInt(columnIds::indexOf).toArray(); + this.context = requireNonNull(context, "context is null"); if (temporalColumnHandle.isPresent() && columnIds.contains(temporalColumnHandle.get().getColumnId())) { temporalColumnIndex = OptionalInt.of(columnIds.indexOf(temporalColumnHandle.get().getColumnId())); @@ -163,7 +173,7 @@ private PageBuffer createPageBuffer(OptionalInt bucketNumber) { return new PageBuffer( maxBufferBytes, - storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, true), + storageManager.createStoragePageSink(context, transactionId, bucketNumber, columnIds, columnTypes, true), columnTypes, sortFields, sortOrders, @@ -266,6 +276,10 @@ private void flushIfNecessary() long maxBytes = 0; PageBuffer maxBuffer = null; + if (pageStores.size() > maxAllowedFilesPerWriter) { + throw new PrestoException(RAPTOR_TOO_MANY_FILES_CREATED, format("Number of files created: %s , has exceeded the limit of %s files created per worker per query", pageStores.size(), maxAllowedFilesPerWriter)); + } + for (PageStore store : pageStores.values()) { long bytes = store.getUsedMemoryBytes(); totalBytes += bytes; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSinkProvider.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSinkProvider.java index f8cd8e359c429..a2d66251455e6 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSinkProvider.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSinkProvider.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.raptor; +import com.facebook.presto.raptor.filesystem.FileSystemContext; import com.facebook.presto.raptor.storage.StorageManager; import com.facebook.presto.raptor.storage.StorageManagerConfig; import com.facebook.presto.raptor.storage.organization.TemporalFunction; @@ -41,6 +42,7 @@ public class RaptorPageSinkProvider private final PageSorter pageSorter; private final TemporalFunction temporalFunction; private final DataSize maxBufferSize; + private final int maxAllowedFilesPerWriter; @Inject public RaptorPageSinkProvider(StorageManager storageManager, PageSorter pageSorter, TemporalFunction temporalFunction, StorageManagerConfig config) @@ -49,6 +51,7 @@ public RaptorPageSinkProvider(StorageManager storageManager, PageSorter pageSort this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); this.temporalFunction = requireNonNull(temporalFunction, "temporalFunction is null"); this.maxBufferSize = config.getMaxBufferSize(); + this.maxAllowedFilesPerWriter = config.getMaxAllowedFilesPerWriter(); } @Override @@ -58,6 +61,7 @@ public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHa RaptorOutputTableHandle handle = (RaptorOutputTableHandle) tableHandle; return new RaptorPageSink( + new FileSystemContext(session), pageSorter, storageManager, temporalFunction, @@ -69,7 +73,8 @@ public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHa handle.getBucketCount(), toColumnIds(handle.getBucketColumnHandles()), handle.getTemporalColumnHandle(), - maxBufferSize); + maxBufferSize, + maxAllowedFilesPerWriter); } @Override @@ -79,6 +84,7 @@ public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHa RaptorInsertTableHandle handle = (RaptorInsertTableHandle) tableHandle; return new RaptorPageSink( + new FileSystemContext(session), pageSorter, storageManager, temporalFunction, @@ -90,7 +96,8 @@ public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHa handle.getBucketCount(), toColumnIds(handle.getBucketColumnHandles()), handle.getTemporalColumnHandle(), - maxBufferSize); + maxBufferSize, + maxAllowedFilesPerWriter); } private static List toColumnIds(List columnHandles) diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSourceProvider.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSourceProvider.java index 4438787ffa394..bb0d1b66bc473 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSourceProvider.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPageSourceProvider.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.raptor; +import com.facebook.presto.raptor.filesystem.FileSystemContext; import com.facebook.presto.raptor.storage.ReaderAttributes; import com.facebook.presto.raptor.storage.StorageManager; import com.facebook.presto.raptor.util.ConcatPageSource; @@ -60,19 +61,22 @@ public ConnectorPageSource createPageSource(ConnectorTransactionHandle transacti OptionalLong transactionId = raptorSplit.getTransactionId(); Optional> columnTypes = raptorSplit.getColumnTypes(); + FileSystemContext context = new FileSystemContext(session); + if (raptorSplit.getShardUuids().size() == 1) { UUID shardUuid = raptorSplit.getShardUuids().iterator().next(); - return createPageSource(shardUuid, bucketNumber, columns, predicate, attributes, transactionId, columnTypes); + return createPageSource(context, shardUuid, bucketNumber, columns, predicate, attributes, transactionId, columnTypes); } Iterator iterator = raptorSplit.getShardUuids().stream() - .map(shardUuid -> createPageSource(shardUuid, bucketNumber, columns, predicate, attributes, transactionId, columnTypes)) + .map(shardUuid -> createPageSource(context, shardUuid, bucketNumber, columns, predicate, attributes, transactionId, columnTypes)) .iterator(); return new ConcatPageSource(iterator); } private ConnectorPageSource createPageSource( + FileSystemContext context, UUID shardUuid, OptionalInt bucketNumber, List columns, @@ -85,6 +89,6 @@ private ConnectorPageSource createPageSource( List columnIds = columnHandles.stream().map(RaptorColumnHandle::getColumnId).collect(toList()); List columnTypes = columnHandles.stream().map(RaptorColumnHandle::getColumnType).collect(toList()); - return storageManager.getPageSource(shardUuid, bucketNumber, columnIds, columnTypes, predicate, attributes, transactionId, allColumnTypes); + return storageManager.getPageSource(context, shardUuid, bucketNumber, columnIds, columnTypes, predicate, attributes, transactionId, allColumnTypes); } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPlugin.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPlugin.java index 66ee02f727a9e..5f08fa97fe685 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPlugin.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorPlugin.java @@ -33,6 +33,7 @@ public class RaptorPlugin { private final String name; private final Module metadataModule; + private final Map fileSystemProviders; private final Map backupProviders; public RaptorPlugin() @@ -42,21 +43,22 @@ public RaptorPlugin() private RaptorPlugin(PluginInfo info) { - this(info.getName(), info.getMetadataModule(), info.getBackupProviders()); + this(info.getName(), info.getMetadataModule(), info.getFileSystemProviders(), info.getBackupProviders()); } - public RaptorPlugin(String name, Module metadataModule, Map backupProviders) + public RaptorPlugin(String name, Module metadataModule, Map fileSystemProviders, Map backupProviders) { checkArgument(!isNullOrEmpty(name), "name is null or empty"); this.name = name; this.metadataModule = requireNonNull(metadataModule, "metadataModule is null"); + this.fileSystemProviders = requireNonNull(fileSystemProviders, "fileSystemProviders is null"); this.backupProviders = ImmutableMap.copyOf(requireNonNull(backupProviders, "backupProviders is null")); } @Override public Iterable getConnectorFactories() { - return ImmutableList.of(new RaptorConnectorFactory(name, metadataModule, backupProviders)); + return ImmutableList.of(new RaptorConnectorFactory(name, metadataModule, fileSystemProviders, backupProviders)); } private static PluginInfo getPluginInfo() diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorProcedureModule.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorProcedureModule.java new file mode 100644 index 0000000000000..1604a56e4ff58 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorProcedureModule.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor; + +import com.facebook.presto.spi.procedure.Procedure; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; +import com.google.inject.multibindings.Multibinder; + +import static com.google.inject.multibindings.Multibinder.newSetBinder; + +public class RaptorProcedureModule + implements Module +{ + @Override + public void configure(Binder binder) + { + Multibinder procedures = newSetBinder(binder, Procedure.class); + procedures.addBinding().toProvider(TriggerBucketBalancerProcedure.class).in(Scopes.SINGLETON); + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSessionProperties.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSessionProperties.java index 2e8249846f94a..b4bea09362fa7 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSessionProperties.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSessionProperties.java @@ -39,6 +39,7 @@ public class RaptorSessionProperties private static final String READER_TINY_STRIPE_THRESHOLD = "reader_tiny_stripe_threshold"; private static final String READER_LAZY_READ_SMALL_RANGES = "reader_lazy_read_small_ranges"; private static final String ONE_SPLIT_PER_BUCKET_THRESHOLD = "one_split_per_bucket_threshold"; + private static final String ORC_ZSTD_JNI_DECOMPRESSION_ENABLED = "orc_ztd_jni_decompression_enabled"; private final List> sessionProperties; @@ -80,7 +81,12 @@ public RaptorSessionProperties(StorageManagerConfig config) ONE_SPLIT_PER_BUCKET_THRESHOLD, "Experimental: Maximum bucket count at which to produce multiple splits per bucket", config.getOneSplitPerBucketThreshold(), - false)); + false), + booleanProperty( + ORC_ZSTD_JNI_DECOMPRESSION_ENABLED, + "use JNI based std decompression for reading ORC files", + config.isZstdJniDecompressionEnabled(), + true)); } public List> getSessionProperties() @@ -123,6 +129,11 @@ public static int getOneSplitPerBucketThreshold(ConnectorSession session) return session.getProperty(ONE_SPLIT_PER_BUCKET_THRESHOLD, Integer.class); } + public static boolean isZstdJniDecompressionEnabled(ConnectorSession session) + { + return session.getProperty(ORC_ZSTD_JNI_DECOMPRESSION_ENABLED, Boolean.class); + } + public static PropertyMetadata dataSizeSessionProperty(String name, String description, DataSize defaultValue, boolean hidden) { return new PropertyMetadata<>( diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplitManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplitManager.java index 1c592fc2c6714..1d14fd599f8ed 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplitManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorSplitManager.java @@ -49,6 +49,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.function.Supplier; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_NO_HOST_FOR_SHARD; import static com.facebook.presto.raptor.RaptorSessionProperties.getOneSplitPerBucketThreshold; import static com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE; @@ -58,7 +59,6 @@ import static com.google.common.base.Verify.verify; import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.collect.Maps.uniqueIndex; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.CompletableFuture.supplyAsync; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableHandle.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableHandle.java index 97a4bd49e6966..5e38729567868 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableHandle.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableHandle.java @@ -43,6 +43,7 @@ public final class RaptorTableHandle private final OptionalLong transactionId; private final Optional> columnTypes; private final boolean delete; + private final boolean tableSupportsDeltaDelete; @JsonCreator public RaptorTableHandle( @@ -56,7 +57,8 @@ public RaptorTableHandle( @JsonProperty("organized") boolean organized, @JsonProperty("transactionId") OptionalLong transactionId, @JsonProperty("columnTypes") Optional> columnTypes, - @JsonProperty("delete") boolean delete) + @JsonProperty("delete") boolean delete, + @JsonProperty("tableSupportsDeltaDelete") boolean tableSupportsDeltaDelete) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.schemaName = checkSchemaName(schemaName); @@ -73,6 +75,7 @@ public RaptorTableHandle( this.columnTypes = requireNonNull(columnTypes, "columnTypes is null"); this.delete = delete; + this.tableSupportsDeltaDelete = tableSupportsDeltaDelete; } public boolean isBucketed() @@ -146,6 +149,12 @@ public boolean isDelete() return delete; } + @JsonProperty + public boolean isTableSupportsDeltaDelete() + { + return tableSupportsDeltaDelete; + } + @Override public String toString() { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableProperties.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableProperties.java index 7fe53f13df955..aea1537fdc28c 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableProperties.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/RaptorTableProperties.java @@ -39,6 +39,7 @@ public class RaptorTableProperties public static final String BUCKETED_ON_PROPERTY = "bucketed_on"; public static final String DISTRIBUTION_NAME_PROPERTY = "distribution_name"; public static final String ORGANIZED_PROPERTY = "organized"; + public static final String TABLE_SUPPORTS_DELTA_DELETE = "table_supports_delta_delete"; private final List> tableProperties; @@ -70,6 +71,11 @@ public RaptorTableProperties(TypeManager typeManager) "Keep the table organized using the sort order", null, false)) + .add(booleanProperty( + TABLE_SUPPORTS_DELTA_DELETE, + "Support delta delete on the table", + null, + false)) .build(); } @@ -110,6 +116,12 @@ public static boolean isOrganized(Map tableProperties) return (value == null) ? false : value; } + public static boolean isTableSupportsDeltaDelete(Map tableProperties) + { + Boolean value = (Boolean) tableProperties.get(TABLE_SUPPORTS_DELTA_DELETE); + return (value == null) ? false : value; + } + public static PropertyMetadata lowerCaseStringSessionProperty(String name, String description) { return new PropertyMetadata<>( diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/TriggerBucketBalancerProcedure.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/TriggerBucketBalancerProcedure.java new file mode 100644 index 0000000000000..c17d1eeadf0d6 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/TriggerBucketBalancerProcedure.java @@ -0,0 +1,57 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor; + +import com.facebook.presto.raptor.storage.BucketBalancer; +import com.facebook.presto.spi.procedure.Procedure; +import com.google.common.collect.ImmutableList; + +import javax.inject.Inject; +import javax.inject.Provider; + +import java.lang.invoke.MethodHandle; + +import static com.facebook.presto.spi.block.MethodHandleUtil.methodHandle; +import static java.util.Objects.requireNonNull; + +public class TriggerBucketBalancerProcedure + implements Provider +{ + private static final MethodHandle TRIGGER_BUCKET_BALANCER = methodHandle( + TriggerBucketBalancerProcedure.class, + "createTriggerBucketBalancer"); + + private final BucketBalancer bucketBalancer; + + @Inject + public TriggerBucketBalancerProcedure(BucketBalancer bucketBalancer) + { + this.bucketBalancer = requireNonNull(bucketBalancer, "bucketBalancer is null"); + } + + @Override + public Procedure get() + { + return new Procedure( + "system", + "trigger_bucket_balancer", + ImmutableList.of(), + TRIGGER_BUCKET_BALANCER.bindTo(this)); + } + + public void createTriggerBucketBalancer() + { + bucketBalancer.startBalanceJob(); + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupConfig.java index 054dcdb24ea7b..4fa25fac689ec 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.raptor.backup; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import io.airlift.units.Duration; import io.airlift.units.MaxDuration; import io.airlift.units.MinDuration; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupManager.java index b8aa54300a802..7be1f3483900e 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupManager.java @@ -13,13 +13,16 @@ */ package com.facebook.presto.raptor.backup; +import com.facebook.airlift.log.Logger; import com.facebook.presto.raptor.storage.BackupStats; +import com.facebook.presto.raptor.storage.OrcDataEnvironment; import com.facebook.presto.raptor.storage.StorageService; import com.facebook.presto.spi.PrestoException; import com.google.common.io.Files; -import io.airlift.log.Logger; import io.airlift.units.DataSize; import io.airlift.units.Duration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; import org.weakref.jmx.Flatten; import org.weakref.jmx.Managed; @@ -35,9 +38,11 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; +import static com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment.tryGetLocalFileSystem; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.google.common.base.Preconditions.checkState; import static io.airlift.units.DataSize.Unit.BYTE; import static java.util.Objects.requireNonNull; import static java.util.concurrent.CompletableFuture.completedFuture; @@ -51,23 +56,27 @@ public class BackupManager private final Optional backupStore; private final StorageService storageService; private final ExecutorService executorService; + private final Optional localFileSystem; private final AtomicInteger pendingBackups = new AtomicInteger(); private final BackupStats stats = new BackupStats(); @Inject - public BackupManager(Optional backupStore, StorageService storageService, BackupConfig config) + public BackupManager(Optional backupStore, StorageService storageService, OrcDataEnvironment environment, BackupConfig config) { - this(backupStore, storageService, config.getBackupThreads()); + this(backupStore, storageService, environment, config.getBackupThreads()); } - public BackupManager(Optional backupStore, StorageService storageService, int backupThreads) + public BackupManager(Optional backupStore, StorageService storageService, OrcDataEnvironment environment, int backupThreads) { checkArgument(backupThreads > 0, "backupThreads must be > 0"); this.backupStore = requireNonNull(backupStore, "backupStore is null"); this.storageService = requireNonNull(storageService, "storageService is null"); this.executorService = newFixedThreadPool(backupThreads, daemonThreadsNamed("background-shard-backup-%s")); + this.localFileSystem = tryGetLocalFileSystem(requireNonNull(environment, "environment is null")); + + checkState((!backupStore.isPresent() || localFileSystem.isPresent()), "cannot support backup for remote file system"); } @PreDestroy @@ -76,7 +85,7 @@ public void shutdown() executorService.shutdownNow(); } - public CompletableFuture submit(UUID uuid, File source) + public CompletableFuture submit(UUID uuid, Path source) { requireNonNull(uuid, "uuid is null"); requireNonNull(source, "source is null"); @@ -87,7 +96,7 @@ public CompletableFuture submit(UUID uuid, File source) // TODO: decrement when the running task is finished (not immediately on cancel) pendingBackups.incrementAndGet(); - CompletableFuture future = runAsync(new BackgroundBackup(uuid, source), executorService); + CompletableFuture future = runAsync(new BackgroundBackup(uuid, localFileSystem.get().pathToFile(source)), executorService); future.whenComplete((none, throwable) -> pendingBackups.decrementAndGet()); return future; } @@ -115,13 +124,13 @@ public void run() backupStore.get().backupShard(uuid, source); stats.addCopyShardDataRate(new DataSize(source.length(), BYTE), Duration.nanosSince(start)); - File restored = new File(storageService.getStagingFile(uuid) + ".validate"); + File restored = new File(localFileSystem.get().pathToFile(storageService.getStagingFile(uuid)) + ".validate"); backupStore.get().restoreShard(uuid, restored); if (!filesEqual(source, restored)) { stats.incrementBackupCorruption(); - File quarantineBase = storageService.getQuarantineFile(uuid); + File quarantineBase = localFileSystem.get().pathToFile(storageService.getQuarantineFile(uuid)); File quarantineOriginal = new File(quarantineBase.getPath() + ".original"); File quarantineRestored = new File(quarantineBase.getPath() + ".restored"); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupModule.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupModule.java index 981314fc88b9b..ebb13865b9137 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupModule.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupModule.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.raptor.backup; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.airlift.configuration.ConfigurationAwareModule; import com.facebook.presto.raptor.RaptorConnectorId; import com.google.common.collect.ImmutableMap; import com.google.inject.Binder; @@ -20,9 +23,6 @@ import com.google.inject.Provides; import com.google.inject.Scopes; import com.google.inject.util.Providers; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.airlift.configuration.ConfigurationAwareModule; import org.weakref.jmx.MBeanExporter; import javax.annotation.Nullable; @@ -31,7 +31,7 @@ import java.util.Map; import java.util.Optional; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static org.weakref.jmx.ObjectNames.generatedNameOf; public class BackupModule diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupOperationStats.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupOperationStats.java index 9eec70c046d21..cb837f77e1924 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupOperationStats.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/BackupOperationStats.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.raptor.backup; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.TimeStat; import com.facebook.presto.spi.PrestoException; -import io.airlift.stats.CounterStat; -import io.airlift.stats.TimeStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/FileBackupConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/FileBackupConfig.java index fa51b767d9a41..a5de05a512fe8 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/FileBackupConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/FileBackupConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.raptor.backup; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import javax.validation.constraints.NotNull; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/FileBackupModule.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/FileBackupModule.java index 3b69e6f416e70..830ead7b7d722 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/FileBackupModule.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/FileBackupModule.java @@ -17,7 +17,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; public class FileBackupModule implements Module diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/FileBackupStore.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/FileBackupStore.java index 62e7189537c9b..c7fa5d5370c59 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/FileBackupStore.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/FileBackupStore.java @@ -29,7 +29,7 @@ import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_BACKUP_ERROR; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_BACKUP_NOT_FOUND; -import static com.facebook.presto.raptor.storage.FileStorageService.getFileSystemPath; +import static com.facebook.presto.raptor.filesystem.LocalFileStorageService.getFileSystemPath; import static java.nio.file.Files.deleteIfExists; import static java.util.Objects.requireNonNull; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/HttpBackupConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/HttpBackupConfig.java index 18b102478f33f..3a8209a9ab304 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/HttpBackupConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/HttpBackupConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.raptor.backup; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import javax.validation.constraints.NotNull; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/HttpBackupModule.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/HttpBackupModule.java index 376acb9d68570..299057a5c2b2b 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/HttpBackupModule.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/HttpBackupModule.java @@ -13,19 +13,19 @@ */ package com.facebook.presto.raptor.backup; +import com.facebook.airlift.node.NodeInfo; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.Scopes; -import io.airlift.node.NodeInfo; import javax.inject.Singleton; import java.net.URI; import java.util.function.Supplier; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.http.client.HttpClientBinder.httpClientBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; public class HttpBackupModule implements Module diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/HttpBackupStore.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/HttpBackupStore.java index 64b794be6dbf7..4643908b272a4 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/HttpBackupStore.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/HttpBackupStore.java @@ -13,16 +13,16 @@ */ package com.facebook.presto.raptor.backup; +import com.facebook.airlift.http.client.BodyGenerator; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpStatus; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.Response; +import com.facebook.airlift.http.client.ResponseHandler; +import com.facebook.airlift.http.client.StatusResponseHandler.StatusResponse; import com.facebook.presto.spi.PrestoException; import com.google.common.io.ByteStreams; import com.google.common.io.Files; -import io.airlift.http.client.BodyGenerator; -import io.airlift.http.client.HttpClient; -import io.airlift.http.client.HttpStatus; -import io.airlift.http.client.Request; -import io.airlift.http.client.Response; -import io.airlift.http.client.ResponseHandler; -import io.airlift.http.client.StatusResponseHandler.StatusResponse; import io.airlift.slice.XxHash64; import javax.inject.Inject; @@ -37,16 +37,16 @@ import java.util.UUID; import java.util.function.Supplier; +import static com.facebook.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static com.facebook.airlift.http.client.Request.Builder.prepareDelete; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.airlift.http.client.Request.Builder.prepareHead; +import static com.facebook.airlift.http.client.Request.Builder.preparePut; +import static com.facebook.airlift.http.client.ResponseHandlerUtils.propagate; +import static com.facebook.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_BACKUP_ERROR; import static com.google.common.net.HttpHeaders.CONTENT_TYPE; import static com.google.common.net.MediaType.APPLICATION_BINARY; -import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; -import static io.airlift.http.client.Request.Builder.prepareDelete; -import static io.airlift.http.client.Request.Builder.prepareGet; -import static io.airlift.http.client.Request.Builder.prepareHead; -import static io.airlift.http.client.Request.Builder.preparePut; -import static io.airlift.http.client.ResponseHandlerUtils.propagate; -import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; import static java.lang.String.format; import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/ManagedBackupStore.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/ManagedBackupStore.java index 34fab89e9bc1d..360969d0060b0 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/ManagedBackupStore.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/ManagedBackupStore.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.raptor.backup; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/TimeoutBackupStore.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/TimeoutBackupStore.java index 01da4fa34b741..259f72850536d 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/TimeoutBackupStore.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/backup/TimeoutBackupStore.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.raptor.backup; +import com.facebook.airlift.concurrent.BoundedExecutor; +import com.facebook.airlift.concurrent.ExecutorServiceAdapter; import com.facebook.presto.spi.PrestoException; import com.google.common.util.concurrent.SimpleTimeLimiter; import com.google.common.util.concurrent.TimeLimiter; import com.google.common.util.concurrent.UncheckedTimeoutException; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.concurrent.ExecutorServiceAdapter; import io.airlift.units.Duration; import javax.annotation.PreDestroy; @@ -27,8 +27,8 @@ import java.util.UUID; import java.util.concurrent.ExecutorService; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_BACKUP_TIMEOUT; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.TimeUnit.MILLISECONDS; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/FileSystemContext.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/FileSystemContext.java new file mode 100644 index 0000000000000..36d33b49c5011 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/FileSystemContext.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.security.ConnectorIdentity; + +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +// TODO: Add schema name and table name to context +public class FileSystemContext +{ + public static final FileSystemContext DEFAULT_RAPTOR_CONTEXT = new FileSystemContext(new ConnectorIdentity("presto-raptor", Optional.empty(), Optional.empty())); + + private final ConnectorIdentity identity; + private final Optional source; + private final Optional queryId; + private final Optional clientInfo; + + public FileSystemContext(ConnectorIdentity identity) + { + this.identity = requireNonNull(identity, "identity is null"); + this.source = Optional.empty(); + this.queryId = Optional.empty(); + this.clientInfo = Optional.empty(); + } + + public FileSystemContext(ConnectorSession session) + { + requireNonNull(session, "session is null"); + this.identity = requireNonNull(session.getIdentity(), "session.getIdentity() is null"); + this.source = requireNonNull(session.getSource(), "session.getSource()"); + this.queryId = Optional.of(session.getQueryId()); + this.clientInfo = session.getClientInfo(); + } + + public ConnectorIdentity getIdentity() + { + return identity; + } + + public Optional getSource() + { + return source; + } + + public Optional getQueryId() + { + return queryId; + } + + public Optional getClientInfo() + { + return clientInfo; + } + + @Override + public String toString() + { + return toStringHelper(this) + .omitNullValues() + .add("user", identity) + .add("source", source.orElse(null)) + .add("queryId", queryId.orElse(null)) + .add("clientInfo", clientInfo.orElse(null)) + .toString(); + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/FileSystemModule.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/FileSystemModule.java new file mode 100644 index 0000000000000..ea73787385f2a --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/FileSystemModule.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.airlift.configuration.ConfigurationAwareModule; +import com.facebook.presto.raptor.storage.StorageManagerConfig; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Binder; +import com.google.inject.Module; + +import java.util.Map; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; + +public class FileSystemModule + extends AbstractConfigurationAwareModule +{ + private final Map providers; + + public FileSystemModule(Map providers) + { + this.providers = ImmutableMap.builder() + .put("file", new LocalFileSystemModule()) + .put("hdfs", new HdfsModule()) + .putAll(providers) + .build(); + } + + @Override + protected void setup(Binder binder) + { + configBinder(binder).bindConfig(StorageManagerConfig.class); + + String fileSystemProvider = buildConfigObject(StorageManagerConfig.class).getFileSystemProvider(); + Module module = providers.get(fileSystemProvider); + + if (module == null) { + binder.addError("Unsupported file system: %s", fileSystemProvider); + } + else if (module instanceof ConfigurationAwareModule) { + install(module); + } + else { + binder.install(module); + } + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/FileSystemUtil.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/FileSystemUtil.java new file mode 100644 index 0000000000000..404df2523e5e7 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/FileSystemUtil.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import com.facebook.presto.spi.PrestoException; +import io.airlift.slice.XxHash64; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; + +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; + +public final class FileSystemUtil +{ + private static final Configuration INITIAL_CONFIGURATION; + + static { + Configuration.addDefaultResource("hdfs-default.xml"); + Configuration.addDefaultResource("hdfs-site.xml"); + + // must not be transitively reloaded during the future loading of various Hadoop modules + // all the required default resources must be declared above + INITIAL_CONFIGURATION = new Configuration(false); + Configuration defaultConfiguration = new Configuration(); + FileSystemUtil.copy(defaultConfiguration, FileSystemUtil.INITIAL_CONFIGURATION); + } + + private FileSystemUtil() {} + + public static Configuration getInitialConfiguration() + { + return copy(INITIAL_CONFIGURATION); + } + + public static Configuration copy(Configuration configuration) + { + Configuration copy = new Configuration(false); + copy(configuration, copy); + return copy; + } + + public static void copy(Configuration from, Configuration to) + { + for (Map.Entry entry : from) { + to.set(entry.getKey(), entry.getValue()); + } + } + + public static long xxhash64(FileSystem fileSystem, Path file) + { + try (InputStream in = fileSystem.open(file)) { + return XxHash64.hash(in); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_ERROR, "Failed to read file: " + file, e); + } + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsModule.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsModule.java new file mode 100644 index 0000000000000..04586ef9222a8 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsModule.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.presto.cache.CacheConfig; +import com.facebook.presto.cache.CacheStats; +import com.facebook.presto.cache.ForCachingFileSystem; +import com.facebook.presto.raptor.storage.OrcDataEnvironment; +import com.facebook.presto.raptor.storage.StorageManagerConfig; +import com.facebook.presto.raptor.storage.StorageService; +import com.google.inject.Binder; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import org.apache.hadoop.fs.Path; + +import javax.inject.Singleton; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static org.weakref.jmx.guice.ExportBinder.newExporter; + +public class HdfsModule + extends AbstractConfigurationAwareModule +{ + @Override + public void setup(Binder binder) + { + configBinder(binder).bindConfig(StorageManagerConfig.class); + configBinder(binder).bindConfig(RaptorHdfsConfig.class); + + CacheConfig cacheConfig = buildConfigObject(CacheConfig.class); + if (cacheConfig.getBaseDirectory() != null) { + binder.bind(CacheStats.class).in(Scopes.SINGLETON); + newExporter(binder).export(CacheStats.class).withGeneratedName(); + + binder.bind(RaptorHdfsConfiguration.class).annotatedWith(ForCachingFileSystem.class).to(RaptorHiveHdfsConfiguration.class).in(Scopes.SINGLETON); + binder.bind(RaptorHdfsConfiguration.class).to(RaptorCachingHdfsConfiguration.class).in(Scopes.SINGLETON); + } + else { + binder.bind(RaptorHdfsConfiguration.class).to(RaptorHiveHdfsConfiguration.class).in(Scopes.SINGLETON); + } + + binder.bind(StorageService.class).to(HdfsStorageService.class).in(Scopes.SINGLETON); + binder.bind(OrcDataEnvironment.class).to(HdfsOrcDataEnvironment.class).in(Scopes.SINGLETON); + } + + @Singleton + @Provides + public Path createBaseLocation(StorageManagerConfig config) + { + return new Path(config.getDataDirectory()); + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsOrcDataEnvironment.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsOrcDataEnvironment.java new file mode 100644 index 0000000000000..2d12dc614d30a --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsOrcDataEnvironment.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import com.facebook.presto.orc.OrcDataSink; +import com.facebook.presto.orc.OrcDataSource; +import com.facebook.presto.orc.OrcDataSourceId; +import com.facebook.presto.orc.OutputStreamOrcDataSink; +import com.facebook.presto.raptor.storage.OrcDataEnvironment; +import com.facebook.presto.raptor.storage.ReaderAttributes; +import com.facebook.presto.spi.PrestoException; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import javax.inject.Inject; + +import java.io.IOException; + +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_FILE_SYSTEM_ERROR; +import static java.util.Objects.requireNonNull; + +public class HdfsOrcDataEnvironment + implements OrcDataEnvironment +{ + private final Path baseLocation; + private final RaptorHdfsConfiguration configuration; + + @Inject + public HdfsOrcDataEnvironment(Path baseLocation, RaptorHdfsConfiguration configuration) + { + this.baseLocation = requireNonNull(baseLocation, "baseLocation is null"); + this.configuration = requireNonNull(configuration, "configuration is null"); + } + + @Override + public FileSystem getFileSystem(FileSystemContext fileSystemContext) + { + try { + return baseLocation.getFileSystem(configuration.getConfiguration(fileSystemContext, baseLocation.toUri())); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_FILE_SYSTEM_ERROR, "Raptor cannot create HDFS file system", e); + } + } + + @Override + public OrcDataSource createOrcDataSource(FileSystem fileSystem, Path path, ReaderAttributes readerAttributes) + throws IOException + { + return new HdfsOrcDataSource( + new OrcDataSourceId(path.toString()), + fileSystem.getFileStatus(path).getLen(), + readerAttributes.getMaxMergeDistance(), + readerAttributes.getMaxReadSize(), + readerAttributes.getStreamBufferSize(), + readerAttributes.isLazyReadSmallRanges(), + fileSystem.open(path)); + } + + @Override + public OrcDataSink createOrcDataSink(FileSystem fileSystem, Path path) + throws IOException + { + return new OutputStreamOrcDataSink(fileSystem.create(path)); + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsOrcDataSource.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsOrcDataSource.java new file mode 100644 index 0000000000000..907a4a019134e --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsOrcDataSource.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import com.facebook.presto.orc.AbstractOrcDataSource; +import com.facebook.presto.orc.OrcDataSourceId; +import com.facebook.presto.spi.PrestoException; +import io.airlift.units.DataSize; +import org.apache.hadoop.fs.FSDataInputStream; + +import java.io.IOException; + +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class HdfsOrcDataSource + extends AbstractOrcDataSource +{ + private final FSDataInputStream inputStream; + + public HdfsOrcDataSource( + OrcDataSourceId id, + long size, + DataSize maxMergeDistance, + DataSize maxReadSize, + DataSize streamBufferSize, + boolean lazyReadSmallRanges, + FSDataInputStream inputStream) + { + super(id, size, maxMergeDistance, maxReadSize, streamBufferSize, lazyReadSmallRanges); + this.inputStream = requireNonNull(inputStream, "inputStream is null"); + } + + @Override + public void close() + throws IOException + { + inputStream.close(); + } + + @Override + protected void readInternal(long position, byte[] buffer, int bufferOffset, int bufferLength) + { + try { + inputStream.readFully(position, buffer, bufferOffset, bufferLength); + } + catch (PrestoException e) { + // just in case there is a Presto wrapper or hook + throw e; + } + catch (Exception e) { + String message = format("Error reading from %s at position %s", this, position); + if (e.getClass().getSimpleName().equals("BlockMissingException")) { + throw new PrestoException(RAPTOR_ERROR, message, e); + } + if (e instanceof IOException) { + throw new PrestoException(RAPTOR_ERROR, message, e); + } + throw new PrestoException(RAPTOR_ERROR, message, e); + } + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsStorageService.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsStorageService.java new file mode 100644 index 0000000000000..54c173fe788f2 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/HdfsStorageService.java @@ -0,0 +1,132 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import com.facebook.presto.raptor.storage.OrcDataEnvironment; +import com.facebook.presto.raptor.storage.StorageService; +import com.facebook.presto.spi.PrestoException; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; + +import java.io.IOException; +import java.util.Set; +import java.util.UUID; + +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_FILE_SYSTEM_ERROR; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +// TODO: this only works for hard file affinity; but good for a prototype +// TODO: need to handle race condition on all workers +// TODO: Staging dir and storage dir are the same in this StorageService implementation +public class HdfsStorageService + implements StorageService +{ + private static final String FILE_EXTENSION = ".orc"; + + private final Path baseStorageDir; + private final Path baseQuarantineDir; + private final OrcDataEnvironment environment; + + @Inject + public HdfsStorageService(OrcDataEnvironment environment, Path baseLocation) + { + requireNonNull(baseLocation, "baseLocation is null"); + this.baseStorageDir = new Path(baseLocation, "storage"); + this.baseQuarantineDir = new Path(baseLocation, "quarantine"); + this.environment = requireNonNull(environment, "environment is null"); + } + + @Override + @PostConstruct + public void start() + { + createDirectory(baseStorageDir); + createDirectory(baseQuarantineDir); + } + + @Override + public long getAvailableBytes() + { + return Long.MAX_VALUE; + } + + @Override + public Path getStorageFile(UUID shardUuid) + { + return getFileSystemPath(baseStorageDir, shardUuid); + } + + @Override + public Path getStagingFile(UUID shardUuid) + { + // Deliberately returned storage file path because we don't have a stage directory here + return getStorageFile(shardUuid); + } + + @Override + public Path getQuarantineFile(UUID shardUuid) + { + throw new PrestoException(RAPTOR_ERROR, "Possible data corruption is detected in metadata or remote storage"); + } + + @Override + public Set getStorageShards() + { + // Bug: This prevent ShardCleaner from cleaning up any unused shards so currently storage will grow indefinitely + // This can only be solved until we re-design the metadata layout for disagg Raptor + throw new UnsupportedOperationException("HDFS storage does not support list directory on purpose"); + } + + @Override + public void createParents(Path file) + { + checkArgument(file.getParent().equals(baseStorageDir) || file.getParent().equals(baseQuarantineDir)); + // No need to create parent as we only have 2 levels of directory now + // TODO: This may change based on metadata redesign + } + + @Override + public void promoteFromStagingToStorage(UUID shardUuid) + { + // Nothing to do as we don't have staging directory + } + + private static Path getFileSystemPath(Path base, UUID shardUuid) + { + String uuid = shardUuid.toString().toLowerCase(ENGLISH); + return new Path(base, uuid + FILE_EXTENSION); + } + + private void createDirectory(Path directory) + { + boolean madeDirectory; + try { + FileSystem fileSystem = environment.getFileSystem(FileSystemContext.DEFAULT_RAPTOR_CONTEXT); + madeDirectory = fileSystem.mkdirs(directory) && fileSystem.isDirectory(directory); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_FILE_SYSTEM_ERROR, "Failed creating directories: " + directory, e); + } + + if (!madeDirectory) { + throw new PrestoException(RAPTOR_FILE_SYSTEM_ERROR, "Failed creating directories: " + directory); + } + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileStorageService.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/LocalFileStorageService.java similarity index 65% rename from presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileStorageService.java rename to presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/LocalFileStorageService.java index 5ca493964a2ac..69699a8995e8f 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileStorageService.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/LocalFileStorageService.java @@ -11,12 +11,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor.storage; +package com.facebook.presto.raptor.filesystem; +import com.facebook.airlift.log.Logger; +import com.facebook.presto.raptor.storage.OrcDataEnvironment; +import com.facebook.presto.raptor.storage.StorageManagerConfig; +import com.facebook.presto.raptor.storage.StorageService; import com.facebook.presto.spi.PrestoException; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -25,6 +30,7 @@ import java.io.File; import java.io.FileFilter; import java.io.IOException; +import java.net.URI; import java.nio.file.Files; import java.util.List; import java.util.Optional; @@ -33,30 +39,39 @@ import java.util.regex.Pattern; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; +import static com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment.tryGetLocalFileSystem; +import static com.google.common.base.Preconditions.checkState; import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; -public class FileStorageService +public class LocalFileStorageService implements StorageService { - private static final Logger log = Logger.get(FileStorageService.class); + private static final Logger log = Logger.get(LocalFileStorageService.class); private static final Pattern HEX_DIRECTORY = Pattern.compile("[0-9a-f]{2}"); private static final String FILE_EXTENSION = ".orc"; + private final RawLocalFileSystem localFileSystem; + private final File baseStorageDir; private final File baseStagingDir; private final File baseQuarantineDir; @Inject - public FileStorageService(StorageManagerConfig config) + public LocalFileStorageService(OrcDataEnvironment environment, StorageManagerConfig storageManagerConfig) { - this(config.getDataDirectory()); + this(environment, storageManagerConfig.getDataDirectory()); } - public FileStorageService(File dataDirectory) + public LocalFileStorageService(OrcDataEnvironment environment, URI dataDirectory) { - File baseDataDir = requireNonNull(dataDirectory, "dataDirectory is null"); + Optional fileSystem = tryGetLocalFileSystem(requireNonNull(environment, "environment is null")); + checkState(fileSystem.isPresent(), "LocalFileStorageService has to have local file system"); + checkState(dataDirectory.isAbsolute(), "dataDirectory URI is not absolute"); + checkState(dataDirectory.getScheme().equals("file"), "dataDirectory URI is not pointing to local file system"); + this.localFileSystem = fileSystem.get(); + File baseDataDir = requireNonNull(localFileSystem.pathToFile(new Path(dataDirectory)), "dataDirectory is null"); this.baseStorageDir = new File(baseDataDir, "storage"); this.baseStagingDir = new File(baseDataDir, "staging"); this.baseQuarantineDir = new File(baseDataDir, "quarantine"); @@ -67,9 +82,9 @@ public FileStorageService(File dataDirectory) public void start() { deleteStagingFilesAsync(); - createParents(new File(baseStagingDir, ".")); - createParents(new File(baseStorageDir, ".")); - createParents(new File(baseQuarantineDir, ".")); + createDirectory(new Path(baseStagingDir.toURI())); + createDirectory(new Path(baseStorageDir.toURI())); + createDirectory(new Path(baseQuarantineDir.toURI())); } @Override @@ -86,31 +101,31 @@ public void stop() } @Override - public File getStorageFile(UUID shardUuid) + public Path getStorageFile(UUID shardUuid) { - return getFileSystemPath(baseStorageDir, shardUuid); + return new Path(getFileSystemPath(baseStorageDir, shardUuid).toString()); } @Override - public File getStagingFile(UUID shardUuid) + public Path getStagingFile(UUID shardUuid) { String name = getFileSystemPath(new File("/"), shardUuid).getName(); - return new File(baseStagingDir, name); + return new Path(baseStagingDir.toString(), name); } @Override - public File getQuarantineFile(UUID shardUuid) + public Path getQuarantineFile(UUID shardUuid) { String name = getFileSystemPath(new File("/"), shardUuid).getName(); - return new File(baseQuarantineDir, name); + return new Path(baseQuarantineDir.toString(), name); } @Override public Set getStorageShards() { ImmutableSet.Builder shards = ImmutableSet.builder(); - for (File level1 : listFiles(baseStorageDir, FileStorageService::isHexDirectory)) { - for (File level2 : listFiles(level1, FileStorageService::isHexDirectory)) { + for (File level1 : listFiles(baseStorageDir, LocalFileStorageService::isHexDirectory)) { + for (File level2 : listFiles(level1, LocalFileStorageService::isHexDirectory)) { for (File file : listFiles(level2, path -> true)) { if (file.isFile()) { uuidFromFileName(file.getName()).ifPresent(shards::add); @@ -122,11 +137,24 @@ public Set getStorageShards() } @Override - public void createParents(File file) + public void createParents(Path file) { - File dir = file.getParentFile(); - if (!dir.mkdirs() && !dir.isDirectory()) { - throw new PrestoException(RAPTOR_ERROR, "Failed creating directories: " + dir); + createDirectory(file.getParent()); + } + + @Override + public void promoteFromStagingToStorage(UUID shardUuid) + { + Path stagingFile = getStagingFile(shardUuid); + Path storageFile = getStorageFile(shardUuid); + + createParents(storageFile); + + try { + localFileSystem.rename(stagingFile, storageFile); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_ERROR, "Failed to move shard file", e); } } @@ -185,6 +213,14 @@ private static void deleteDirectory(File dir) Files.deleteIfExists(dir.toPath()); } + private void createDirectory(Path file) + { + File directory = localFileSystem.pathToFile(file); + if (!directory.mkdirs() && !directory.isDirectory()) { + throw new PrestoException(RAPTOR_ERROR, "Failed creating directories: " + directory); + } + } + private static List listFiles(File dir, FileFilter filter) { File[] files = dir.listFiles(filter); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/LocalFileSystemModule.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/LocalFileSystemModule.java new file mode 100644 index 0000000000000..dda8558334318 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/LocalFileSystemModule.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import com.facebook.presto.raptor.storage.OrcDataEnvironment; +import com.facebook.presto.raptor.storage.StorageService; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +public class LocalFileSystemModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(StorageService.class).to(LocalFileStorageService.class).in(Scopes.SINGLETON); + binder.bind(OrcDataEnvironment.class).to(LocalOrcDataEnvironment.class).in(Scopes.SINGLETON); + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/LocalOrcDataEnvironment.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/LocalOrcDataEnvironment.java new file mode 100644 index 0000000000000..286fc5adbf842 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/LocalOrcDataEnvironment.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import com.facebook.presto.orc.FileOrcDataSource; +import com.facebook.presto.orc.OrcDataSink; +import com.facebook.presto.orc.OrcDataSource; +import com.facebook.presto.orc.OutputStreamOrcDataSink; +import com.facebook.presto.raptor.storage.OrcDataEnvironment; +import com.facebook.presto.raptor.storage.ReaderAttributes; +import com.facebook.presto.spi.PrestoException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; + +import javax.inject.Inject; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Optional; + +import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_FILE_SYSTEM_ERROR; + +public class LocalOrcDataEnvironment + implements OrcDataEnvironment +{ + private static final Configuration CONFIGURATION = new Configuration(); + private final RawLocalFileSystem localFileSystem; + + @Inject + public LocalOrcDataEnvironment() + { + try { + this.localFileSystem = new RaptorLocalFileSystem(CONFIGURATION); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_FILE_SYSTEM_ERROR, "Raptor cannot create local file system", e); + } + } + + @Override + public FileSystem getFileSystem(FileSystemContext ignore) + { + return localFileSystem; + } + + public RawLocalFileSystem getLocalFileSystem() + { + return localFileSystem; + } + + @Override + public OrcDataSource createOrcDataSource(FileSystem ignore, Path path, ReaderAttributes readerAttributes) + throws IOException + { + return new FileOrcDataSource( + localFileSystem.pathToFile(path), + readerAttributes.getMaxMergeDistance(), + readerAttributes.getMaxReadSize(), + readerAttributes.getStreamBufferSize(), + readerAttributes.isLazyReadSmallRanges()); + } + + @Override + public OrcDataSink createOrcDataSink(FileSystem fileSystem, Path path) + throws IOException + { + return new OutputStreamOrcDataSink(new FileOutputStream(localFileSystem.pathToFile(path))); + } + + public static Optional tryGetLocalFileSystem(OrcDataEnvironment environment) + { + if (environment instanceof LocalOrcDataEnvironment) { + return Optional.of(((LocalOrcDataEnvironment) environment).getLocalFileSystem()); + } + return Optional.empty(); + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorCachingHdfsConfiguration.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorCachingHdfsConfiguration.java new file mode 100644 index 0000000000000..bc1ff197e8f48 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorCachingHdfsConfiguration.java @@ -0,0 +1,116 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import com.facebook.presto.cache.CacheConfig; +import com.facebook.presto.cache.CacheManager; +import com.facebook.presto.cache.CacheStats; +import com.facebook.presto.cache.CachingFileSystem; +import com.facebook.presto.cache.ForCachingFileSystem; +import com.facebook.presto.cache.LocalRangeCacheManager; +import com.facebook.presto.cache.NoOpCacheManager; +import com.facebook.presto.hadoop.FileSystemFactory; +import com.facebook.presto.spi.PrestoException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.JobConf; + +import javax.inject.Inject; + +import java.io.IOException; +import java.net.URI; +import java.util.Map; +import java.util.function.BiFunction; + +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.Executors.newScheduledThreadPool; + +public class RaptorCachingHdfsConfiguration + implements RaptorHdfsConfiguration +{ + private final RaptorHdfsConfiguration hiveHdfsConfiguration; + private final CacheManager cacheManager; + private final boolean cacheValidationEnabled; + + @Inject + public RaptorCachingHdfsConfiguration( + @ForCachingFileSystem RaptorHdfsConfiguration hiveHdfsConfiguration, + CacheConfig cacheConfig, + CacheStats cacheStats) + { + this.hiveHdfsConfiguration = requireNonNull(hiveHdfsConfiguration, "hiveHdfsConfiguration is null"); + + CacheConfig config = requireNonNull(cacheConfig, "cachingFileSystemConfig is null"); + this.cacheManager = config.getBaseDirectory() == null ? + new NoOpCacheManager() : + new LocalRangeCacheManager( + cacheConfig, + cacheStats, + newScheduledThreadPool(5, daemonThreadsNamed("raptor-cache-flusher-%s")), + newScheduledThreadPool(1, daemonThreadsNamed("raptor-cache-remover-%s"))); + this.cacheValidationEnabled = cacheConfig.isValidationEnabled(); + } + + @Override + public Configuration getConfiguration(FileSystemContext context, URI uri) + { + @SuppressWarnings("resource") + Configuration config = new CachingJobConf((factoryConfig, factoryUri) -> { + try { + return new CachingFileSystem( + factoryUri, + factoryConfig, + cacheManager, + (new Path(uri)).getFileSystem(hiveHdfsConfiguration.getConfiguration(context, uri)), + cacheValidationEnabled); + } + catch (IOException e) { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "cannot create caching FS", e); + } + }); + Configuration defaultConfig = hiveHdfsConfiguration.getConfiguration(context, uri); + + copy(defaultConfig, config); + return config; + } + + private static void copy(Configuration from, Configuration to) + { + for (Map.Entry entry : from) { + to.set(entry.getKey(), entry.getValue()); + } + } + + private static class CachingJobConf + extends JobConf + implements FileSystemFactory + { + private final BiFunction factory; + + public CachingJobConf(BiFunction factory) + { + super(false); + this.factory = requireNonNull(factory, "factory is null"); + } + + @Override + public FileSystem createFileSystem(URI uri) + { + return factory.apply(this, uri); + } + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorHdfsConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorHdfsConfig.java new file mode 100644 index 0000000000000..a6f906a3f78ad --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorHdfsConfig.java @@ -0,0 +1,197 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.LegacyConfig; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.net.HostAndPort; +import io.airlift.units.DataSize; +import io.airlift.units.Duration; +import io.airlift.units.MaxDataSize; +import io.airlift.units.MinDataSize; +import io.airlift.units.MinDuration; +import org.apache.hadoop.fs.Path; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static io.airlift.units.DataSize.Unit.MEGABYTE; + +public class RaptorHdfsConfig +{ + private HostAndPort socksProxy; + + private Duration ipcPingInterval = new Duration(10, TimeUnit.SECONDS); + private Duration dfsTimeout = new Duration(60, TimeUnit.SECONDS); + private Duration dfsConnectTimeout = new Duration(500, TimeUnit.MILLISECONDS); + private int dfsConnectMaxRetries = 5; + private String domainSocketPath; + + private List resourceConfigFiles = ImmutableList.of(); + + private DataSize textMaxLineLength = new DataSize(100, MEGABYTE); + + private boolean hdfsWireEncryptionEnabled; + + private int fileSystemMaxCacheSize = 1000; + + private Path hdfsPath; + + public HostAndPort getSocksProxy() + { + return socksProxy; + } + + @Config("hive.thrift.client.socks-proxy") + public RaptorHdfsConfig setSocksProxy(HostAndPort socksProxy) + { + this.socksProxy = socksProxy; + return this; + } + + @NotNull + public List getResourceConfigFiles() + { + return resourceConfigFiles; + } + + @Config("hive.config.resources") + public RaptorHdfsConfig setResourceConfigFiles(String files) + { + this.resourceConfigFiles = Splitter.on(',').trimResults().omitEmptyStrings().splitToList(files); + return this; + } + + public RaptorHdfsConfig setResourceConfigFiles(List files) + { + this.resourceConfigFiles = ImmutableList.copyOf(files); + return this; + } + + @NotNull + @MinDuration("1ms") + public Duration getIpcPingInterval() + { + return ipcPingInterval; + } + + @Config("hive.dfs.ipc-ping-interval") + public RaptorHdfsConfig setIpcPingInterval(Duration pingInterval) + { + this.ipcPingInterval = pingInterval; + return this; + } + + @NotNull + @MinDuration("1ms") + public Duration getDfsTimeout() + { + return dfsTimeout; + } + + @Config("hive.dfs-timeout") + public RaptorHdfsConfig setDfsTimeout(Duration dfsTimeout) + { + this.dfsTimeout = dfsTimeout; + return this; + } + + @MinDuration("1ms") + @NotNull + public Duration getDfsConnectTimeout() + { + return dfsConnectTimeout; + } + + @Config("hive.dfs.connect.timeout") + public RaptorHdfsConfig setDfsConnectTimeout(Duration dfsConnectTimeout) + { + this.dfsConnectTimeout = dfsConnectTimeout; + return this; + } + + @Min(0) + public int getDfsConnectMaxRetries() + { + return dfsConnectMaxRetries; + } + + @Config("hive.dfs.connect.max-retries") + public RaptorHdfsConfig setDfsConnectMaxRetries(int dfsConnectMaxRetries) + { + this.dfsConnectMaxRetries = dfsConnectMaxRetries; + return this; + } + + public String getDomainSocketPath() + { + return domainSocketPath; + } + + @Config("hive.dfs.domain-socket-path") + @LegacyConfig("dfs.domain-socket-path") + public RaptorHdfsConfig setDomainSocketPath(String domainSocketPath) + { + this.domainSocketPath = domainSocketPath; + return this; + } + + @MinDataSize("1B") + @MaxDataSize("1GB") + @NotNull + public DataSize getTextMaxLineLength() + { + return textMaxLineLength; + } + + @Config("hive.text.max-line-length") + @ConfigDescription("Maximum line length for text files") + public RaptorHdfsConfig setTextMaxLineLength(DataSize textMaxLineLength) + { + this.textMaxLineLength = textMaxLineLength; + return this; + } + + public boolean isHdfsWireEncryptionEnabled() + { + return hdfsWireEncryptionEnabled; + } + + @Config("hive.hdfs.wire-encryption.enabled") + @ConfigDescription("Should be turned on when HDFS wire encryption is enabled") + public RaptorHdfsConfig setHdfsWireEncryptionEnabled(boolean hdfsWireEncryptionEnabled) + { + this.hdfsWireEncryptionEnabled = hdfsWireEncryptionEnabled; + return this; + } + + public int getFileSystemMaxCacheSize() + { + return fileSystemMaxCacheSize; + } + + @Config("hive.fs.cache.max-size") + @ConfigDescription("Hadoop FileSystem cache size") + public RaptorHdfsConfig setFileSystemMaxCacheSize(int fileSystemMaxCacheSize) + { + this.fileSystemMaxCacheSize = fileSystemMaxCacheSize; + return this; + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorHdfsConfiguration.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorHdfsConfiguration.java new file mode 100644 index 0000000000000..6ae951c5b1a55 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorHdfsConfiguration.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import org.apache.hadoop.conf.Configuration; + +import java.net.URI; + +public interface RaptorHdfsConfiguration +{ + Configuration getConfiguration(FileSystemContext context, URI uri); +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorHiveHdfsConfiguration.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorHiveHdfsConfiguration.java new file mode 100644 index 0000000000000..2cc407dade79d --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorHiveHdfsConfiguration.java @@ -0,0 +1,190 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import com.facebook.presto.hadoop.HadoopFileSystemCache; +import com.facebook.presto.hadoop.HadoopNative; +import com.google.common.collect.ImmutableList; +import com.google.common.net.HostAndPort; +import io.airlift.units.Duration; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; +import org.apache.hadoop.net.DNSToSwitchMapping; +import org.apache.hadoop.net.SocksSocketFactory; + +import javax.inject.Inject; +import javax.net.SocketFactory; + +import java.net.URI; +import java.util.List; + +import static com.facebook.presto.raptor.filesystem.FileSystemUtil.copy; +import static com.facebook.presto.raptor.filesystem.FileSystemUtil.getInitialConfiguration; +import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_PING_INTERVAL_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_RPC_PROTECTION; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SOCKS_SERVER_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY; + +public class RaptorHiveHdfsConfiguration + implements RaptorHdfsConfiguration +{ + static { + HadoopNative.requireHadoopNative(); + HadoopFileSystemCache.initialize(); + } + + private static final Configuration INITIAL_CONFIGURATION = getInitialConfiguration(); + + @SuppressWarnings("ThreadLocalNotStaticFinal") + private final ThreadLocal hadoopConfiguration = new ThreadLocal() + { + @Override + protected Configuration initialValue() + { + Configuration configuration = new Configuration(false); + copy(INITIAL_CONFIGURATION, configuration); + initializer.updateConfiguration(configuration); + return configuration; + } + }; + + private final HdfsConfigurationInitializer initializer; + + @Inject + public RaptorHiveHdfsConfiguration(RaptorHdfsConfig config) + { + this.initializer = new HdfsConfigurationInitializer(requireNonNull(config, "config is null")); + } + + // TODO: Support DynamicConfigurationProvider which consumes context and URI + @Override + public Configuration getConfiguration(FileSystemContext context, URI uri) + { + return hadoopConfiguration.get(); + } + + private static class HdfsConfigurationInitializer + { + private final HostAndPort socksProxy; + private final Duration ipcPingInterval; + private final Duration dfsTimeout; + private final Duration dfsConnectTimeout; + private final int dfsConnectMaxRetries; + private final String domainSocketPath; + private final Configuration resourcesConfiguration; + private final int fileSystemMaxCacheSize; + private final boolean isHdfsWireEncryptionEnabled; + private int textMaxLineLength; + + public HdfsConfigurationInitializer(RaptorHdfsConfig config) + { + requireNonNull(config, "config is null"); + checkArgument(config.getDfsTimeout().toMillis() >= 1, "dfsTimeout must be at least 1 ms"); + checkArgument(toIntExact(config.getTextMaxLineLength().toBytes()) >= 1, "textMaxLineLength must be at least 1 byte"); + + this.socksProxy = config.getSocksProxy(); + this.ipcPingInterval = config.getIpcPingInterval(); + this.dfsTimeout = config.getDfsTimeout(); + this.dfsConnectTimeout = config.getDfsConnectTimeout(); + this.dfsConnectMaxRetries = config.getDfsConnectMaxRetries(); + this.domainSocketPath = config.getDomainSocketPath(); + this.resourcesConfiguration = readConfiguration(config.getResourceConfigFiles()); + this.fileSystemMaxCacheSize = config.getFileSystemMaxCacheSize(); + this.isHdfsWireEncryptionEnabled = config.isHdfsWireEncryptionEnabled(); + this.textMaxLineLength = toIntExact(config.getTextMaxLineLength().toBytes()); + } + + private static Configuration readConfiguration(List resourcePaths) + { + Configuration result = new Configuration(false); + + for (String resourcePath : resourcePaths) { + Configuration resourceProperties = new Configuration(false); + resourceProperties.addResource(new Path(resourcePath)); + copy(resourceProperties, result); + } + + return result; + } + + public void updateConfiguration(Configuration config) + { + copy(resourcesConfiguration, config); + + // this is to prevent dfs client from doing reverse DNS lookups to determine whether nodes are rack local + config.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, NoOpDNSToSwitchMapping.class, DNSToSwitchMapping.class); + + if (socksProxy != null) { + config.setClass(HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY, SocksSocketFactory.class, SocketFactory.class); + config.set(HADOOP_SOCKS_SERVER_KEY, socksProxy.toString()); + } + + if (domainSocketPath != null) { + config.setStrings(DFS_DOMAIN_SOCKET_PATH_KEY, domainSocketPath); + } + + // only enable short circuit reads if domain socket path is properly configured + if (!config.get(DFS_DOMAIN_SOCKET_PATH_KEY, "").trim().isEmpty()) { + config.setBooleanIfUnset(DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); + } + + config.setInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, toIntExact(dfsTimeout.toMillis())); + config.setInt(IPC_PING_INTERVAL_KEY, toIntExact(ipcPingInterval.toMillis())); + config.setInt(IPC_CLIENT_CONNECT_TIMEOUT_KEY, toIntExact(dfsConnectTimeout.toMillis())); + config.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, dfsConnectMaxRetries); + + if (isHdfsWireEncryptionEnabled) { + config.set(HADOOP_RPC_PROTECTION, "privacy"); + config.setBoolean("dfs.encrypt.data.transfer", true); + } + + config.setInt("fs.cache.max-size", fileSystemMaxCacheSize); + + config.setInt(LineRecordReader.MAX_LINE_LENGTH, textMaxLineLength); + } + + public static class NoOpDNSToSwitchMapping + implements DNSToSwitchMapping + { + @Override + public List resolve(List names) + { + // dfs client expects an empty list as an indication that the host->switch mapping for the given names are not known + return ImmutableList.of(); + } + + @Override + public void reloadCachedMappings() + { + // no-op + } + + @Override + public void reloadCachedMappings(List names) + { + // no-op + } + } + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorLocalFileSystem.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorLocalFileSystem.java new file mode 100644 index 0000000000000..e1968f37f2de8 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/filesystem/RaptorLocalFileSystem.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.filesystem; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; + +import java.io.IOException; +import java.nio.file.Files; + +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; + +/** + * The class overrides some inefficient methods from local file system + */ +public final class RaptorLocalFileSystem + extends RawLocalFileSystem +{ + public RaptorLocalFileSystem(Configuration configuration) + throws IOException + { + initialize(getUri(), configuration); + } + + @Override + public boolean exists(Path file) + { + return pathToFile(file).exists(); + } + + @Override + public boolean rename(Path from, Path to) + throws IOException + { + Files.move(pathToFile(from).toPath(), pathToFile(to).toPath(), ATOMIC_MOVE); + return true; + } +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/AssignmentLimiter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/AssignmentLimiter.java index a1d87b1abe18b..d7a24834fb357 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/AssignmentLimiter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/AssignmentLimiter.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.raptor.metadata; +import com.facebook.airlift.log.Logger; import com.facebook.presto.raptor.NodeSupplier; import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.google.common.base.Ticker; -import io.airlift.log.Logger; import io.airlift.units.Duration; import javax.annotation.PostConstruct; @@ -33,10 +33,10 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicBoolean; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_NOT_ENOUGH_NODES; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_REASSIGNMENT_DELAY; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_REASSIGNMENT_THROTTLE; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newScheduledThreadPool; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseConfig.java index a6478e830270e..f0425e070b706 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.raptor.metadata; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import javax.validation.constraints.NotNull; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseMetadataModule.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseMetadataModule.java index c64492a95940f..ba36b87b9e682 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseMetadataModule.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseMetadataModule.java @@ -13,6 +13,12 @@ */ package com.facebook.presto.raptor.metadata; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.airlift.dbpool.H2EmbeddedDataSourceModule; +import com.facebook.airlift.dbpool.MySqlDataSource; +import com.facebook.airlift.dbpool.MySqlDataSourceConfig; +import com.facebook.airlift.discovery.client.ServiceDescriptor; +import com.facebook.airlift.discovery.client.testing.StaticServiceSelector; import com.facebook.presto.raptor.util.DaoSupplier; import com.google.common.reflect.TypeParameter; import com.google.common.reflect.TypeToken; @@ -23,12 +29,6 @@ import com.google.inject.Provides; import com.google.inject.Scopes; import com.google.inject.TypeLiteral; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.airlift.dbpool.H2EmbeddedDataSourceModule; -import io.airlift.dbpool.MySqlDataSource; -import io.airlift.dbpool.MySqlDataSourceConfig; -import io.airlift.discovery.client.ServiceDescriptor; -import io.airlift.discovery.client.testing.StaticServiceSelector; import org.skife.jdbi.v2.IDBI; import org.skife.jdbi.v2.tweak.ConnectionFactory; @@ -39,10 +39,10 @@ import java.lang.reflect.Type; +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.discovery.client.ServiceDescriptor.serviceDescriptor; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.configuration.ConditionalModule.installModuleIf; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.discovery.client.ServiceDescriptor.serviceDescriptor; import static java.util.Objects.requireNonNull; public class DatabaseMetadataModule diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseShardManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseShardManager.java index bfb39bd7a2a63..922e179596e40 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseShardManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseShardManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.raptor.metadata; +import com.facebook.airlift.log.Logger; import com.facebook.presto.raptor.NodeSupplier; import com.facebook.presto.raptor.RaptorColumnHandle; import com.facebook.presto.raptor.storage.organization.ShardOrganizerDao; @@ -31,7 +32,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.util.concurrent.UncheckedExecutionException; -import io.airlift.log.Logger; import io.airlift.units.Duration; import org.h2.jdbc.JdbcConnection; import org.skife.jdbi.v2.Handle; @@ -151,7 +151,7 @@ public DatabaseShardManager( } @Override - public void createTable(long tableId, List columns, boolean bucketed, OptionalLong temporalColumnId) + public void createTable(long tableId, List columns, boolean bucketed, OptionalLong temporalColumnId, boolean tableSupportsDeltaDelete) { StringJoiner tableColumns = new StringJoiner(",\n ", " ", ",\n").setEmptyValue(""); @@ -169,43 +169,27 @@ public void createTable(long tableId, List columns, boolean bucketed temporalColumnId.ifPresent(id -> coveringIndexColumns.add(maxColumn(id))); temporalColumnId.ifPresent(id -> coveringIndexColumns.add(minColumn(id))); - String sql; if (bucketed) { - coveringIndexColumns - .add("bucket_number") - .add("shard_id") - .add("shard_uuid"); - - sql = "" + - "CREATE TABLE " + shardIndexTable(tableId) + " (\n" + - " shard_id BIGINT NOT NULL,\n" + - " shard_uuid BINARY(16) NOT NULL,\n" + - " bucket_number INT NOT NULL\n," + - tableColumns + - " PRIMARY KEY (bucket_number, shard_uuid),\n" + - " UNIQUE (shard_id),\n" + - " UNIQUE (shard_uuid),\n" + - " UNIQUE (" + coveringIndexColumns + ")\n" + - ")"; + coveringIndexColumns.add("bucket_number"); } else { - coveringIndexColumns - .add("node_ids") - .add("shard_id") - .add("shard_uuid"); - - sql = "" + - "CREATE TABLE " + shardIndexTable(tableId) + " (\n" + - " shard_id BIGINT NOT NULL,\n" + - " shard_uuid BINARY(16) NOT NULL,\n" + - " node_ids VARBINARY(128) NOT NULL,\n" + - tableColumns + - " PRIMARY KEY (node_ids, shard_uuid),\n" + - " UNIQUE (shard_id),\n" + - " UNIQUE (shard_uuid),\n" + - " UNIQUE (" + coveringIndexColumns + ")\n" + - ")"; + coveringIndexColumns.add("node_ids"); } + coveringIndexColumns + .add("shard_id") + .add("shard_uuid"); + String sql = "" + + "CREATE TABLE " + shardIndexTable(tableId) + " (\n" + + " shard_id BIGINT NOT NULL,\n" + + " shard_uuid BINARY(16) NOT NULL,\n" + + (tableSupportsDeltaDelete ? " delta_shard_uuid BINARY(16) DEFAULT NULL,\n" : "") + + (bucketed ? " bucket_number INT NOT NULL\n," : " node_ids VARBINARY(128) NOT NULL,\n") + + tableColumns + + (bucketed ? " PRIMARY KEY (bucket_number, shard_uuid),\n" : " PRIMARY KEY (node_ids, shard_uuid),\n") + + " UNIQUE (shard_id),\n" + + " UNIQUE (shard_uuid),\n" + + " UNIQUE (" + coveringIndexColumns + ")\n" + + ")"; try (Handle handle = dbi.open()) { handle.execute(sql); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseShardRecorder.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseShardRecorder.java index faf824ddaa6e7..6ccfdb8792181 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseShardRecorder.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/DatabaseShardRecorder.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.raptor.metadata; +import com.facebook.airlift.log.Logger; import com.facebook.presto.raptor.util.DaoSupplier; import com.facebook.presto.spi.PrestoException; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/Distribution.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/Distribution.java index 9687b3e9fe128..88a15666f47cc 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/Distribution.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/Distribution.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.raptor.metadata; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import org.skife.jdbi.v2.StatementContext; import org.skife.jdbi.v2.tweak.ResultSetMapper; @@ -28,9 +28,9 @@ import java.util.List; import java.util.Optional; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; -import static io.airlift.json.JsonCodec.listJsonCodec; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/JdbcDatabaseConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/JdbcDatabaseConfig.java index f206bafe81691..f586f71c48889 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/JdbcDatabaseConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/JdbcDatabaseConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.raptor.metadata; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import javax.validation.constraints.NotNull; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/MetadataConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/MetadataConfig.java index 4c380401cb76e..41e89fcb4f54b 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/MetadataConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/MetadataConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.raptor.metadata; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import io.airlift.units.Duration; import javax.validation.constraints.Min; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/MetadataDao.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/MetadataDao.java index d5276410b8465..ba25904a5d95a 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/MetadataDao.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/MetadataDao.java @@ -27,7 +27,7 @@ public interface MetadataDao { String TABLE_INFORMATION_SELECT = "" + - "SELECT t.table_id, t.distribution_id, d.distribution_name, d.bucket_count, t.temporal_column_id, t.organization_enabled\n" + + "SELECT t.table_id, t.distribution_id, d.distribution_name, d.bucket_count, t.temporal_column_id, t.organization_enabled, t.table_supports_delta_delete\n" + "FROM tables t\n" + "LEFT JOIN distributions d ON (t.distribution_id = d.distribution_id)\n"; @@ -115,11 +115,11 @@ List getViews( @SqlUpdate("INSERT INTO tables (\n" + " schema_name, table_name, compaction_enabled, organization_enabled, distribution_id,\n" + " create_time, update_time, table_version,\n" + - " shard_count, row_count, compressed_size, uncompressed_size)\n" + + " shard_count, row_count, compressed_size, uncompressed_size, table_supports_delta_delete)\n" + "VALUES (\n" + " :schemaName, :tableName, :compactionEnabled, :organizationEnabled, :distributionId,\n" + " :createTime, :createTime, 0,\n" + - " 0, 0, 0, 0)\n") + " 0, 0, 0, 0, :tableSupportsDeltaDelete)\n") @GetGeneratedKeys long insertTable( @Bind("schemaName") String schemaName, @@ -127,7 +127,8 @@ long insertTable( @Bind("compactionEnabled") boolean compactionEnabled, @Bind("organizationEnabled") boolean organizationEnabled, @Bind("distributionId") Long distributionId, - @Bind("createTime") long createTime); + @Bind("createTime") long createTime, + @Bind("tableSupportsDeltaDelete") boolean tableSupportsDeltaDelete); @SqlUpdate("UPDATE tables SET\n" + " update_time = :updateTime\n" + @@ -246,7 +247,7 @@ long insertDistribution( @Bind("columnTypes") String columnTypes, @Bind("bucketCount") int bucketCount); - @SqlQuery("SELECT table_id, schema_name, table_name, temporal_column_id, distribution_name, bucket_count, organization_enabled\n" + + @SqlQuery("SELECT table_id, schema_name, table_name, temporal_column_id, distribution_name, bucket_count, organization_enabled, table_supports_delta_delete\n" + "FROM tables\n" + "LEFT JOIN distributions\n" + "ON tables.distribution_id = distributions.distribution_id\n" + diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/SchemaDao.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/SchemaDao.java index 952dbffcbef2f..5b0597e4452ef 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/SchemaDao.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/SchemaDao.java @@ -38,10 +38,12 @@ public interface SchemaDao " update_time BIGINT NOT NULL,\n" + " table_version BIGINT NOT NULL,\n" + " shard_count BIGINT NOT NULL,\n" + + " delta_count BIGINT NOT NULL DEFAULT 0,\n" + " row_count BIGINT NOT NULL,\n" + " compressed_size BIGINT NOT NULL,\n" + " uncompressed_size BIGINT NOT NULL,\n" + " maintenance_blocked DATETIME,\n" + + " table_supports_delta_delete BOOLEAN NOT NULL DEFAULT false,\n" + " UNIQUE (schema_name, table_name),\n" + " UNIQUE (distribution_id, table_id),\n" + " UNIQUE (maintenance_blocked, table_id),\n" + @@ -49,6 +51,14 @@ public interface SchemaDao ")") void createTableTables(); + @SqlUpdate("ALTER TABLE tables\n" + + " ADD COLUMN table_supports_delta_delete BOOLEAN NOT NULL DEFAULT false") + void alterTableTablesWithDeltaDelete(); + + @SqlUpdate("ALTER TABLE tables\n" + + " ADD COLUMN delta_count BIGINT NOT NULL DEFAULT 0") + void alterTableTablesWithDeltaCount(); + @SqlUpdate("CREATE TABLE IF NOT EXISTS columns (\n" + " table_id BIGINT NOT NULL,\n" + " column_id BIGINT NOT NULL,\n" + @@ -91,6 +101,8 @@ public interface SchemaDao " compressed_size BIGINT NOT NULL,\n" + " uncompressed_size BIGINT NOT NULL,\n" + " xxhash64 BIGINT NOT NULL,\n" + + " is_delta BOOLEAN NOT NULL DEFAULT false,\n" + + " delta_uuid BINARY(16),\n" + " UNIQUE (shard_uuid),\n" + // include a covering index organized by table_id " UNIQUE (table_id, bucket_number, shard_id, shard_uuid, create_time, row_count, compressed_size, uncompressed_size, xxhash64),\n" + @@ -98,6 +110,14 @@ public interface SchemaDao ")") void createTableShards(); + @SqlUpdate("ALTER TABLE shards\n" + + " ADD COLUMN is_delta BOOLEAN NOT NULL DEFAULT false") + void alterTableShardsWithIsDelta(); + + @SqlUpdate("ALTER TABLE shards\n" + + " ADD COLUMN delta_uuid BINARY(16)") + void alterTableShardsWithDeltaUuid(); + @SqlUpdate("CREATE TABLE IF NOT EXISTS shard_nodes (\n" + " shard_id BIGINT NOT NULL,\n" + " node_id INT NOT NULL,\n" + diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/SchemaDaoUtil.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/SchemaDaoUtil.java index 001aa93d96d61..0a93fc1467f40 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/SchemaDaoUtil.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/SchemaDaoUtil.java @@ -13,12 +13,18 @@ */ package com.facebook.presto.raptor.metadata; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import io.airlift.units.Duration; import org.skife.jdbi.v2.Handle; import org.skife.jdbi.v2.IDBI; import org.skife.jdbi.v2.exceptions.UnableToObtainConnectionException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.SQLTransientException; +import java.sql.Statement; import java.util.concurrent.TimeUnit; public final class SchemaDaoUtil @@ -33,12 +39,16 @@ public static void createTablesWithRetry(IDBI dbi) while (true) { try (Handle handle = dbi.open()) { createTables(handle.attach(SchemaDao.class)); + alterTables(handle.getConnection(), handle.attach(SchemaDao.class)); return; } - catch (UnableToObtainConnectionException e) { + catch (UnableToObtainConnectionException | SQLTransientException e) { log.warn("Failed to connect to database. Will retry again in %s. Exception: %s", delay, e.getMessage()); sleep(delay); } + catch (SQLException e) { + throw new RuntimeException(e); + } } } @@ -59,6 +69,40 @@ private static void createTables(SchemaDao dao) dao.createTableShardOrganizerJobs(); } + private static void alterTables(Connection connection, SchemaDao dao) + throws SQLException + { + // for upgrading compatibility + alterTable(dao::alterTableTablesWithDeltaDelete, "tables", "table_supports_delta_delete", connection); + alterTable(dao::alterTableTablesWithDeltaCount, "tables", "delta_count", connection); + alterTable(dao::alterTableShardsWithIsDelta, "shards", "is_delta", connection); + alterTable(dao::alterTableShardsWithDeltaUuid, "shards", "delta_uuid", connection); + } + + private static void alterTable(Runnable alterTableFunction, String tableName, String columnName, Connection connection) + throws SQLException + { + if (!findColumnName(tableName, columnName, connection)) { + log.info("Alter table %s, add column %s", tableName, columnName); + alterTableFunction.run(); + } + } + + private static boolean findColumnName(String tableName, String columnName, Connection connection) + throws SQLException + { + Statement statement = connection.createStatement(); + ResultSet results = statement.executeQuery("SELECT * FROM " + tableName + " LIMIT 0"); + ResultSetMetaData metadata = results.getMetaData(); + int columnCount = metadata.getColumnCount(); + for (int i = 0; i < columnCount; i++) { + if (columnName.equalsIgnoreCase(metadata.getColumnName(i + 1))) { + return true; + } + } + return false; + } + private static void sleep(Duration duration) { try { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardCleaner.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardCleaner.java index 755f0da958d06..5faa855898de9 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardCleaner.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardCleaner.java @@ -13,7 +13,11 @@ */ package com.facebook.presto.raptor.metadata; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.raptor.backup.BackupStore; +import com.facebook.presto.raptor.filesystem.FileSystemContext; +import com.facebook.presto.raptor.storage.OrcDataEnvironment; import com.facebook.presto.raptor.storage.StorageService; import com.facebook.presto.raptor.util.DaoSupplier; import com.facebook.presto.spi.NodeManager; @@ -21,9 +25,8 @@ import com.google.common.base.Ticker; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; import io.airlift.units.Duration; +import org.apache.hadoop.fs.Path; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -32,7 +35,7 @@ import javax.annotation.concurrent.GuardedBy; import javax.inject.Inject; -import java.io.File; +import java.io.IOException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Collection; @@ -50,11 +53,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.raptor.metadata.ShardDao.CLEANABLE_SHARDS_BATCH_SIZE; import static com.facebook.presto.raptor.metadata.ShardDao.CLEANUP_TRANSACTIONS_BATCH_SIZE; import static com.google.common.collect.Sets.difference; import static com.google.common.collect.Sets.newConcurrentHashSet; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.Math.min; import static java.util.Objects.requireNonNull; import static java.util.concurrent.CompletableFuture.runAsync; @@ -83,6 +86,7 @@ public class ShardCleaner private final Duration backupCleanTime; private final ScheduledExecutorService scheduler; private final ExecutorService backupExecutor; + private final OrcDataEnvironment orcDataEnvironment; private final Duration maxCompletedTransactionAge; private final AtomicBoolean started = new AtomicBoolean(); @@ -104,6 +108,7 @@ public ShardCleaner( NodeManager nodeManager, StorageService storageService, Optional backupStore, + OrcDataEnvironment environment, ShardCleanerConfig config) { this( @@ -113,6 +118,7 @@ public ShardCleaner( ticker, storageService, backupStore, + environment, config.getMaxTransactionAge(), config.getTransactionCleanerInterval(), config.getLocalCleanerInterval(), @@ -130,6 +136,7 @@ public ShardCleaner( Ticker ticker, StorageService storageService, Optional backupStore, + OrcDataEnvironment environment, Duration maxTransactionAge, Duration transactionCleanerInterval, Duration localCleanerInterval, @@ -153,6 +160,7 @@ public ShardCleaner( this.backupCleanTime = requireNonNull(backupCleanTime, "backupCleanTime is null"); this.scheduler = newScheduledThreadPool(2, daemonThreadsNamed("shard-cleaner-%s")); this.backupExecutor = newFixedThreadPool(backupDeletionThreads, daemonThreadsNamed("shard-cleaner-backup-%s")); + this.orcDataEnvironment = requireNonNull(environment, "environment is null"); this.maxCompletedTransactionAge = requireNonNull(maxCompletedTransactionAge, "maxCompletedTransactionAge is null"); } @@ -368,6 +376,7 @@ synchronized Set getLocalShards() @VisibleForTesting synchronized void cleanLocalShardsImmediately(Set local) + throws IOException { // get shards assigned to the local node Set assigned = dao.getNodeShards(currentNode, null).stream() @@ -387,6 +396,7 @@ synchronized void cleanLocalShardsImmediately(Set local) @VisibleForTesting synchronized void cleanLocalShards() + throws IOException { // find all files on the local node Set local = getLocalShards(); @@ -513,8 +523,9 @@ private static Timestamp maxTimestamp(Duration duration) } @SuppressWarnings("ResultOfMethodCallIgnored") - private static void deleteFile(File file) + private void deleteFile(Path file) + throws IOException { - file.delete(); + orcDataEnvironment.getFileSystem(FileSystemContext.DEFAULT_RAPTOR_CONTEXT).delete(file, false); } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardCleanerConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardCleanerConfig.java index 10595d5b50af6..c48a446b5b01a 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardCleanerConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardCleanerConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.raptor.metadata; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import io.airlift.units.Duration; import io.airlift.units.MaxDuration; import io.airlift.units.MinDuration; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardIterator.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardIterator.java index a8f6ed2a49668..005c0a2b36a7b 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardIterator.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardIterator.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.raptor.metadata; +import com.facebook.airlift.log.Logger; import com.facebook.presto.raptor.RaptorColumnHandle; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.predicate.TupleDomain; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; import org.skife.jdbi.v2.IDBI; import org.skife.jdbi.v2.ResultIterator; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardManager.java index 2bdf8b779c5c4..797bec3a2e488 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardManager.java @@ -30,7 +30,7 @@ public interface ShardManager /** * Create a table. */ - void createTable(long tableId, List columns, boolean bucketed, OptionalLong temporalColumnId); + void createTable(long tableId, List columns, boolean bucketed, OptionalLong temporalColumnId, boolean tableSupportsDeltaDelete); /** * Drop a table. diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/Table.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/Table.java index 8780df97643cd..53cd6203b610d 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/Table.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/Table.java @@ -35,6 +35,7 @@ public final class Table private final OptionalInt bucketCount; private final OptionalLong temporalColumnId; private final boolean organized; + private final boolean tableSupportsDeltaDelete; public Table( long tableId, @@ -42,7 +43,8 @@ public Table( Optional distributionName, OptionalInt bucketCount, OptionalLong temporalColumnId, - boolean organized) + boolean organized, + boolean tableSupportsDeltaDelete) { this.tableId = tableId; this.distributionId = requireNonNull(distributionId, "distributionId is null"); @@ -50,6 +52,7 @@ public Table( this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); this.temporalColumnId = requireNonNull(temporalColumnId, "temporalColumnId is null"); this.organized = organized; + this.tableSupportsDeltaDelete = tableSupportsDeltaDelete; } public long getTableId() @@ -82,6 +85,11 @@ public boolean isOrganized() return organized; } + public boolean isTableSupportsDeltaDelete() + { + return tableSupportsDeltaDelete; + } + @Override public String toString() { @@ -91,6 +99,7 @@ public String toString() .add("bucketCount", bucketCount.isPresent() ? bucketCount.getAsInt() : null) .add("temporalColumnId", temporalColumnId.isPresent() ? temporalColumnId.getAsLong() : null) .add("organized", organized) + .add("tableSupportsDeltaDelete", tableSupportsDeltaDelete) .omitNullValues() .toString(); } @@ -108,7 +117,8 @@ public Table map(int index, ResultSet r, StatementContext ctx) Optional.ofNullable(r.getString("distribution_name")), getOptionalInt(r, "bucket_count"), getOptionalLong(r, "temporal_column_id"), - r.getBoolean("organization_enabled")); + r.getBoolean("organization_enabled"), + r.getBoolean("table_supports_delta_delete")); } } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/TableMetadataRow.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/TableMetadataRow.java index 50f00161b6a36..6ef08bc887f20 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/TableMetadataRow.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/TableMetadataRow.java @@ -35,6 +35,7 @@ public class TableMetadataRow private final Optional distributionName; private final OptionalInt bucketCount; private final boolean organized; + private final boolean tableSupportsDeltaDelete; public TableMetadataRow( long tableId, @@ -43,7 +44,8 @@ public TableMetadataRow( OptionalLong temporalColumnId, Optional distributionName, OptionalInt bucketCount, - boolean organized) + boolean organized, + boolean tableSupportsDeltaDelete) { this.tableId = tableId; this.schemaName = requireNonNull(schemaName, "schemaName is null"); @@ -52,6 +54,7 @@ public TableMetadataRow( this.distributionName = requireNonNull(distributionName, "distributionName is null"); this.bucketCount = requireNonNull(bucketCount, "bucketCount is null"); this.organized = organized; + this.tableSupportsDeltaDelete = tableSupportsDeltaDelete; } public long getTableId() @@ -89,6 +92,11 @@ public boolean isOrganized() return organized; } + public boolean isTableSupportsDeltaDelete() + { + return tableSupportsDeltaDelete; + } + public static class Mapper implements ResultSetMapper { @@ -103,7 +111,8 @@ public TableMetadataRow map(int index, ResultSet rs, StatementContext context) getOptionalLong(rs, "temporal_column_id"), Optional.ofNullable(rs.getString("distribution_name")), getOptionalInt(rs, "bucket_count"), - rs.getBoolean("organization_enabled")); + rs.getBoolean("organization_enabled"), + rs.getBoolean("table_supports_delta_delete")); } } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/security/RaptorSecurityConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/security/RaptorSecurityConfig.java index 2ba8d94980625..559fb0c081a90 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/security/RaptorSecurityConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/security/RaptorSecurityConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.raptor.security; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import javax.validation.constraints.NotNull; import javax.validation.constraints.Pattern; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/security/RaptorSecurityModule.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/security/RaptorSecurityModule.java index e385b51fe1e6c..a588291543f1a 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/security/RaptorSecurityModule.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/security/RaptorSecurityModule.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.raptor.security; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.facebook.presto.plugin.base.security.AllowAllAccessControlModule; import com.facebook.presto.plugin.base.security.FileBasedAccessControlModule; import com.facebook.presto.plugin.base.security.ReadOnlySecurityModule; import com.google.inject.Binder; import com.google.inject.Module; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import static io.airlift.configuration.ConditionalModule.installModuleIf; +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; public class RaptorSecurityModule extends AbstractConfigurationAwareModule diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/BackupStats.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/BackupStats.java index d4e697de4b9af..e702639929e88 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/BackupStats.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/BackupStats.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.raptor.storage; -import io.airlift.stats.CounterStat; -import io.airlift.stats.DistributionStat; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.DistributionStat; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.weakref.jmx.Managed; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/BucketBalancer.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/BucketBalancer.java index 5214f58c39bf9..599fab079222e 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/BucketBalancer.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/BucketBalancer.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.raptor.NodeSupplier; import com.facebook.presto.raptor.RaptorConnectorId; import com.facebook.presto.raptor.backup.BackupService; @@ -31,8 +33,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; import com.google.common.collect.Multiset; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; import io.airlift.units.Duration; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -50,7 +50,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicBoolean; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.String.format; import static java.util.Comparator.comparingInt; import static java.util.Objects.requireNonNull; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/BucketBalancerConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/BucketBalancerConfig.java index 17138bfc5858d..b6bfc069ba025 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/BucketBalancerConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/BucketBalancerConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.raptor.storage; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import io.airlift.units.Duration; import java.util.concurrent.TimeUnit; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcDataEnvironment.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcDataEnvironment.java new file mode 100644 index 0000000000000..a76eeaee30930 --- /dev/null +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcDataEnvironment.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.storage; + +import com.facebook.presto.orc.OrcDataSink; +import com.facebook.presto.orc.OrcDataSource; +import com.facebook.presto.raptor.filesystem.FileSystemContext; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; + +public interface OrcDataEnvironment +{ + FileSystem getFileSystem(FileSystemContext context); + + OrcDataSource createOrcDataSource(FileSystem fileSystem, Path path, ReaderAttributes readerAttributes) + throws IOException; + + OrcDataSink createOrcDataSink(FileSystem fileSystem, Path path) + throws IOException; +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java similarity index 80% rename from presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java rename to presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java index 54e591a310cbd..1238491d7abed 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageFileRewriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java @@ -13,12 +13,16 @@ */ package com.facebook.presto.raptor.storage; -import com.facebook.presto.orc.FileOrcDataSource; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.facebook.presto.orc.OrcBatchRecordReader; +import com.facebook.presto.orc.OrcDataSource; import com.facebook.presto.orc.OrcReader; +import com.facebook.presto.orc.OrcReaderOptions; import com.facebook.presto.orc.OrcWriter; import com.facebook.presto.orc.OrcWriterStats; -import com.facebook.presto.orc.OutputStreamOrcDataSink; +import com.facebook.presto.orc.StripeMetadataSource; +import com.facebook.presto.orc.cache.OrcFileTailSource; import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.raptor.util.Closer; import com.facebook.presto.spi.Page; @@ -29,12 +33,9 @@ import com.facebook.presto.spi.type.TypeSignature; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InterruptedIOException; import java.util.BitSet; @@ -44,6 +45,7 @@ import java.util.TreeMap; import java.util.stream.IntStream; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.OrcEncoding.ORC; import static com.facebook.presto.orc.OrcPredicate.TRUE; @@ -54,16 +56,14 @@ import static com.facebook.presto.raptor.storage.OrcStorageManager.HUGE_MAX_READ_BLOCK_SIZE; import static com.facebook.presto.raptor.util.Closer.closer; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.units.Duration.nanosSince; import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toMap; -public final class OrcPageFileRewriter - implements FileRewriter +public final class OrcFileRewriter { - private static final Logger log = Logger.get(OrcPageFileRewriter.class); + private static final Logger log = Logger.get(OrcFileRewriter.class); private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); private final ReaderAttributes readerAttributes; @@ -71,33 +71,41 @@ public final class OrcPageFileRewriter private final OrcWriterStats stats; private final TypeManager typeManager; private final CompressionKind compression; + private final OrcDataEnvironment orcDataEnvironment; + private final OrcFileTailSource orcFileTailSource; + private final StripeMetadataSource stripeMetadataSource; - OrcPageFileRewriter( + OrcFileRewriter( ReaderAttributes readerAttributes, boolean validate, OrcWriterStats stats, TypeManager typeManager, - CompressionKind compression) + OrcDataEnvironment orcDataEnvironment, + CompressionKind compression, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource) { this.readerAttributes = requireNonNull(readerAttributes, "readerAttributes is null"); this.validate = validate; this.stats = requireNonNull(stats, "stats is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.orcDataEnvironment = requireNonNull(orcDataEnvironment, "orcDataEnvironment is null"); this.compression = requireNonNull(compression, "compression is null"); + this.orcFileTailSource = requireNonNull(orcFileTailSource, "orcFileTailSource is null"); + this.stripeMetadataSource = requireNonNull(stripeMetadataSource, "stripeMetadataSource is null"); } - @Override - public OrcFileInfo rewrite(Map allColumnTypes, File input, File output, BitSet rowsToDelete) + public OrcFileInfo rewrite(FileSystem fileSystem, Map allColumnTypes, Path input, Path output, BitSet rowsToDelete) throws IOException { - try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader())) { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader()); + OrcDataSource dataSource = orcDataEnvironment.createOrcDataSource(fileSystem, input, readerAttributes)) { OrcReader reader = new OrcReader( - new FileOrcDataSource(input, readerAttributes.getMaxMergeDistance(), readerAttributes.getMaxReadSize(), readerAttributes.getStreamBufferSize(), readerAttributes.isLazyReadSmallRanges()), + dataSource, ORC, - readerAttributes.getMaxMergeDistance(), - readerAttributes.getMaxReadSize(), - readerAttributes.getTinyStripeThreshold(), - HUGE_MAX_READ_BLOCK_SIZE); + orcFileTailSource, + stripeMetadataSource, + new OrcReaderOptions(readerAttributes.getMaxMergeDistance(), readerAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE, readerAttributes.isZstdJniDecompressionEnabled())); if (reader.getFooter().getNumberOfRows() < rowsToDelete.length()) { throw new IOException("File has fewer rows than deletion vector"); @@ -156,9 +164,12 @@ public OrcFileInfo rewrite(Map allColumnTypes, File input, File ou } userMetadata = ImmutableMap.of(OrcFileMetadata.KEY, METADATA_CODEC.toJson(new OrcFileMetadata(metadataBuilder.build()))); } - try (Closer recordReader = closer(reader.createBatchRecordReader(readerColumns, TRUE, DEFAULT_STORAGE_TIMEZONE, newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE), OrcBatchRecordReader::close); + + StorageTypeConverter storageTypeConverter = new StorageTypeConverter(typeManager); + + try (Closer recordReader = closer(reader.createBatchRecordReader(storageTypeConverter.toStorageTypes(readerColumns), TRUE, DEFAULT_STORAGE_TIMEZONE, newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE), OrcBatchRecordReader::close); Closer writer = closer(new OrcWriter( - new OutputStreamOrcDataSink(new FileOutputStream(output)), + orcDataEnvironment.createOrcDataSink(fileSystem, output), writerColumnIds, writerStorageTypes, ORC, @@ -206,7 +217,7 @@ private static OrcFileInfo rewrite( Block[] blocks = new Block[types.size()]; for (int i = 0; i < types.size(); i++) { // read from existing columns - blocks[i] = reader.readBlock(types.get(i), readerColumnIndex.get(i)); + blocks[i] = reader.readBlock(readerColumnIndex.get(i)); } row = toIntExact(reader.getFilePosition()); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java index c3c735b4306c5..231a6a5e87e76 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileWriter.java @@ -13,24 +13,20 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.orc.OrcDataSink; import com.facebook.presto.orc.OrcWriter; import com.facebook.presto.orc.OrcWriterOptions; import com.facebook.presto.orc.OrcWriterStats; -import com.facebook.presto.orc.OutputStreamOrcDataSink; import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import org.apache.hadoop.fs.FileSystem; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collection; @@ -38,14 +34,13 @@ import java.util.List; import java.util.Map; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.orc.OrcEncoding.ORC; import static com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode.HASHED; -import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_WRITER_DATA_ERROR; import static com.facebook.presto.raptor.storage.OrcStorageManager.DEFAULT_STORAGE_TIMEZONE; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.json.JsonCodec.jsonCodec; import static java.util.Objects.requireNonNull; public class OrcFileWriter @@ -60,7 +55,7 @@ public class OrcFileWriter private long rowCount; private long uncompressedSize; - public OrcFileWriter(List columnIds, List columnTypes, File target, boolean validate, OrcWriterStats stats, TypeManager typeManager, CompressionKind compression) + public OrcFileWriter(List columnIds, List columnTypes, OrcDataSink target, boolean validate, OrcWriterStats stats, TypeManager typeManager, CompressionKind compression) { this(columnIds, columnTypes, target, true, validate, stats, typeManager, compression); } @@ -69,7 +64,7 @@ public OrcFileWriter(List columnIds, List columnTypes, File target, OrcFileWriter( List columnIds, List columnTypes, - File target, + OrcDataSink target, boolean writeMetadata, boolean validate, OrcWriterStats stats, @@ -85,32 +80,27 @@ public OrcFileWriter(List columnIds, List columnTypes, File target, .collect(toImmutableList()); List columnNames = columnIds.stream().map(Object::toString).collect(toImmutableList()); - try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader())) { - Map userMetadata = ImmutableMap.of(); - if (writeMetadata) { - ImmutableMap.Builder columnTypesMap = ImmutableMap.builder(); - for (int i = 0; i < columnIds.size(); i++) { - columnTypesMap.put(columnIds.get(i), columnTypes.get(i).getTypeSignature()); - } - userMetadata = ImmutableMap.of(OrcFileMetadata.KEY, METADATA_CODEC.toJson(new OrcFileMetadata(columnTypesMap.build()))); + Map userMetadata = ImmutableMap.of(); + if (writeMetadata) { + ImmutableMap.Builder columnTypesMap = ImmutableMap.builder(); + for (int i = 0; i < columnIds.size(); i++) { + columnTypesMap.put(columnIds.get(i), columnTypes.get(i).getTypeSignature()); } - - orcWriter = new OrcWriter( - new OutputStreamOrcDataSink(new FileOutputStream(target)), - columnNames, - storageTypes, - ORC, - requireNonNull(compression, "compression is null"), - DEFAULT_OPTION, - userMetadata, - DEFAULT_STORAGE_TIMEZONE, - validate, - HASHED, - stats); - } - catch (IOException e) { - throw new PrestoException(RAPTOR_ERROR, "Failed to create writer", e); + userMetadata = ImmutableMap.of(OrcFileMetadata.KEY, METADATA_CODEC.toJson(new OrcFileMetadata(columnTypesMap.build()))); } + + orcWriter = new OrcWriter( + target, + columnNames, + storageTypes, + ORC, + requireNonNull(compression, "compression is null"), + DEFAULT_OPTION, + userMetadata, + DEFAULT_STORAGE_TIMEZONE, + validate, + HASHED, + stats); } @Override diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageSource.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageSource.java index 743a3f7b03562..783674fd00485 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageSource.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcPageSource.java @@ -72,6 +72,7 @@ public class OrcPageSource private final AggregatedMemoryContext systemMemoryContext; private int batchId; + private long completedPositions; private boolean closed; public OrcPageSource( @@ -130,6 +131,12 @@ public long getCompletedBytes() return orcDataSource.getReadBytes(); } + @Override + public long getCompletedPositions() + { + return completedPositions; + } + @Override public long getReadTimeNanos() { @@ -152,11 +159,13 @@ public Page getNextPage() close(); return null; } + + completedPositions += batchSize; + long filePosition = recordReader.getFilePosition(); Block[] blocks = new Block[columnIndexes.length]; for (int fieldId = 0; fieldId < blocks.length; fieldId++) { - Type type = types.get(fieldId); if (constantBlocks[fieldId] != null) { blocks[fieldId] = constantBlocks[fieldId].getRegion(0, batchSize); } @@ -164,7 +173,7 @@ else if (columnIndexes[fieldId] == ROWID_COLUMN) { blocks[fieldId] = buildSequenceBlock(filePosition, batchSize); } else { - blocks[fieldId] = new LazyBlock(batchSize, new OrcBlockLoader(columnIndexes[fieldId], type)); + blocks[fieldId] = new LazyBlock(batchSize, new OrcBlockLoader(columnIndexes[fieldId])); } } @@ -254,13 +263,11 @@ private final class OrcBlockLoader { private final int expectedBatchId = batchId; private final int columnIndex; - private final Type type; private boolean loaded; - public OrcBlockLoader(int columnIndex, Type type) + public OrcBlockLoader(int columnIndex) { this.columnIndex = columnIndex; - this.type = requireNonNull(type, "type is null"); } @Override @@ -273,7 +280,7 @@ public final void load(LazyBlock lazyBlock) checkState(batchId == expectedBatchId); try { - Block block = recordReader.readBlock(type, columnIndex); + Block block = recordReader.readBlock(columnIndex); lazyBlock.setBlock(block); } catch (IOException e) { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java deleted file mode 100644 index cb0c62cb99ec3..0000000000000 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordFileRewriter.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.raptor.storage; - -import com.facebook.presto.raptor.util.Closer; -import com.facebook.presto.raptor.util.SyncingFileSystem; -import com.facebook.presto.spi.classloader.ThreadContextClassLoader; -import com.facebook.presto.spi.type.Type; -import io.airlift.log.Logger; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.io.orc.NullMemoryManager; -import org.apache.hadoop.hive.ql.io.orc.OrcFile.WriterOptions; -import org.apache.hadoop.hive.ql.io.orc.OrcStruct; -import org.apache.hadoop.hive.ql.io.orc.OrcWriterOptions; -import org.apache.hadoop.hive.ql.io.orc.Reader; -import org.apache.hadoop.hive.ql.io.orc.RecordReader; -import org.apache.hadoop.hive.ql.io.orc.Writer; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; -import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; -import org.apache.hadoop.hive.serde2.io.ShortWritable; -import org.apache.hadoop.io.BooleanWritable; -import org.apache.hadoop.io.ByteWritable; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.FloatWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; - -import java.io.File; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.nio.ByteBuffer; -import java.util.BitSet; -import java.util.List; -import java.util.Map; - -import static com.facebook.presto.raptor.util.Closer.closer; -import static io.airlift.slice.SizeOf.SIZE_OF_BYTE; -import static io.airlift.slice.SizeOf.SIZE_OF_DOUBLE; -import static io.airlift.slice.SizeOf.SIZE_OF_FLOAT; -import static io.airlift.slice.SizeOf.SIZE_OF_INT; -import static io.airlift.slice.SizeOf.SIZE_OF_LONG; -import static io.airlift.slice.SizeOf.SIZE_OF_SHORT; -import static io.airlift.units.Duration.nanosSince; -import static java.lang.Math.toIntExact; -import static org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader; -import static org.apache.hadoop.hive.ql.io.orc.OrcFile.createWriter; -import static org.apache.hadoop.hive.ql.io.orc.OrcUtil.getFieldValue; - -public final class OrcRecordFileRewriter - implements FileRewriter -{ - private static final Logger log = Logger.get(OrcRecordFileRewriter.class); - private static final Configuration CONFIGURATION = new Configuration(); - - @Override - public OrcFileInfo rewrite(Map columns, File input, File output, BitSet rowsToDelete) - throws IOException - { - try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader()); - FileSystem fileSystem = new SyncingFileSystem(CONFIGURATION)) { - Reader reader = createReader(fileSystem, path(input)); - - if (reader.getNumberOfRows() < rowsToDelete.length()) { - throw new IOException("File has fewer rows than deletion vector"); - } - int deleteRowCount = rowsToDelete.cardinality(); - if (reader.getNumberOfRows() == deleteRowCount) { - return new OrcFileInfo(0, 0); - } - if (reader.getNumberOfRows() >= Integer.MAX_VALUE) { - throw new IOException("File has too many rows"); - } - int inputRowCount = toIntExact(reader.getNumberOfRows()); - - WriterOptions writerOptions = new OrcWriterOptions(CONFIGURATION) - .memory(new NullMemoryManager(CONFIGURATION)) - .fileSystem(fileSystem) - .compress(reader.getCompression()) - .inspector(reader.getObjectInspector()); - - long start = System.nanoTime(); - try (Closer recordReader = closer(reader.rows(), RecordReader::close); - Closer writer = closer(createWriter(path(output), writerOptions), Writer::close)) { - if (reader.hasMetadataValue(OrcFileMetadata.KEY)) { - ByteBuffer orcFileMetadata = reader.getMetadataValue(OrcFileMetadata.KEY); - writer.get().addUserMetadata(OrcFileMetadata.KEY, orcFileMetadata); - } - OrcFileInfo fileInfo = rewrite(recordReader.get(), writer.get(), rowsToDelete, inputRowCount); - log.debug("Rewrote file %s in %s (input rows: %s, output rows: %s)", input.getName(), nanosSince(start), inputRowCount, inputRowCount - deleteRowCount); - return fileInfo; - } - } - } - - private static OrcFileInfo rewrite(RecordReader reader, Writer writer, BitSet rowsToDelete, int inputRowCount) - throws IOException - { - Object object = null; - int row = 0; - long rowCount = 0; - long uncompressedSize = 0; - - row = rowsToDelete.nextClearBit(row); - if (row < inputRowCount) { - reader.seekToRow(row); - } - - while (row < inputRowCount) { - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedIOException(); - } - - // seekToRow() is extremely expensive - if (reader.getRowNumber() < row) { - reader.next(object); - continue; - } - - object = reader.next(object); - writer.addRow(object); - rowCount++; - uncompressedSize += uncompressedSize(object); - - row = rowsToDelete.nextClearBit(row + 1); - } - return new OrcFileInfo(rowCount, uncompressedSize); - } - - private static Path path(File input) - { - return new Path(input.toURI()); - } - - private static int uncompressedSize(Object object) - throws IOException - { - if (object instanceof OrcStruct) { - OrcStruct struct = (OrcStruct) object; - int size = 0; - for (int i = 0; i < struct.getNumFields(); i++) { - size += uncompressedSize(getFieldValue(struct, i)); - } - return size; - } - if ((object == null) || (object instanceof BooleanWritable)) { - return SIZE_OF_BYTE; - } - if (object instanceof LongWritable) { - return SIZE_OF_LONG; - } - if (object instanceof DoubleWritable) { - return SIZE_OF_DOUBLE; - } - if (object instanceof HiveDecimalWritable) { - return SIZE_OF_LONG; - } - if (object instanceof Text) { - return ((Text) object).getLength(); - } - if (object instanceof BytesWritable) { - return ((BytesWritable) object).getLength(); - } - if (object instanceof ByteWritable) { - return SIZE_OF_BYTE; - } - if (object instanceof HiveVarcharWritable) { - return ((HiveVarcharWritable) object).getTextValue().getLength(); - } - if (object instanceof DateWritable) { - return SIZE_OF_INT; - } - if (object instanceof IntWritable) { - return SIZE_OF_INT; - } - if (object instanceof ShortWritable) { - return SIZE_OF_SHORT; - } - if (object instanceof FloatWritable) { - return SIZE_OF_FLOAT; - } - if (object instanceof List) { - int size = 0; - for (Object element : (Iterable) object) { - size += uncompressedSize(element); - } - return size; - } - if (object instanceof Map) { - int size = 0; - for (Map.Entry entry : ((Map) object).entrySet()) { - size += uncompressedSize(entry.getKey()); - size += uncompressedSize(entry.getValue()); - } - return size; - } - throw new IOException("Unhandled ORC object: " + object.getClass().getName()); - } -} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java deleted file mode 100644 index ba565c72d8673..0000000000000 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcRecordWriter.java +++ /dev/null @@ -1,347 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.raptor.storage; - -import com.facebook.presto.orc.metadata.CompressionKind; -import com.facebook.presto.raptor.util.SyncingFileSystem; -import com.facebook.presto.spi.Page; -import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.classloader.ThreadContextClassLoader; -import com.facebook.presto.spi.type.DecimalType; -import com.facebook.presto.spi.type.TimestampWithTimeZoneType; -import com.facebook.presto.spi.type.Type; -import com.facebook.presto.spi.type.TypeSignature; -import com.facebook.presto.spi.type.VarbinaryType; -import com.facebook.presto.spi.type.VarcharType; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.slice.Slice; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.io.orc.NullMemoryManager; -import org.apache.hadoop.hive.ql.io.orc.OrcFile; -import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcSerde; -import org.apache.hadoop.hive.ql.io.orc.OrcWriterOptions; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Properties; - -import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; -import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_UNSUPPORTED_COMPRESSION_KIND; -import static com.facebook.presto.raptor.storage.Row.extractRow; -import static com.facebook.presto.raptor.storage.StorageType.arrayOf; -import static com.facebook.presto.raptor.storage.StorageType.mapOf; -import static com.facebook.presto.raptor.util.Types.isArrayType; -import static com.facebook.presto.raptor.util.Types.isMapType; -import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; -import static com.google.common.base.Functions.toStringFunction; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.Iterables.transform; -import static io.airlift.json.JsonCodec.jsonCodec; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS; -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES; -import static org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; -import static org.apache.hadoop.hive.ql.io.orc.CompressionKind.NONE; -import static org.apache.hadoop.hive.ql.io.orc.CompressionKind.SNAPPY; -import static org.apache.hadoop.hive.ql.io.orc.CompressionKind.ZLIB; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.LIST; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.MAP; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardListObjectInspector; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector; -import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector; -import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getPrimitiveTypeInfo; - -public class OrcRecordWriter - implements FileWriter -{ - private static final Configuration CONFIGURATION = new Configuration(); - private static final Constructor WRITER_CONSTRUCTOR = getOrcWriterConstructor(); - private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); - - private final List columnTypes; - - private final OrcSerde serializer; - private final RecordWriter recordWriter; - private final SettableStructObjectInspector tableInspector; - private final List structFields; - private final Object orcRow; - - private boolean closed; - private long rowCount; - private long uncompressedSize; - - public OrcRecordWriter(List columnIds, List columnTypes, File target, CompressionKind compression) - { - this(columnIds, columnTypes, target, compression, true); - } - - @VisibleForTesting - OrcRecordWriter(List columnIds, List columnTypes, File target, CompressionKind compression, boolean writeMetadata) - { - this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); - checkArgument(columnIds.size() == columnTypes.size(), "ids and types mismatch"); - checkArgument(isUnique(columnIds), "ids must be unique"); - - List storageTypes = ImmutableList.copyOf(toStorageTypes(columnTypes)); - Iterable hiveTypeNames = storageTypes.stream().map(StorageType::getHiveTypeName).collect(toList()); - List columnNames = ImmutableList.copyOf(transform(columnIds, toStringFunction())); - - Properties properties = new Properties(); - properties.setProperty(META_TABLE_COLUMNS, Joiner.on(',').join(columnNames)); - properties.setProperty(META_TABLE_COLUMN_TYPES, Joiner.on(':').join(hiveTypeNames)); - - serializer = createSerializer(properties); - recordWriter = createRecordWriter(new Path(target.toURI()), columnIds, columnTypes, requireNonNull(compression, "compression is null"), writeMetadata); - - tableInspector = getStandardStructObjectInspector(columnNames, getJavaObjectInspectors(storageTypes)); - structFields = ImmutableList.copyOf(tableInspector.getAllStructFieldRefs()); - orcRow = tableInspector.create(); - } - - @Override - public void appendPages(List pages) - { - for (Page page : pages) { - for (int position = 0; position < page.getPositionCount(); position++) { - appendRow(extractRow(page, position, columnTypes)); - } - } - } - - @Override - public void appendPages(List inputPages, int[] pageIndexes, int[] positionIndexes) - { - checkArgument(pageIndexes.length == positionIndexes.length, "pageIndexes and positionIndexes do not match"); - for (int i = 0; i < pageIndexes.length; i++) { - Page page = inputPages.get(pageIndexes[i]); - appendRow(extractRow(page, positionIndexes[i], columnTypes)); - } - } - - @Override - public void close() - throws IOException - { - if (closed) { - return; - } - closed = true; - - recordWriter.close(false); - } - - @Override - public long getRowCount() - { - return rowCount; - } - - @Override - public long getUncompressedSize() - { - return uncompressedSize; - } - - private void appendRow(Row row) - { - List columns = row.getColumns(); - checkArgument(columns.size() == columnTypes.size()); - for (int channel = 0; channel < columns.size(); channel++) { - tableInspector.setStructFieldData(orcRow, structFields.get(channel), columns.get(channel)); - } - try { - recordWriter.write(serializer.serialize(orcRow, tableInspector)); - } - catch (IOException e) { - throw new PrestoException(RAPTOR_ERROR, "Failed to write record", e); - } - rowCount++; - uncompressedSize += row.getSizeInBytes(); - } - - private static OrcSerde createSerializer(Properties properties) - { - OrcSerde serde = new OrcSerde(); - serde.initialize(CONFIGURATION, properties); - return serde; - } - - private static RecordWriter createRecordWriter(Path target, List columnIds, List columnTypes, CompressionKind compression, boolean writeMetadata) - { - try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader()); - FileSystem fileSystem = new SyncingFileSystem(CONFIGURATION)) { - OrcFile.WriterOptions options = new OrcWriterOptions(CONFIGURATION) - .memory(new NullMemoryManager(CONFIGURATION)) - .fileSystem(fileSystem) - .compress(toCompressionKind(compression)); - - if (writeMetadata) { - options.callback(createFileMetadataCallback(columnIds, columnTypes)); - } - - return WRITER_CONSTRUCTOR.newInstance(target, options); - } - catch (ReflectiveOperationException | IOException e) { - throw new PrestoException(RAPTOR_ERROR, "Failed to create writer", e); - } - } - - private static org.apache.hadoop.hive.ql.io.orc.CompressionKind toCompressionKind(CompressionKind compression) - { - switch (compression) { - case NONE: - return NONE; - case SNAPPY: - return SNAPPY; - case ZLIB: - return ZLIB; - default: - throw new PrestoException(RAPTOR_UNSUPPORTED_COMPRESSION_KIND, "Found unsupported compression kind: " + compression); - } - } - - private static OrcFile.WriterCallback createFileMetadataCallback(List columnIds, List columnTypes) - { - return new OrcFile.WriterCallback() - { - @Override - public void preStripeWrite(OrcFile.WriterContext context) - {} - - @Override - public void preFooterWrite(OrcFile.WriterContext context) - { - ImmutableMap.Builder columnTypesMap = ImmutableMap.builder(); - for (int i = 0; i < columnIds.size(); i++) { - columnTypesMap.put(columnIds.get(i), columnTypes.get(i).getTypeSignature()); - } - byte[] bytes = METADATA_CODEC.toJsonBytes(new OrcFileMetadata(columnTypesMap.build())); - context.getWriter().addUserMetadata(OrcFileMetadata.KEY, ByteBuffer.wrap(bytes)); - } - }; - } - - private static Constructor getOrcWriterConstructor() - { - try { - String writerClassName = OrcOutputFormat.class.getName() + "$OrcRecordWriter"; - Constructor constructor = OrcOutputFormat.class.getClassLoader() - .loadClass(writerClassName).asSubclass(RecordWriter.class) - .getDeclaredConstructor(Path.class, OrcFile.WriterOptions.class); - constructor.setAccessible(true); - return constructor; - } - catch (ReflectiveOperationException e) { - throw new RuntimeException(e); - } - } - - private static List getJavaObjectInspectors(List types) - { - return types.stream() - .map(StorageType::getHiveTypeName) - .map(TypeInfoUtils::getTypeInfoFromTypeString) - .map(OrcRecordWriter::getJavaObjectInspector) - .collect(toList()); - } - - private static ObjectInspector getJavaObjectInspector(TypeInfo typeInfo) - { - Category category = typeInfo.getCategory(); - if (category == PRIMITIVE) { - return getPrimitiveJavaObjectInspector(getPrimitiveTypeInfo(typeInfo.getTypeName())); - } - if (category == LIST) { - ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; - return getStandardListObjectInspector(getJavaObjectInspector(listTypeInfo.getListElementTypeInfo())); - } - if (category == MAP) { - MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; - return getStandardMapObjectInspector( - getJavaObjectInspector(mapTypeInfo.getMapKeyTypeInfo()), - getJavaObjectInspector(mapTypeInfo.getMapValueTypeInfo())); - } - throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unhandled storage type: " + category); - } - - private static boolean isUnique(Collection items) - { - return new HashSet<>(items).size() == items.size(); - } - - private static List toStorageTypes(List columnTypes) - { - return columnTypes.stream().map(OrcRecordWriter::toStorageType).collect(toList()); - } - - private static StorageType toStorageType(Type type) - { - if (type instanceof DecimalType) { - DecimalType decimalType = (DecimalType) type; - return StorageType.decimal(decimalType.getPrecision(), decimalType.getScale()); - } - if (type instanceof TimestampWithTimeZoneType) { - throw new PrestoException(NOT_SUPPORTED, "The current ORC writer does not support type: " + type); - } - Class javaType = type.getJavaType(); - if (javaType == boolean.class) { - return StorageType.BOOLEAN; - } - if (javaType == long.class) { - return StorageType.LONG; - } - if (javaType == double.class) { - return StorageType.DOUBLE; - } - if (javaType == Slice.class) { - if (type instanceof VarcharType) { - return StorageType.STRING; - } - if (type.equals(VarbinaryType.VARBINARY)) { - return StorageType.BYTES; - } - } - if (isArrayType(type)) { - return arrayOf(toStorageType(type.getTypeParameters().get(0))); - } - if (isMapType(type)) { - return mapOf(toStorageType(type.getTypeParameters().get(0)), toStorageType(type.getTypeParameters().get(1))); - } - throw new PrestoException(NOT_SUPPORTED, "No storage type for type: " + type); - } -} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java index ea9a46b4018ae..74d9eb5c8a330 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java @@ -13,21 +13,26 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.memory.context.AggregatedMemoryContext; -import com.facebook.presto.orc.FileOrcDataSource; import com.facebook.presto.orc.OrcBatchRecordReader; +import com.facebook.presto.orc.OrcDataSink; import com.facebook.presto.orc.OrcDataSource; import com.facebook.presto.orc.OrcPredicate; import com.facebook.presto.orc.OrcReader; +import com.facebook.presto.orc.OrcReaderOptions; import com.facebook.presto.orc.OrcWriterStats; +import com.facebook.presto.orc.StripeMetadataSource; import com.facebook.presto.orc.TupleDomainOrcPredicate; import com.facebook.presto.orc.TupleDomainOrcPredicate.ColumnReference; +import com.facebook.presto.orc.cache.OrcFileTailSource; import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.orc.metadata.OrcType; import com.facebook.presto.raptor.RaptorColumnHandle; import com.facebook.presto.raptor.RaptorConnectorId; import com.facebook.presto.raptor.backup.BackupManager; import com.facebook.presto.raptor.backup.BackupStore; +import com.facebook.presto.raptor.filesystem.FileSystemContext; import com.facebook.presto.raptor.metadata.ColumnInfo; import com.facebook.presto.raptor.metadata.ColumnStats; import com.facebook.presto.raptor.metadata.ShardDelta; @@ -39,10 +44,14 @@ import com.facebook.presto.spi.Page; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.DecimalType; +import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.NamedTypeSignature; import com.facebook.presto.spi.type.RowFieldName; +import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.TimestampType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; @@ -51,24 +60,19 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; import io.airlift.slice.Slice; import io.airlift.slice.Slices; -import io.airlift.slice.XxHash64; import io.airlift.units.DataSize; import io.airlift.units.Duration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.joda.time.DateTimeZone; import javax.annotation.PreDestroy; import javax.inject.Inject; import java.io.Closeable; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; import java.util.ArrayList; import java.util.BitSet; import java.util.Collection; @@ -86,6 +90,10 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import static com.facebook.airlift.concurrent.MoreFutures.allAsList; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.OrcEncoding.ORC; import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE; @@ -97,12 +105,12 @@ import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_LOCAL_DISK_FULL; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_RECOVERY_ERROR; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_RECOVERY_TIMEOUT; +import static com.facebook.presto.raptor.filesystem.FileSystemUtil.xxhash64; import static com.facebook.presto.raptor.storage.OrcPageSource.BUCKET_NUMBER_COLUMN; import static com.facebook.presto.raptor.storage.OrcPageSource.NULL_COLUMN; import static com.facebook.presto.raptor.storage.OrcPageSource.ROWID_COLUMN; import static com.facebook.presto.raptor.storage.OrcPageSource.SHARD_UUID_COLUMN; import static com.facebook.presto.raptor.storage.ShardStats.computeColumnStats; -import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED; import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED_AND_VALIDATED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -114,13 +122,10 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Throwables.throwIfInstanceOf; -import static io.airlift.concurrent.MoreFutures.allAsList; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.units.DataSize.Unit.PETABYTE; import static java.lang.Math.min; -import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.CompletableFuture.completedFuture; import static java.util.concurrent.CompletableFuture.supplyAsync; @@ -161,8 +166,11 @@ public class OrcStorageManager private final TypeManager typeManager; private final ExecutorService deletionExecutor; private final ExecutorService commitExecutor; - private final FileRewriter fileRewriter; + private final OrcDataEnvironment orcDataEnvironment; + private final OrcFileRewriter fileRewriter; private final OrcWriterStats stats = new OrcWriterStats(); + private final OrcFileTailSource orcFileTailSource; + private final StripeMetadataSource stripeMetadataSource; @Inject public OrcStorageManager( @@ -175,7 +183,10 @@ public OrcStorageManager( BackupManager backgroundBackupManager, ShardRecoveryManager recoveryManager, ShardRecorder shardRecorder, - TypeManager typeManager) + TypeManager typeManager, + OrcDataEnvironment orcDataEnvironment, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource) { this(nodeManager.getCurrentNode().getNodeIdentifier(), storageService, @@ -185,6 +196,7 @@ public OrcStorageManager( recoveryManager, shardRecorder, typeManager, + orcDataEnvironment, connectorId.toString(), config.getDeletionThreads(), config.getShardRecoveryTimeout(), @@ -192,7 +204,9 @@ public OrcStorageManager( config.getMaxShardSize(), config.getMinAvailableSpace(), config.getOrcCompressionKind(), - config.getOrcOptimizedWriterStage()); + config.getOrcOptimizedWriterStage(), + orcFileTailSource, + stripeMetadataSource); } public OrcStorageManager( @@ -204,6 +218,7 @@ public OrcStorageManager( ShardRecoveryManager recoveryManager, ShardRecorder shardRecorder, TypeManager typeManager, + OrcDataEnvironment orcDataEnvironment, String connectorId, int deletionThreads, Duration shardRecoveryTimeout, @@ -211,7 +226,9 @@ public OrcStorageManager( DataSize maxShardSize, DataSize minAvailableSpace, CompressionKind compression, - OrcOptimizedWriterStage orcOptimizedWriterStage) + OrcOptimizedWriterStage orcOptimizedWriterStage, + OrcFileTailSource orcFileTailSource, + StripeMetadataSource stripeMetadataSource) { this.nodeId = requireNonNull(nodeId, "nodeId is null"); this.storageService = requireNonNull(storageService, "storageService is null"); @@ -232,12 +249,18 @@ public OrcStorageManager( this.commitExecutor = newCachedThreadPool(daemonThreadsNamed("raptor-commit-" + connectorId + "-%s")); this.compression = requireNonNull(compression, "compression is null"); this.orcOptimizedWriterStage = requireNonNull(orcOptimizedWriterStage, "orcOptimizedWriterStage is null"); - if (orcOptimizedWriterStage.ordinal() >= ENABLED.ordinal()) { - this.fileRewriter = new OrcPageFileRewriter(readerAttributes, orcOptimizedWriterStage.equals(ENABLED_AND_VALIDATED), stats, typeManager, compression); - } - else { - this.fileRewriter = new OrcRecordFileRewriter(); - } + this.orcDataEnvironment = requireNonNull(orcDataEnvironment, "orcDataEnvironment is null"); + this.fileRewriter = new OrcFileRewriter( + readerAttributes, + orcOptimizedWriterStage.equals(ENABLED_AND_VALIDATED), + stats, + typeManager, + orcDataEnvironment, + compression, + orcFileTailSource, + stripeMetadataSource); + this.orcFileTailSource = requireNonNull(orcFileTailSource, "orcFileTailSource is null"); + this.stripeMetadataSource = requireNonNull(stripeMetadataSource, "stripeMetadataSource is null"); } @PreDestroy @@ -249,6 +272,7 @@ public void shutdown() @Override public ConnectorPageSource getPageSource( + FileSystemContext fileSystemContext, UUID shardUuid, OptionalInt bucketNumber, List columnIds, @@ -258,12 +282,18 @@ public ConnectorPageSource getPageSource( OptionalLong transactionId, Optional> allColumnTypes) { - OrcDataSource dataSource = openShard(shardUuid, readerAttributes); + FileSystem fileSystem = orcDataEnvironment.getFileSystem(fileSystemContext); + OrcDataSource dataSource = openShard(fileSystem, shardUuid, readerAttributes); AggregatedMemoryContext systemMemoryUsage = newSimpleAggregatedMemoryContext(); try { - OrcReader reader = new OrcReader(dataSource, ORC, readerAttributes.getMaxMergeDistance(), readerAttributes.getMaxReadSize(), readerAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE); + OrcReader reader = new OrcReader( + dataSource, + ORC, + orcFileTailSource, + stripeMetadataSource, + new OrcReaderOptions(readerAttributes.getMaxMergeDistance(), readerAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE, readerAttributes.isZstdJniDecompressionEnabled())); Map indexMap = columnIdIndex(reader.getColumnNames()); ImmutableMap.Builder includedColumns = ImmutableMap.builder(); @@ -281,18 +311,20 @@ public ConnectorPageSource getPageSource( } else { columnIndexes.add(index); - includedColumns.put(index, columnTypes.get(i)); + includedColumns.put(index, toOrcFileType(columnTypes.get(i), typeManager)); } } OrcPredicate predicate = getPredicate(effectivePredicate, indexMap); - OrcBatchRecordReader recordReader = reader.createBatchRecordReader(includedColumns.build(), predicate, DEFAULT_STORAGE_TIMEZONE, systemMemoryUsage, INITIAL_BATCH_SIZE); + StorageTypeConverter storageTypeConverter = new StorageTypeConverter(typeManager); + + OrcBatchRecordReader recordReader = reader.createBatchRecordReader(storageTypeConverter.toStorageTypes(includedColumns.build()), predicate, DEFAULT_STORAGE_TIMEZONE, systemMemoryUsage, INITIAL_BATCH_SIZE); Optional shardRewriter = Optional.empty(); if (transactionId.isPresent()) { checkState(allColumnTypes.isPresent()); - shardRewriter = Optional.of(createShardRewriter(transactionId.getAsLong(), bucketNumber, shardUuid, allColumnTypes.get())); + shardRewriter = Optional.of(createShardRewriter(fileSystem, transactionId.getAsLong(), bucketNumber, shardUuid, allColumnTypes.get())); } return new OrcPageSource(shardRewriter, recordReader, dataSource, columnIds, columnTypes, columnIndexes.build(), shardUuid, bucketNumber, systemMemoryUsage); @@ -322,21 +354,27 @@ private static int toSpecialIndex(long columnId) } @Override - public StoragePageSink createStoragePageSink(long transactionId, OptionalInt bucketNumber, List columnIds, List columnTypes, boolean checkSpace) + public StoragePageSink createStoragePageSink( + FileSystemContext fileSystemContext, + long transactionId, + OptionalInt bucketNumber, + List columnIds, + List columnTypes, + boolean checkSpace) { if (checkSpace && storageService.getAvailableBytes() < minAvailableSpace.toBytes()) { throw new PrestoException(RAPTOR_LOCAL_DISK_FULL, "Local disk is full on node " + nodeId); } - return new OrcStoragePageSink(transactionId, columnIds, columnTypes, bucketNumber); + return new OrcStoragePageSink(orcDataEnvironment.getFileSystem(fileSystemContext), transactionId, columnIds, columnTypes, bucketNumber); } - private ShardRewriter createShardRewriter(long transactionId, OptionalInt bucketNumber, UUID shardUuid, Map columns) + private ShardRewriter createShardRewriter(FileSystem fileSystem, long transactionId, OptionalInt bucketNumber, UUID shardUuid, Map columns) { return rowsToDelete -> { if (rowsToDelete.isEmpty()) { return completedFuture(ImmutableList.of()); } - return supplyAsync(() -> rewriteShard(transactionId, bucketNumber, shardUuid, columns, rowsToDelete), deletionExecutor); + return supplyAsync(() -> rewriteShard(fileSystem, transactionId, bucketNumber, shardUuid, columns, rowsToDelete), deletionExecutor); }; } @@ -346,25 +384,23 @@ private void writeShard(UUID shardUuid) throw new PrestoException(RAPTOR_ERROR, "Backup does not exist after write"); } - File stagingFile = storageService.getStagingFile(shardUuid); - File storageFile = storageService.getStorageFile(shardUuid); + storageService.promoteFromStagingToStorage(shardUuid); + } - storageService.createParents(storageFile); + @VisibleForTesting + OrcDataSource openShard(FileSystem fileSystem, UUID shardUuid, ReaderAttributes readerAttributes) + { + Path file = storageService.getStorageFile(shardUuid); + boolean exists; try { - Files.move(stagingFile.toPath(), storageFile.toPath(), ATOMIC_MOVE); + exists = fileSystem.exists(file); } catch (IOException e) { - throw new PrestoException(RAPTOR_ERROR, "Failed to move shard file", e); + throw new PrestoException(RAPTOR_ERROR, "Error locating file " + file, e.getCause()); } - } - - @VisibleForTesting - OrcDataSource openShard(UUID shardUuid, ReaderAttributes readerAttributes) - { - File file = storageService.getStorageFile(shardUuid).getAbsoluteFile(); - if (!file.exists() && backupStore.isPresent()) { + if (!exists && backupStore.isPresent()) { try { Future future = recoveryManager.recoverShard(shardUuid); future.get(recoveryTimeout.toMillis(), TimeUnit.MILLISECONDS); @@ -385,32 +421,36 @@ OrcDataSource openShard(UUID shardUuid, ReaderAttributes readerAttributes) } try { - return fileOrcDataSource(readerAttributes, file); + return orcDataEnvironment.createOrcDataSource(fileSystem, file, readerAttributes); } catch (IOException e) { throw new PrestoException(RAPTOR_ERROR, "Failed to open shard file: " + file, e); } } - private static FileOrcDataSource fileOrcDataSource(ReaderAttributes readerAttributes, File file) - throws FileNotFoundException + private ShardInfo createShardInfo(FileSystem fileSystem, UUID shardUuid, OptionalInt bucketNumber, Path file, Set nodes, long rowCount, long uncompressedSize) { - return new FileOrcDataSource(file, readerAttributes.getMaxMergeDistance(), readerAttributes.getMaxReadSize(), readerAttributes.getStreamBufferSize(), readerAttributes.isLazyReadSmallRanges()); - } - - private ShardInfo createShardInfo(UUID shardUuid, OptionalInt bucketNumber, File file, Set nodes, long rowCount, long uncompressedSize) - { - return new ShardInfo(shardUuid, bucketNumber, nodes, computeShardStats(file), rowCount, file.length(), uncompressedSize, xxhash64(file)); + try { + return new ShardInfo(shardUuid, bucketNumber, nodes, computeShardStats(fileSystem, file), rowCount, fileSystem.getFileStatus(file).getLen(), uncompressedSize, xxhash64(fileSystem, file)); + } + catch (IOException e) { + throw new PrestoException(RAPTOR_ERROR, "Failed to get file status: " + file, e); + } } - private List computeShardStats(File file) + private List computeShardStats(FileSystem fileSystem, Path file) { - try (OrcDataSource dataSource = fileOrcDataSource(defaultReaderAttributes, file)) { - OrcReader reader = new OrcReader(dataSource, ORC, defaultReaderAttributes.getMaxMergeDistance(), defaultReaderAttributes.getMaxReadSize(), defaultReaderAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE); + try (OrcDataSource dataSource = orcDataEnvironment.createOrcDataSource(fileSystem, file, defaultReaderAttributes)) { + OrcReader reader = new OrcReader( + dataSource, + ORC, + orcFileTailSource, + stripeMetadataSource, + new OrcReaderOptions(defaultReaderAttributes.getMaxMergeDistance(), defaultReaderAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE, defaultReaderAttributes.isZstdJniDecompressionEnabled())); ImmutableList.Builder list = ImmutableList.builder(); for (ColumnInfo info : getColumnInfo(reader)) { - computeColumnStats(reader, info.getColumnId(), info.getType()).ifPresent(list::add); + computeColumnStats(reader, info.getColumnId(), info.getType(), typeManager).ifPresent(list::add); } return list.build(); } @@ -420,17 +460,17 @@ private List computeShardStats(File file) } @VisibleForTesting - Collection rewriteShard(long transactionId, OptionalInt bucketNumber, UUID shardUuid, Map columns, BitSet rowsToDelete) + Collection rewriteShard(FileSystem fileSystem, long transactionId, OptionalInt bucketNumber, UUID shardUuid, Map columns, BitSet rowsToDelete) { if (rowsToDelete.isEmpty()) { return ImmutableList.of(); } UUID newShardUuid = UUID.randomUUID(); - File input = storageService.getStorageFile(shardUuid); - File output = storageService.getStagingFile(newShardUuid); + Path input = storageService.getStorageFile(shardUuid); + Path output = storageService.getStagingFile(newShardUuid); - OrcFileInfo info = rewriteFile(columns, input, output, rowsToDelete); + OrcFileInfo info = rewriteFile(fileSystem, columns, input, output, rowsToDelete); long rowCount = info.getRowCount(); if (rowCount == 0) { @@ -445,7 +485,7 @@ Collection rewriteShard(long transactionId, OptionalInt bucketNumber, UUI Set nodes = ImmutableSet.of(nodeId); long uncompressedSize = info.getUncompressedSize(); - ShardInfo shard = createShardInfo(newShardUuid, bucketNumber, output, nodes, rowCount, uncompressedSize); + ShardInfo shard = createShardInfo(fileSystem, newShardUuid, bucketNumber, output, nodes, rowCount, uncompressedSize); writeShard(newShardUuid); @@ -459,10 +499,10 @@ private static Collection shardDelta(UUID oldShardUuid, Optional columns, File input, File output, BitSet rowsToDelete) + private OrcFileInfo rewriteFile(FileSystem fileSystem, Map columns, Path input, Path output, BitSet rowsToDelete) { try { - return fileRewriter.rewrite(columns, input, output, rowsToDelete); + return fileRewriter.rewrite(fileSystem, columns, input, output, rowsToDelete); } catch (IOException e) { throw new PrestoException(RAPTOR_ERROR, "Failed to rewrite shard file: " + input, e); @@ -494,16 +534,6 @@ private List getColumnInfoFromOrcColumnTypes(List orcColumnN return list.build(); } - static long xxhash64(File file) - { - try (InputStream in = new FileInputStream(file)) { - return XxHash64.hash(in); - } - catch (IOException e) { - throw new PrestoException(RAPTOR_ERROR, "Failed to read file: " + file, e); - } - } - private static Optional getOrcFileMetadata(OrcReader reader) { return Optional.ofNullable(reader.getFooter().getUserMetadata().get(OrcFileMetadata.KEY)) @@ -559,6 +589,30 @@ private Type getType(List types, int index) throw new PrestoException(RAPTOR_ERROR, "Unhandled ORC type: " + type); } + static Type toOrcFileType(Type raptorType, TypeManager typeManager) + { + // TIMESTAMPS are stored as BIGINT to void the poor encoding in ORC + if (raptorType == TimestampType.TIMESTAMP) { + return BIGINT; + } + if (raptorType instanceof ArrayType) { + Type elementType = toOrcFileType(((ArrayType) raptorType).getElementType(), typeManager); + return new ArrayType(elementType); + } + if (raptorType instanceof MapType) { + TypeSignature keyType = toOrcFileType(((MapType) raptorType).getKeyType(), typeManager).getTypeSignature(); + TypeSignature valueType = toOrcFileType(((MapType) raptorType).getValueType(), typeManager).getTypeSignature(); + return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.of(keyType), TypeSignatureParameter.of(valueType))); + } + if (raptorType instanceof RowType) { + List fields = ((RowType) raptorType).getFields().stream() + .map(field -> new RowType.Field(field.getName(), toOrcFileType(field.getType(), typeManager))) + .collect(toImmutableList()); + return RowType.from(fields); + } + return raptorType; + } + private static OrcPredicate getPredicate(TupleDomain effectivePredicate, Map indexMap) { ImmutableList.Builder> columns = ImmutableList.builder(); @@ -588,20 +642,23 @@ private class OrcStoragePageSink private final List columnTypes; private final OptionalInt bucketNumber; - private final List stagingFiles = new ArrayList<>(); + private final List stagingFiles = new ArrayList<>(); private final List shards = new ArrayList<>(); private final List> futures = new ArrayList<>(); + private final FileSystem fileSystem; private boolean committed; private FileWriter writer; private UUID shardUuid; public OrcStoragePageSink( + FileSystem fileSystem, long transactionId, List columnIds, List columnTypes, OptionalInt bucketNumber) { + this.fileSystem = requireNonNull(fileSystem, "fileSystem is null"); this.transactionId = transactionId; this.columnIds = ImmutableList.copyOf(requireNonNull(columnIds, "columnIds is null")); this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); @@ -644,14 +701,14 @@ public void flush() shardRecorder.recordCreatedShard(transactionId, shardUuid); - File stagingFile = storageService.getStagingFile(shardUuid); + Path stagingFile = storageService.getStagingFile(shardUuid); futures.add(backupManager.submit(shardUuid, stagingFile)); Set nodes = ImmutableSet.of(nodeId); long rowCount = writer.getRowCount(); long uncompressedSize = writer.getUncompressedSize(); - shards.add(createShardInfo(shardUuid, bucketNumber, stagingFile, nodes, rowCount, uncompressedSize)); + shards.add(createShardInfo(fileSystem, shardUuid, bucketNumber, stagingFile, nodes, rowCount, uncompressedSize)); writer = null; shardUuid = null; @@ -692,8 +749,13 @@ public void rollback() } } finally { - for (File file : stagingFiles) { - file.delete(); + for (Path file : stagingFiles) { + try { + fileSystem.delete(file, false); + } + catch (IOException e) { + // ignore + } } // cancel incomplete backup jobs @@ -712,15 +774,17 @@ private void createWriterIfNecessary() { if (writer == null) { shardUuid = UUID.randomUUID(); - File stagingFile = storageService.getStagingFile(shardUuid); + Path stagingFile = storageService.getStagingFile(shardUuid); storageService.createParents(stagingFile); stagingFiles.add(stagingFile); - if (orcOptimizedWriterStage.ordinal() >= ENABLED.ordinal()) { - writer = new OrcFileWriter(columnIds, columnTypes, stagingFile, orcOptimizedWriterStage.equals(ENABLED_AND_VALIDATED), stats, typeManager, compression); + OrcDataSink sink; + try { + sink = orcDataEnvironment.createOrcDataSink(fileSystem, stagingFile); } - else { - writer = new OrcRecordWriter(columnIds, columnTypes, stagingFile, compression); + catch (IOException e) { + throw new PrestoException(RAPTOR_ERROR, format("Failed to create staging file %s", stagingFile), e); } + writer = new OrcFileWriter(columnIds, columnTypes, sink, orcOptimizedWriterStage.equals(ENABLED_AND_VALIDATED), stats, typeManager, compression); } } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ReaderAttributes.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ReaderAttributes.java index e2d00a55207dc..8c574dc00edae 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ReaderAttributes.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ReaderAttributes.java @@ -25,23 +25,25 @@ public class ReaderAttributes { private final DataSize maxMergeDistance; private final DataSize maxReadSize; - private final DataSize streamBufferSize; private final DataSize tinyStripeThreshold; + private final DataSize streamBufferSize; private final boolean lazyReadSmallRanges; + private final boolean zstdJniDecompressionEnabled; @Inject public ReaderAttributes(StorageManagerConfig config) { - this(config.getOrcMaxMergeDistance(), config.getOrcMaxReadSize(), config.getOrcStreamBufferSize(), config.getOrcTinyStripeThreshold(), config.isOrcLazyReadSmallRanges()); + this(config.getOrcMaxMergeDistance(), config.getOrcMaxReadSize(), config.getOrcStreamBufferSize(), config.getOrcTinyStripeThreshold(), config.isOrcLazyReadSmallRanges(), config.isZstdJniDecompressionEnabled()); } - public ReaderAttributes(DataSize maxMergeDistance, DataSize maxReadSize, DataSize streamBufferSize, DataSize tinyStripeThreshold, boolean lazyReadSmallRanges) + public ReaderAttributes(DataSize maxMergeDistance, DataSize maxReadSize, DataSize streamBufferSize, DataSize tinyStripeThreshold, boolean lazyReadSmallRanges, boolean zstdJniDecompressionEnabled) { this.maxMergeDistance = requireNonNull(maxMergeDistance, "maxMergeDistance is null"); this.maxReadSize = requireNonNull(maxReadSize, "maxReadSize is null"); - this.streamBufferSize = requireNonNull(streamBufferSize, "streamBufferSize is null"); this.tinyStripeThreshold = requireNonNull(tinyStripeThreshold, "tinyStripeThreshold is null"); + this.streamBufferSize = requireNonNull(streamBufferSize, "streamBufferSize is null"); this.lazyReadSmallRanges = lazyReadSmallRanges; + this.zstdJniDecompressionEnabled = zstdJniDecompressionEnabled; } public DataSize getMaxMergeDistance() @@ -69,6 +71,11 @@ public boolean isLazyReadSmallRanges() return lazyReadSmallRanges; } + public boolean isZstdJniDecompressionEnabled() + { + return zstdJniDecompressionEnabled; + } + public static ReaderAttributes from(ConnectorSession session) { return new ReaderAttributes( @@ -76,6 +83,7 @@ public static ReaderAttributes from(ConnectorSession session) RaptorSessionProperties.getReaderMaxReadSize(session), RaptorSessionProperties.getReaderStreamBufferSize(session), RaptorSessionProperties.getReaderTinyStripeThreshold(session), - RaptorSessionProperties.isReaderLazyReadSmallRanges(session)); + RaptorSessionProperties.isReaderLazyReadSmallRanges(session), + RaptorSessionProperties.isZstdJniDecompressionEnabled(session)); } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardEjector.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardEjector.java index 104dea569cdcf..3b2ec0c97d219 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardEjector.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardEjector.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.raptor.NodeSupplier; import com.facebook.presto.raptor.RaptorConnectorId; import com.facebook.presto.raptor.backup.BackupStore; @@ -21,9 +23,9 @@ import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeManager; import com.google.common.annotations.VisibleForTesting; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; import io.airlift.units.Duration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -31,7 +33,7 @@ import javax.annotation.PreDestroy; import javax.inject.Inject; -import java.io.File; +import java.io.IOException; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; @@ -46,11 +48,12 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicBoolean; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment.tryGetLocalFileSystem; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Maps.filterKeys; import static com.google.common.collect.Maps.filterValues; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.Math.round; import static java.util.Comparator.comparingLong; import static java.util.Objects.requireNonNull; @@ -71,6 +74,7 @@ public class ShardEjector private final Duration interval; private final Optional backupStore; private final ScheduledExecutorService executor; + private final Optional localFileSystem; private final AtomicBoolean started = new AtomicBoolean(); @@ -85,6 +89,7 @@ public ShardEjector( StorageService storageService, StorageManagerConfig config, Optional backupStore, + OrcDataEnvironment environment, RaptorConnectorId connectorId) { this(nodeManager.getCurrentNode().getNodeIdentifier(), @@ -93,6 +98,7 @@ public ShardEjector( storageService, config.getShardEjectorInterval(), backupStore, + environment, connectorId.toString()); } @@ -103,6 +109,7 @@ public ShardEjector( StorageService storageService, Duration interval, Optional backupStore, + OrcDataEnvironment environment, String connectorId) { this.currentNode = requireNonNull(currentNode, "currentNode is null"); @@ -112,6 +119,9 @@ public ShardEjector( this.interval = requireNonNull(interval, "interval is null"); this.backupStore = requireNonNull(backupStore, "backupStore is null"); this.executor = newScheduledThreadPool(1, daemonThreadsNamed("shard-ejector-" + connectorId)); + this.localFileSystem = tryGetLocalFileSystem(requireNonNull(environment, "environment is null")); + + checkState((!backupStore.isPresent() || localFileSystem.isPresent()), "cannot support backup for remote file system"); } @PostConstruct @@ -166,6 +176,7 @@ private void startJob() @VisibleForTesting void process() + throws IOException { checkState(backupStore.isPresent(), "backup store must be present"); @@ -242,8 +253,8 @@ void process() shardManager.replaceShardAssignment(shard.getTableId(), shardUuid, target, false); // delete local file - File file = storageService.getStorageFile(shardUuid); - if (file.exists() && !file.delete()) { + Path file = storageService.getStorageFile(shardUuid); + if (localFileSystem.get().exists(file) && !localFileSystem.get().delete(file, false)) { log.warn("Failed to delete shard file: %s", file); } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardRecoveryManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardRecoveryManager.java index 53684b8765d9e..7775d73603bba 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardRecoveryManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardRecoveryManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.airlift.log.Logger; import com.facebook.presto.raptor.backup.BackupStore; import com.facebook.presto.raptor.metadata.ShardManager; import com.facebook.presto.raptor.metadata.ShardMetadata; @@ -25,9 +26,10 @@ import com.google.common.cache.LoadingCache; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.log.Logger; import io.airlift.units.DataSize; import io.airlift.units.Duration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; import org.weakref.jmx.Flatten; import org.weakref.jmx.Managed; @@ -54,14 +56,16 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import static com.facebook.airlift.concurrent.MoreFutures.addExceptionCallback; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_RECOVERY_ERROR; -import static com.facebook.presto.raptor.storage.OrcStorageManager.xxhash64; +import static com.facebook.presto.raptor.filesystem.FileSystemUtil.xxhash64; +import static com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment.tryGetLocalFileSystem; import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.addExceptionCallback; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.succinctBytes; import static io.airlift.units.DataSize.succinctDataSize; @@ -79,6 +83,7 @@ public class ShardRecoveryManager private final StorageService storageService; private final Optional backupStore; + private final Optional localFileSystem; private final String nodeIdentifier; private final ShardManager shardManager; private final Duration missingShardDiscoveryInterval; @@ -94,12 +99,14 @@ public class ShardRecoveryManager public ShardRecoveryManager( StorageService storageService, Optional backupStore, + OrcDataEnvironment environment, NodeManager nodeManager, ShardManager shardManager, StorageManagerConfig config) { this(storageService, backupStore, + environment, nodeManager, shardManager, config.getMissingShardDiscoveryInterval(), @@ -109,6 +116,7 @@ public ShardRecoveryManager( public ShardRecoveryManager( StorageService storageService, Optional backupStore, + OrcDataEnvironment environment, NodeManager nodeManager, ShardManager shardManager, Duration missingShardDiscoveryInterval, @@ -116,6 +124,8 @@ public ShardRecoveryManager( { this.storageService = requireNonNull(storageService, "storageService is null"); this.backupStore = requireNonNull(backupStore, "backupStore is null"); + this.localFileSystem = tryGetLocalFileSystem(requireNonNull(environment, "environment is null")); + checkState((!backupStore.isPresent() || localFileSystem.isPresent()), "cannot support backup for remote file system"); this.nodeIdentifier = requireNonNull(nodeManager, "nodeManager is null").getCurrentNode().getNodeIdentifier(); this.shardManager = requireNonNull(shardManager, "shardManager is null"); this.missingShardDiscoveryInterval = requireNonNull(missingShardDiscoveryInterval, "missingShardDiscoveryInterval is null"); @@ -185,7 +195,7 @@ private Set getMissingShards() private boolean shardNeedsRecovery(UUID shardUuid, long shardSize) { - File storageFile = storageService.getStorageFile(shardUuid); + File storageFile = localFileSystem.get().pathToFile(storageService.getStorageFile(shardUuid)); return !storageFile.exists() || (storageFile.length() != shardSize); } @@ -203,7 +213,7 @@ public Future recoverShard(UUID shardUuid) @VisibleForTesting void restoreFromBackup(UUID shardUuid, long shardSize, OptionalLong shardXxhash64) { - File storageFile = storageService.getStorageFile(shardUuid); + File storageFile = localFileSystem.get().pathToFile(storageService.getStorageFile(shardUuid)); if (!backupStore.get().shardExists(shardUuid)) { stats.incrementShardRecoveryBackupNotFound(); @@ -219,8 +229,8 @@ void restoreFromBackup(UUID shardUuid, long shardSize, OptionalLong shardXxhash6 } // create a temporary file in the staging directory - File stagingFile = temporarySuffix(storageService.getStagingFile(shardUuid)); - storageService.createParents(stagingFile); + File stagingFile = temporarySuffix(localFileSystem.get().pathToFile(storageService.getStagingFile(shardUuid))); + storageService.createParents(new Path(stagingFile.toURI())); // copy to temporary file log.info("Copying shard %s from backup...", shardUuid); @@ -243,7 +253,7 @@ void restoreFromBackup(UUID shardUuid, long shardSize, OptionalLong shardXxhash6 log.info("Copied shard %s from backup in %s (%s at %s/s)", shardUuid, duration, size, rate); // move to final location - storageService.createParents(storageFile); + storageService.createParents(new Path(storageFile.toURI())); try { Files.move(stagingFile.toPath(), storageFile.toPath(), ATOMIC_MOVE); } @@ -275,7 +285,7 @@ void restoreFromBackup(UUID shardUuid, long shardSize, OptionalLong shardXxhash6 private void quarantineFile(UUID shardUuid, File file, String message) { - File quarantine = new File(storageService.getQuarantineFile(shardUuid).getPath() + ".corrupt"); + File quarantine = new File(localFileSystem.get().pathToFile(storageService.getQuarantineFile(shardUuid)) + ".corrupt"); if (quarantine.exists()) { log.warn("%s Quarantine already exists: %s", message, quarantine); return; @@ -291,9 +301,9 @@ private void quarantineFile(UUID shardUuid, File file, String message) } } - private static boolean isFileCorrupt(File file, long size, OptionalLong xxhash64) + private boolean isFileCorrupt(File file, long size, OptionalLong xxhash64) { - return (file.length() != size) || (xxhash64.isPresent() && (xxhash64(file) != xxhash64.getAsLong())); + return (file.length() != size) || (xxhash64.isPresent() && (xxhash64(localFileSystem.get(), new Path(file.toURI())) != xxhash64.getAsLong())); } @VisibleForTesting diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardRecoveryStats.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardRecoveryStats.java index 1afd4ef75fd1f..e535d98da6a0f 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardRecoveryStats.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardRecoveryStats.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.raptor.storage; -import io.airlift.stats.CounterStat; -import io.airlift.stats.DistributionStat; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.DistributionStat; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.weakref.jmx.Managed; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardStats.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardStats.java index 129f37d28a625..6c1f4702bc658 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardStats.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/ShardStats.java @@ -20,12 +20,11 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.BigintType; -import com.facebook.presto.spi.type.BooleanType; import com.facebook.presto.spi.type.DateType; -import com.facebook.presto.spi.type.DoubleType; import com.facebook.presto.spi.type.TimeType; import com.facebook.presto.spi.type.TimestampType; import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.VarcharType; import com.google.common.collect.ImmutableMap; import io.airlift.slice.Slice; @@ -37,6 +36,9 @@ import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_ERROR; +import static com.facebook.presto.raptor.storage.OrcStorageManager.toOrcFileType; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static java.lang.Double.isInfinite; import static java.lang.Double.isNaN; import static org.joda.time.DateTimeZone.UTC; @@ -58,20 +60,27 @@ public static Slice truncateIndexValue(Slice slice) return slice; } - public static Optional computeColumnStats(OrcReader orcReader, long columnId, Type type) + public static Optional computeColumnStats(OrcReader orcReader, long columnId, Type type, TypeManager typeManager) throws IOException { - return Optional.ofNullable(doComputeColumnStats(orcReader, columnId, type)); + return Optional.ofNullable(doComputeColumnStats(orcReader, columnId, type, typeManager)); } - private static ColumnStats doComputeColumnStats(OrcReader orcReader, long columnId, Type type) + private static ColumnStats doComputeColumnStats(OrcReader orcReader, long columnId, Type type, TypeManager typeManager) throws IOException { + StorageTypeConverter storageTypeConverter = new StorageTypeConverter(typeManager); + int columnIndex = columnIndex(orcReader.getColumnNames(), columnId); - OrcBatchRecordReader reader = orcReader.createBatchRecordReader(ImmutableMap.of(columnIndex, type), OrcPredicate.TRUE, UTC, newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); + OrcBatchRecordReader reader = orcReader.createBatchRecordReader( + storageTypeConverter.toStorageTypes(ImmutableMap.of(columnIndex, toOrcFileType(type, typeManager))), + OrcPredicate.TRUE, + UTC, + newSimpleAggregatedMemoryContext(), + INITIAL_BATCH_SIZE); - if (type.equals(BooleanType.BOOLEAN)) { - return indexBoolean(type, reader, columnIndex, columnId); + if (type.equals(BOOLEAN)) { + return indexBoolean(reader, columnIndex, columnId); } if (type.equals(BigintType.BIGINT) || type.equals(DateType.DATE) || @@ -79,8 +88,8 @@ private static ColumnStats doComputeColumnStats(OrcReader orcReader, long column type.equals(TimestampType.TIMESTAMP)) { return indexLong(type, reader, columnIndex, columnId); } - if (type.equals(DoubleType.DOUBLE)) { - return indexDouble(type, reader, columnIndex, columnId); + if (type.equals(DOUBLE)) { + return indexDouble(reader, columnIndex, columnId); } if (type instanceof VarcharType) { return indexString(type, reader, columnIndex, columnId); @@ -97,7 +106,7 @@ private static int columnIndex(List columnNames, long columnId) return index; } - private static ColumnStats indexBoolean(Type type, OrcBatchRecordReader reader, int columnIndex, long columnId) + private static ColumnStats indexBoolean(OrcBatchRecordReader reader, int columnIndex, long columnId) throws IOException { boolean minSet = false; @@ -110,13 +119,13 @@ private static ColumnStats indexBoolean(Type type, OrcBatchRecordReader reader, if (batchSize <= 0) { break; } - Block block = reader.readBlock(type, columnIndex); + Block block = reader.readBlock(columnIndex); for (int i = 0; i < batchSize; i++) { if (block.isNull(i)) { continue; } - boolean value = type.getBoolean(block, i); + boolean value = BOOLEAN.getBoolean(block, i); if (!minSet || Boolean.compare(value, min) < 0) { minSet = true; min = value; @@ -146,7 +155,7 @@ private static ColumnStats indexLong(Type type, OrcBatchRecordReader reader, int if (batchSize <= 0) { break; } - Block block = reader.readBlock(type, columnIndex); + Block block = reader.readBlock(columnIndex); for (int i = 0; i < batchSize; i++) { if (block.isNull(i)) { @@ -169,7 +178,7 @@ private static ColumnStats indexLong(Type type, OrcBatchRecordReader reader, int maxSet ? max : null); } - private static ColumnStats indexDouble(Type type, OrcBatchRecordReader reader, int columnIndex, long columnId) + private static ColumnStats indexDouble(OrcBatchRecordReader reader, int columnIndex, long columnId) throws IOException { boolean minSet = false; @@ -182,13 +191,13 @@ private static ColumnStats indexDouble(Type type, OrcBatchRecordReader reader, i if (batchSize <= 0) { break; } - Block block = reader.readBlock(type, columnIndex); + Block block = reader.readBlock(columnIndex); for (int i = 0; i < batchSize; i++) { if (block.isNull(i)) { continue; } - double value = type.getDouble(block, i); + double value = DOUBLE.getDouble(block, i); if (isNaN(value)) { continue; } @@ -231,7 +240,7 @@ private static ColumnStats indexString(Type type, OrcBatchRecordReader reader, i if (batchSize <= 0) { break; } - Block block = reader.readBlock(type, columnIndex); + Block block = reader.readBlock(columnIndex); for (int i = 0; i < batchSize; i++) { if (block.isNull(i)) { diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManager.java index 47e5bd981e3be..f2147cd285a38 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.raptor.storage; import com.facebook.presto.raptor.RaptorColumnHandle; +import com.facebook.presto.raptor.filesystem.FileSystemContext; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.Type; @@ -28,6 +29,7 @@ public interface StorageManager { default ConnectorPageSource getPageSource( + FileSystemContext fileSystemContext, UUID shardUuid, OptionalInt bucketNumber, List columnIds, @@ -35,10 +37,11 @@ default ConnectorPageSource getPageSource( TupleDomain effectivePredicate, ReaderAttributes readerAttributes) { - return getPageSource(shardUuid, bucketNumber, columnIds, columnTypes, effectivePredicate, readerAttributes, OptionalLong.empty(), Optional.empty()); + return getPageSource(fileSystemContext, shardUuid, bucketNumber, columnIds, columnTypes, effectivePredicate, readerAttributes, OptionalLong.empty(), Optional.empty()); } ConnectorPageSource getPageSource( + FileSystemContext fileSystemContext, UUID shardUuid, OptionalInt bucketNumber, List columnIds, @@ -49,6 +52,7 @@ ConnectorPageSource getPageSource( Optional> allColumnTypes); StoragePageSink createStoragePageSink( + FileSystemContext fileSystemContext, long transactionId, OptionalInt bucketNumber, List columnIds, diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java index fe3d773240737..1a209de852cce 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.DefunctConfig; +import com.facebook.airlift.configuration.LegacyConfig; import com.facebook.presto.orc.metadata.CompressionKind; import com.facebook.presto.spi.type.TimeZoneKey; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.DefunctConfig; -import io.airlift.configuration.LegacyConfig; import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.airlift.units.MaxDataSize; @@ -30,7 +30,7 @@ import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; -import java.io.File; +import java.net.URI; import java.util.TimeZone; import java.util.concurrent.TimeUnit; @@ -44,7 +44,8 @@ @DefunctConfig("storage.backup-directory") public class StorageManagerConfig { - private File dataDirectory; + private URI dataDirectory; + private String fileSystemProvider = "file"; private DataSize minAvailableSpace = new DataSize(0, BYTE); private Duration shardRecoveryTimeout = new Duration(30, TimeUnit.SECONDS); private Duration missingShardDiscoveryInterval = new Duration(5, TimeUnit.MINUTES); @@ -70,18 +71,34 @@ public class StorageManagerConfig private DataSize maxBufferSize = new DataSize(256, MEGABYTE); private int oneSplitPerBucketThreshold; private String shardDayBoundaryTimeZone = TimeZoneKey.UTC_KEY.getId(); + private int maxAllowedFilesPerWriter = Integer.MAX_VALUE; + private boolean zstdJniDecompressionEnabled; @NotNull - public File getDataDirectory() + public URI getDataDirectory() { return dataDirectory; } @Config("storage.data-directory") - @ConfigDescription("Base directory to use for storing shard data") - public StorageManagerConfig setDataDirectory(File dataDirectory) + @ConfigDescription("Base URI to use for storing shard data") + public StorageManagerConfig setDataDirectory(URI dataURI) { - this.dataDirectory = dataDirectory; + this.dataDirectory = dataURI; + return this; + } + + @NotNull + public String getFileSystemProvider() + { + return fileSystemProvider; + } + + @Config("storage.file-system") + @ConfigDescription("File system used for storage (e.g. file, hdfs)") + public StorageManagerConfig setFileSystemProvider(String fileSystemProvider) + { + this.fileSystemProvider = fileSystemProvider; return this; } @@ -168,7 +185,7 @@ public StorageManagerConfig setOrcLazyReadSmallRanges(boolean orcLazyReadSmallRa public enum OrcOptimizedWriterStage { - DISABLED, ENABLED, ENABLED_AND_VALIDATED + ENABLED, ENABLED_AND_VALIDATED } public OrcOptimizedWriterStage getOrcOptimizedWriterStage() @@ -368,6 +385,20 @@ public StorageManagerConfig setMaxBufferSize(DataSize maxBufferSize) return this; } + @Min(1) + public int getMaxAllowedFilesPerWriter() + { + return maxAllowedFilesPerWriter; + } + + @Config("storage.max-allowed-files-per-writer") + @ConfigDescription("Maximum number of files that can be created per writer for a query. Default value is Integer.MAX_VALUE") + public StorageManagerConfig setMaxAllowedFilesPerWriter(int maxAllowedFilesPerWriter) + { + this.maxAllowedFilesPerWriter = maxAllowedFilesPerWriter; + return this; + } + public boolean isCompactionEnabled() { return compactionEnabled; @@ -417,4 +448,16 @@ public StorageManagerConfig setShardDayBoundaryTimeZone(String timeZone) this.shardDayBoundaryTimeZone = timeZone; return this; } + + public boolean isZstdJniDecompressionEnabled() + { + return zstdJniDecompressionEnabled; + } + + @Config("storage.zstd-jni-decompression-enabled") + public StorageManagerConfig setZstdJniDecompressionEnabled(boolean zstdJniDecompressionEnabled) + { + this.zstdJniDecompressionEnabled = zstdJniDecompressionEnabled; + return this; + } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageModule.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageModule.java index 0178699678abd..f87248cab72a1 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageModule.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageModule.java @@ -13,6 +13,18 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.presto.orc.CacheStatsMBean; +import com.facebook.presto.orc.CachingStripeMetadataSource; +import com.facebook.presto.orc.OrcDataSourceId; +import com.facebook.presto.orc.StorageStripeMetadataSource; +import com.facebook.presto.orc.StripeMetadataSource; +import com.facebook.presto.orc.StripeReader.StripeId; +import com.facebook.presto.orc.StripeReader.StripeStreamId; +import com.facebook.presto.orc.cache.CachingOrcFileTailSource; +import com.facebook.presto.orc.cache.OrcCacheConfig; +import com.facebook.presto.orc.cache.OrcFileTailSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; +import com.facebook.presto.orc.metadata.OrcFileTail; import com.facebook.presto.raptor.backup.BackupManager; import com.facebook.presto.raptor.metadata.AssignmentLimiter; import com.facebook.presto.raptor.metadata.DatabaseShardManager; @@ -30,11 +42,20 @@ import com.facebook.presto.raptor.storage.organization.ShardOrganizer; import com.facebook.presto.raptor.storage.organization.TemporalFunction; import com.google.common.base.Ticker; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.inject.Binder; import com.google.inject.Module; +import com.google.inject.Provides; import com.google.inject.Scopes; +import io.airlift.slice.Slice; +import org.weakref.jmx.MBeanExporter; -import static io.airlift.configuration.ConfigBinder.configBinder; +import javax.inject.Singleton; + +import java.util.concurrent.TimeUnit; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; import static org.weakref.jmx.ObjectNames.generatedNameOf; import static org.weakref.jmx.guice.ExportBinder.newExporter; @@ -56,11 +77,11 @@ public void configure(Binder binder) configBinder(binder).bindConfig(BucketBalancerConfig.class); configBinder(binder).bindConfig(ShardCleanerConfig.class); configBinder(binder).bindConfig(MetadataConfig.class); + configBinder(binder).bindConfig(OrcCacheConfig.class, connectorId); binder.bind(Ticker.class).toInstance(Ticker.systemTicker()); binder.bind(StorageManager.class).to(OrcStorageManager.class).in(Scopes.SINGLETON); - binder.bind(StorageService.class).to(FileStorageService.class).in(Scopes.SINGLETON); binder.bind(ShardManager.class).to(DatabaseShardManager.class).in(Scopes.SINGLETON); binder.bind(ShardRecorder.class).to(DatabaseShardRecorder.class).in(Scopes.SINGLETON); binder.bind(DatabaseShardManager.class).in(Scopes.SINGLETON); @@ -90,4 +111,50 @@ public void configure(Binder binder) newExporter(binder).export(BucketBalancer.class).as(generatedNameOf(BucketBalancer.class, connectorId)); newExporter(binder).export(JobFactory.class).withGeneratedName(); } + + @Singleton + @Provides + public OrcFileTailSource createOrcFileTailSource(OrcCacheConfig orcCacheConfig, MBeanExporter exporter) + { + OrcFileTailSource orcFileTailSource = new StorageOrcFileTailSource(); + if (orcCacheConfig.isFileTailCacheEnabled()) { + Cache cache = CacheBuilder.newBuilder() + .maximumWeight(orcCacheConfig.getFileTailCacheSize().toBytes()) + .weigher((id, tail) -> ((OrcFileTail) tail).getFooterSize() + ((OrcFileTail) tail).getMetadataSize()) + .expireAfterAccess(orcCacheConfig.getFileTailCacheTtlSinceLastAccess().toMillis(), TimeUnit.MILLISECONDS) + .recordStats() + .build(); + CacheStatsMBean cacheStatsMBean = new CacheStatsMBean(cache); + orcFileTailSource = new CachingOrcFileTailSource(orcFileTailSource, cache); + exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_OrcFileTail"), cacheStatsMBean); + } + return orcFileTailSource; + } + + @Singleton + @Provides + public StripeMetadataSource createStripeMetadataSource(OrcCacheConfig orcCacheConfig, MBeanExporter exporter) + { + StripeMetadataSource stripeMetadataSource = new StorageStripeMetadataSource(); + if (orcCacheConfig.isStripeMetadataCacheEnabled()) { + Cache footerCache = CacheBuilder.newBuilder() + .maximumWeight(orcCacheConfig.getStripeFooterCacheSize().toBytes()) + .weigher((id, footer) -> ((Slice) footer).length()) + .expireAfterAccess(orcCacheConfig.getStripeFooterCacheTtlSinceLastAccess().toMillis(), TimeUnit.MILLISECONDS) + .recordStats() + .build(); + Cache streamCache = CacheBuilder.newBuilder() + .maximumWeight(orcCacheConfig.getStripeStreamCacheSize().toBytes()) + .weigher((id, stream) -> ((Slice) stream).length()) + .expireAfterAccess(orcCacheConfig.getStripeStreamCacheTtlSinceLastAccess().toMillis(), TimeUnit.MILLISECONDS) + .recordStats() + .build(); + CacheStatsMBean footerCacheStatsMBean = new CacheStatsMBean(footerCache); + CacheStatsMBean streamCacheStatsMBean = new CacheStatsMBean(streamCache); + stripeMetadataSource = new CachingStripeMetadataSource(stripeMetadataSource, footerCache, streamCache); + exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_StripeFooter"), footerCacheStatsMBean); + exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_StripeStream"), streamCacheStatsMBean); + } + return stripeMetadataSource; + } } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageService.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageService.java index a035e0ddc4432..98bdd62307e8f 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageService.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageService.java @@ -13,7 +13,8 @@ */ package com.facebook.presto.raptor.storage; -import java.io.File; +import org.apache.hadoop.fs.Path; + import java.util.Set; import java.util.UUID; @@ -23,13 +24,15 @@ public interface StorageService long getAvailableBytes(); - void createParents(File file); + void createParents(Path file); - File getStorageFile(UUID shardUuid); + Path getStorageFile(UUID shardUuid); - File getStagingFile(UUID shardUuid); + Path getStagingFile(UUID shardUuid); - File getQuarantineFile(UUID shardUuid); + Path getQuarantineFile(UUID shardUuid); Set getStorageShards(); + + void promoteFromStagingToStorage(UUID shardUuid); } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageTypeConverter.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageTypeConverter.java index 71ff53c91088f..ce224f388ee81 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageTypeConverter.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageTypeConverter.java @@ -22,6 +22,8 @@ import com.facebook.presto.spi.type.TypeSignatureParameter; import com.google.common.collect.ImmutableList; +import java.util.Map; + import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -39,6 +41,7 @@ import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.spi.type.Varchars.isVarcharType; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.util.Objects.requireNonNull; public class StorageTypeConverter @@ -90,6 +93,11 @@ else if (type instanceof MapType) { return storageType; } + public Map toStorageTypes(Map includedColumns) + { + return includedColumns.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, entry -> toStorageType(entry.getValue()))); + } + private Type mapType(Type keyType, Type valueType) { return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of( diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/OrganizationJob.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/OrganizationJob.java index b18d443bd4e49..3dfbc7449ec58 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/OrganizationJob.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/OrganizationJob.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.raptor.storage.organization; +import com.facebook.airlift.log.Logger; import com.facebook.presto.raptor.metadata.ColumnInfo; import com.facebook.presto.raptor.metadata.MetadataDao; import com.facebook.presto.raptor.metadata.ShardInfo; import com.facebook.presto.raptor.metadata.ShardManager; import com.facebook.presto.raptor.metadata.TableColumn; import com.facebook.presto.raptor.metadata.TableMetadata; -import io.airlift.log.Logger; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactionManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactionManager.java index b85cb27138201..651801c571a2f 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactionManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactionManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.raptor.storage.organization; +import com.facebook.airlift.log.Logger; import com.facebook.presto.raptor.metadata.ForMetadata; import com.facebook.presto.raptor.metadata.MetadataDao; import com.facebook.presto.raptor.metadata.ShardManager; @@ -25,7 +26,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ListMultimap; import com.google.common.collect.Multimaps; -import io.airlift.log.Logger; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.skife.jdbi.v2.IDBI; @@ -44,12 +44,12 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.raptor.storage.organization.ShardOrganizerUtil.getOrganizationEligibleShards; import static com.facebook.presto.raptor.util.DatabaseUtil.onDemandDao; import static com.facebook.presto.spi.type.DateType.DATE; import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newScheduledThreadPool; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactor.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactor.java index 6c99abfcd0dbd..0ceebd0c76668 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactor.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardCompactor.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.raptor.storage.organization; +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.DistributionStat; +import com.facebook.presto.raptor.filesystem.FileSystemContext; import com.facebook.presto.raptor.metadata.ColumnInfo; import com.facebook.presto.raptor.metadata.ShardInfo; import com.facebook.presto.raptor.storage.ReaderAttributes; @@ -25,8 +28,6 @@ import com.facebook.presto.spi.predicate.TupleDomain; import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableList; -import io.airlift.stats.CounterStat; -import io.airlift.stats.DistributionStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -43,8 +44,8 @@ import java.util.Set; import java.util.UUID; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static io.airlift.units.Duration.nanosSince; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; @@ -75,7 +76,7 @@ public List compact(long transactionId, OptionalInt bucketNumber, Set List columnIds = columns.stream().map(ColumnInfo::getColumnId).collect(toList()); List columnTypes = columns.stream().map(ColumnInfo::getType).collect(toList()); - StoragePageSink storagePageSink = storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, false); + StoragePageSink storagePageSink = storageManager.createStoragePageSink(FileSystemContext.DEFAULT_RAPTOR_CONTEXT, transactionId, bucketNumber, columnIds, columnTypes, false); List shardInfos; try { @@ -94,7 +95,7 @@ private List compact(StoragePageSink storagePageSink, OptionalInt buc throws IOException { for (UUID uuid : uuids) { - try (ConnectorPageSource pageSource = storageManager.getPageSource(uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), readerAttributes)) { + try (ConnectorPageSource pageSource = storageManager.getPageSource(FileSystemContext.DEFAULT_RAPTOR_CONTEXT, uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), readerAttributes)) { while (!pageSource.isFinished()) { Page page = pageSource.getNextPage(); if (isNullOrEmptyPage(page)) { @@ -127,10 +128,10 @@ public List compactSorted(long transactionId, OptionalInt bucketNumbe .collect(toList()); Queue rowSources = new PriorityQueue<>(); - StoragePageSink outputPageSink = storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, false); + StoragePageSink outputPageSink = storageManager.createStoragePageSink(FileSystemContext.DEFAULT_RAPTOR_CONTEXT, transactionId, bucketNumber, columnIds, columnTypes, false); try { for (UUID uuid : uuids) { - ConnectorPageSource pageSource = storageManager.getPageSource(uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), readerAttributes); + ConnectorPageSource pageSource = storageManager.getPageSource(FileSystemContext.DEFAULT_RAPTOR_CONTEXT, uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), readerAttributes); SortedPageSource rowSource = new SortedPageSource(pageSource, columnTypes, sortIndexes, sortOrders); rowSources.add(rowSource); } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardOrganizationManager.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardOrganizationManager.java index ab327e94720d0..025e5f3865c17 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardOrganizationManager.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardOrganizationManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.raptor.storage.organization; +import com.facebook.airlift.log.Logger; import com.facebook.presto.raptor.metadata.ForMetadata; import com.facebook.presto.raptor.metadata.MetadataDao; import com.facebook.presto.raptor.metadata.ShardManager; @@ -25,7 +26,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; -import io.airlift.log.Logger; import io.airlift.units.Duration; import org.skife.jdbi.v2.IDBI; @@ -44,19 +44,18 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static com.facebook.airlift.concurrent.MoreFutures.allAsList; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.raptor.storage.organization.ShardOrganizerUtil.createOrganizationSet; import static com.facebook.presto.raptor.storage.organization.ShardOrganizerUtil.getOrganizationEligibleShards; import static com.facebook.presto.raptor.storage.organization.ShardOrganizerUtil.getShardsByDaysBuckets; import static com.facebook.presto.raptor.util.DatabaseUtil.onDemandDao; import static com.google.common.collect.Sets.difference; import static com.google.common.collect.Sets.newConcurrentHashSet; -import static io.airlift.concurrent.MoreFutures.allAsList; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.lang.Math.max; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newScheduledThreadPool; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; @@ -148,7 +147,7 @@ private void startDiscovery() discoveryService.scheduleWithFixedDelay(() -> { try { // jitter to avoid overloading database and overloading the backup store - SECONDS.sleep(ThreadLocalRandom.current().nextLong(1, organizationDiscoveryIntervalMillis)); + MILLISECONDS.sleep(ThreadLocalRandom.current().nextLong(1, organizationDiscoveryIntervalMillis)); log.info("Running shard organizer..."); submitJobs(discoverAndInitializeTablesToOrganize()); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardOrganizer.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardOrganizer.java index 2259ae4b9d452..e85fed5722119 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardOrganizer.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/organization/ShardOrganizer.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.raptor.storage.organization; +import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.raptor.storage.StorageManagerConfig; -import io.airlift.concurrent.ThreadPoolExecutorMBean; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -30,9 +30,9 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.Sets.newConcurrentHashSet; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static java.util.Objects.requireNonNull; import static java.util.concurrent.CompletableFuture.runAsync; import static java.util.concurrent.Executors.newFixedThreadPool; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/systemtables/TableMetadataSystemTable.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/systemtables/TableMetadataSystemTable.java index d00390b97d3cf..ae46c726216d6 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/systemtables/TableMetadataSystemTable.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/systemtables/TableMetadataSystemTable.java @@ -89,7 +89,8 @@ public TableMetadataSystemTable(@ForMetadata IDBI dbi, TypeManager typeManager) new ColumnMetadata("distribution_name", VARCHAR), new ColumnMetadata("bucket_count", BIGINT), new ColumnMetadata("bucketing_columns", arrayOfVarchar), - new ColumnMetadata("organized", BOOLEAN))); + new ColumnMetadata("organized", BOOLEAN), + new ColumnMetadata("table_supports_delta_delete", BOOLEAN))); } @Override @@ -190,6 +191,9 @@ private static List buildPages(MetadataDao dao, ConnectorTableMetadata tab // organized BOOLEAN.writeBoolean(pageBuilder.nextBlockBuilder(), tableRow.isOrganized()); + + // delta delete enabled + BOOLEAN.writeBoolean(pageBuilder.nextBlockBuilder(), tableRow.isTableSupportsDeltaDelete()); } return pageBuilder.build(); diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/util/ConcatPageSource.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/util/ConcatPageSource.java index b24406e8e600e..cfdea869fd966 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/util/ConcatPageSource.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/util/ConcatPageSource.java @@ -28,6 +28,7 @@ public class ConcatPageSource private ConnectorPageSource current; private long completedBytes; + private long completedPositions; private long readTimeNanos; public ConcatPageSource(Iterator iterator) @@ -42,6 +43,13 @@ public long getCompletedBytes() return completedBytes + ((current != null) ? current.getCompletedBytes() : 0); } + @Override + public long getCompletedPositions() + { + setup(); + return completedPositions; + } + @Override public long getReadTimeNanos() { @@ -70,6 +78,7 @@ public Page getNextPage() } completedBytes += current.getCompletedBytes(); + completedPositions += current.getCompletedPositions(); readTimeNanos += current.getReadTimeNanos(); current = null; } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/util/PrioritizedFifoExecutor.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/util/PrioritizedFifoExecutor.java index eaaed6d5b8b9d..28749fb196383 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/util/PrioritizedFifoExecutor.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/util/PrioritizedFifoExecutor.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.raptor.util; +import com.facebook.airlift.log.Logger; import com.google.common.collect.ComparisonChain; import com.google.common.util.concurrent.ExecutionList; import com.google.common.util.concurrent.ListenableFuture; -import io.airlift.log.Logger; import javax.annotation.concurrent.ThreadSafe; @@ -34,7 +34,7 @@ import static java.util.Objects.requireNonNull; /** - * This class is based on io.airlift.concurrent.BoundedExecutor + * This class is based on com.facebook.airlift.concurrent.BoundedExecutor */ @ThreadSafe public class PrioritizedFifoExecutor diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/util/RebindSafeMBeanServer.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/util/RebindSafeMBeanServer.java index bece1dfaf56d9..4111faba0ab28 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/util/RebindSafeMBeanServer.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/util/RebindSafeMBeanServer.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.raptor.util; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import javax.annotation.concurrent.ThreadSafe; import javax.management.Attribute; diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/util/SyncingFileSystem.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/util/SyncingFileSystem.java deleted file mode 100644 index 420d8052baa73..0000000000000 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/util/SyncingFileSystem.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.raptor.util; - -import io.airlift.slice.XxHash64; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RawLocalFileSystem; -import org.apache.hadoop.util.Progressable; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -import static java.util.Objects.requireNonNull; - -public final class SyncingFileSystem - extends RawLocalFileSystem -{ - public SyncingFileSystem(Configuration configuration) - throws IOException - { - initialize(getUri(), configuration); - } - - @Override - public FSDataOutputStream create(Path path, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) - throws IOException - { - if (exists(path) && !overwrite) { - throw new IOException("file already exists: " + path); - } - Path parent = path.getParent(); - if ((parent != null) && !mkdirs(parent)) { - throw new IOException("mkdirs failed to create " + parent.toString()); - } - return new FSDataOutputStream( - new BufferedOutputStream(new LocalFileOutputStream(pathToFile(path)), bufferSize), - statistics); - } - - private static class LocalFileOutputStream - extends OutputStream - { - private final byte[] oneByte = new byte[1]; - private final XxHash64 hash = new XxHash64(); - private final File file; - private final FileOutputStream out; - private boolean closed; - - private LocalFileOutputStream(File file) - throws IOException - { - this.file = requireNonNull(file, "file is null"); - this.out = new FileOutputStream(file); - } - - @Override - public void close() - throws IOException - { - if (closed) { - return; - } - closed = true; - - flush(); - out.getFD().sync(); - out.close(); - - // extremely paranoid code to detect a broken local file system - try (InputStream in = new FileInputStream(file)) { - if (hash.hash() != XxHash64.hash(in)) { - throw new IOException("File is corrupt after write"); - } - } - } - - @Override - public void flush() - throws IOException - { - out.flush(); - } - - @Override - public void write(byte[] b, int off, int len) - throws IOException - { - out.write(b, off, len); - hash.update(b, off, len); - } - - @SuppressWarnings("NumericCastThatLosesPrecision") - @Override - public void write(int b) - throws IOException - { - oneByte[0] = (byte) (b & 0xFF); - write(oneByte, 0, 1); - } - } -} diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/RaptorQueryRunner.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/RaptorQueryRunner.java index 89e79da4db541..ba30e4790c582 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/RaptorQueryRunner.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/RaptorQueryRunner.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.raptor; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.log.Logging; import com.facebook.presto.Session; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.metadata.SessionPropertyManager; @@ -22,8 +24,6 @@ import com.facebook.presto.tests.DistributedQueryRunner; import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import io.airlift.log.Logging; import io.airlift.tpch.TpchTable; import org.intellij.lang.annotations.Language; @@ -43,16 +43,17 @@ public final class RaptorQueryRunner private RaptorQueryRunner() {} - public static DistributedQueryRunner createRaptorQueryRunner(Map extraProperties, boolean loadTpch, boolean bucketed) + public static DistributedQueryRunner createRaptorQueryRunner(Map extraProperties, boolean loadTpch, boolean bucketed, boolean useHdfs) throws Exception { - return createRaptorQueryRunner(extraProperties, loadTpch, bucketed, ImmutableMap.of()); + return createRaptorQueryRunner(extraProperties, loadTpch, bucketed, useHdfs, ImmutableMap.of()); } public static DistributedQueryRunner createRaptorQueryRunner( Map extraProperties, boolean loadTpch, boolean bucketed, + boolean useHdfs, Map extraRaptorProperties) throws Exception { @@ -63,16 +64,29 @@ public static DistributedQueryRunner createRaptorQueryRunner( queryRunner.installPlugin(new RaptorPlugin()); File baseDir = queryRunner.getCoordinator().getBaseDataDir().toFile(); - Map raptorProperties = ImmutableMap.builder() - .putAll(extraRaptorProperties) + + ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.putAll(extraRaptorProperties) .put("metadata.db.type", "h2") .put("metadata.db.connections.max", "100") .put("metadata.db.filename", new File(baseDir, "db").getAbsolutePath()) - .put("storage.data-directory", new File(baseDir, "data").getAbsolutePath()) - .put("storage.max-shard-rows", "2000") - .put("backup.provider", "file") - .put("backup.directory", new File(baseDir, "backup").getAbsolutePath()) - .build(); + .put("storage.max-shard-rows", "2000"); + + if (useHdfs) { + builder.put("storage.file-system", "hdfs") + .put("cache.base-directory", "file://" + new File(baseDir, "cache").getAbsolutePath()) + .put("cache.max-in-memory-cache-size", "100MB") + .put("cache.validation-enabled", "true") + .put("storage.data-directory", queryRunner.getCoordinator().getBaseDataDir().resolve("hive_data").toFile().toURI().toString()); + } + else { + builder.put("backup.provider", "file") + .put("backup.directory", new File(baseDir, "backup").getAbsolutePath()) + .put("storage.file-system", "file") + .put("storage.data-directory", new File(baseDir, "data").toURI().toString()); + } + + Map raptorProperties = builder.build(); queryRunner.createCatalog("raptor", "raptor", raptorProperties); @@ -145,7 +159,7 @@ public static void main(String[] args) { Logging.initialize(); Map properties = ImmutableMap.of("http-server.http.port", "8080"); - DistributedQueryRunner queryRunner = createRaptorQueryRunner(properties, false, false); + DistributedQueryRunner queryRunner = createRaptorQueryRunner(properties, true, false, false); Thread.sleep(10); Logger log = Logger.get(RaptorQueryRunner.class); log.info("======== SERVER STARTED ========"); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorConnector.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorConnector.java index 5c28c9a16d324..4dfe3395ee5e5 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorConnector.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorConnector.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.raptor; +import com.facebook.airlift.bootstrap.LifeCycleManager; import com.facebook.presto.PagesIndexPageSorter; import com.facebook.presto.operator.PagesIndex; import com.facebook.presto.plugin.base.security.AllowAllAccessControl; @@ -45,7 +46,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.io.Files; -import io.airlift.bootstrap.LifeCycleManager; import io.airlift.slice.Slice; import org.joda.time.DateTimeZone; import org.skife.jdbi.v2.DBI; @@ -58,6 +58,7 @@ import java.util.Collection; import java.util.Optional; +import static com.facebook.presto.raptor.RaptorTableProperties.TABLE_SUPPORTS_DELTA_DELETE; import static com.facebook.presto.raptor.RaptorTableProperties.TEMPORAL_COLUMN_PROPERTY; import static com.facebook.presto.raptor.metadata.SchemaDaoUtil.createTablesWithRetry; import static com.facebook.presto.raptor.metadata.TestDatabaseShardManager.createShardManager; @@ -101,12 +102,12 @@ public void setup() NodeManager nodeManager = new TestingNodeManager(); NodeSupplier nodeSupplier = nodeManager::getWorkerNodes; ShardManager shardManager = createShardManager(dbi); - StorageManager storageManager = createOrcStorageManager(dbi, dataDir, true); + StorageManager storageManager = createOrcStorageManager(dbi, dataDir); StorageManagerConfig config = new StorageManagerConfig(); connector = new RaptorConnector( new LifeCycleManager(ImmutableList.of(), null), new TestingNodeManager(), - new RaptorMetadataFactory(connectorId, dbi, shardManager), + new RaptorMetadataFactory(connectorId, dbi, shardManager, new TypeRegistry()), new RaptorSplitManager(connectorId, nodeSupplier, shardManager, false), new RaptorPageSourceProvider(storageManager), new RaptorPageSinkProvider(storageManager, @@ -118,7 +119,8 @@ public void setup() new RaptorTableProperties(typeRegistry), ImmutableSet.of(), new AllowAllAccessControl(), - dbi); + dbi, + ImmutableSet.of()); } @AfterMethod(alwaysRun = true) @@ -224,7 +226,8 @@ private void assertSplitShard(Type temporalType, String min, String max, String System.currentTimeMillis(), new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties(), ImmutableMap.of(), - true); + true, + Optional.empty()); ConnectorTransactionHandle transaction = connector.beginTransaction(READ_COMMITTED, false); connector.getMetadata(transaction).createTable( @@ -232,7 +235,7 @@ private void assertSplitShard(Type temporalType, String min, String max, String new ConnectorTableMetadata( new SchemaTableName("test", "test"), ImmutableList.of(new ColumnMetadata("id", BIGINT), new ColumnMetadata("time", temporalType)), - ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "time")), + ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "time", TABLE_SUPPORTS_DELTA_DELETE, false)), false); connector.commit(transaction); @@ -273,7 +276,8 @@ private long createTable(String name) SESSION, new ConnectorTableMetadata( new SchemaTableName("test", name), - ImmutableList.of(new ColumnMetadata("id", BIGINT))), + ImmutableList.of(new ColumnMetadata("id", BIGINT)), + ImmutableMap.of(TABLE_SUPPORTS_DELTA_DELETE, false)), false); connector.commit(transaction); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorPlugin.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorPlugin.java index 5fe6cb3187c1a..a09a2897ca3bd 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorPlugin.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/TestRaptorPlugin.java @@ -24,10 +24,10 @@ import java.util.Map; import java.util.ServiceLoader; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.testing.Assertions.assertInstanceOf; public class TestRaptorPlugin { @@ -45,7 +45,7 @@ public void testPlugin() Map config = ImmutableMap.builder() .put("metadata.db.type", "h2") .put("metadata.db.filename", tmpDir.getAbsolutePath()) - .put("storage.data-directory", tmpDir.getAbsolutePath()) + .put("storage.data-directory", tmpDir.toURI().toString()) .build(); factory.create("test", config, new TestingConnectorContext()); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestBackupConfig.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestBackupConfig.java index a902346bb22aa..cfce304a1eb90 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestBackupConfig.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestBackupConfig.java @@ -19,9 +19,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestBackupManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestBackupManager.java index f3423b1bb5674..2edf1f9325f61 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestBackupManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestBackupManager.java @@ -13,10 +13,12 @@ */ package com.facebook.presto.raptor.backup; +import com.facebook.presto.raptor.filesystem.LocalFileStorageService; +import com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment; import com.facebook.presto.raptor.storage.BackupStats; -import com.facebook.presto.raptor.storage.FileStorageService; import com.facebook.presto.spi.PrestoException; import com.google.common.io.Files; +import org.apache.hadoop.fs.Path; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -54,7 +56,7 @@ public class TestBackupManager private File temporary; private BackupStore backupStore; - private FileStorageService storageService; + private LocalFileStorageService storageService; private BackupManager backupManager; @BeforeMethod @@ -66,10 +68,10 @@ public void setup() fileStore.start(); backupStore = new TestingBackupStore(fileStore); - storageService = new FileStorageService(new File(temporary, "data")); + storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), new File(temporary, "data").toURI()); storageService.start(); - backupManager = new BackupManager(Optional.of(backupStore), storageService, 5); + backupManager = new BackupManager(Optional.of(backupStore), storageService, new LocalOrcDataEnvironment(), 5); } @AfterMethod(alwaysRun = true) @@ -94,7 +96,7 @@ public void testSimple() Files.write("hello world", file, UTF_8); uuids.add(randomUUID()); - futures.add(backupManager.submit(uuids.get(i), file)); + futures.add(backupManager.submit(uuids.get(i), path(file))); } futures.forEach(CompletableFuture::join); for (UUID uuid : uuids) { @@ -116,7 +118,7 @@ public void testFailure() Files.write("hello world", file, UTF_8); try { - backupManager.submit(FAILURE_UUID, file).get(1, SECONDS); + backupManager.submit(FAILURE_UUID, path(file)).get(1, SECONDS); fail("expected exception"); } catch (ExecutionException wrapper) { @@ -140,7 +142,7 @@ public void testCorruption() Files.write("hello world", file, UTF_8); try { - backupManager.submit(CORRUPTION_UUID, file).get(1, SECONDS); + backupManager.submit(CORRUPTION_UUID, path(file)).get(1, SECONDS); fail("expected exception"); } catch (ExecutionException wrapper) { @@ -149,7 +151,7 @@ public void testCorruption() assertEquals(e.getMessage(), "Backup is corrupt after write: " + CORRUPTION_UUID); } - File quarantineBase = storageService.getQuarantineFile(CORRUPTION_UUID); + File quarantineBase = new File(storageService.getQuarantineFile(CORRUPTION_UUID).toString()); assertFile(new File(quarantineBase.getPath() + ".original")); assertFile(new File(quarantineBase.getPath() + ".restored")); @@ -159,7 +161,7 @@ public void testCorruption() private void assertEmptyStagingDirectory() { - File staging = storageService.getStagingFile(randomUUID()).getParentFile(); + File staging = new File(storageService.getStagingFile(randomUUID()).getParent().toString()); assertEquals(staging.list(), new String[] {}); } @@ -171,6 +173,11 @@ private void assertBackupStats(int successCount, int failureCount, int corruptio assertEquals(stats.getBackupCorruption().getTotalCount(), corruptionCount); } + private static Path path(File file) + { + return new Path(file.toURI()); + } + private static class TestingBackupStore implements BackupStore { diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestFileBackupConfig.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestFileBackupConfig.java index 20063402d6d7d..f366b04f5eb70 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestFileBackupConfig.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestFileBackupConfig.java @@ -19,9 +19,9 @@ import java.io.File; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestFileBackupConfig { diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestFileBackupStore.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestFileBackupStore.java index 8ddf8ad6a13b2..8351b09a6fb26 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestFileBackupStore.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestFileBackupStore.java @@ -26,6 +26,7 @@ import static java.lang.String.format; import static org.testng.Assert.assertEquals; +@Test(singleThreaded = true) public class TestFileBackupStore extends AbstractTestBackupStore { diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestHttpBackupConfig.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestHttpBackupConfig.java index e26c7c42ef503..1af513307ead4 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestHttpBackupConfig.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestHttpBackupConfig.java @@ -19,9 +19,9 @@ import java.net.URI; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestHttpBackupConfig { diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestHttpBackupStore.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestHttpBackupStore.java index a931b421ff925..78ef14b487ed7 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestHttpBackupStore.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestHttpBackupStore.java @@ -13,18 +13,18 @@ */ package com.facebook.presto.raptor.backup; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.http.server.HttpServerInfo; +import com.facebook.airlift.http.server.testing.TestingHttpServerModule; +import com.facebook.airlift.jaxrs.JaxrsModule; +import com.facebook.airlift.json.JsonModule; +import com.facebook.airlift.node.testing.TestingNodeModule; import com.google.common.collect.ImmutableMap; import com.google.inject.Binder; import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.Provides; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.http.server.HttpServerInfo; -import io.airlift.http.server.testing.TestingHttpServerModule; -import io.airlift.jaxrs.JaxrsModule; -import io.airlift.json.JsonModule; -import io.airlift.node.testing.TestingNodeModule; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -35,11 +35,11 @@ import java.util.Map; import java.util.function.Supplier; +import static com.facebook.airlift.jaxrs.JaxrsBinder.jaxrsBinder; import static com.google.common.io.Files.createTempDir; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static com.google.inject.util.Modules.override; -import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; @Test(singleThreaded = true) public class TestHttpBackupStore diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestingHttpBackupResource.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestingHttpBackupResource.java index d33c9ee32f7d0..b7b8f4fe6158b 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestingHttpBackupResource.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/backup/TestingHttpBackupResource.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.raptor.backup; -import io.airlift.node.NodeInfo; +import com.facebook.airlift.node.NodeInfo; import io.airlift.slice.Slices; import io.airlift.slice.XxHash64; diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueries.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueries.java index 62467a93d98fe..7b6be430cb662 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueries.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueries.java @@ -24,7 +24,7 @@ public class TestRaptorDistributedQueries @SuppressWarnings("unused") public TestRaptorDistributedQueries() { - this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); + this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false, false, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); } protected TestRaptorDistributedQueries(QueryRunnerSupplier supplier) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueriesBucketed.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueriesBucketed.java index e1dec16bc3a53..413df5135347d 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueriesBucketed.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorDistributedQueriesBucketed.java @@ -22,7 +22,7 @@ public class TestRaptorDistributedQueriesBucketed { public TestRaptorDistributedQueriesBucketed() { - super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); + super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true, false, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); } @Override diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java index 2c644b0f21c03..9f13968abd028 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTest.java @@ -34,6 +34,10 @@ import java.util.UUID; import java.util.stream.IntStream; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static com.facebook.presto.SystemSessionProperties.COLOCATED_JOIN; import static com.facebook.presto.raptor.RaptorColumnHandle.SHARD_UUID_COLUMN_TYPE; import static com.facebook.presto.raptor.RaptorQueryRunner.createRaptorQueryRunner; @@ -44,10 +48,6 @@ import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertGreaterThanOrEqual; -import static io.airlift.testing.Assertions.assertInstanceOf; -import static io.airlift.testing.Assertions.assertLessThan; import static java.lang.String.format; import static java.util.Arrays.asList; import static java.util.function.Function.identity; @@ -62,7 +62,12 @@ public class TestRaptorIntegrationSmokeTest @SuppressWarnings("unused") public TestRaptorIntegrationSmokeTest() { - this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false, ImmutableMap.of("storage.orc.optimized-writer-stage", "DISABLED"))); + this(() -> createRaptorQueryRunner( + ImmutableMap.of(), + true, + false, + false, + ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); } protected TestRaptorIntegrationSmokeTest(QueryRunnerSupplier supplier) @@ -78,6 +83,13 @@ public void testCreateArrayTable() assertUpdate("DROP TABLE array_test"); } + @Test + public void testCreateTableUnsupportedType() + { + assertQueryFails("CREATE TABLE rowtype_test AS SELECT row(1) AS c", "Type not supported: row\\(integer\\)"); + assertQueryFails("CREATE TABLE rowtype_test(row_type_field row(s varchar))", "Type not supported: row\\(s varchar\\)"); + } + @Test public void testMapTable() { @@ -362,6 +374,8 @@ public void testTableProperties() { computeActual("CREATE TABLE test_table_properties_1 (foo BIGINT, bar BIGINT, ds DATE) WITH (ordering=array['foo','bar'], temporal_column='ds')"); computeActual("CREATE TABLE test_table_properties_2 (foo BIGINT, bar BIGINT, ds DATE) WITH (ORDERING=array['foo','bar'], TEMPORAL_COLUMN='ds')"); + computeActual("CREATE TABLE test_table_properties_3 (foo BIGINT, bar BIGINT, ds DATE) WITH (TABLE_SUPPORTS_DELTA_DELETE=false)"); + computeActual("CREATE TABLE test_table_properties_4 (foo BIGINT, bar BIGINT, ds DATE) WITH (table_supports_delta_delete=true)"); } @Test @@ -577,6 +591,7 @@ public void testShowCreateTable() " bucket_count = 32,\n" + " bucketed_on = ARRAY['c1','c6'],\n" + " ordering = ARRAY['c6','c1'],\n" + + " table_supports_delta_delete = true,\n" + " temporal_column = 'c7'\n" + ")", getSession().getCatalog().get(), getSession().getSchema().get(), "test_show_create_table"); @@ -606,7 +621,8 @@ public void testShowCreateTable() " bucket_count = 32,\n" + " bucketed_on = ARRAY['c1','c6'],\n" + " ordering = ARRAY['c6','c1'],\n" + - " organized = true\n" + + " organized = true,\n" + + " table_supports_delta_delete = true\n" + ")", getSession().getCatalog().get(), getSession().getSchema().get(), "test_show_create_table_organized"); assertUpdate(createTableSql); @@ -628,10 +644,10 @@ public void testShowCreateTable() " \"c'4\" array(bigint),\n" + " c5 map(bigint, varchar)\n" + ")", - getSession().getCatalog().get(), getSession().getSchema().get(), "\"test_show_create_table\"\"2\""); + getSession().getCatalog().get(), getSession().getSchema().get(), "test_show_create_table_default"); assertUpdate(createTableSql); - actualResult = computeActual("SHOW CREATE TABLE \"test_show_create_table\"\"2\""); + actualResult = computeActual("SHOW CREATE TABLE \"test_show_create_table_default\""); assertEquals(getOnlyElement(actualResult.getOnlyColumnAsSet()), createTableSql); } @@ -655,6 +671,9 @@ public void testTablesSystemTable() assertUpdate("" + "CREATE TABLE system_tables_test5 (c50 timestamp, c51 varchar, c52 double, c53 bigint, c54 bigint) " + "WITH (ordering = ARRAY['c51', 'c52'], distribution_name = 'test_distribution', bucket_count = 50, bucketed_on = ARRAY ['c53', 'c54'], organized = true)"); + assertUpdate("" + + "CREATE TABLE system_tables_test6 (c60 timestamp, c61 varchar, c62 double, c63 bigint, c64 bigint) " + + "WITH (ordering = ARRAY['c61', 'c62'], distribution_name = 'test_distribution', bucket_count = 50, bucketed_on = ARRAY ['c63', 'c64'], organized = true, table_supports_delta_delete = true)"); MaterializedResult actualResults = computeActual("SELECT * FROM system.tables"); assertEquals( @@ -668,35 +687,39 @@ public void testTablesSystemTable() .add(BIGINT) // bucket_count .add(new ArrayType(VARCHAR)) // bucket_columns .add(BOOLEAN) // organized + .add(BOOLEAN) // table_supports_delta_delete .build()); Map map = actualResults.getMaterializedRows().stream() .filter(row -> ((String) row.getField(1)).startsWith("system_tables_test")) .collect(toImmutableMap(row -> ((String) row.getField(1)), identity())); - assertEquals(map.size(), 6); + assertEquals(map.size(), 7); assertEquals( map.get("system_tables_test0").getFields(), - asList("tpch", "system_tables_test0", null, null, null, null, null, Boolean.FALSE)); + asList("tpch", "system_tables_test0", null, null, null, null, null, Boolean.FALSE, Boolean.FALSE)); assertEquals( map.get("system_tables_test1").getFields(), - asList("tpch", "system_tables_test1", "c10", null, null, null, null, Boolean.FALSE)); + asList("tpch", "system_tables_test1", "c10", null, null, null, null, Boolean.FALSE, Boolean.FALSE)); assertEquals( map.get("system_tables_test2").getFields(), - asList("tpch", "system_tables_test2", "c20", ImmutableList.of("c22", "c21"), null, null, null, Boolean.FALSE)); + asList("tpch", "system_tables_test2", "c20", ImmutableList.of("c22", "c21"), null, null, null, Boolean.FALSE, Boolean.FALSE)); assertEquals( map.get("system_tables_test3").getFields(), - asList("tpch", "system_tables_test3", "c30", null, null, 40L, ImmutableList.of("c34", "c33"), Boolean.FALSE)); + asList("tpch", "system_tables_test3", "c30", null, null, 40L, ImmutableList.of("c34", "c33"), Boolean.FALSE, Boolean.FALSE)); assertEquals( map.get("system_tables_test4").getFields(), - asList("tpch", "system_tables_test4", "c40", ImmutableList.of("c41", "c42"), "test_distribution", 50L, ImmutableList.of("c43", "c44"), Boolean.FALSE)); + asList("tpch", "system_tables_test4", "c40", ImmutableList.of("c41", "c42"), "test_distribution", 50L, ImmutableList.of("c43", "c44"), Boolean.FALSE, Boolean.FALSE)); assertEquals( map.get("system_tables_test5").getFields(), - asList("tpch", "system_tables_test5", null, ImmutableList.of("c51", "c52"), "test_distribution", 50L, ImmutableList.of("c53", "c54"), Boolean.TRUE)); + asList("tpch", "system_tables_test5", null, ImmutableList.of("c51", "c52"), "test_distribution", 50L, ImmutableList.of("c53", "c54"), Boolean.TRUE, Boolean.FALSE)); + assertEquals( + map.get("system_tables_test6").getFields(), + asList("tpch", "system_tables_test6", null, ImmutableList.of("c61", "c62"), "test_distribution", 50L, ImmutableList.of("c63", "c64"), Boolean.TRUE, Boolean.TRUE)); actualResults = computeActual("SELECT * FROM system.tables WHERE table_schema = 'tpch'"); long actualRowCount = actualResults.getMaterializedRows().stream() .filter(row -> ((String) row.getField(1)).startsWith("system_tables_test")) .count(); - assertEquals(actualRowCount, 6); + assertEquals(actualRowCount, 7); actualResults = computeActual("SELECT * FROM system.tables WHERE table_name = 'system_tables_test3'"); assertEquals(actualResults.getMaterializedRows().size(), 1); @@ -717,6 +740,7 @@ public void testTablesSystemTable() assertUpdate("DROP TABLE system_tables_test3"); assertUpdate("DROP TABLE system_tables_test4"); assertUpdate("DROP TABLE system_tables_test5"); + assertUpdate("DROP TABLE system_tables_test6"); assertEquals(computeActual("SELECT * FROM system.tables WHERE table_schema IN ('foo', 'bar')").getRowCount(), 0); } @@ -838,6 +862,14 @@ public void testAlterTable() assertUpdate("DROP TABLE test_alter_table"); } + @Test + public void testAlterTableUnsupportedType() + { + assertUpdate("CREATE TABLE test_alter_table_unsupported_type (c1 bigint, c2 bigint)"); + assertQueryFails("ALTER TABLE test_alter_table_unsupported_type ADD COLUMN c3 row(bigint)", "Type not supported: row\\(bigint\\)"); + assertUpdate("DROP TABLE test_alter_table_unsupported_type"); + } + @Test public void testDelete() { @@ -861,4 +893,10 @@ public void testDelete() assertUpdate("DROP TABLE test_delete_table"); } + + @Test + public void testTriggerBucketBalancer() + { + assertUpdate("CALL system.trigger_bucket_balancer()"); + } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketed.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketed.java index ef58207d9dc73..d2863f3f38f23 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketed.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketed.java @@ -23,7 +23,7 @@ public class TestRaptorIntegrationSmokeTestBucketed { public TestRaptorIntegrationSmokeTestBucketed() { - super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true, ImmutableMap.of("storage.orc.optimized-writer-stage", "DISABLED"))); + super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true, false, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); } @Test diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestHdfs.java similarity index 84% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter.java rename to presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestHdfs.java index d2b1b8597fd3e..5b74cf271f600 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestHdfs.java @@ -18,12 +18,12 @@ import static com.facebook.presto.raptor.RaptorQueryRunner.createRaptorQueryRunner; -public class TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter +public class TestRaptorIntegrationSmokeTestHdfs extends TestRaptorIntegrationSmokeTest { - public TestRaptorIntegrationSmokeTestBucketedWithOptimizedWriter() + public TestRaptorIntegrationSmokeTestHdfs() { - super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); + super(() -> createRaptorQueryRunner(ImmutableMap.of(), true, true, true, ImmutableMap.of())); } @Test diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestMySql.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestMySql.java index 609c89db151c9..eba1e9c8df190 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestMySql.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestMySql.java @@ -14,17 +14,18 @@ package com.facebook.presto.raptor.integration; import com.facebook.presto.raptor.RaptorPlugin; +import com.facebook.presto.testing.mysql.MySqlOptions; +import com.facebook.presto.testing.mysql.TestingMySqlServer; import com.facebook.presto.tests.DistributedQueryRunner; import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.testing.mysql.MySqlOptions; -import io.airlift.testing.mysql.TestingMySqlServer; import io.airlift.units.Duration; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; import java.io.File; +import java.io.IOException; import java.util.Map; import static com.facebook.presto.raptor.RaptorQueryRunner.copyTables; @@ -55,6 +56,7 @@ private TestRaptorIntegrationSmokeTestMySql(TestingMySqlServer mysqlServer) @AfterClass(alwaysRun = true) public final void destroy() + throws IOException { mysqlServer.close(); } @@ -72,7 +74,7 @@ private static DistributedQueryRunner createRaptorMySqlQueryRunner(String mysqlU Map raptorProperties = ImmutableMap.builder() .put("metadata.db.type", "mysql") .put("metadata.db.url", mysqlUrl) - .put("storage.data-directory", new File(baseDir, "data").getAbsolutePath()) + .put("storage.data-directory", new File(baseDir, "data").toURI().toString()) .put("storage.max-shard-rows", "2000") .put("backup.provider", "file") .put("backup.directory", new File(baseDir, "backup").getAbsolutePath()) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestAssignmentLimiter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestAssignmentLimiter.java index 76453d31bac7e..75701986cb469 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestAssignmentLimiter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestAssignmentLimiter.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.raptor.metadata; +import com.facebook.airlift.testing.TestingTicker; import com.facebook.presto.client.NodeVersion; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.Node; import com.facebook.presto.spi.PrestoException; import com.google.common.collect.ImmutableSet; -import io.airlift.testing.TestingTicker; import io.airlift.units.Duration; import org.testng.annotations.Test; diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseConfig.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseConfig.java index 94fc59e22301c..13334472a95f3 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseConfig.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseConfig.java @@ -18,9 +18,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestDatabaseConfig { diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java index 13a0a21702463..e826f9e1fb7ea 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.raptor.metadata; +import com.facebook.airlift.testing.TestingTicker; import com.facebook.presto.client.NodeVersion; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.raptor.NodeSupplier; @@ -33,7 +34,6 @@ import com.google.common.collect.Multimap; import com.google.common.io.Files; import io.airlift.slice.Slice; -import io.airlift.testing.TestingTicker; import io.airlift.units.Duration; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.Handle; @@ -47,6 +47,7 @@ import java.io.IOException; import java.net.URI; import java.sql.ResultSet; +import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.time.LocalDate; @@ -93,6 +94,7 @@ import static java.util.stream.Collectors.toSet; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.fail; @Test(singleThreaded = true) @@ -121,6 +123,42 @@ public void teardown() deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); } + @Test + public void testCreateTable() + throws SQLException + { + long tableId = createTable("test"); + List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); + + Statement statement = dummyHandle.getConnection().createStatement(); + ResultSet resultSet = statement.executeQuery("select * from " + shardIndexTable(tableId)); + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(metaData.getColumnLabel(1), "SHARD_ID"); + assertEquals(metaData.getColumnLabel(2), "SHARD_UUID"); + assertNotEquals(metaData.getColumnLabel(3), "DELTA_SHARD_UUID"); + resultSet.close(); + statement.close(); + } + + @Test + public void testCreateTableWithDeltaDelete() + throws SQLException + { + long tableId = createTable("test"); + List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), true); + + Statement statement = dummyHandle.getConnection().createStatement(); + ResultSet resultSet = statement.executeQuery("select * from " + shardIndexTable(tableId)); + ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(metaData.getColumnLabel(1), "SHARD_ID"); + assertEquals(metaData.getColumnLabel(2), "SHARD_UUID"); + assertEquals(metaData.getColumnLabel(3), "DELTA_SHARD_UUID"); + resultSet.close(); + statement.close(); + } + @Test public void testCommit() { @@ -134,7 +172,7 @@ public void testCommit() List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); long transactionId = shardManager.beginTransaction(); shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); @@ -150,7 +188,7 @@ public void testRollback() List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); List shards = ImmutableList.of(shardInfo(UUID.randomUUID(), "node1")); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); long transactionId = shardManager.beginTransaction(); shardManager.rollbackTransaction(transactionId); @@ -172,7 +210,7 @@ public void testAssignShard() List shardNodes = ImmutableList.of(shardInfo(shard, "node1")); List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); long transactionId = shardManager.beginTransaction(); shardManager.commitShards(transactionId, tableId, columns, shardNodes, Optional.empty(), 0); @@ -214,7 +252,7 @@ public void testGetNodeBytes() new ShardInfo(shard2, bucketNumber, ImmutableSet.of("node1"), ImmutableList.of(), 5, 55, 555, 0)); List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); long transactionId = shardManager.beginTransaction(); shardManager.commitShards(transactionId, tableId, columns, shardNodes, Optional.empty(), 0); @@ -249,7 +287,7 @@ public void testGetNodeTableShards() inputShards.add(shardInfo(uuid, node)); } - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); long transactionId = shardManager.beginTransaction(); shardManager.commitShards(transactionId, tableId, columns, inputShards.build(), Optional.empty(), 0); @@ -271,7 +309,7 @@ public void testGetExistingShards() List shardNodes = ImmutableList.of(shardInfo(shard1, "node1"), shardInfo(shard2, "node1")); List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); long transactionId = shardManager.beginTransaction(); shardManager.commitShards(transactionId, tableId, columns, shardNodes, Optional.empty(), 0); @@ -294,7 +332,7 @@ public void testReplaceShardUuids() .add(shardInfo(originalUuids.get(2), nodes.get(2))) .build(); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); long transactionId = shardManager.beginTransaction(); shardManager.commitShards(transactionId, tableId, columns, oldShards, Optional.empty(), 0); @@ -350,7 +388,7 @@ public void testExternalBatches() List shards = ImmutableList.of(shardInfo(UUID.randomUUID(), "node1")); List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); long transactionId = shardManager.beginTransaction(); shardManager.commitShards(transactionId, tableId, columns, shards, externalBatchId, 0); @@ -417,7 +455,7 @@ public void testEmptyTable() { long tableId = createTable("test"); List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); try (ResultIterator iterator = shardManager.getShardNodes(tableId, TupleDomain.all())) { assertFalse(iterator.hasNext()); @@ -429,7 +467,7 @@ public void testEmptyTableBucketed() { long tableId = createTable("test"); List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, true, OptionalLong.empty()); + shardManager.createTable(tableId, columns, true, OptionalLong.empty(), false); try (ResultIterator iterator = shardManager.getShardNodesBucketed(tableId, true, ImmutableList.of(), TupleDomain.all())) { assertFalse(iterator.hasNext()); @@ -441,11 +479,11 @@ public void testTemporalColumnTableCreation() { long tableId = createTable("test"); List columns = ImmutableList.of(new ColumnInfo(1, TIMESTAMP)); - shardManager.createTable(tableId, columns, false, OptionalLong.of(1)); + shardManager.createTable(tableId, columns, false, OptionalLong.of(1), false); long tableId2 = createTable("test2"); List columns2 = ImmutableList.of(new ColumnInfo(1, TIMESTAMP)); - shardManager.createTable(tableId2, columns2, true, OptionalLong.of(1)); + shardManager.createTable(tableId2, columns2, true, OptionalLong.of(1), false); } @Test @@ -511,7 +549,7 @@ public void testShardPruning() RaptorColumnHandle c6 = new RaptorColumnHandle("raptor", "c6", 6, BOOLEAN); long tableId = createTable("test"); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); long transactionId = shardManager.beginTransaction(); shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); @@ -609,7 +647,7 @@ public void testShardPruningTruncatedValues() RaptorColumnHandle c1 = new RaptorColumnHandle("raptor", "c1", 1, createVarcharType(10)); long tableId = createTable("test"); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); long transactionId = shardManager.beginTransaction(); shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); @@ -644,7 +682,7 @@ public void testShardPruningNoStats() List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); RaptorColumnHandle c1 = new RaptorColumnHandle("raptor", "c1", 1, BIGINT); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); long transactionId = shardManager.beginTransaction(); shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); @@ -659,7 +697,7 @@ public void testAddNewColumn() { long tableId = createTable("test"); List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); int before = columnCount(tableId); ColumnInfo newColumn = new ColumnInfo(2, BIGINT); @@ -675,7 +713,7 @@ public void testAddDuplicateColumn() { long tableId = createTable("test"); List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); int before = columnCount(tableId); shardManager.addColumn(tableId, columns.get(0)); @@ -713,7 +751,7 @@ private Set getShardNodes(long tableId, TupleDomain fields = (ImmutableList.of(new RowType.Field(Optional.of("field_1"), BIGINT))); + + try { + metadata.addColumn( + SESSION, + raptorTableHandle, + new ColumnMetadata("new_col", RowType.from(fields))); + fail(); + } + catch (PrestoException e) { + assertEquals(e.getErrorCode(), NOT_SUPPORTED.toErrorCode()); + } + } + private boolean transactionExists(long transactionId) { try (Handle handle = dbi.open()) { diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestRaptorSplitManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestRaptorSplitManager.java index 50f6f6d23cc7c..9bf0a587e2624 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestRaptorSplitManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestRaptorSplitManager.java @@ -54,6 +54,9 @@ import java.util.Optional; import java.util.UUID; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; +import static com.facebook.presto.raptor.RaptorTableProperties.TABLE_SUPPORTS_DELTA_DELETE; import static com.facebook.presto.raptor.metadata.DatabaseShardManager.shardIndexTable; import static com.facebook.presto.raptor.metadata.SchemaDaoUtil.createTablesWithRetry; import static com.facebook.presto.raptor.metadata.TestDatabaseShardManager.shardInfo; @@ -66,8 +69,6 @@ import static com.google.common.io.Files.createTempDir; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.testing.Assertions.assertInstanceOf; import static java.lang.String.format; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.stream.Collectors.toList; @@ -80,6 +81,7 @@ public class TestRaptorSplitManager .column("ds", createVarcharType(10)) .column("foo", createVarcharType(10)) .column("bar", BigintType.BIGINT) + .property(TABLE_SUPPORTS_DELTA_DELETE, false) .build(); private Handle dummyHandle; @@ -109,7 +111,7 @@ public void setup() nodeManager.addNode(new InternalNode(nodeName, new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN, false)); RaptorConnectorId connectorId = new RaptorConnectorId("raptor"); - metadata = new RaptorMetadata(connectorId.toString(), dbi, shardManager); + metadata = new RaptorMetadata(connectorId.toString(), dbi, shardManager, new TypeRegistry()); metadata.createTable(SESSION, TEST_TABLE, false); tableHandle = metadata.getTableHandle(SESSION, TEST_TABLE.getTable()); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardCleaner.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardCleaner.java index 5276c94255a9d..d3e501c333f38 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardCleaner.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardCleaner.java @@ -13,15 +13,17 @@ */ package com.facebook.presto.raptor.metadata; +import com.facebook.airlift.testing.TestingTicker; import com.facebook.presto.raptor.backup.BackupStore; import com.facebook.presto.raptor.backup.FileBackupStore; -import com.facebook.presto.raptor.storage.FileStorageService; +import com.facebook.presto.raptor.filesystem.LocalFileStorageService; +import com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment; import com.facebook.presto.raptor.storage.StorageService; import com.facebook.presto.raptor.util.DaoSupplier; import com.facebook.presto.raptor.util.UuidUtil.UuidArgumentFactory; import com.google.common.collect.ImmutableSet; -import io.airlift.testing.TestingTicker; import io.airlift.units.Duration; +import org.apache.hadoop.fs.Path; import org.intellij.lang.annotations.Language; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.Handle; @@ -43,12 +45,12 @@ import java.util.Set; import java.util.UUID; +import static com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static com.facebook.presto.raptor.metadata.SchemaDaoUtil.createTablesWithRetry; import static com.facebook.presto.raptor.util.UuidUtil.uuidFromBytes; import static com.google.common.io.Files.createTempDir; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.testing.Assertions.assertEqualsIgnoreOrder; import static java.util.Arrays.asList; import static java.util.UUID.randomUUID; import static java.util.concurrent.TimeUnit.HOURS; @@ -77,7 +79,7 @@ public void setup() temporary = createTempDir(); File directory = new File(temporary, "data"); - storageService = new FileStorageService(directory); + storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), directory.toURI()); storageService.start(); File backupDirectory = new File(temporary, "backup"); @@ -94,6 +96,7 @@ public void setup() ticker, storageService, Optional.of(backupStore), + new LocalOrcDataEnvironment(), config.getMaxTransactionAge(), config.getTransactionCleanerInterval(), config.getLocalCleanerInterval(), @@ -194,7 +197,7 @@ public void testCleanLocalShardsImmediately() TestingShardDao shardDao = dbi.onDemand(TestingShardDao.class); MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - long tableId = metadataDao.insertTable("test", "test", false, false, null, 0); + long tableId = metadataDao.insertTable("test", "test", false, false, null, 0, false); UUID shard1 = randomUUID(); UUID shard2 = randomUUID(); @@ -240,7 +243,7 @@ public void testCleanLocalShards() TestingShardDao shardDao = dbi.onDemand(TestingShardDao.class); MetadataDao metadataDao = dbi.onDemand(MetadataDao.class); - long tableId = metadataDao.insertTable("test", "test", false, false, null, 0); + long tableId = metadataDao.insertTable("test", "test", false, false, null, 0, false); UUID shard1 = randomUUID(); UUID shard2 = randomUUID(); @@ -387,14 +390,14 @@ public void testDeleteOldCompletedTransactions() private boolean shardFileExists(UUID uuid) { - return storageService.getStorageFile(uuid).exists(); + return new File(storageService.getStorageFile(uuid).toString()).exists(); } private void createShardFile(UUID uuid) throws IOException { - File file = storageService.getStorageFile(uuid); - storageService.createParents(file); + File file = new File(storageService.getStorageFile(uuid).toString()); + storageService.createParents(new Path(file.toURI())); assertTrue(file.createNewFile()); } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardCleanerConfig.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardCleanerConfig.java index 59fdcaaf92671..67a2760a49465 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardCleanerConfig.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardCleanerConfig.java @@ -19,9 +19,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static java.util.concurrent.TimeUnit.DAYS; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardDao.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardDao.java index b40179e01519d..2208432b4d5e0 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardDao.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardDao.java @@ -29,8 +29,8 @@ import java.util.Set; import java.util.UUID; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.raptor.metadata.SchemaDaoUtil.createTablesWithRetry; -import static io.airlift.testing.Assertions.assertInstanceOf; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -167,8 +167,8 @@ public void testNodeShards() dao.insertBuckets(distributionId, ImmutableList.of(i), ImmutableList.of(nodeId)); } - long plainTableId = metadataDao.insertTable("test", "plain", false, false, null, 0); - long bucketedTableId = metadataDao.insertTable("test", "bucketed", false, false, distributionId, 0); + long plainTableId = metadataDao.insertTable("test", "plain", false, false, null, 0, false); + long bucketedTableId = metadataDao.insertTable("test", "bucketed", false, false, distributionId, 0, false); long shardId1 = dao.insertShard(shardUuid1, plainTableId, null, 1, 11, 111, 888_111); long shardId2 = dao.insertShard(shardUuid2, plainTableId, null, 2, 22, 222, 888_222); @@ -278,7 +278,7 @@ public void testShardSelection() private long createTable(String name) { - return dbi.onDemand(MetadataDao.class).insertTable("test", name, false, false, null, 0); + return dbi.onDemand(MetadataDao.class).insertTable("test", name, false, false, null, 0, false); } private static void assertContainsShardNode(Set nodes, String nodeName, UUID shardUuid) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestUpgradeMetadata.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestUpgradeMetadata.java new file mode 100644 index 0000000000000..e96e3eaa52d87 --- /dev/null +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestUpgradeMetadata.java @@ -0,0 +1,164 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.metadata; + +import org.skife.jdbi.v2.DBI; +import org.skife.jdbi.v2.Handle; +import org.skife.jdbi.v2.IDBI; +import org.skife.jdbi.v2.exceptions.UnableToCreateStatementException; +import org.skife.jdbi.v2.sqlobject.SqlUpdate; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.OptionalLong; +import java.util.UUID; + +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.presto.raptor.metadata.SchemaDaoUtil.createTablesWithRetry; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + +/** + * Test the correctness of metadata after upgrading + * from environment without delta delete to environment with delta delete + */ +@Test(singleThreaded = true) +public class TestUpgradeMetadata +{ + private IDBI dbi; + private MetadataDao dao; + private Handle dummyHandle; + + @BeforeMethod + public void setup() + { + dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime()); + dummyHandle = dbi.open(); + dao = dbi.onDemand(MetadataDao.class); + createLegacyTables(dummyHandle.attach(SchemaDao.class), dummyHandle.attach(LegacySchemaDao.class)); + } + + private void createLegacyTables(SchemaDao dao, LegacySchemaDao legacySchemaDao) + { + dao.createTableDistributions(); + legacySchemaDao.createTableTablesForTest(); + dao.createTableColumns(); + dao.createTableViews(); + dao.createTableNodes(); + legacySchemaDao.createTableShardsForTest(); + dao.createTableShardNodes(); + dao.createTableExternalBatches(); + dao.createTableTransactions(); + dao.createTableCreatedShards(); + dao.createTableDeletedShards(); + dao.createTableBuckets(); + dao.createTableShardOrganizerJobs(); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() + { + dummyHandle.close(); + } + + @Test(expectedExceptions = UnableToCreateStatementException.class) + public void testTableTablesWithoutUpgrading() + { + // without perform upgrading operation would cause metadata incorrect + dao.insertTable("schema1", "table1", true, false, null, 0, false); + } + + @Test + public void testTableTables() + { + // perform upgrading operation on metadata + createTablesWithRetry(dbi); + long tableId = dao.insertTable("schema1", "table1", true, false, null, 0, false); + assertGreaterThan(tableId, 0L); + } + + @Test + public void testTableShards() + { + // perform upgrading operation on metadata + createTablesWithRetry(dbi); + TestingShardDao testingShardDao = dbi.onDemand(TestingShardDao.class); + long tableId = createTable("test"); + UUID shardUuid = UUID.randomUUID(); + UUID deltaUuid = UUID.randomUUID(); + long shardId = testingShardDao.insertShardWithDelta(shardUuid, tableId, false, deltaUuid, null, 13, 42, 84, 1234); + + ShardMetadata shard = testingShardDao.getShard(shardUuid); + assertNotNull(shard); + assertEquals(shard.getTableId(), tableId); + assertEquals(shard.getShardId(), shardId); + assertEquals(shard.getShardUuid(), shardUuid); + assertEquals(shard.getRowCount(), 13); + assertEquals(shard.getCompressedSize(), 42); + assertEquals(shard.getUncompressedSize(), 84); + assertEquals(shard.getXxhash64(), OptionalLong.of(1234)); + } + + private long createTable(String name) + { + return dbi.onDemand(MetadataDao.class).insertTable("test", name, false, false, null, 0, false); + } + + /** + * This class is for generating the schema for environment without delta delete + */ + interface LegacySchemaDao + { + @SqlUpdate("CREATE TABLE IF NOT EXISTS tables (\n" + + " table_id BIGINT PRIMARY KEY AUTO_INCREMENT,\n" + + " schema_name VARCHAR(255) NOT NULL,\n" + + " table_name VARCHAR(255) NOT NULL,\n" + + " temporal_column_id BIGINT,\n" + + " compaction_enabled BOOLEAN NOT NULL,\n" + + " organization_enabled BOOLEAN NOT NULL,\n" + + " distribution_id BIGINT,\n" + + " create_time BIGINT NOT NULL,\n" + + " update_time BIGINT NOT NULL,\n" + + " table_version BIGINT NOT NULL,\n" + + " shard_count BIGINT NOT NULL,\n" + + " row_count BIGINT NOT NULL,\n" + + " compressed_size BIGINT NOT NULL,\n" + + " uncompressed_size BIGINT NOT NULL,\n" + + " maintenance_blocked DATETIME,\n" + + " UNIQUE (schema_name, table_name),\n" + + " UNIQUE (distribution_id, table_id),\n" + + " UNIQUE (maintenance_blocked, table_id),\n" + + " FOREIGN KEY (distribution_id) REFERENCES distributions (distribution_id)\n" + + ")") + void createTableTablesForTest(); + + @SqlUpdate("CREATE TABLE IF NOT EXISTS shards (\n" + + " shard_id BIGINT PRIMARY KEY AUTO_INCREMENT,\n" + + " shard_uuid BINARY(16) NOT NULL,\n" + + " table_id BIGINT NOT NULL,\n" + + " bucket_number INT,\n" + + " create_time DATETIME NOT NULL,\n" + + " row_count BIGINT NOT NULL,\n" + + " compressed_size BIGINT NOT NULL,\n" + + " uncompressed_size BIGINT NOT NULL,\n" + + " xxhash64 BIGINT NOT NULL,\n" + + " UNIQUE (shard_uuid),\n" + + // include a covering index organized by table_id + " UNIQUE (table_id, bucket_number, shard_id, shard_uuid, create_time, row_count, compressed_size, uncompressed_size, xxhash64),\n" + + " FOREIGN KEY (table_id) REFERENCES tables (table_id)\n" + + ")") + void createTableShardsForTest(); + } +} diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestingShardDao.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestingShardDao.java index b90bb5249377d..ac3d329d30c78 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestingShardDao.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestingShardDao.java @@ -57,6 +57,20 @@ long insertShard( @Bind("uncompressedSize") long uncompressedSize, @Bind("xxhash64") long xxhash64); + @SqlUpdate("INSERT INTO shards (shard_uuid, table_id, is_delta, delta_uuid, bucket_number, create_time, row_count, compressed_size, uncompressed_size, xxhash64)\n" + + "VALUES (:shardUuid, :tableId, :isDelta, :deltaUuid, :bucketNumber, CURRENT_TIMESTAMP, :rowCount, :compressedSize, :uncompressedSize, :xxhash64)") + @GetGeneratedKeys + long insertShardWithDelta( + @Bind("shardUuid") UUID shardUuid, + @Bind("tableId") long tableId, + @Bind("isDelta") boolean isDelta, + @Bind("deltaUuid") UUID deltaUuid, + @Bind("bucketNumber") Integer bucketNumber, + @Bind("rowCount") long rowCount, + @Bind("compressedSize") long compressedSize, + @Bind("uncompressedSize") long uncompressedSize, + @Bind("xxhash64") long xxhash64); + @SqlUpdate("INSERT INTO shard_nodes (shard_id, node_id)\n" + "VALUES (:shardId, :nodeId)\n") void insertShardNode(@Bind("shardId") long shardId, @Bind("nodeId") int nodeId); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/security/TestRaptorFileBasedSecurity.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/security/TestRaptorFileBasedSecurity.java index 6ed3076905731..8e4a7e1e3144b 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/security/TestRaptorFileBasedSecurity.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/security/TestRaptorFileBasedSecurity.java @@ -39,6 +39,7 @@ public void setUp() ImmutableMap.of(), true, false, + false, ImmutableMap.of("security.config-file", path, "raptor.security", "file")); } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/security/TestRaptorReadOnlySecurity.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/security/TestRaptorReadOnlySecurity.java index fba26fb2ec90f..af424a162a5d7 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/security/TestRaptorReadOnlySecurity.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/security/TestRaptorReadOnlySecurity.java @@ -29,7 +29,7 @@ public class TestRaptorReadOnlySecurity public void setUp() throws Exception { - queryRunner = createRaptorQueryRunner(ImmutableMap.of(), false, false, ImmutableMap.of("raptor.security", "read-only")); + queryRunner = createRaptorQueryRunner(ImmutableMap.of(), false, false, false, ImmutableMap.of("raptor.security", "read-only")); } @AfterClass(alwaysRun = true) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/security/TestRaptorSecurityConfig.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/security/TestRaptorSecurityConfig.java index ccbac2567cca2..f8df810b061b7 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/security/TestRaptorSecurityConfig.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/security/TestRaptorSecurityConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.raptor.security; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import org.testng.annotations.Test; import java.util.Map; diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java index 6273fcc2d75cb..c6cf179aff33f 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java @@ -15,21 +15,28 @@ import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.orc.FileOrcDataSource; import com.facebook.presto.orc.OrcBatchRecordReader; +import com.facebook.presto.orc.OrcCorruptionException; import com.facebook.presto.orc.OrcDataSource; import com.facebook.presto.orc.OrcPredicate; import com.facebook.presto.orc.OrcReader; +import com.facebook.presto.orc.OrcReaderOptions; import com.facebook.presto.orc.OrcWriterStats; +import com.facebook.presto.orc.OutputStreamOrcDataSink; +import com.facebook.presto.orc.StorageStripeMetadataSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.spi.type.Type; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.type.TypeRegistry; -import com.google.common.collect.ImmutableMap; import io.airlift.units.DataSize; import org.joda.time.DateTimeZone; import java.io.File; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.util.HashMap; import java.util.List; @@ -38,7 +45,6 @@ import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.orc.OrcEncoding.ORC; import static com.facebook.presto.orc.OrcReader.MAX_BATCH_SIZE; -import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; import static com.facebook.presto.orc.metadata.CompressionKind.ZSTD; import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.units.DataSize.Unit.MEGABYTE; @@ -57,7 +63,12 @@ public static OrcDataSource fileOrcDataSource(File file) public static OrcBatchRecordReader createReader(OrcDataSource dataSource, List columnIds, List types) throws IOException { - OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); + OrcReader orcReader = new OrcReader( + dataSource, + ORC, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource(), + createDefaultTestConfig()); List columnNames = orcReader.getColumnNames(); assertEquals(columnNames.size(), columnIds.size()); @@ -73,19 +84,15 @@ public static OrcBatchRecordReader createReader(OrcDataSource dataSource, List includedColumns) + throws OrcCorruptionException { - return orcReader.createBatchRecordReader(includedColumns, OrcPredicate.TRUE, DateTimeZone.UTC, newSimpleAggregatedMemoryContext(), MAX_BATCH_SIZE); + Metadata metadata = MetadataManager.createTestMetadataManager(); + FunctionManager functionManager = metadata.getFunctionManager(); + TypeRegistry typeRegistry = new TypeRegistry(); + typeRegistry.setFunctionManager(functionManager); + StorageTypeConverter storageTypeConverter = new StorageTypeConverter(typeRegistry); + return orcReader.createBatchRecordReader(storageTypeConverter.toStorageTypes(includedColumns), OrcPredicate.TRUE, DateTimeZone.UTC, newSimpleAggregatedMemoryContext(), MAX_BATCH_SIZE); } public static byte[] octets(int... values) @@ -104,13 +111,20 @@ public static byte octet(int b) return (byte) b; } - public static FileWriter createFileWriter(List columnIds, List columnTypes, File file, boolean useOptimizedOrcWriter) + public static FileWriter createFileWriter(List columnIds, List columnTypes, File file) + throws IOException { - if (useOptimizedOrcWriter) { - TypeRegistry typeManager = new TypeRegistry(); - new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - return new OrcFileWriter(columnIds, columnTypes, file, true, true, new OrcWriterStats(), typeManager, ZSTD); - } - return new OrcRecordWriter(columnIds, columnTypes, file, SNAPPY, true); + TypeRegistry typeManager = new TypeRegistry(); + new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + return new OrcFileWriter(columnIds, columnTypes, new OutputStreamOrcDataSink(new FileOutputStream(file)), true, true, new OrcWriterStats(), typeManager, ZSTD); + } + + public static OrcReaderOptions createDefaultTestConfig() + { + return new OrcReaderOptions( + new DataSize(1, MEGABYTE), + new DataSize(1, MEGABYTE), + new DataSize(1, MEGABYTE), + false); } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancer.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancer.java index 2d94b8c8f59ac..2688feadf34a0 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancer.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancer.java @@ -42,13 +42,13 @@ import java.util.OptionalLong; import java.util.stream.Collectors; +import static com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual; +import static com.facebook.airlift.testing.Assertions.assertLessThanOrEqual; import static com.facebook.presto.raptor.metadata.Distribution.serializeColumnTypes; import static com.facebook.presto.raptor.metadata.SchemaDaoUtil.createTablesWithRetry; import static com.facebook.presto.raptor.metadata.TestDatabaseShardManager.createShardManager; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.testing.Assertions.assertGreaterThanOrEqual; -import static io.airlift.testing.Assertions.assertLessThanOrEqual; import static java.util.concurrent.TimeUnit.DAYS; import static org.testng.Assert.assertEquals; @@ -275,9 +275,9 @@ private long createBucketedTable(String tableName, long distributionId) private long createBucketedTable(String tableName, long distributionId, DataSize compressedSize) { MetadataDao dao = dbi.onDemand(MetadataDao.class); - long tableId = dao.insertTable("test", tableName, false, false, distributionId, 0); + long tableId = dao.insertTable("test", tableName, false, false, distributionId, 0, false); List columnsA = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columnsA, false, OptionalLong.empty()); + shardManager.createTable(tableId, columnsA, false, OptionalLong.empty(), false); metadataDao.updateTableStats(tableId, 1024, 1024 * 1024 * 1024, compressedSize.toBytes(), compressedSize.toBytes() * 2); return tableId; diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancerConfig.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancerConfig.java index b0dd5ab899df4..7779dc16a5c88 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancerConfig.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestBucketBalancerConfig.java @@ -19,9 +19,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static java.util.concurrent.TimeUnit.HOURS; public class TestBucketBalancerConfig diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestFileStorageService.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestLocalFileStorageService.java similarity index 79% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestFileStorageService.java rename to presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestLocalFileStorageService.java index c2cd3f7632b74..d2efac0653faa 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestFileStorageService.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestLocalFileStorageService.java @@ -13,7 +13,10 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.presto.raptor.filesystem.LocalFileStorageService; +import com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment; import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.fs.Path; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -22,7 +25,7 @@ import java.util.Set; import java.util.UUID; -import static com.facebook.presto.raptor.storage.FileStorageService.getFileSystemPath; +import static com.facebook.presto.raptor.filesystem.LocalFileStorageService.getFileSystemPath; import static com.google.common.io.Files.createTempDir; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; @@ -35,16 +38,16 @@ import static org.testng.FileAssert.assertFile; @Test(singleThreaded = true) -public class TestFileStorageService +public class TestLocalFileStorageService { private File temporary; - private FileStorageService store; + private LocalFileStorageService store; @BeforeMethod public void setup() { temporary = createTempDir(); - store = new FileStorageService(temporary); + store = new LocalFileStorageService(new LocalOrcDataEnvironment(), temporary.toURI()); store.start(); } @@ -70,9 +73,9 @@ public void testFilePaths() File staging = new File(temporary, format("staging/%s.orc", uuid)); File storage = new File(temporary, format("storage/70/1e/%s.orc", uuid)); File quarantine = new File(temporary, format("quarantine/%s.orc", uuid)); - assertEquals(store.getStagingFile(uuid), staging); - assertEquals(store.getStorageFile(uuid), storage); - assertEquals(store.getQuarantineFile(uuid), quarantine); + assertEquals(new File(store.getStagingFile(uuid).toString()), staging); + assertEquals(new File(store.getStorageFile(uuid).toString()), storage); + assertEquals(new File(store.getQuarantineFile(uuid).toString()), quarantine); } @Test @@ -87,8 +90,8 @@ public void testStop() assertDirectory(storage); assertDirectory(quarantine); - File file = store.getStagingFile(randomUUID()); - store.createParents(file); + File file = new File(store.getStagingFile(randomUUID()).toString()); + store.createParents(new Path(file.toURI())); assertFalse(file.exists()); assertTrue(file.createNewFile()); assertFile(file); @@ -111,8 +114,8 @@ public void testGetStorageShards() .build(); for (UUID shard : shards) { - File file = store.getStorageFile(shard); - store.createParents(file); + File file = new File(store.getStorageFile(shard).toString()); + store.createParents(new Path(file.toURI())); assertTrue(file.createNewFile()); } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java index 7d6c5fd588ec0..dff12aed8ae5a 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcFileRewriter.java @@ -13,12 +13,18 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.orc.OrcBatchRecordReader; import com.facebook.presto.orc.OrcDataSource; import com.facebook.presto.orc.OrcReader; import com.facebook.presto.orc.OrcWriterStats; +import com.facebook.presto.orc.OutputStreamOrcDataSink; +import com.facebook.presto.orc.StorageStripeMetadataSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; +import com.facebook.presto.raptor.filesystem.FileSystemContext; +import com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment; import com.facebook.presto.raptor.metadata.TableColumn; import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.Page; @@ -36,16 +42,18 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.Files; -import io.airlift.json.JsonCodec; import io.airlift.units.DataSize; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.Handle; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.math.BigDecimal; import java.util.BitSet; import java.util.List; @@ -55,11 +63,12 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.testing.Assertions.assertBetweenInclusive; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.orc.OrcEncoding.ORC; -import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; import static com.facebook.presto.orc.metadata.CompressionKind.ZSTD; -import static com.facebook.presto.raptor.storage.FileStorageService.getFileSystemPath; +import static com.facebook.presto.raptor.filesystem.LocalFileStorageService.getFileSystemPath; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReader; import static com.facebook.presto.raptor.storage.OrcTestingUtil.fileOrcDataSource; import static com.facebook.presto.raptor.storage.TestOrcStorageManager.createOrcStorageManager; @@ -76,9 +85,7 @@ import static com.google.common.io.Files.createTempDir; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.slice.Slices.utf8Slice; -import static io.airlift.testing.Assertions.assertBetweenInclusive; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.nio.file.Files.readAllBytes; import static java.util.UUID.randomUUID; @@ -90,17 +97,11 @@ @Test(singleThreaded = true) public class TestOrcFileRewriter { - private static final ReaderAttributes READER_ATTRIBUTES = new ReaderAttributes(new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); + private static final ReaderAttributes READER_ATTRIBUTES = new ReaderAttributes(new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true, false); private static final JsonCodec METADATA_CODEC = jsonCodec(OrcFileMetadata.class); private File temporary; - @DataProvider(name = "useOptimizedOrcWriter") - public static Object[][] useOptimizedOrcWriter() - { - return new Object[][] {{true}, {false}}; - } - @BeforeClass public void setup() { @@ -114,8 +115,8 @@ public void tearDown() deleteRecursively(temporary.toPath(), ALLOW_INSECURE); } - @Test(dataProvider = "useOptimizedOrcWriter") - public void testRewrite(boolean useOptimizedOrcWriter) + @Test + public void testRewrite() throws Exception { TypeManager typeManager = new TypeRegistry(); @@ -133,7 +134,7 @@ public void testRewrite(boolean useOptimizedOrcWriter) List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), arrayType, mapType, arrayOfArrayType, decimalType); File file = new File(temporary, randomUUID().toString()); - try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { + try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file)) { List pages = rowPagesBuilder(columnTypes) .row(123L, "hello", arrayBlockOf(BIGINT, 1, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)), new BigDecimal("2.3")) .row(777L, "sky", arrayBlockOf(BIGINT, 3, 4), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 6)), new BigDecimal("2.3")) @@ -153,7 +154,7 @@ public void testRewrite(boolean useOptimizedOrcWriter) assertEquals(reader.nextBatch(), 5); - Block column0 = reader.readBlock(BIGINT, 0); + Block column0 = reader.readBlock(0); assertEquals(column0.getPositionCount(), 5); for (int i = 0; i < 5; i++) { assertEquals(column0.isNull(i), false); @@ -164,7 +165,7 @@ public void testRewrite(boolean useOptimizedOrcWriter) assertEquals(BIGINT.getLong(column0, 3), 888L); assertEquals(BIGINT.getLong(column0, 4), 999L); - Block column1 = reader.readBlock(createVarcharType(20), 1); + Block column1 = reader.readBlock(1); assertEquals(column1.getPositionCount(), 5); for (int i = 0; i < 5; i++) { assertEquals(column1.isNull(i), false); @@ -175,7 +176,7 @@ public void testRewrite(boolean useOptimizedOrcWriter) assertEquals(createVarcharType(20).getSlice(column1, 3), utf8Slice("world")); assertEquals(createVarcharType(20).getSlice(column1, 4), utf8Slice("done")); - Block column2 = reader.readBlock(arrayType, 2); + Block column2 = reader.readBlock(2); assertEquals(column2.getPositionCount(), 5); for (int i = 0; i < 5; i++) { assertEquals(column2.isNull(i), false); @@ -186,7 +187,7 @@ public void testRewrite(boolean useOptimizedOrcWriter) assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 3), arrayBlockOf(BIGINT, 7, 8))); assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 4), arrayBlockOf(BIGINT, 9, 10))); - Block column3 = reader.readBlock(mapType, 3); + Block column3 = reader.readBlock(3); assertEquals(column3.getPositionCount(), 5); for (int i = 0; i < 5; i++) { assertEquals(column3.isNull(i), false); @@ -197,7 +198,7 @@ public void testRewrite(boolean useOptimizedOrcWriter) assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column3, 3), mapBlockOf(createVarcharType(5), BOOLEAN, "k4", true))); assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column3, 4), mapBlockOf(createVarcharType(5), BOOLEAN, "k5", true))); - Block column4 = reader.readBlock(arrayOfArrayType, 4); + Block column4 = reader.readBlock(4); assertEquals(column4.getPositionCount(), 5); for (int i = 0; i < 5; i++) { assertEquals(column4.isNull(i), false); @@ -227,7 +228,8 @@ public void testRewrite(boolean useOptimizedOrcWriter) rowsToDelete.set(4); File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, rowsToDelete); + FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(FileSystemContext.DEFAULT_RAPTOR_CONTEXT); + OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(columnIds, columnTypes), path(file), path(newFile), rowsToDelete); assertEquals(info.getRowCount(), 2); assertBetweenInclusive(info.getUncompressedSize(), 94L, 118L * 2); @@ -240,7 +242,7 @@ public void testRewrite(boolean useOptimizedOrcWriter) assertEquals(reader.nextBatch(), 2); - Block column0 = reader.readBlock(BIGINT, 0); + Block column0 = reader.readBlock(0); assertEquals(column0.getPositionCount(), 2); for (int i = 0; i < 2; i++) { assertEquals(column0.isNull(i), false); @@ -248,7 +250,7 @@ public void testRewrite(boolean useOptimizedOrcWriter) assertEquals(BIGINT.getLong(column0, 0), 123L); assertEquals(BIGINT.getLong(column0, 1), 456L); - Block column1 = reader.readBlock(createVarcharType(20), 1); + Block column1 = reader.readBlock(1); assertEquals(column1.getPositionCount(), 2); for (int i = 0; i < 2; i++) { assertEquals(column1.isNull(i), false); @@ -256,7 +258,7 @@ public void testRewrite(boolean useOptimizedOrcWriter) assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("hello")); assertEquals(createVarcharType(20).getSlice(column1, 1), utf8Slice("bye")); - Block column2 = reader.readBlock(arrayType, 2); + Block column2 = reader.readBlock(2); assertEquals(column2.getPositionCount(), 2); for (int i = 0; i < 2; i++) { assertEquals(column2.isNull(i), false); @@ -264,7 +266,7 @@ public void testRewrite(boolean useOptimizedOrcWriter) assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 0), arrayBlockOf(BIGINT, 1, 2))); assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column2, 1), arrayBlockOf(BIGINT, 5, 6))); - Block column3 = reader.readBlock(mapType, 3); + Block column3 = reader.readBlock(3); assertEquals(column3.getPositionCount(), 2); for (int i = 0; i < 2; i++) { assertEquals(column3.isNull(i), false); @@ -272,7 +274,7 @@ public void testRewrite(boolean useOptimizedOrcWriter) assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column3, 0), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true))); assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column3, 1), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", true))); - Block column4 = reader.readBlock(arrayOfArrayType, 4); + Block column4 = reader.readBlock(4); assertEquals(column4.getPositionCount(), 2); for (int i = 0; i < 2; i++) { assertEquals(column4.isNull(i), false); @@ -294,15 +296,15 @@ public void testRewrite(boolean useOptimizedOrcWriter) } } - @Test(dataProvider = "useOptimizedOrcWriter") - public void testRewriteWithoutMetadata(boolean useOptimizedOrcWriter) + @Test + public void testRewriteWithoutMetadata() throws Exception { List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20)); File file = new File(temporary, randomUUID().toString()); - try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false, useOptimizedOrcWriter)) { + try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false)) { List pages = rowPagesBuilder(columnTypes) .row(123L, "hello") .row(777L, "sky") @@ -319,7 +321,7 @@ public void testRewriteWithoutMetadata(boolean useOptimizedOrcWriter) assertEquals(reader.nextBatch(), 2); - Block column0 = reader.readBlock(BIGINT, 0); + Block column0 = reader.readBlock(0); assertEquals(column0.getPositionCount(), 2); for (int i = 0; i < 2; i++) { assertEquals(column0.isNull(i), false); @@ -327,7 +329,7 @@ public void testRewriteWithoutMetadata(boolean useOptimizedOrcWriter) assertEquals(BIGINT.getLong(column0, 0), 123L); assertEquals(BIGINT.getLong(column0, 1), 777L); - Block column1 = reader.readBlock(createVarcharType(20), 1); + Block column1 = reader.readBlock(1); assertEquals(column1.getPositionCount(), 2); for (int i = 0; i < 2; i++) { assertEquals(column1.isNull(i), false); @@ -342,7 +344,8 @@ public void testRewriteWithoutMetadata(boolean useOptimizedOrcWriter) rowsToDelete.set(1); File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, rowsToDelete); + FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(FileSystemContext.DEFAULT_RAPTOR_CONTEXT); + OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(columnIds, columnTypes), path(file), path(newFile), rowsToDelete); assertEquals(info.getRowCount(), 1); assertBetweenInclusive(info.getUncompressedSize(), 13L, 13L * 2); @@ -355,12 +358,12 @@ public void testRewriteWithoutMetadata(boolean useOptimizedOrcWriter) assertEquals(reader.nextBatch(), 1); - Block column0 = reader.readBlock(BIGINT, 0); + Block column0 = reader.readBlock(0); assertEquals(column0.getPositionCount(), 1); assertEquals(column0.isNull(0), false); assertEquals(BIGINT.getLong(column0, 0), 123L); - Block column1 = reader.readBlock(createVarcharType(20), 1); + Block column1 = reader.readBlock(1); assertEquals(column1.getPositionCount(), 1); assertEquals(column1.isNull(0), false); assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("hello")); @@ -369,15 +372,15 @@ public void testRewriteWithoutMetadata(boolean useOptimizedOrcWriter) } } - @Test(dataProvider = "useOptimizedOrcWriter") - public void testRewriteAllRowsDeleted(boolean useOptimizedOrcWriter) + @Test + public void testRewriteAllRowsDeleted() throws Exception { List columnIds = ImmutableList.of(3L); List columnTypes = ImmutableList.of(BIGINT); File file = new File(temporary, randomUUID().toString()); - try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { + try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file)) { writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build()); } @@ -386,43 +389,45 @@ public void testRewriteAllRowsDeleted(boolean useOptimizedOrcWriter) rowsToDelete.set(1); File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, rowsToDelete); + FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(FileSystemContext.DEFAULT_RAPTOR_CONTEXT); + OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(columnIds, columnTypes), path(file), path(newFile), rowsToDelete); assertEquals(info.getRowCount(), 0); assertEquals(info.getUncompressedSize(), 0); assertFalse(newFile.exists()); } - @Test(dataProvider = "useOptimizedOrcWriter") - public void testRewriteNoRowsDeleted(boolean useOptimizedOrcWriter) + @Test + public void testRewriteNoRowsDeleted() throws Exception { List columnIds = ImmutableList.of(3L); List columnTypes = ImmutableList.of(BIGINT); File file = new File(temporary, randomUUID().toString()); - try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { + try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file)) { writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build()); } BitSet rowsToDelete = new BitSet(); File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, rowsToDelete); + FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(FileSystemContext.DEFAULT_RAPTOR_CONTEXT); + OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(columnIds, columnTypes), path(file), path(newFile), rowsToDelete); assertEquals(info.getRowCount(), 2); assertBetweenInclusive(info.getUncompressedSize(), 16L, 16L * 2); assertEquals(readAllBytes(newFile.toPath()), readAllBytes(file.toPath())); } - @Test(dataProvider = "useOptimizedOrcWriter") - public void testUncompressedSize(boolean useOptimizedOrcWriter) + @Test + public void testUncompressedSize() throws Exception { List columnIds = ImmutableList.of(1L, 2L, 3L, 4L, 5L); List columnTypes = ImmutableList.of(BOOLEAN, BIGINT, DOUBLE, createVarcharType(10), VARBINARY); File file = new File(temporary, randomUUID().toString()); - try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { + try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file)) { List pages = rowPagesBuilder(columnTypes) .row(true, 123L, 98.7, "hello", utf8Slice("abc")) .row(false, 456L, 65.4, "world", utf8Slice("xyz")) @@ -432,7 +437,8 @@ public void testUncompressedSize(boolean useOptimizedOrcWriter) } File newFile = new File(temporary, randomUUID().toString()); - OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite(getColumnTypes(columnIds, columnTypes), file, newFile, new BitSet()); + FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(FileSystemContext.DEFAULT_RAPTOR_CONTEXT); + OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(columnIds, columnTypes), path(file), path(newFile), new BitSet()); assertEquals(info.getRowCount(), 3); assertBetweenInclusive(info.getUncompressedSize(), 55L, 55L * 2); } @@ -440,8 +446,8 @@ public void testUncompressedSize(boolean useOptimizedOrcWriter) /** * The following test add or drop different columns */ - @Test(dataProvider = "useOptimizedOrcWriter") - public void testRewriterDropThenAddDifferentColumns(boolean useOptimizedOrcWriter) + @Test + public void testRewriterDropThenAddDifferentColumns() throws Exception { TypeRegistry typeRegistry = new TypeRegistry(); @@ -450,13 +456,13 @@ public void testRewriterDropThenAddDifferentColumns(boolean useOptimizedOrcWrite Handle dummyHandle = dbi.open(); File dataDir = Files.createTempDir(); - StorageManager storageManager = createOrcStorageManager(dbi, dataDir, useOptimizedOrcWriter); + StorageManager storageManager = createOrcStorageManager(dbi, dataDir); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20)); File file = new File(temporary, randomUUID().toString()); - try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false, useOptimizedOrcWriter)) { + try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false)) { List pages = rowPagesBuilder(columnTypes) .row(1L, "1") .row(2L, "2") @@ -468,38 +474,37 @@ public void testRewriterDropThenAddDifferentColumns(boolean useOptimizedOrcWrite // Add a column File newFile1 = new File(temporary, randomUUID().toString()); - OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite( + FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(FileSystemContext.DEFAULT_RAPTOR_CONTEXT); + OrcFileInfo info = createFileRewriter().rewrite( + fileSystem, getColumnTypes(ImmutableList.of(3L, 7L, 10L), ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE)), - file, - newFile1, + path(file), + path(newFile1), new BitSet(5)); assertEquals(info.getRowCount(), 4); assertEquals(readAllBytes(file.toPath()), readAllBytes(newFile1.toPath())); // Drop a column File newFile2 = new File(temporary, randomUUID().toString()); - info = createFileRewriter(useOptimizedOrcWriter).rewrite( + info = createFileRewriter().rewrite( + fileSystem, getColumnTypes(ImmutableList.of(7L, 10L), ImmutableList.of(createVarcharType(20), DOUBLE)), - newFile1, - newFile2, + path(newFile1), + path(newFile2), new BitSet(5)); assertEquals(info.getRowCount(), 4); - if (useOptimizedOrcWriter) { - // Optimized writer will keep the only column - OrcReader orcReader = new OrcReader(fileOrcDataSource(newFile2), ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); - orcReader.getColumnNames().equals(ImmutableList.of("7")); - } - else { - // Apache writer does not drop columns - assertEquals(readAllBytes(newFile1.toPath()), readAllBytes(newFile2.toPath())); - } + + // Optimized writer will keep the only column + OrcReader orcReader = new OrcReader(fileOrcDataSource(newFile2), ORC, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), OrcTestingUtil.createDefaultTestConfig()); + orcReader.getColumnNames().equals(ImmutableList.of("7")); // Add a column with the different ID with different type File newFile3 = new File(temporary, randomUUID().toString()); - info = createFileRewriter(useOptimizedOrcWriter).rewrite( + info = createFileRewriter().rewrite( + fileSystem, getColumnTypes(ImmutableList.of(7L, 10L, 13L), ImmutableList.of(createVarcharType(20), DOUBLE, createVarcharType(5))), - newFile2, - newFile3, + path(newFile2), + path(newFile3), new BitSet(5)); assertEquals(info.getRowCount(), 4); assertEquals(readAllBytes(newFile2.toPath()), readAllBytes(newFile3.toPath())); @@ -507,25 +512,26 @@ public void testRewriterDropThenAddDifferentColumns(boolean useOptimizedOrcWrite // Get prepared for the final file; make sure it is accessible from storage manager UUID uuid = randomUUID(); File newFile4 = getFileSystemPath(new File(dataDir, "data/storage"), uuid); - if (useOptimizedOrcWriter) { - // Apache ORC writer creates the file itself - newFile4.getParentFile().mkdirs(); - newFile4.createNewFile(); - } + + // Optimized ORC writer does not create the file itself + newFile4.getParentFile().mkdirs(); + newFile4.createNewFile(); // Drop a column and add a column; also delete 3 rows BitSet rowsToDelete = new BitSet(5); rowsToDelete.set(0); rowsToDelete.set(1); rowsToDelete.set(3); - info = createFileRewriter(useOptimizedOrcWriter).rewrite( + info = createFileRewriter().rewrite( + fileSystem, getColumnTypes(ImmutableList.of(7L, 13L, 18L), ImmutableList.of(createVarcharType(20), createVarcharType(5), INTEGER)), - newFile3, - newFile4, + path(newFile3), + path(newFile4), rowsToDelete); assertEquals(info.getRowCount(), 1); ConnectorPageSource source = storageManager.getPageSource( + FileSystemContext.DEFAULT_RAPTOR_CONTEXT, uuid, OptionalInt.empty(), ImmutableList.of(13L, 7L, 18L), @@ -553,20 +559,16 @@ public void testRewriterDropThenAddDifferentColumns(boolean useOptimizedOrcWrite // Remove all the columns File newFile5 = new File(temporary, randomUUID().toString()); - info = createFileRewriter(useOptimizedOrcWriter).rewrite( + info = createFileRewriter().rewrite( + fileSystem, getColumnTypes(ImmutableList.of(13L, 18L), ImmutableList.of(createVarcharType(5), INTEGER)), - newFile4, - newFile5, + path(newFile4), + path(newFile5), new BitSet(5)); - if (useOptimizedOrcWriter) { - // Optimized writer will drop the file - assertEquals(info.getRowCount(), 0); - assertFalse(newFile5.exists()); - } - else { - assertEquals(info.getRowCount(), 1); - assertEquals(readAllBytes(newFile4.toPath()), readAllBytes(newFile5.toPath())); - } + + // Optimized writer will drop the file + assertEquals(info.getRowCount(), 0); + assertFalse(newFile5.exists()); dummyHandle.close(); deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); @@ -576,8 +578,8 @@ public void testRewriterDropThenAddDifferentColumns(boolean useOptimizedOrcWrite * The following test drop and add the same columns; the legacy ORC rewriter will fail due to unchanged schema. * However, if we enforce the newly added column to always have the largest ID, this won't happen. */ - @Test(dataProvider = "useOptimizedOrcWriter") - public void testRewriterDropThenAddSameColumns(boolean useOptimizedOrcWriter) + @Test + public void testRewriterDropThenAddSameColumns() throws Exception { TypeRegistry typeRegistry = new TypeRegistry(); @@ -586,13 +588,13 @@ public void testRewriterDropThenAddSameColumns(boolean useOptimizedOrcWriter) Handle dummyHandle = dbi.open(); File dataDir = Files.createTempDir(); - StorageManager storageManager = createOrcStorageManager(dbi, dataDir, useOptimizedOrcWriter); + StorageManager storageManager = createOrcStorageManager(dbi, dataDir); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20)); File file = new File(temporary, randomUUID().toString()); - try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false, useOptimizedOrcWriter)) { + try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false)) { List pages = rowPagesBuilder(columnTypes) .row(2L, "2") .build(); @@ -601,54 +603,59 @@ public void testRewriterDropThenAddSameColumns(boolean useOptimizedOrcWriter) // Add a column File newFile1 = new File(temporary, randomUUID().toString()); - OrcFileInfo info = createFileRewriter(useOptimizedOrcWriter).rewrite( + FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(FileSystemContext.DEFAULT_RAPTOR_CONTEXT); + OrcFileInfo info = createFileRewriter().rewrite( + fileSystem, getColumnTypes(ImmutableList.of(3L, 7L, 10L), ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE)), - file, - newFile1, + path(file), + path(newFile1), new BitSet(5)); assertEquals(info.getRowCount(), 1); // Drop a column File newFile2 = new File(temporary, randomUUID().toString()); - info = createFileRewriter(useOptimizedOrcWriter).rewrite( + info = createFileRewriter().rewrite( + fileSystem, getColumnTypes(ImmutableList.of(7L, 10L), ImmutableList.of(createVarcharType(20), DOUBLE)), - newFile1, - newFile2, + path(newFile1), + path(newFile2), new BitSet(5)); assertEquals(info.getRowCount(), 1); // Add a column with the same ID but different type File newFile3 = new File(temporary, randomUUID().toString()); - info = createFileRewriter(useOptimizedOrcWriter).rewrite( + info = createFileRewriter().rewrite( + fileSystem, getColumnTypes(ImmutableList.of(7L, 10L, 3L), ImmutableList.of(createVarcharType(20), DOUBLE, createVarcharType(5))), - newFile2, - newFile3, + path(newFile2), + path(newFile3), new BitSet(5)); assertEquals(info.getRowCount(), 1); // Get prepared for the final file; make sure it is accessible from storage manager UUID uuid = randomUUID(); File newFile4 = getFileSystemPath(new File(dataDir, "data/storage"), uuid); - if (useOptimizedOrcWriter) { - // Apache ORC writer creates the file itself - newFile4.getParentFile().mkdirs(); - newFile4.createNewFile(); - } + + // Optimized ORC writer does not create the file itself + newFile4.getParentFile().mkdirs(); + newFile4.createNewFile(); // Drop a column and add a column - info = createFileRewriter(useOptimizedOrcWriter).rewrite( + info = createFileRewriter().rewrite( + fileSystem, getColumnTypes(ImmutableList.of(7L, 3L, 8L), ImmutableList.of(createVarcharType(20), createVarcharType(5), INTEGER)), - newFile3, - newFile4, + path(newFile3), + path(newFile4), new BitSet(5)); assertEquals(info.getRowCount(), 1); ConnectorPageSource source = storageManager.getPageSource( + FileSystemContext.DEFAULT_RAPTOR_CONTEXT, uuid, OptionalInt.empty(), ImmutableList.of(3L, 7L, 8L), ImmutableList.of(createVarcharType(5), createVarcharType(20), INTEGER), - TupleDomain.all(), + TupleDomain.all(), READER_ATTRIBUTES); Page page = null; @@ -672,40 +679,49 @@ public void testRewriterDropThenAddSameColumns(boolean useOptimizedOrcWriter) dummyHandle.close(); deleteRecursively(dataDir.toPath(), ALLOW_INSECURE); - - if (!useOptimizedOrcWriter) { - // Apache ORC rewriter will not respect the schema - fail(); - } } catch (UnsupportedOperationException e) { - if (useOptimizedOrcWriter) { - // Optimized ORC rewriter will respect the schema - fail(); - } + // Optimized ORC rewriter will respect the schema + fail(); } } - private static FileWriter createFileWriter(List columnIds, List columnTypes, File file, boolean writeMetadata, boolean useOptimizedOrcWriter) + private static FileWriter createFileWriter(List columnIds, List columnTypes, File file, boolean writeMetadata) + throws FileNotFoundException { - if (useOptimizedOrcWriter) { - return new OrcFileWriter(columnIds, columnTypes, file, writeMetadata, true, new OrcWriterStats(), new TypeRegistry(), ZSTD); - } - return new OrcRecordWriter(columnIds, columnTypes, file, SNAPPY, writeMetadata); + return new OrcFileWriter( + columnIds, + columnTypes, + new OutputStreamOrcDataSink(new FileOutputStream(file)), + writeMetadata, + true, + new OrcWriterStats(), + new TypeRegistry(), + ZSTD); } - private static FileRewriter createFileRewriter(boolean useOptimizedOrcWriter) + private static OrcFileRewriter createFileRewriter() { - if (useOptimizedOrcWriter) { - TypeRegistry typeManager = new TypeRegistry(); - new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); - return new OrcPageFileRewriter(READER_ATTRIBUTES, true, new OrcWriterStats(), typeManager, ZSTD); - } - return new OrcRecordFileRewriter(); + TypeRegistry typeManager = new TypeRegistry(); + new FunctionManager(typeManager, new BlockEncodingManager(typeManager), new FeaturesConfig()); + return new OrcFileRewriter( + READER_ATTRIBUTES, + true, + new OrcWriterStats(), + typeManager, + new LocalOrcDataEnvironment(), + ZSTD, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource()); } private static Map getColumnTypes(List columnIds, List columnTypes) { return IntStream.range(0, columnIds.size()).boxed().collect(Collectors.toMap(index -> String.valueOf(columnIds.get(index)), columnTypes::get)); } + + private static Path path(File file) + { + return new Path(file.toURI()); + } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java index eafdc29c4fe3b..0572e6c0b14ca 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestOrcStorageManager.java @@ -15,10 +15,16 @@ import com.facebook.presto.orc.OrcBatchRecordReader; import com.facebook.presto.orc.OrcDataSource; +import com.facebook.presto.orc.StorageStripeMetadataSource; +import com.facebook.presto.orc.cache.StorageOrcFileTailSource; import com.facebook.presto.raptor.RaptorColumnHandle; import com.facebook.presto.raptor.backup.BackupManager; import com.facebook.presto.raptor.backup.BackupStore; import com.facebook.presto.raptor.backup.FileBackupStore; +import com.facebook.presto.raptor.filesystem.FileSystemContext; +import com.facebook.presto.raptor.filesystem.LocalFileStorageService; +import com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment; +import com.facebook.presto.raptor.filesystem.RaptorLocalFileSystem; import com.facebook.presto.raptor.metadata.ColumnStats; import com.facebook.presto.raptor.metadata.ShardDelta; import com.facebook.presto.raptor.metadata.ShardInfo; @@ -45,6 +51,9 @@ import io.airlift.slice.Slice; import io.airlift.units.DataSize; import io.airlift.units.Duration; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.joda.time.DateTime; import org.joda.time.Days; import org.joda.time.chrono.ISOChronology; @@ -53,11 +62,11 @@ import org.skife.jdbi.v2.IDBI; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.File; import java.io.IOException; +import java.net.URI; import java.util.Arrays; import java.util.BitSet; import java.util.Collection; @@ -70,14 +79,15 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.RowPagesBuilder.rowPagesBuilder; import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; +import static com.facebook.presto.raptor.filesystem.FileSystemUtil.xxhash64; import static com.facebook.presto.raptor.metadata.SchemaDaoUtil.createTablesWithRetry; import static com.facebook.presto.raptor.metadata.TestDatabaseShardManager.createShardManager; -import static com.facebook.presto.raptor.storage.OrcStorageManager.xxhash64; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReader; import static com.facebook.presto.raptor.storage.OrcTestingUtil.octets; -import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.DISABLED; import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED_AND_VALIDATED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; @@ -98,8 +108,6 @@ import static com.google.common.io.Files.hash; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.slice.Slices.wrappedBuffer; import static io.airlift.units.DataSize.Unit.BYTE; @@ -128,7 +136,7 @@ public class TestOrcStorageManager private static final int MAX_SHARD_ROWS = 100; private static final DataSize MAX_FILE_SIZE = new DataSize(1, MEGABYTE); private static final Duration MISSING_SHARD_DISCOVERY = new Duration(5, TimeUnit.MINUTES); - private static final ReaderAttributes READER_ATTRIBUTES = new ReaderAttributes(new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); + private static final ReaderAttributes READER_ATTRIBUTES = new ReaderAttributes(new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true, false); private final NodeManager nodeManager = new TestingNodeManager(); private Handle dummyHandle; @@ -139,18 +147,12 @@ public class TestOrcStorageManager private Optional backupStore; private InMemoryShardRecorder shardRecorder; - @DataProvider(name = "useOptimizedOrcWriter") - public static Object[][] useOptimizedOrcWriter() - { - return new Object[][] {{true}, {false}}; - } - @BeforeMethod public void setup() { temporary = createTempDir(); - File directory = new File(temporary, "data"); - storageService = new FileStorageService(directory); + URI directory = new File(temporary, "data").toURI(); + storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), directory); storageService.start(); File backupDirectory = new File(temporary, "backup"); @@ -164,7 +166,7 @@ public void setup() ShardManager shardManager = createShardManager(dbi); Duration discoveryInterval = new Duration(5, TimeUnit.MINUTES); - recoveryManager = new ShardRecoveryManager(storageService, backupStore, nodeManager, shardManager, discoveryInterval, 10); + recoveryManager = new ShardRecoveryManager(storageService, backupStore, new LocalOrcDataEnvironment(), nodeManager, shardManager, discoveryInterval, 10); shardRecorder = new InMemoryShardRecorder(); } @@ -179,11 +181,11 @@ public void tearDown() deleteRecursively(temporary.toPath(), ALLOW_INSECURE); } - @Test(dataProvider = "useOptimizedOrcWriter") - public void testWriter(boolean useOptimizedOrcWriter) + @Test + public void testWriter() throws Exception { - OrcStorageManager manager = createOrcStorageManager(useOptimizedOrcWriter); + OrcStorageManager manager = createOrcStorageManager(); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); @@ -210,7 +212,7 @@ public void testWriter(boolean useOptimizedOrcWriter) ShardInfo shardInfo = Iterables.getOnlyElement(shards); UUID shardUuid = shardInfo.getShardUuid(); - File file = storageService.getStorageFile(shardUuid); + File file = new File(storageService.getStorageFile(shardUuid).toString()); File backupFile = fileBackupStore.getBackupFile(shardUuid); assertEquals(recordedShards.get(0).getTransactionId(), TRANSACTION_ID); @@ -218,7 +220,7 @@ public void testWriter(boolean useOptimizedOrcWriter) assertEquals(shardInfo.getRowCount(), 2); assertEquals(shardInfo.getCompressedSize(), file.length()); - assertEquals(shardInfo.getXxhash64(), xxhash64(file)); + assertEquals(shardInfo.getXxhash64(), xxhash64(new RaptorLocalFileSystem(new Configuration()), new Path(file.toURI()))); // verify primary and backup shard exist assertFile(file, "primary shard"); @@ -233,18 +235,19 @@ public void testWriter(boolean useOptimizedOrcWriter) recoveryManager.restoreFromBackup(shardUuid, shardInfo.getCompressedSize(), OptionalLong.of(shardInfo.getXxhash64())); - try (OrcDataSource dataSource = manager.openShard(shardUuid, READER_ATTRIBUTES)) { + FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(FileSystemContext.DEFAULT_RAPTOR_CONTEXT); + try (OrcDataSource dataSource = manager.openShard(fileSystem, shardUuid, READER_ATTRIBUTES)) { OrcBatchRecordReader reader = createReader(dataSource, columnIds, columnTypes); assertEquals(reader.nextBatch(), 2); - Block column0 = reader.readBlock(BIGINT, 0); + Block column0 = reader.readBlock(0); assertEquals(column0.isNull(0), false); assertEquals(column0.isNull(1), false); assertEquals(BIGINT.getLong(column0, 0), 123L); assertEquals(BIGINT.getLong(column0, 1), 456L); - Block column1 = reader.readBlock(createVarcharType(10), 1); + Block column1 = reader.readBlock(1); assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello")); assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("bye")); @@ -252,11 +255,11 @@ public void testWriter(boolean useOptimizedOrcWriter) } } - @Test(dataProvider = "useOptimizedOrcWriter") - public void testReader(boolean useOptimizedOrcWriter) + @Test + public void testReader() throws Exception { - OrcStorageManager manager = createOrcStorageManager(useOptimizedOrcWriter); + OrcStorageManager manager = createOrcStorageManager(); List columnIds = ImmutableList.of(2L, 4L, 6L, 7L, 8L, 9L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DATE, BOOLEAN, DOUBLE); @@ -326,11 +329,12 @@ public void testReader(boolean useOptimizedOrcWriter) } } - @Test(dataProvider = "useOptimizedOrcWriter") - public void testRewriter(boolean useOptimizedOrcWriter) + @Test + public void testRewriter() throws Exception { - OrcStorageManager manager = createOrcStorageManager(useOptimizedOrcWriter); + OrcStorageManager manager = createOrcStorageManager(); + FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(FileSystemContext.DEFAULT_RAPTOR_CONTEXT); long transactionId = TRANSACTION_ID; List columnIds = ImmutableList.of(3L, 7L); @@ -351,6 +355,7 @@ public void testRewriter(boolean useOptimizedOrcWriter) BitSet rowsToDelete = new BitSet(); rowsToDelete.set(0); Collection fragments = manager.rewriteShard( + fileSystem, transactionId, OptionalInt.empty(), shards.get(0).getShardUuid(), @@ -365,7 +370,7 @@ public void testRewriter(boolean useOptimizedOrcWriter) assertEquals(shardInfo.getRowCount(), 1); // check that storage file is same as backup file - File storageFile = storageService.getStorageFile(shardInfo.getShardUuid()); + File storageFile = new File(storageService.getStorageFile(shardInfo.getShardUuid()).toString()); File backupFile = fileBackupStore.getBackupFile(shardInfo.getShardUuid()); assertFileEquals(storageFile, backupFile); @@ -376,8 +381,7 @@ public void testRewriter(boolean useOptimizedOrcWriter) assertEquals(recordedShards.get(1).getShardUuid(), shardInfo.getShardUuid()); } - @Test(dataProvider = "useOptimizedOrcWriter") - public void testWriterRollback(boolean useOptimizedOrcWriter) + public void testWriterRollback() { // verify staging directory is empty File staging = new File(new File(temporary, "data"), "staging"); @@ -385,7 +389,7 @@ public void testWriterRollback(boolean useOptimizedOrcWriter) assertEquals(staging.list(), new String[] {}); // create a shard in staging - OrcStorageManager manager = createOrcStorageManager(useOptimizedOrcWriter); + OrcStorageManager manager = createOrcStorageManager(); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); @@ -530,10 +534,10 @@ public void testShardStatsTime() assertColumnStats(columnStats, 1, minTime, maxTime); } - @Test(dataProvider = "useOptimizedOrcWriter") - public void testMaxShardRows(boolean useOptimizedOrcWriter) + @Test + public void testMaxShardRows() { - OrcStorageManager manager = createOrcStorageManager(2, new DataSize(2, MEGABYTE), useOptimizedOrcWriter); + OrcStorageManager manager = createOrcStorageManager(2, new DataSize(2, MEGABYTE)); List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(10)); @@ -547,8 +551,8 @@ public void testMaxShardRows(boolean useOptimizedOrcWriter) assertTrue(sink.isFull()); } - @Test(dataProvider = "useOptimizedOrcWriter") - public void testMaxFileSize(boolean useOptimizedOrcWriter) + @Test + public void testMaxFileSize() { List columnIds = ImmutableList.of(3L, 7L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(5)); @@ -559,7 +563,7 @@ public void testMaxFileSize(boolean useOptimizedOrcWriter) .build(); // Set maxFileSize to 1 byte, so adding any page makes the StoragePageSink full - OrcStorageManager manager = createOrcStorageManager(20, new DataSize(1, BYTE), useOptimizedOrcWriter); + OrcStorageManager manager = createOrcStorageManager(20, new DataSize(1, BYTE)); StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); sink.appendPages(pages); assertTrue(sink.isFull()); @@ -572,34 +576,34 @@ private static ConnectorPageSource getPageSource( UUID uuid, TupleDomain tupleDomain) { - return manager.getPageSource(uuid, OptionalInt.empty(), columnIds, columnTypes, tupleDomain, READER_ATTRIBUTES); + return manager.getPageSource(FileSystemContext.DEFAULT_RAPTOR_CONTEXT, uuid, OptionalInt.empty(), columnIds, columnTypes, tupleDomain, READER_ATTRIBUTES); } private static StoragePageSink createStoragePageSink(StorageManager manager, List columnIds, List columnTypes) { long transactionId = TRANSACTION_ID; - return manager.createStoragePageSink(transactionId, OptionalInt.empty(), columnIds, columnTypes, false); + return manager.createStoragePageSink(FileSystemContext.DEFAULT_RAPTOR_CONTEXT, transactionId, OptionalInt.empty(), columnIds, columnTypes, false); } - private OrcStorageManager createOrcStorageManager(boolean useOptimizedWriter) + private OrcStorageManager createOrcStorageManager() { - return createOrcStorageManager(MAX_SHARD_ROWS, MAX_FILE_SIZE, useOptimizedWriter); + return createOrcStorageManager(MAX_SHARD_ROWS, MAX_FILE_SIZE); } - private OrcStorageManager createOrcStorageManager(int maxShardRows, DataSize maxFileSize, boolean useOptimizedWriter) + private OrcStorageManager createOrcStorageManager(int maxShardRows, DataSize maxFileSize) { - return createOrcStorageManager(storageService, backupStore, recoveryManager, shardRecorder, maxShardRows, maxFileSize, useOptimizedWriter); + return createOrcStorageManager(storageService, backupStore, recoveryManager, shardRecorder, maxShardRows, maxFileSize); } - public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary, boolean useOptimizedWriter) + public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary) { - return createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS, useOptimizedWriter); + return createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS); } - public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary, int maxShardRows, boolean useOptimizedWriter) + public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary, int maxShardRows) { - File directory = new File(temporary, "data"); - StorageService storageService = new FileStorageService(directory); + URI directory = new File(temporary, "data").toURI(); + StorageService storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), directory); storageService.start(); File backupDirectory = new File(temporary, "backup"); @@ -611,6 +615,7 @@ public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary ShardRecoveryManager recoveryManager = new ShardRecoveryManager( storageService, backupStore, + new LocalOrcDataEnvironment(), new TestingNodeManager(), shardManager, MISSING_SHARD_DISCOVERY, @@ -621,8 +626,7 @@ public static OrcStorageManager createOrcStorageManager(IDBI dbi, File temporary recoveryManager, new InMemoryShardRecorder(), maxShardRows, - MAX_FILE_SIZE, - useOptimizedWriter); + MAX_FILE_SIZE); } public static OrcStorageManager createOrcStorageManager( @@ -631,18 +635,18 @@ public static OrcStorageManager createOrcStorageManager( ShardRecoveryManager recoveryManager, ShardRecorder shardRecorder, int maxShardRows, - DataSize maxFileSize, - boolean useOptimizedWriter) + DataSize maxFileSize) { return new OrcStorageManager( CURRENT_NODE, storageService, backupStore, READER_ATTRIBUTES, - new BackupManager(backupStore, storageService, 1), + new BackupManager(backupStore, storageService, new LocalOrcDataEnvironment(), 1), recoveryManager, shardRecorder, new TypeRegistry(), + new LocalOrcDataEnvironment(), CONNECTOR_ID, DELETION_THREADS, SHARD_RECOVERY_TIMEOUT, @@ -650,7 +654,9 @@ public static OrcStorageManager createOrcStorageManager( maxFileSize, new DataSize(0, BYTE), SNAPPY, - useOptimizedWriter ? ENABLED_AND_VALIDATED : DISABLED); + ENABLED_AND_VALIDATED, + new StorageOrcFileTailSource(), + new StorageStripeMetadataSource()); } private static void assertFileEquals(File actual, File expected) @@ -704,7 +710,7 @@ private List columnStats(List columnTypes, Object[]... rows) private List columnStats(boolean useOptimizedWriter, List columnIds, List columnTypes, Object[]... rows) { - OrcStorageManager manager = createOrcStorageManager(useOptimizedWriter); + OrcStorageManager manager = createOrcStorageManager(); StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); sink.appendPages(rowPagesBuilder(columnTypes).rows(rows).build()); List shards = getFutureValue(sink.commit()); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestRaptorHdfsConfig.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestRaptorHdfsConfig.java new file mode 100644 index 0000000000000..22697cfc467ce --- /dev/null +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestRaptorHdfsConfig.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.raptor.storage; + +import com.facebook.airlift.configuration.testing.ConfigAssertions; +import com.facebook.presto.raptor.filesystem.RaptorHdfsConfig; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.net.HostAndPort; +import io.airlift.units.DataSize; +import io.airlift.units.DataSize.Unit; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class TestRaptorHdfsConfig +{ + @Test + public void testDefaults() + { + ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(RaptorHdfsConfig.class) + .setSocksProxy(null) + .setDfsTimeout(new Duration(60, TimeUnit.SECONDS)) + .setIpcPingInterval(new Duration(10, TimeUnit.SECONDS)) + .setDfsConnectTimeout(new Duration(500, TimeUnit.MILLISECONDS)) + .setDfsConnectMaxRetries(5) + .setDomainSocketPath(null) + .setTextMaxLineLength(new DataSize(100, Unit.MEGABYTE)) + .setResourceConfigFiles("") + .setFileSystemMaxCacheSize(1000) + .setHdfsWireEncryptionEnabled(false)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("hive.thrift.client.socks-proxy", "localhost:1080") + .put("hive.dfs.ipc-ping-interval", "34s") + .put("hive.dfs-timeout", "33s") + .put("hive.dfs.connect.timeout", "20s") + .put("hive.dfs.connect.max-retries", "10") + .put("hive.dfs.domain-socket-path", "/foo") + .put("hive.text.max-line-length", "13MB") + .put("hive.fs.cache.max-size", "1010") + .put("hive.hdfs.wire-encryption.enabled", "true") + .put("hive.config.resources", "a,b,c") + .build(); + + RaptorHdfsConfig expected = new RaptorHdfsConfig() + .setSocksProxy(HostAndPort.fromParts("localhost", 1080)) + .setIpcPingInterval(new Duration(34, TimeUnit.SECONDS)) + .setDfsTimeout(new Duration(33, TimeUnit.SECONDS)) + .setDfsConnectTimeout(new Duration(20, TimeUnit.SECONDS)) + .setDfsConnectMaxRetries(10) + .setDomainSocketPath("/foo") + .setTextMaxLineLength(new DataSize(13, Unit.MEGABYTE)) + .setFileSystemMaxCacheSize(1010) + .setResourceConfigFiles(ImmutableList.of("a", "b", "c")) + .setHdfsWireEncryptionEnabled(true); + + ConfigAssertions.assertFullMapping(properties, expected); + } +} diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardEjector.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardEjector.java index 2d34aa7d13197..6da0b736e8255 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardEjector.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardEjector.java @@ -16,6 +16,8 @@ import com.facebook.presto.client.NodeVersion; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.raptor.backup.BackupStore; +import com.facebook.presto.raptor.filesystem.LocalFileStorageService; +import com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment; import com.facebook.presto.raptor.metadata.ColumnInfo; import com.facebook.presto.raptor.metadata.MetadataDao; import com.facebook.presto.raptor.metadata.ShardInfo; @@ -28,6 +30,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import io.airlift.units.Duration; +import org.apache.hadoop.fs.Path; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.Handle; import org.skife.jdbi.v2.IDBI; @@ -75,7 +78,7 @@ public void setup() shardManager = createShardManager(dbi); dataDir = createTempDir(); - storageService = new FileStorageService(dataDir); + storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), dataDir.toURI()); storageService.start(); } @@ -104,6 +107,7 @@ public void testEjector() storageService, new Duration(1, HOURS), Optional.of(new TestingBackupStore()), + new LocalOrcDataEnvironment(), "test"); List shards = ImmutableList.builder() @@ -126,14 +130,14 @@ public void testEjector() long tableId = createTable("test"); List columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); - shardManager.createTable(tableId, columns, false, OptionalLong.empty()); + shardManager.createTable(tableId, columns, false, OptionalLong.empty(), false); long transactionId = shardManager.beginTransaction(); shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty(), 0); for (ShardInfo shard : shards.subList(0, 8)) { - File file = storageService.getStorageFile(shard.getShardUuid()); - storageService.createParents(file); + File file = new File(storageService.getStorageFile(shard.getShardUuid()).toString()); + storageService.createParents(new Path(file.toURI())); assertTrue(file.createNewFile()); } @@ -152,12 +156,12 @@ public void testEjector() for (UUID uuid : ejectedShards) { assertFalse(remaining.contains(uuid)); - assertFalse(storageService.getStorageFile(uuid).exists()); + assertFalse(new File(storageService.getStorageFile(uuid).toString()).exists()); } assertEquals(remaining, keptShards); for (UUID uuid : keptShards) { - assertTrue(storageService.getStorageFile(uuid).exists()); + assertTrue(new File(storageService.getStorageFile(uuid).toString()).exists()); } Set others = ImmutableSet.builder() @@ -172,7 +176,7 @@ public void testEjector() private long createTable(String name) { - return dbi.onDemand(MetadataDao.class).insertTable("test", name, false, false, null, 0); + return dbi.onDemand(MetadataDao.class).insertTable("test", name, false, false, null, 0, false); } private static Set uuids(Set metadata) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardRecovery.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardRecovery.java index e105d5dc1c9bf..4fea67745ff00 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardRecovery.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardRecovery.java @@ -15,12 +15,17 @@ import com.facebook.presto.raptor.backup.BackupStore; import com.facebook.presto.raptor.backup.FileBackupStore; +import com.facebook.presto.raptor.filesystem.LocalFileStorageService; +import com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment; +import com.facebook.presto.raptor.filesystem.RaptorLocalFileSystem; import com.facebook.presto.raptor.metadata.ShardManager; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.testing.TestingNodeManager; import com.google.common.collect.ImmutableList; import com.google.common.io.Files; import io.airlift.units.Duration; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.Handle; import org.skife.jdbi.v2.IDBI; @@ -35,9 +40,9 @@ import java.util.UUID; import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_BACKUP_CORRUPTION; +import static com.facebook.presto.raptor.filesystem.FileSystemUtil.xxhash64; import static com.facebook.presto.raptor.metadata.SchemaDaoUtil.createTablesWithRetry; import static com.facebook.presto.raptor.metadata.TestDatabaseShardManager.createShardManager; -import static com.facebook.presto.raptor.storage.OrcStorageManager.xxhash64; import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.io.Files.createTempDir; import static com.google.common.io.MoreFiles.deleteRecursively; @@ -69,7 +74,7 @@ public void setup() File backupDirectory = new File(temporary, "backup"); backupStore = new FileBackupStore(backupDirectory); backupStore.start(); - storageService = new FileStorageService(directory); + storageService = new LocalFileStorageService(new LocalOrcDataEnvironment(), directory.toURI()); storageService.start(); IDBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime()); @@ -95,7 +100,7 @@ public void testShardRecovery() throws Exception { UUID shardUuid = UUID.randomUUID(); - File file = storageService.getStorageFile(shardUuid); + File file = new File(storageService.getStorageFile(shardUuid).toString()); File tempFile = createTempFile("tmp", null, temporary); Files.write("test data", tempFile, UTF_8); @@ -129,8 +134,8 @@ public void testShardRecoveryExistingFileSizeMismatch() assertTrue(Files.equal(tempFile, backupFile)); // write corrupt storage file with wrong length - File storageFile = storageService.getStorageFile(shardUuid); - storageService.createParents(storageFile); + File storageFile = new File(storageService.getStorageFile(shardUuid).toString()); + storageService.createParents(new Path(storageFile.toURI())); Files.write("bad data", storageFile, UTF_8); @@ -145,7 +150,7 @@ public void testShardRecoveryExistingFileSizeMismatch() assertTrue(Files.equal(storageFile, tempFile)); // verify quarantine exists - List quarantined = listFiles(storageService.getQuarantineFile(shardUuid).getParentFile()); + List quarantined = listFiles(new File(storageService.getQuarantineFile(shardUuid).getParent().toString())); assertEquals(quarantined.size(), 1); assertTrue(getOnlyElement(quarantined).startsWith(shardUuid + ".orc.corrupt")); } @@ -167,8 +172,8 @@ public void testShardRecoveryExistingFileChecksumMismatch() assertTrue(Files.equal(tempFile, backupFile)); // write corrupt storage file with wrong data - File storageFile = storageService.getStorageFile(shardUuid); - storageService.createParents(storageFile); + File storageFile = new File(storageService.getStorageFile(shardUuid).toString()); + storageService.createParents(new Path(storageFile.toURI())); Files.write("test xata", storageFile, UTF_8); @@ -177,13 +182,13 @@ public void testShardRecoveryExistingFileChecksumMismatch() assertFalse(Files.equal(storageFile, tempFile)); // restore from backup and verify - recoveryManager.restoreFromBackup(shardUuid, tempFile.length(), OptionalLong.of(xxhash64(tempFile))); + recoveryManager.restoreFromBackup(shardUuid, tempFile.length(), OptionalLong.of(xxhash64(new RaptorLocalFileSystem(new Configuration()), new Path(tempFile.toURI())))); assertTrue(storageFile.exists()); assertTrue(Files.equal(storageFile, tempFile)); // verify quarantine exists - List quarantined = listFiles(storageService.getQuarantineFile(shardUuid).getParentFile()); + List quarantined = listFiles(new File(storageService.getQuarantineFile(shardUuid).getParent().toString())); assertEquals(quarantined.size(), 1); assertTrue(getOnlyElement(quarantined).startsWith(shardUuid + ".orc.corrupt")); } @@ -195,13 +200,13 @@ public void testShardRecoveryBackupChecksumMismatch() UUID shardUuid = UUID.randomUUID(); // write storage file - File storageFile = storageService.getStorageFile(shardUuid); - storageService.createParents(storageFile); + File storageFile = new File(storageService.getStorageFile(shardUuid).toString()); + storageService.createParents(new Path(storageFile.toURI())); Files.write("test data", storageFile, UTF_8); long size = storageFile.length(); - long xxhash64 = xxhash64(storageFile); + long xxhash64 = xxhash64(new RaptorLocalFileSystem(new Configuration()), new Path(storageFile.toURI())); // backup and verify backupStore.backupShard(shardUuid, storageFile); @@ -232,7 +237,7 @@ public void testShardRecoveryBackupChecksumMismatch() } // verify quarantine exists - List quarantined = listFiles(storageService.getQuarantineFile(shardUuid).getParentFile()); + List quarantined = listFiles(new File(storageService.getQuarantineFile(shardUuid).getParent().toString())); assertEquals(quarantined.size(), 1); assertTrue(getOnlyElement(quarantined).startsWith(shardUuid + ".orc.corrupt")); } @@ -251,6 +256,7 @@ public static ShardRecoveryManager createShardRecoveryManager( return new ShardRecoveryManager( storageService, backupStore, + new LocalOrcDataEnvironment(), new TestingNodeManager(), shardManager, new Duration(5, MINUTES), diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java index 2ff35d9a61dac..cfdcde5c34ffb 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestShardWriter.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.raptor.storage; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.RowPagesBuilder; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.metadata.FunctionManager; @@ -34,7 +35,6 @@ import com.facebook.presto.type.TypeRegistry; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; import org.joda.time.DateTime; import org.joda.time.Days; import org.joda.time.chrono.ISOChronology; @@ -46,9 +46,9 @@ import java.io.File; import java.util.List; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createFileWriter; import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReader; -import static com.facebook.presto.raptor.storage.OrcTestingUtil.createReaderNoRows; import static com.facebook.presto.raptor.storage.OrcTestingUtil.fileOrcDataSource; import static com.facebook.presto.raptor.storage.OrcTestingUtil.octets; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -70,7 +70,6 @@ import static com.google.common.io.Files.createTempDir; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.json.JsonCodec.jsonCodec; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.slice.Slices.wrappedBuffer; import static java.util.Locale.ENGLISH; @@ -136,7 +135,7 @@ public void testWriter(boolean useOptimizedOrcWriter) .row(456L, "bye \u2603", wrappedBuffer(bytes3), Double.NaN, false, arrayBlockOf(BIGINT), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT)), timestampValue, timeValue, dateValue); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader()); - FileWriter writer = createFileWriter(columnIds, columnTypes, file, useOptimizedOrcWriter)) { + FileWriter writer = createFileWriter(columnIds, columnTypes, file)) { writer.appendPages(rowPagesBuilder.build()); } @@ -151,24 +150,24 @@ public void testWriter(boolean useOptimizedOrcWriter) assertEquals(reader.getReaderPosition(), 0); assertEquals(reader.getFilePosition(), reader.getFilePosition()); - Block column0 = reader.readBlock(BIGINT, 0); + Block column0 = reader.readBlock(0); assertEquals(column0.isNull(0), false); assertEquals(column0.isNull(1), true); assertEquals(column0.isNull(2), false); assertEquals(BIGINT.getLong(column0, 0), 123L); assertEquals(BIGINT.getLong(column0, 2), 456L); - Block column1 = reader.readBlock(createVarcharType(10), 1); + Block column1 = reader.readBlock(1); assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello")); assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("world")); assertEquals(createVarcharType(10).getSlice(column1, 2), utf8Slice("bye \u2603")); - Block column2 = reader.readBlock(VARBINARY, 2); + Block column2 = reader.readBlock(2); assertEquals(VARBINARY.getSlice(column2, 0), wrappedBuffer(bytes1)); assertEquals(column2.isNull(1), true); assertEquals(VARBINARY.getSlice(column2, 2), wrappedBuffer(bytes3)); - Block column3 = reader.readBlock(DOUBLE, 3); + Block column3 = reader.readBlock(3); assertEquals(column3.isNull(0), false); assertEquals(column3.isNull(1), false); assertEquals(column3.isNull(2), false); @@ -176,21 +175,21 @@ public void testWriter(boolean useOptimizedOrcWriter) assertEquals(DOUBLE.getDouble(column3, 1), Double.POSITIVE_INFINITY); assertEquals(DOUBLE.getDouble(column3, 2), Double.NaN); - Block column4 = reader.readBlock(BOOLEAN, 4); + Block column4 = reader.readBlock(4); assertEquals(column4.isNull(0), false); assertEquals(column4.isNull(1), true); assertEquals(column4.isNull(2), false); assertEquals(BOOLEAN.getBoolean(column4, 0), true); assertEquals(BOOLEAN.getBoolean(column4, 2), false); - Block column5 = reader.readBlock(arrayType, 5); + Block column5 = reader.readBlock(5); assertEquals(column5.getPositionCount(), 3); assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 0), arrayBlockOf(BIGINT, 1, 2))); assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 1), arrayBlockOf(BIGINT, 3, null))); assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 2), arrayBlockOf(BIGINT))); - Block column6 = reader.readBlock(mapType, 6); + Block column6 = reader.readBlock(6); assertEquals(column6.getPositionCount(), 3); assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column6, 0), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true))); @@ -199,24 +198,24 @@ public void testWriter(boolean useOptimizedOrcWriter) assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, object, k2)); assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column6, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false))); - Block column7 = reader.readBlock(arrayOfArrayType, 7); + Block column7 = reader.readBlock(7); assertEquals(column7.getPositionCount(), 3); assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 0), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)))); assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 1), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7)))); assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 2), arrayBlockOf(arrayType, arrayBlockOf(BIGINT)))); - Block column8 = reader.readBlock(TIMESTAMP, 8); + Block column8 = reader.readBlock(8); assertEquals(TIMESTAMP.getLong(column8, 0), timestampValue); assertEquals(TIMESTAMP.getLong(column8, 1), timestampValue); assertEquals(TIMESTAMP.getLong(column8, 2), timestampValue); - Block column9 = reader.readBlock(TIME, 9); + Block column9 = reader.readBlock(9); assertEquals(TIME.getLong(column9, 0), timeValue); assertEquals(TIME.getLong(column9, 1), timeValue); assertEquals(TIME.getLong(column9, 2), timeValue); - Block column10 = reader.readBlock(DATE, 10); + Block column10 = reader.readBlock(10); assertEquals(DATE.getLong(column10, 0), dateValue); assertEquals(DATE.getLong(column10, 1), dateValue); assertEquals(DATE.getLong(column10, 2), dateValue); @@ -247,7 +246,7 @@ public void testWriter(boolean useOptimizedOrcWriter) // Test unsupported types for (Type type : ImmutableList.of(TIMESTAMP_WITH_TIME_ZONE, RowType.anonymous(ImmutableList.of(BIGINT, DOUBLE)))) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader())) { - createFileWriter(ImmutableList.of(1L), ImmutableList.of(type), file, useOptimizedOrcWriter); + createFileWriter(ImmutableList.of(1L), ImmutableList.of(type), file); fail(); } catch (PrestoException e) { @@ -266,20 +265,8 @@ public void testWriterZeroRows() File file = new File(directory, System.nanoTime() + ".orc"); - try (FileWriter ignored = createFileWriter(columnIds, columnTypes, file, false)) { - // no rows - } - - try (OrcDataSource dataSource = fileOrcDataSource(file)) { - OrcBatchRecordReader reader = createReaderNoRows(dataSource); - assertEquals(reader.getReaderRowCount(), 0); - assertEquals(reader.getReaderPosition(), 0); - - assertEquals(reader.nextBatch(), -1); - } - // optimized ORC writer will flush metadata on close - try (FileWriter ignored = createFileWriter(columnIds, columnTypes, file, true)) { + try (FileWriter ignored = createFileWriter(columnIds, columnTypes, file)) { // no rows } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java index d9dc2e5d5a3f9..f55f1dbf66a25 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/TestStorageManagerConfig.java @@ -19,19 +19,16 @@ import io.airlift.units.Duration; import org.testng.annotations.Test; -import javax.validation.constraints.NotNull; - -import java.io.File; +import java.net.URI; import java.util.Map; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static com.facebook.presto.orc.metadata.CompressionKind.SNAPPY; import static com.facebook.presto.orc.metadata.CompressionKind.ZSTD; import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.DISABLED; import static com.facebook.presto.raptor.storage.StorageManagerConfig.OrcOptimizedWriterStage.ENABLED_AND_VALIDATED; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static io.airlift.testing.ValidationAssertions.assertFailsValidation; import static io.airlift.units.DataSize.Unit.BYTE; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.KILOBYTE; @@ -50,6 +47,7 @@ public void testDefaults() { assertRecordedDefaults(recordDefaults(StorageManagerConfig.class) .setDataDirectory(null) + .setFileSystemProvider("file") .setMinAvailableSpace(new DataSize(0, BYTE)) .setOrcMaxMergeDistance(new DataSize(1, MEGABYTE)) .setOrcMaxReadSize(new DataSize(8, MEGABYTE)) @@ -73,14 +71,18 @@ public void testDefaults() .setMaxShardSize(new DataSize(256, MEGABYTE)) .setMaxBufferSize(new DataSize(256, MEGABYTE)) .setOneSplitPerBucketThreshold(0) - .setShardDayBoundaryTimeZone(TimeZoneKey.UTC_KEY.getId())); + .setShardDayBoundaryTimeZone(TimeZoneKey.UTC_KEY.getId()) + .setZstdJniDecompressionEnabled(false) + .setMaxAllowedFilesPerWriter(Integer.MAX_VALUE)); } @Test public void testExplicitPropertyMappings() + throws Exception { Map properties = new ImmutableMap.Builder() - .put("storage.data-directory", "/data") + .put("storage.data-directory", "file:///data") + .put("storage.file-system", "hdfs") .put("storage.min-available-space", "123GB") .put("storage.orc.max-merge-distance", "16kB") .put("storage.orc.max-read-size", "16kB") @@ -105,10 +107,13 @@ public void testExplicitPropertyMappings() .put("storage.max-buffer-size", "512MB") .put("storage.one-split-per-bucket-threshold", "4") .put("storage.shard-day-boundary-time-zone", "PST") + .put("storage.max-allowed-files-per-writer", "50") + .put("storage.zstd-jni-decompression-enabled", "true") .build(); StorageManagerConfig expected = new StorageManagerConfig() - .setDataDirectory(new File("/data")) + .setDataDirectory(new URI("file:///data")) + .setFileSystemProvider("hdfs") .setMinAvailableSpace(new DataSize(123, GIGABYTE)) .setOrcMaxMergeDistance(new DataSize(16, KILOBYTE)) .setOrcMaxReadSize(new DataSize(16, KILOBYTE)) @@ -132,14 +137,10 @@ public void testExplicitPropertyMappings() .setMaxShardSize(new DataSize(10, MEGABYTE)) .setMaxBufferSize(new DataSize(512, MEGABYTE)) .setOneSplitPerBucketThreshold(4) - .setShardDayBoundaryTimeZone("PST"); + .setShardDayBoundaryTimeZone("PST") + .setMaxAllowedFilesPerWriter(50) + .setZstdJniDecompressionEnabled(true); assertFullMapping(properties, expected); } - - @Test - public void testValidations() - { - assertFailsValidation(new StorageManagerConfig().setDataDirectory(null), "dataDirectory", "may not be null", NotNull.class); - } } diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestCompactionSetCreator.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestCompactionSetCreator.java index 0ec4924562d55..dd7de5dbaecb3 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestCompactionSetCreator.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestCompactionSetCreator.java @@ -40,10 +40,10 @@ public class TestCompactionSetCreator { private static final long MAX_SHARD_ROWS = 100; private static final DataSize MAX_SHARD_SIZE = new DataSize(100, DataSize.Unit.BYTE); - private static final Table tableInfo = new Table(1L, OptionalLong.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.empty(), false); - private static final Table temporalTableInfo = new Table(1L, OptionalLong.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.of(1), false); - private static final Table bucketedTableInfo = new Table(1L, OptionalLong.empty(), Optional.empty(), OptionalInt.of(3), OptionalLong.empty(), false); - private static final Table bucketedTemporalTableInfo = new Table(1L, OptionalLong.empty(), Optional.empty(), OptionalInt.of(3), OptionalLong.of(1), false); + private static final Table tableInfo = new Table(1L, OptionalLong.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.empty(), false, false); + private static final Table temporalTableInfo = new Table(1L, OptionalLong.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.of(1), false, false); + private static final Table bucketedTableInfo = new Table(1L, OptionalLong.empty(), Optional.empty(), OptionalInt.of(3), OptionalLong.empty(), false, false); + private static final Table bucketedTemporalTableInfo = new Table(1L, OptionalLong.empty(), Optional.empty(), OptionalInt.of(3), OptionalLong.of(1), false, false); private final CompactionSetCreator compactionSetCreator = new CompactionSetCreator(new TemporalFunction(UTC), MAX_SHARD_SIZE, MAX_SHARD_ROWS); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardCompactor.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardCompactor.java index 9cc835ba8bca6..c0c53fba077d8 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardCompactor.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardCompactor.java @@ -16,6 +16,7 @@ import com.facebook.presto.PagesIndexPageSorter; import com.facebook.presto.SequencePageBuilder; import com.facebook.presto.operator.PagesIndex; +import com.facebook.presto.raptor.filesystem.FileSystemContext; import com.facebook.presto.raptor.metadata.ColumnInfo; import com.facebook.presto.raptor.metadata.ShardInfo; import com.facebook.presto.raptor.storage.ReaderAttributes; @@ -47,6 +48,7 @@ import java.util.Set; import java.util.UUID; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.raptor.storage.TestOrcStorageManager.createOrcStorageManager; import static com.facebook.presto.spi.block.SortOrder.ASC_NULLS_FIRST; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -61,7 +63,6 @@ import static com.google.common.io.Files.createTempDir; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static java.util.Collections.nCopies; import static java.util.stream.Collectors.toList; @@ -72,7 +73,7 @@ public class TestShardCompactor { private static final int MAX_SHARD_ROWS = 1000; private static final PagesIndexPageSorter PAGE_SORTER = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)); - private static final ReaderAttributes READER_ATTRIBUTES = new ReaderAttributes(new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); + private static final ReaderAttributes READER_ATTRIBUTES = new ReaderAttributes(new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true, false); private File temporary; private IDBI dbi; @@ -106,7 +107,7 @@ public void tearDown() public void testShardCompactor(boolean useOptimizedOrcWriter) throws Exception { - StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS, useOptimizedOrcWriter); + StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS); List columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE, DATE, TIMESTAMP); @@ -133,7 +134,7 @@ public void testShardCompactor(boolean useOptimizedOrcWriter) public void testShardCompactorSorted(boolean useOptimizedOrcWriter) throws Exception { - StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS, useOptimizedOrcWriter); + StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS); List columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DATE, TIMESTAMP, DOUBLE); List columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L); List sortColumnIds = ImmutableList.of(1L, 2L, 3L, 5L, 7L); @@ -263,7 +264,7 @@ private MaterializedResult getMaterializedRows(StorageManager storageManager, Li private ConnectorPageSource getPageSource(StorageManager storageManager, List columnIds, List columnTypes, UUID uuid) { - return storageManager.getPageSource(uuid, OptionalInt.empty(), columnIds, columnTypes, TupleDomain.all(), READER_ATTRIBUTES); + return storageManager.getPageSource(FileSystemContext.DEFAULT_RAPTOR_CONTEXT, uuid, OptionalInt.empty(), columnIds, columnTypes, TupleDomain.all(), READER_ATTRIBUTES); } private static List createSortedShards(StorageManager storageManager, List columnIds, List columnTypes, List sortChannels, List sortOrders, int shardCount) @@ -307,7 +308,7 @@ private static List createShards(StorageManager storageManager, List< private static StoragePageSink createStoragePageSink(StorageManager manager, List columnIds, List columnTypes) { long transactionId = 1; - return manager.createStoragePageSink(transactionId, OptionalInt.empty(), columnIds, columnTypes, false); + return manager.createStoragePageSink(FileSystemContext.DEFAULT_RAPTOR_CONTEXT, transactionId, OptionalInt.empty(), columnIds, columnTypes, false); } private static List createPages(List columnTypes) diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardOrganizationManager.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardOrganizationManager.java index e65d4483e2752..9f746196be6c7 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardOrganizationManager.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardOrganizationManager.java @@ -58,8 +58,8 @@ public class TestShardOrganizationManager private MetadataDao metadataDao; private ShardOrganizerDao organizerDao; - private static final Table tableInfo = new Table(1L, OptionalLong.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.empty(), true); - private static final Table temporalTableInfo = new Table(1L, OptionalLong.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.of(1), true); + private static final Table tableInfo = new Table(1L, OptionalLong.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.empty(), true, false); + private static final Table temporalTableInfo = new Table(1L, OptionalLong.empty(), Optional.empty(), OptionalInt.empty(), OptionalLong.of(1), true, false); private static final List types = ImmutableList.of(BIGINT, VARCHAR, DATE, TIMESTAMP); private static final TemporalFunction TEMPORAL_FUNCTION = new TemporalFunction(UTC); @@ -84,11 +84,11 @@ public void teardown() @Test public void testOrganizationEligibleTables() { - long table1 = metadataDao.insertTable("schema", "table1", false, true, null, 0); + long table1 = metadataDao.insertTable("schema", "table1", false, true, null, 0, false); metadataDao.insertColumn(table1, 1, "foo", 1, "bigint", 1, null); - metadataDao.insertTable("schema", "table2", false, true, null, 0); - metadataDao.insertTable("schema", "table3", false, false, null, 0); + metadataDao.insertTable("schema", "table2", false, true, null, 0, false); + metadataDao.insertTable("schema", "table3", false, false, null, 0, false); assertEquals(metadataDao.getOrganizationEligibleTables(), ImmutableSet.of(table1)); } @@ -96,13 +96,13 @@ public void testOrganizationEligibleTables() public void testTableDiscovery() throws Exception { - long table1 = metadataDao.insertTable("schema", "table1", false, true, null, 0); + long table1 = metadataDao.insertTable("schema", "table1", false, true, null, 0, false); metadataDao.insertColumn(table1, 1, "foo", 1, "bigint", 1, null); - long table2 = metadataDao.insertTable("schema", "table2", false, true, null, 0); + long table2 = metadataDao.insertTable("schema", "table2", false, true, null, 0, false); metadataDao.insertColumn(table2, 1, "foo", 1, "bigint", 1, null); - metadataDao.insertTable("schema", "table3", false, false, null, 0); + metadataDao.insertTable("schema", "table3", false, false, null, 0, false); long intervalMillis = 100; ShardOrganizationManager organizationManager = createShardOrganizationManager(intervalMillis); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardOrganizerUtil.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardOrganizerUtil.java index df61e4f0a9070..5f291c0e375fd 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardOrganizerUtil.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestShardOrganizerUtil.java @@ -83,7 +83,7 @@ public void setup() createTablesWithRetry(dbi); dataDir = Files.createTempDir(); - metadata = new RaptorMetadata("raptor", dbi, createShardManager(dbi)); + metadata = new RaptorMetadata("raptor", dbi, createShardManager(dbi), new TypeRegistry()); metadataDao = dbi.onDemand(MetadataDao.class); shardManager = createShardManager(dbi); @@ -110,6 +110,7 @@ public void testGetOrganizationEligibleShards() .column("orderstatus", createVarcharType(3)) .property("ordering", ImmutableList.of("orderstatus", "orderkey")) .property("temporal_column", "orderdate") + .property("table_supports_delta_delete", false) .build(), false); Table tableInfo = metadataDao.getTableInformation(tableName.getSchemaName(), tableName.getTableName()); diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestTuple.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestTuple.java index bb6f9ccd59129..725bdeba97a69 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestTuple.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestTuple.java @@ -19,6 +19,8 @@ import java.util.List; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DateType.DATE; @@ -26,8 +28,6 @@ import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.spi.type.VarcharType.createVarcharType; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertLessThan; import static org.testng.Assert.assertEquals; public class TestTuple diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/systemtables/TestShardMetadataRecordCursor.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/systemtables/TestShardMetadataRecordCursor.java index 2df831840b39e..b0efee13f09a8 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/systemtables/TestShardMetadataRecordCursor.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/systemtables/TestShardMetadataRecordCursor.java @@ -49,6 +49,7 @@ import java.util.UUID; import static com.facebook.presto.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder; +import static com.facebook.presto.raptor.RaptorTableProperties.TABLE_SUPPORTS_DELTA_DELETE; import static com.facebook.presto.raptor.metadata.SchemaDaoUtil.createTablesWithRetry; import static com.facebook.presto.raptor.metadata.TestDatabaseShardManager.createShardManager; import static com.facebook.presto.raptor.systemtables.ShardMetadataRecordCursor.SHARD_METADATA; @@ -79,13 +80,14 @@ public void setup() this.dbi.registerMapper(new TableColumn.Mapper(new TypeRegistry())); this.dummyHandle = dbi.open(); createTablesWithRetry(dbi); - this.metadata = new RaptorMetadata("raptor", dbi, createShardManager(dbi)); + this.metadata = new RaptorMetadata("raptor", dbi, createShardManager(dbi), new TypeRegistry()); // Create table ConnectorTableMetadata table = tableMetadataBuilder(DEFAULT_TEST_ORDERS) .column("orderkey", BIGINT) .column("orderdate", DATE) .property("temporal_column", "orderdate") + .property(TABLE_SUPPORTS_DELTA_DELETE, false) .build(); createTable(table); } @@ -157,11 +159,13 @@ public void testNoSchemaFilter() // Create "orders" table in a different schema createTable(tableMetadataBuilder(new SchemaTableName("other", "orders")) .column("orderkey", BIGINT) + .property(TABLE_SUPPORTS_DELTA_DELETE, false) .build()); // Create another table that should not be selected createTable(tableMetadataBuilder(new SchemaTableName("schema1", "foo")) .column("orderkey", BIGINT) + .property(TABLE_SUPPORTS_DELTA_DELETE, false) .build()); TupleDomain tupleDomain = TupleDomain.withColumnDomains( @@ -183,11 +187,13 @@ public void testNoTableFilter() // Create "orders" table in a different schema createTable(tableMetadataBuilder(new SchemaTableName("test", "orders2")) .column("orderkey", BIGINT) + .property(TABLE_SUPPORTS_DELTA_DELETE, false) .build()); // Create another table that should not be selected createTable(tableMetadataBuilder(new SchemaTableName("schema1", "foo")) .column("orderkey", BIGINT) + .property(TABLE_SUPPORTS_DELTA_DELETE, false) .build()); TupleDomain tupleDomain = TupleDomain.withColumnDomains( diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index 7e23bc2f29e07..94d1753a1d834 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-rcfile @@ -86,7 +86,7 @@ - io.airlift + com.facebook.airlift testing test @@ -104,7 +104,7 @@ - io.airlift + com.facebook.airlift json test diff --git a/presto-rcfile/src/main/java/com/facebook/presto/rcfile/binary/ListEncoding.java b/presto-rcfile/src/main/java/com/facebook/presto/rcfile/binary/ListEncoding.java index a9a3e20270fe4..65916461599b1 100644 --- a/presto-rcfile/src/main/java/com/facebook/presto/rcfile/binary/ListEncoding.java +++ b/presto-rcfile/src/main/java/com/facebook/presto/rcfile/binary/ListEncoding.java @@ -38,7 +38,7 @@ public ListEncoding(Type type, BinaryColumnEncoding elementEncoding) @Override public void encodeValue(Block block, int position, SliceOutput output) { - Block list = block.getObject(position, Block.class); + Block list = block.getBlock(position); writeVInt(output, list.getPositionCount()); // write null bits diff --git a/presto-rcfile/src/main/java/com/facebook/presto/rcfile/binary/MapEncoding.java b/presto-rcfile/src/main/java/com/facebook/presto/rcfile/binary/MapEncoding.java index 1df20f89f8abc..74dbce590ac8e 100644 --- a/presto-rcfile/src/main/java/com/facebook/presto/rcfile/binary/MapEncoding.java +++ b/presto-rcfile/src/main/java/com/facebook/presto/rcfile/binary/MapEncoding.java @@ -42,7 +42,7 @@ public MapEncoding(Type type, BinaryColumnEncoding keyReader, BinaryColumnEncodi @Override public void encodeValue(Block block, int position, SliceOutput output) { - Block map = block.getObject(position, Block.class); + Block map = block.getBlock(position); // write entry count writeVInt(output, map.getPositionCount() / 2); diff --git a/presto-rcfile/src/main/java/com/facebook/presto/rcfile/binary/StructEncoding.java b/presto-rcfile/src/main/java/com/facebook/presto/rcfile/binary/StructEncoding.java index 07f88f46e115f..58f61784dc081 100644 --- a/presto-rcfile/src/main/java/com/facebook/presto/rcfile/binary/StructEncoding.java +++ b/presto-rcfile/src/main/java/com/facebook/presto/rcfile/binary/StructEncoding.java @@ -36,7 +36,7 @@ public StructEncoding(Type type, List structFields) @Override public void encodeValue(Block block, int position, SliceOutput output) { - Block row = block.getObject(position, Block.class); + Block row = block.getBlock(position); // write values for (int batchStart = 0; batchStart < row.getPositionCount(); batchStart += 8) { diff --git a/presto-rcfile/src/main/java/com/facebook/presto/rcfile/text/ListEncoding.java b/presto-rcfile/src/main/java/com/facebook/presto/rcfile/text/ListEncoding.java index 3f9410858dc9b..f565503546b24 100644 --- a/presto-rcfile/src/main/java/com/facebook/presto/rcfile/text/ListEncoding.java +++ b/presto-rcfile/src/main/java/com/facebook/presto/rcfile/text/ListEncoding.java @@ -37,7 +37,7 @@ public void encodeValueInto(int depth, Block block, int position, SliceOutput ou { byte separator = getSeparator(depth); - Block list = block.getObject(position, Block.class); + Block list = block.getBlock(position); for (int elementIndex = 0; elementIndex < list.getPositionCount(); elementIndex++) { if (elementIndex > 0) { output.writeByte(separator); diff --git a/presto-rcfile/src/main/java/com/facebook/presto/rcfile/text/MapEncoding.java b/presto-rcfile/src/main/java/com/facebook/presto/rcfile/text/MapEncoding.java index 61890f09f1d41..6aca98854f43d 100644 --- a/presto-rcfile/src/main/java/com/facebook/presto/rcfile/text/MapEncoding.java +++ b/presto-rcfile/src/main/java/com/facebook/presto/rcfile/text/MapEncoding.java @@ -46,7 +46,7 @@ public void encodeValueInto(int depth, Block block, int position, SliceOutput ou byte elementSeparator = getSeparator(depth); byte keyValueSeparator = getSeparator(depth + 1); - Block map = block.getObject(position, Block.class); + Block map = block.getBlock(position); boolean first = true; for (int elementIndex = 0; elementIndex < map.getPositionCount(); elementIndex += 2) { if (map.isNull(elementIndex)) { diff --git a/presto-rcfile/src/main/java/com/facebook/presto/rcfile/text/StructEncoding.java b/presto-rcfile/src/main/java/com/facebook/presto/rcfile/text/StructEncoding.java index 2d5b5804578d0..354763030e67e 100644 --- a/presto-rcfile/src/main/java/com/facebook/presto/rcfile/text/StructEncoding.java +++ b/presto-rcfile/src/main/java/com/facebook/presto/rcfile/text/StructEncoding.java @@ -45,7 +45,7 @@ public void encodeValueInto(int depth, Block block, int position, SliceOutput ou { byte separator = getSeparator(depth); - Block row = block.getObject(position, Block.class); + Block row = block.getBlock(position); for (int fieldIndex = 0; fieldIndex < structFields.size(); fieldIndex++) { if (fieldIndex > 0) { output.writeByte(separator); diff --git a/presto-rcfile/src/test/java/com/facebook/presto/rcfile/TestBufferedOutputStreamSliceOutput.java b/presto-rcfile/src/test/java/com/facebook/presto/rcfile/TestBufferedOutputStreamSliceOutput.java index 2795319602e82..a6d9c6b0dddc1 100644 --- a/presto-rcfile/src/test/java/com/facebook/presto/rcfile/TestBufferedOutputStreamSliceOutput.java +++ b/presto-rcfile/src/test/java/com/facebook/presto/rcfile/TestBufferedOutputStreamSliceOutput.java @@ -19,7 +19,7 @@ import java.io.ByteArrayOutputStream; -import static io.airlift.testing.Assertions.assertLessThanOrEqual; +import static com.facebook.airlift.testing.Assertions.assertLessThanOrEqual; import static org.testng.Assert.assertEquals; public class TestBufferedOutputStreamSliceOutput diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index b0c2822985cdf..486b61356257a 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-record-decoder @@ -37,7 +37,7 @@ - io.airlift + com.facebook.airlift json @@ -94,7 +94,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-record-decoder/src/test/java/com/facebook/presto/decoder/json/JsonFieldDecoderTester.java b/presto-record-decoder/src/test/java/com/facebook/presto/decoder/json/JsonFieldDecoderTester.java index 48293f9765126..4f587ef98d982 100644 --- a/presto-record-decoder/src/test/java/com/facebook/presto/decoder/json/JsonFieldDecoderTester.java +++ b/presto-record-decoder/src/test/java/com/facebook/presto/decoder/json/JsonFieldDecoderTester.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.decoder.json; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.decoder.DecoderColumnHandle; import com.facebook.presto.decoder.DecoderTestColumnHandle; import com.facebook.presto.decoder.FieldValueProvider; @@ -20,7 +21,6 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableSet; -import io.airlift.json.ObjectMapperProvider; import io.airlift.slice.Slice; import java.util.Map; diff --git a/presto-record-decoder/src/test/java/com/facebook/presto/decoder/json/TestCustomDateTimeJsonFieldDecoder.java b/presto-record-decoder/src/test/java/com/facebook/presto/decoder/json/TestCustomDateTimeJsonFieldDecoder.java index ad87565026ab9..5b05d4b0299de 100644 --- a/presto-record-decoder/src/test/java/com/facebook/presto/decoder/json/TestCustomDateTimeJsonFieldDecoder.java +++ b/presto-record-decoder/src/test/java/com/facebook/presto/decoder/json/TestCustomDateTimeJsonFieldDecoder.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.decoder.json; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.decoder.DecoderTestColumnHandle; import com.facebook.presto.spi.PrestoException; import com.google.common.collect.ImmutableSet; -import io.airlift.json.ObjectMapperProvider; import org.testng.annotations.Test; import static com.facebook.presto.spi.type.DateTimeEncoding.packDateTimeWithZone; diff --git a/presto-record-decoder/src/test/java/com/facebook/presto/decoder/json/TestJsonDecoder.java b/presto-record-decoder/src/test/java/com/facebook/presto/decoder/json/TestJsonDecoder.java index 4b6867d5606c7..a75f72ee40b0d 100644 --- a/presto-record-decoder/src/test/java/com/facebook/presto/decoder/json/TestJsonDecoder.java +++ b/presto-record-decoder/src/test/java/com/facebook/presto/decoder/json/TestJsonDecoder.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.decoder.json; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.decoder.DecoderColumnHandle; import com.facebook.presto.decoder.DecoderTestColumnHandle; import com.facebook.presto.decoder.FieldValueProvider; @@ -21,7 +22,6 @@ import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableSet; import com.google.common.io.ByteStreams; -import io.airlift.json.ObjectMapperProvider; import org.assertj.core.api.ThrowableAssert.ThrowingCallable; import org.testng.annotations.Test; diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 774114b6ba74b..528bdefc36cad 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-redis @@ -18,22 +18,22 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift configuration @@ -126,7 +126,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorConfig.java b/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorConfig.java index 4e72af58a5c95..e9086f8980929 100644 --- a/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorConfig.java +++ b/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorConfig.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.redis; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigSecuritySensitive; import com.facebook.presto.spi.HostAddress; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableSet; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigSecuritySensitive; import io.airlift.units.Duration; import io.airlift.units.MinDuration; diff --git a/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorFactory.java b/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorFactory.java index adfa182f3f232..4f36678504038 100644 --- a/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorFactory.java +++ b/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorFactory.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.redis; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.SchemaTableName; @@ -23,8 +25,6 @@ import com.google.inject.Injector; import com.google.inject.Scopes; import com.google.inject.TypeLiteral; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; import java.util.Optional; diff --git a/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorModule.java b/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorModule.java index 923c2b66da3f8..96c1aec36dd34 100644 --- a/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorModule.java +++ b/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorModule.java @@ -23,11 +23,11 @@ import javax.inject.Inject; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.util.Objects.requireNonNull; /** diff --git a/presto-redis/src/main/java/com/facebook/presto/redis/RedisJedisManager.java b/presto-redis/src/main/java/com/facebook/presto/redis/RedisJedisManager.java index a2976eadb028a..35c815c2d26a5 100644 --- a/presto-redis/src/main/java/com/facebook/presto/redis/RedisJedisManager.java +++ b/presto-redis/src/main/java/com/facebook/presto/redis/RedisJedisManager.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.redis; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.NodeManager; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import io.airlift.log.Logger; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; diff --git a/presto-redis/src/main/java/com/facebook/presto/redis/RedisMetadata.java b/presto-redis/src/main/java/com/facebook/presto/redis/RedisMetadata.java index 57a277b51ff39..3e15900c5a936 100644 --- a/presto-redis/src/main/java/com/facebook/presto/redis/RedisMetadata.java +++ b/presto-redis/src/main/java/com/facebook/presto/redis/RedisMetadata.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.redis; +import com.facebook.airlift.log.Logger; import com.facebook.presto.decoder.dummy.DummyRowDecoder; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; @@ -31,7 +32,6 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-redis/src/main/java/com/facebook/presto/redis/RedisRecordCursor.java b/presto-redis/src/main/java/com/facebook/presto/redis/RedisRecordCursor.java index 2ab3a5eb8242d..deb78451976c8 100644 --- a/presto-redis/src/main/java/com/facebook/presto/redis/RedisRecordCursor.java +++ b/presto-redis/src/main/java/com/facebook/presto/redis/RedisRecordCursor.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.redis; +import com.facebook.airlift.log.Logger; import com.facebook.presto.decoder.DecoderColumnHandle; import com.facebook.presto.decoder.FieldValueProvider; import com.facebook.presto.decoder.RowDecoder; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.RecordCursor; import com.facebook.presto.spi.type.Type; -import io.airlift.log.Logger; import io.airlift.slice.Slice; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; diff --git a/presto-redis/src/main/java/com/facebook/presto/redis/RedisTableDescriptionSupplier.java b/presto-redis/src/main/java/com/facebook/presto/redis/RedisTableDescriptionSupplier.java index ccaec1bc2f5ed..73b8f41569b03 100644 --- a/presto-redis/src/main/java/com/facebook/presto/redis/RedisTableDescriptionSupplier.java +++ b/presto-redis/src/main/java/com/facebook/presto/redis/RedisTableDescriptionSupplier.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.redis; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.facebook.presto.decoder.dummy.DummyRowDecoder; import com.facebook.presto.spi.SchemaTableName; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import javax.inject.Inject; diff --git a/presto-redis/src/test/java/com/facebook/presto/redis/RedisQueryRunner.java b/presto-redis/src/test/java/com/facebook/presto/redis/RedisQueryRunner.java index 32e01c36fc6cf..261d583a1ab40 100644 --- a/presto-redis/src/test/java/com/facebook/presto/redis/RedisQueryRunner.java +++ b/presto-redis/src/test/java/com/facebook/presto/redis/RedisQueryRunner.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.redis; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.QualifiedObjectName; @@ -25,17 +27,15 @@ import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import io.airlift.tpch.TpchTable; import java.util.Map; +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; import static com.facebook.presto.redis.util.RedisTestUtils.installRedisPlugin; import static com.facebook.presto.redis.util.RedisTestUtils.loadTpchTableDescription; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static io.airlift.testing.Closeables.closeAllSuppress; import static io.airlift.units.Duration.nanosSince; import static java.util.Locale.ENGLISH; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisConnectorConfig.java b/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisConnectorConfig.java index cde2af7ed8f44..7431e4f47824d 100644 --- a/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisConnectorConfig.java +++ b/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisConnectorConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.redis; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import org.testng.annotations.Test; import java.io.File; diff --git a/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisPlugin.java b/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisPlugin.java index 3da08f75ddc43..e432771529264 100644 --- a/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisPlugin.java +++ b/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisPlugin.java @@ -19,8 +19,8 @@ import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.testing.Assertions.assertInstanceOf; import static org.testng.Assert.assertNotNull; @Test diff --git a/presto-redis/src/test/java/com/facebook/presto/redis/util/CodecSupplier.java b/presto-redis/src/test/java/com/facebook/presto/redis/util/CodecSupplier.java index 007da3b45a512..e29f989e74080 100644 --- a/presto-redis/src/test/java/com/facebook/presto/redis/util/CodecSupplier.java +++ b/presto-redis/src/test/java/com/facebook/presto/redis/util/CodecSupplier.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.redis.util; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; import java.util.function.Supplier; diff --git a/presto-redis/src/test/java/com/facebook/presto/redis/util/RedisTestUtils.java b/presto-redis/src/test/java/com/facebook/presto/redis/util/RedisTestUtils.java index a88f1e789eddf..f3e34dc379aa9 100644 --- a/presto-redis/src/test/java/com/facebook/presto/redis/util/RedisTestUtils.java +++ b/presto-redis/src/test/java/com/facebook/presto/redis/util/RedisTestUtils.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.redis.util; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.metadata.QualifiedObjectName; import com.facebook.presto.redis.RedisPlugin; import com.facebook.presto.redis.RedisTableDescription; @@ -22,7 +23,6 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; -import io.airlift.json.JsonCodec; import java.io.IOException; import java.io.InputStream; diff --git a/presto-redshift/pom.xml b/presto-redshift/pom.xml index c7d55ec15342e..b972fec8f13c4 100644 --- a/presto-redshift/pom.xml +++ b/presto-redshift/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-redshift @@ -23,7 +23,7 @@ - io.airlift + com.facebook.airlift configuration diff --git a/presto-redshift/src/main/java/com/facebook/presto/plugin/redshift/RedshiftClient.java b/presto-redshift/src/main/java/com/facebook/presto/plugin/redshift/RedshiftClient.java index 9a706ac1004a4..629af1428d859 100644 --- a/presto-redshift/src/main/java/com/facebook/presto/plugin/redshift/RedshiftClient.java +++ b/presto-redshift/src/main/java/com/facebook/presto/plugin/redshift/RedshiftClient.java @@ -17,6 +17,7 @@ import com.facebook.presto.plugin.jdbc.BaseJdbcConfig; import com.facebook.presto.plugin.jdbc.DriverConnectionFactory; import com.facebook.presto.plugin.jdbc.JdbcConnectorId; +import com.facebook.presto.plugin.jdbc.JdbcIdentity; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; import org.postgresql.Driver; @@ -50,10 +51,10 @@ public PreparedStatement getPreparedStatement(Connection connection, String sql) } @Override - protected void renameTable(String catalogName, SchemaTableName oldTable, SchemaTableName newTable) + protected void renameTable(JdbcIdentity identity, String catalogName, SchemaTableName oldTable, SchemaTableName newTable) { // Redshift does not allow qualifying the target of a rename - try (Connection connection = connectionFactory.openConnection()) { + try (Connection connection = connectionFactory.openConnection(identity)) { String sql = format( "ALTER TABLE %s RENAME TO %s", quoted(catalogName, oldTable.getSchemaName(), oldTable.getTableName()), diff --git a/presto-redshift/src/main/java/com/facebook/presto/plugin/redshift/RedshiftClientModule.java b/presto-redshift/src/main/java/com/facebook/presto/plugin/redshift/RedshiftClientModule.java index 68433a970302e..2ad46ce369230 100644 --- a/presto-redshift/src/main/java/com/facebook/presto/plugin/redshift/RedshiftClientModule.java +++ b/presto-redshift/src/main/java/com/facebook/presto/plugin/redshift/RedshiftClientModule.java @@ -19,7 +19,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; public class RedshiftClientModule implements Module diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index a3d5371804ea1..ab1a8e77f4525 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-resource-group-managers @@ -28,32 +28,32 @@ - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift configuration - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift stats @@ -147,7 +147,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupConfig.java b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupConfig.java index 686d411227fac..c1107a9533237 100644 --- a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupConfig.java +++ b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.resourceGroups; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import javax.validation.constraints.NotNull; diff --git a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupConfigurationManager.java b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupConfigurationManager.java index fd4391cb2b207..2ab98f70f900f 100644 --- a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupConfigurationManager.java +++ b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupConfigurationManager.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.resourceGroups; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.memory.ClusterMemoryPoolManager; import com.facebook.presto.spi.resourceGroups.ResourceGroup; import com.facebook.presto.spi.resourceGroups.SelectionContext; @@ -20,9 +23,6 @@ import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; import com.google.common.annotations.VisibleForTesting; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; import io.airlift.units.Duration; import javax.inject.Inject; diff --git a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupConfigurationManagerFactory.java b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupConfigurationManagerFactory.java index 60ee551d7f3cf..0745677798d9d 100644 --- a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupConfigurationManagerFactory.java +++ b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupConfigurationManagerFactory.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.resourceGroups; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.spi.memory.ClusterMemoryPoolManager; import com.facebook.presto.spi.resourceGroups.ResourceGroupConfigurationManager; import com.facebook.presto.spi.resourceGroups.ResourceGroupConfigurationManagerContext; import com.facebook.presto.spi.resourceGroups.ResourceGroupConfigurationManagerFactory; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; diff --git a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupsModule.java b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupsModule.java index b423a68eea216..5efcc313e9124 100644 --- a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupsModule.java +++ b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/FileResourceGroupsModule.java @@ -17,7 +17,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; public class FileResourceGroupsModule implements Module diff --git a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupConfig.java b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupConfig.java index 7c489b113d70f..07e44a4eec3c4 100644 --- a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupConfig.java +++ b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.resourceGroups.db; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import io.airlift.units.Duration; import io.airlift.units.MinDuration; diff --git a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupConfigurationManager.java b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupConfigurationManager.java index 091dec94eacb0..1e81be6a08a60 100644 --- a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupConfigurationManager.java +++ b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupConfigurationManager.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.resourceGroups.db; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.resourceGroups.AbstractResourceConfigurationManager; import com.facebook.presto.resourceGroups.ManagerSpec; import com.facebook.presto.resourceGroups.ResourceGroupIdTemplate; @@ -30,8 +32,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import io.airlift.log.Logger; -import io.airlift.stats.CounterStat; import io.airlift.units.Duration; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; @@ -58,13 +58,15 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.spi.StandardErrorCode.CONFIGURATION_INVALID; import static com.facebook.presto.spi.StandardErrorCode.CONFIGURATION_UNAVAILABLE; import static com.google.common.base.Preconditions.checkState; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.units.Duration.succinctNanos; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; +import static java.util.concurrent.TimeUnit.MILLISECONDS; public class DbResourceGroupConfigurationManager extends AbstractResourceConfigurationManager @@ -116,9 +118,8 @@ protected Optional getCpuQuotaPeriod() @Override protected List getRootGroups() { - if (lastRefresh.get() == 0) { - throw new PrestoException(CONFIGURATION_UNAVAILABLE, "Root groups cannot be fetched from database"); - } + checkMaxRefreshInterval(); + if (this.selectors.get().isEmpty()) { throw new PrestoException(CONFIGURATION_INVALID, "No root groups are configured"); } @@ -156,9 +157,8 @@ public void configure(ResourceGroup group, SelectionContext criteri @Override public Optional> match(SelectionCriteria criteria) { - if (lastRefresh.get() == 0) { - throw new PrestoException(CONFIGURATION_UNAVAILABLE, "Selectors cannot be fetched from database"); - } + checkMaxRefreshInterval(); + if (selectors.get().isEmpty()) { throw new PrestoException(CONFIGURATION_INVALID, "No selectors are configured"); } @@ -173,9 +173,8 @@ public Optional> match(SelectionCriteria criteria) @VisibleForTesting public List getSelectors() { - if (lastRefresh.get() == 0) { - throw new PrestoException(CONFIGURATION_UNAVAILABLE, "Selectors cannot be fetched from database"); - } + checkMaxRefreshInterval(); + if (selectors.get().isEmpty()) { throw new PrestoException(CONFIGURATION_INVALID, "No selectors are configured"); } @@ -237,11 +236,11 @@ public synchronized void load() lastRefresh.set(System.nanoTime()); } catch (Throwable e) { - if (succinctNanos(System.nanoTime() - lastRefresh.get()).compareTo(maxRefreshInterval) > 0) { - lastRefresh.set(0); - } refreshFailures.update(1); log.error(e, "Error loading configuration from db"); + if (lastRefresh.get() != 0) { + log.debug("Last successful configuration loading was %s ago", succinctNanos(System.nanoTime() - lastRefresh.get()).toString()); + } } } @@ -367,6 +366,18 @@ private ResourceGroup getRootGroup(ResourceGroupId groupId) return groups.get(groupId); } + private void checkMaxRefreshInterval() + { + if (System.nanoTime() - lastRefresh.get() > maxRefreshInterval.toMillis() * MILLISECONDS.toNanos(1)) { + String message = "Resource group configuration cannot be fetched from database."; + if (lastRefresh.get() != 0) { + message += format(" Current resource group configuration is loaded %s ago", succinctNanos(System.nanoTime() - lastRefresh.get()).toString()); + } + + throw new PrestoException(CONFIGURATION_UNAVAILABLE, message); + } + } + @Managed @Nested public CounterStat getRefreshFailures() diff --git a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupConfigurationManagerFactory.java b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupConfigurationManagerFactory.java index 412d1bc78e244..4f50c887616cc 100644 --- a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupConfigurationManagerFactory.java +++ b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupConfigurationManagerFactory.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.resourceGroups.db; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.resourceGroups.VariableMap; import com.facebook.presto.spi.memory.ClusterMemoryPoolManager; import com.facebook.presto.spi.resourceGroups.ResourceGroupConfigurationManager; import com.facebook.presto.spi.resourceGroups.ResourceGroupConfigurationManagerContext; import com.facebook.presto.spi.resourceGroups.ResourceGroupConfigurationManagerFactory; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; diff --git a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupsModule.java b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupsModule.java index c9f410435cfd9..5eabec3cc81ed 100644 --- a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupsModule.java +++ b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbResourceGroupsModule.java @@ -18,7 +18,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static org.weakref.jmx.guice.ExportBinder.newExporter; public class DbResourceGroupsModule diff --git a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbSourceExactMatchSelector.java b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbSourceExactMatchSelector.java index b38a0c404397f..d137bcb1576eb 100644 --- a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbSourceExactMatchSelector.java +++ b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/DbSourceExactMatchSelector.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.resourceGroups.db; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; import com.facebook.presto.resourceGroups.ResourceGroupSelector; import com.facebook.presto.resourceGroups.VariableMap; import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import com.facebook.presto.spi.resourceGroups.SelectionContext; import com.facebook.presto.spi.resourceGroups.SelectionCriteria; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; import org.jdbi.v3.core.JdbiException; import java.util.Optional; diff --git a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/SelectorRecord.java b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/SelectorRecord.java index afa4409eca7c5..0fc56afb2401a 100644 --- a/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/SelectorRecord.java +++ b/presto-resource-group-managers/src/main/java/com/facebook/presto/resourceGroups/db/SelectorRecord.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.resourceGroups.db; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.resourceGroups.SelectorResourceEstimate; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import org.jdbi.v3.core.mapper.RowMapper; import org.jdbi.v3.core.statement.StatementContext; @@ -25,8 +25,8 @@ import java.util.Optional; import java.util.regex.Pattern; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.json.JsonCodec.listJsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; import static java.util.Objects.requireNonNull; public class SelectorRecord diff --git a/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/TestFileResourceGroupConfig.java b/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/TestFileResourceGroupConfig.java index 7baf35d8d7245..72c1f1a021f3a 100644 --- a/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/TestFileResourceGroupConfig.java +++ b/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/TestFileResourceGroupConfig.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.resourceGroups; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import org.testng.annotations.Test; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; public class TestFileResourceGroupConfig { diff --git a/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestDbResourceGroupConfig.java b/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestDbResourceGroupConfig.java index d4dfa1c7f6bcb..789c6b2171e52 100644 --- a/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestDbResourceGroupConfig.java +++ b/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestDbResourceGroupConfig.java @@ -13,15 +13,15 @@ */ package com.facebook.presto.resourceGroups.db; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.Duration; import org.testng.annotations.Test; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; diff --git a/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestDbResourceGroupConfigurationManager.java b/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestDbResourceGroupConfigurationManager.java index 22f6024b79a77..89d3e4c38ee14 100644 --- a/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestDbResourceGroupConfigurationManager.java +++ b/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestDbResourceGroupConfigurationManager.java @@ -329,7 +329,7 @@ public void testRefreshInterval() fail("Expected unavailable configuration exception"); } catch (Exception e) { - assertEquals(e.getMessage(), "Selectors cannot be fetched from database"); + assertTrue(e.getMessage().startsWith("Resource group configuration cannot be fetched from database.")); } try { @@ -337,7 +337,7 @@ public void testRefreshInterval() fail("Expected unavailable configuration exception"); } catch (Exception e) { - assertEquals(e.getMessage(), "Root groups cannot be fetched from database"); + assertTrue(e.getMessage().startsWith("Resource group configuration cannot be fetched from database.")); } manager.destroy(); diff --git a/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestDbSourceExactMatchSelector.java b/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestDbSourceExactMatchSelector.java index c31ab50234a25..08da3d5792d35 100644 --- a/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestDbSourceExactMatchSelector.java +++ b/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestDbSourceExactMatchSelector.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.resourceGroups.db; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import com.facebook.presto.spi.resourceGroups.SelectionContext; import com.facebook.presto.spi.resourceGroups.SelectionCriteria; import com.facebook.presto.spi.session.ResourceEstimates; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; diff --git a/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestResourceGroupsDao.java b/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestResourceGroupsDao.java index c268f8b4cada8..03bd93d4579a5 100644 --- a/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestResourceGroupsDao.java +++ b/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/db/TestResourceGroupsDao.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.resourceGroups.db; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.resourceGroups.ResourceGroupNameTemplate; import com.facebook.presto.resourceGroups.SelectorResourceEstimate; import com.facebook.presto.resourceGroups.SelectorResourceEstimate.Range; import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.h2.jdbc.JdbcSQLException; @@ -31,12 +31,12 @@ import java.util.Optional; import java.util.regex.Pattern; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; import static com.facebook.presto.spi.resourceGroups.QueryType.DELETE; import static com.facebook.presto.spi.resourceGroups.QueryType.EXPLAIN; import static com.facebook.presto.spi.resourceGroups.QueryType.INSERT; import static com.facebook.presto.spi.resourceGroups.QueryType.SELECT; -import static io.airlift.json.JsonCodec.jsonCodec; -import static io.airlift.json.JsonCodec.listJsonCodec; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index 40b115ea11cb9..fbeef3220f6ed 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index c63b4d8123f9a..b3fc72f623440 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-server @@ -34,7 +34,7 @@ - io.airlift + com.facebook.airlift launcher ${dep.packaging.version} bin @@ -43,7 +43,7 @@ - io.airlift + com.facebook.airlift launcher ${dep.packaging.version} properties @@ -92,6 +92,14 @@ provided + + com.facebook.presto + presto-pinot + ${project.version} + zip + provided + + com.facebook.presto presto-example-http diff --git a/presto-server/src/main/assembly/presto.xml b/presto-server/src/main/assembly/presto.xml index efd14285cc6c8..7df731e7e30e6 100644 --- a/presto-server/src/main/assembly/presto.xml +++ b/presto-server/src/main/assembly/presto.xml @@ -72,6 +72,10 @@ ${project.build.directory}/dependency/presto-cassandra-${project.version} plugin/cassandra + + ${project.build.directory}/dependency/presto-pinot-${project.version} + plugin/pinot + ${project.build.directory}/dependency/presto-example-http-${project.version} plugin/example-http diff --git a/presto-session-property-managers/pom.xml b/presto-session-property-managers/pom.xml index c86f865aa1879..9fa5f58b1bc67 100644 --- a/presto-session-property-managers/pom.xml +++ b/presto-session-property-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-session-property-managers @@ -18,17 +18,17 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift configuration @@ -89,7 +89,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManager.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManager.java index 65ef81a51a5e6..273dd57359981 100644 --- a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManager.java +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManager.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.session; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.session.SessionConfigurationContext; import com.facebook.presto.spi.session.SessionPropertyConfigurationManager; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; import com.google.common.collect.ImmutableMap; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; import javax.inject.Inject; diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerConfig.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerConfig.java index abd169f27eaf1..67b1c656ee81b 100644 --- a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerConfig.java +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.session; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import javax.validation.constraints.NotNull; diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerFactory.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerFactory.java index 80d46e03d1fe9..a2156dd78636c 100644 --- a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerFactory.java +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerFactory.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.session; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; import com.facebook.presto.spi.resourceGroups.SessionPropertyConfigurationManagerContext; import com.facebook.presto.spi.session.SessionPropertyConfigurationManager; import com.facebook.presto.spi.session.SessionPropertyConfigurationManagerFactory; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; import java.util.Map; diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerModule.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerModule.java index 205be8f8c18e1..03fee8d5207dc 100644 --- a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerModule.java +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerModule.java @@ -17,7 +17,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; public class FileSessionPropertyManagerModule implements Module diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManager.java b/presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManager.java index 4ab299e6f8a10..9b03fd9fb8e56 100644 --- a/presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManager.java +++ b/presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.session; +import com.facebook.airlift.testing.TempFile; import com.facebook.presto.spi.resourceGroups.QueryType; import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import com.facebook.presto.spi.session.SessionConfigurationContext; @@ -21,7 +22,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.testing.TempFile; import org.testng.annotations.Test; import java.io.IOException; diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManagerConfig.java b/presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManagerConfig.java index 9d5f13dae78ab..b555593fa09a0 100644 --- a/presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManagerConfig.java +++ b/presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManagerConfig.java @@ -19,9 +19,9 @@ import java.io.File; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestFileSessionPropertyManagerConfig { diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index e8da9f0fde593..34151658d5c77 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-spi @@ -51,7 +51,7 @@ - io.airlift + com.facebook.airlift testing test @@ -93,7 +93,7 @@ - io.airlift + com.facebook.airlift json test diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/CatalogSchemaName.java b/presto-spi/src/main/java/com/facebook/presto/spi/CatalogSchemaName.java index b96897b5ddbfa..78c3de9e4b60d 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/CatalogSchemaName.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/CatalogSchemaName.java @@ -13,7 +13,10 @@ */ package com.facebook.presto.spi; +import com.facebook.presto.spi.function.CatalogSchemaPrefix; + import java.util.Objects; +import java.util.Optional; import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; @@ -39,6 +42,11 @@ public String getSchemaName() return schemaName; } + public CatalogSchemaPrefix asCatalogSchemaPrefix() + { + return new CatalogSchemaPrefix(catalogName, Optional.of(schemaName)); + } + @Override public boolean equals(Object obj) { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorPageSource.java b/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorPageSource.java index e5fabb211a35d..12e83f5f5c94b 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorPageSource.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorPageSource.java @@ -28,6 +28,12 @@ public interface ConnectorPageSource */ long getCompletedBytes(); + /** + * Gets the number of input rows processed by this page source so far. + * If number is not available, this method should return zero. + */ + long getCompletedPositions(); + /** * Gets the wall time this page source spent reading data from the input. * If read time is not available, this method should return zero. diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorPlanOptimizer.java b/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorPlanOptimizer.java index 6d24cbdf18a1d..1915138d33c03 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorPlanOptimizer.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorPlanOptimizer.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.spi; -import com.facebook.presto.spi.api.Experimental; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; @@ -27,10 +26,7 @@ * There could be multiple PlanNodes satisfying the above conditions. * All of them will be processed with the given implementation of ConnectorPlanOptimizer. * Each optimization is processed exactly once at the end of logical planning (i.e. right before AddExchanges). - * TODO: currently, optimization is processed at the end of physical planning given only filter and table scan nodes are supported. - * TODO: the connector optimization will be moved to the end of logical planning after #12828 */ -@Experimental public interface ConnectorPlanOptimizer { PlanNode optimize( diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorSession.java b/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorSession.java index df0b400ed22ec..c186ceb7e3958 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorSession.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorSession.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.security.ConnectorIdentity; import com.facebook.presto.spi.type.TimeZoneKey; @@ -32,16 +33,29 @@ default String getUser() ConnectorIdentity getIdentity(); + /** + * @deprecated In favor of {@link com.facebook.presto.spi.function.SqlFunctionProperties#getTimeZoneKey()} + * @return + */ + @Deprecated TimeZoneKey getTimeZoneKey(); Locale getLocale(); Optional getTraceToken(); + Optional getClientInfo(); + long getStartTime(); + /** + * @deprecated In favor of {@link com.facebook.presto.spi.function.SqlFunctionProperties#isLegacyTimestamp()} + * @return + */ @Deprecated boolean isLegacyTimestamp(); + SqlFunctionProperties getSqlFunctionProperties(); + T getProperty(String name, Class type); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/FixedPageSource.java b/presto-spi/src/main/java/com/facebook/presto/spi/FixedPageSource.java index 5f2dc59d32877..f0a73fd6a72a9 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/FixedPageSource.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/FixedPageSource.java @@ -24,6 +24,7 @@ public class FixedPageSource private final Iterator pages; private long completedBytes; + private long completedPositions; private long memoryUsageBytes; private boolean closed; @@ -51,6 +52,12 @@ public long getCompletedBytes() return completedBytes; } + @Override + public long getCompletedPositions() + { + return completedPositions; + } + @Override public long getReadTimeNanos() { @@ -71,6 +78,7 @@ public Page getNextPage() } Page page = pages.next(); completedBytes += page.getSizeInBytes(); + completedPositions += page.getPositionCount(); return page; } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/InMemoryRecordSet.java b/presto-spi/src/main/java/com/facebook/presto/spi/InMemoryRecordSet.java index e2dd8bd74166f..50988d494ae06 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/InMemoryRecordSet.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/InMemoryRecordSet.java @@ -236,6 +236,10 @@ else if (type.getTypeSignature().getBase().equals("array")) { checkArgument(value instanceof Block, "Expected value %d to be an instance of Block, but is a %s", i, value.getClass().getSimpleName()); } + else if (type.getTypeSignature().getBase().equals("row")) { + checkArgument(value instanceof Block, + "Expected value %d to be an instance of Block, but is a %s", i, value.getClass().getSimpleName()); + } else { throw new IllegalStateException("Unsupported column type " + types.get(i)); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java b/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java index c87c27736113f..28e0c5c46e024 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java @@ -16,6 +16,7 @@ import com.facebook.presto.spi.block.BlockEncoding; import com.facebook.presto.spi.connector.ConnectorFactory; import com.facebook.presto.spi.eventlistener.EventListenerFactory; +import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory; import com.facebook.presto.spi.resourceGroups.ResourceGroupConfigurationManagerFactory; import com.facebook.presto.spi.security.PasswordAuthenticatorFactory; import com.facebook.presto.spi.security.SystemAccessControlFactory; @@ -79,4 +80,9 @@ default Iterable getSessionPropertyC { return emptyList(); } + + default Iterable getFunctionNamespaceManagerFactories() + { + return emptyList(); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/RecordPageSource.java b/presto-spi/src/main/java/com/facebook/presto/spi/RecordPageSource.java index 5b411573c713e..6b7b609f5b4f1 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/RecordPageSource.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/RecordPageSource.java @@ -30,6 +30,7 @@ public class RecordPageSource private final RecordCursor cursor; private final List types; private final PageBuilder pageBuilder; + private long completedPositions; private boolean closed; public RecordPageSource(RecordSet recordSet) @@ -55,6 +56,12 @@ public long getCompletedBytes() return cursor.getCompletedBytes(); } + @Override + public long getCompletedPositions() + { + return completedPositions; + } + @Override public long getReadTimeNanos() { @@ -95,6 +102,8 @@ public Page getNextPage() break; } + completedPositions++; + pageBuilder.declarePosition(); for (int column = 0; column < types.size(); column++) { BlockBuilder output = pageBuilder.getBlockBuilder(column); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java b/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java index 29912529aecd7..8401c1ac557f4 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java @@ -92,6 +92,8 @@ public enum StandardErrorCode INVALID_RESOURCE_GROUP(0x0001_0019, INTERNAL_ERROR), GENERIC_RECOVERY_ERROR(0x0001_001A, INTERNAL_ERROR), TOO_MANY_TASK_FAILED(0x0001_001B, INTERNAL_ERROR), + INDEX_LOADER_TIMEOUT(0x0001_001C, INTERNAL_ERROR), + EXCEEDED_TASK_UPDATE_SIZE_LIMIT(0x0001_001D, INTERNAL_ERROR), GENERIC_INSUFFICIENT_RESOURCES(0x0002_0000, INSUFFICIENT_RESOURCES), EXCEEDED_GLOBAL_MEMORY_LIMIT(0x0002_0001, INSUFFICIENT_RESOURCES), diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/Subfield.java b/presto-spi/src/main/java/com/facebook/presto/spi/Subfield.java index 6582925a659a9..ed2d606bc1d42 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/Subfield.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/Subfield.java @@ -262,6 +262,15 @@ public boolean isPrefix(Subfield other) return false; } + public Subfield tail(String name) + { + if (path.isEmpty()) { + throw new IllegalStateException("path is empty"); + } + + return new Subfield(name, path.subList(1, path.size())); + } + @JsonValue public String serialize() { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/SubfieldTokenizer.java b/presto-spi/src/main/java/com/facebook/presto/spi/SubfieldTokenizer.java index 76a8cd7da0058..94be9a340978c 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/SubfieldTokenizer.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/SubfieldTokenizer.java @@ -148,7 +148,7 @@ private Subfield.PathElement matchWildcardSubscript() private static boolean isUnquotedPathCharacter(char c) { - return c == ':' || c == '$' || isUnquotedSubscriptCharacter(c); + return c == ':' || c == '$' || c == '-' || c == '/' || isUnquotedSubscriptCharacter(c); } private Subfield.PathElement matchUnquotedSubscript() diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractArrayBlock.java index 12f326030b56f..834a4026e7c57 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractArrayBlock.java @@ -145,16 +145,13 @@ public Block copyRegion(int position, int length) } @Override - public T getObject(int position, Class clazz) + public Block getBlock(int position) { - if (clazz != Block.class) { - throw new IllegalArgumentException("clazz must be Block.class"); - } checkReadablePosition(position); int startValueOffset = getOffset(position); int endValueOffset = getOffset(position + 1); - return clazz.cast(getRawElementBlock().getRegion(startValueOffset, endValueOffset - startValueOffset)); + return getRawElementBlock().getRegion(startValueOffset, endValueOffset - startValueOffset); } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java index 9f3629002f4bd..4366751bd7582 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractMapBlock.java @@ -144,7 +144,7 @@ public Block copyPositions(int[] positions, int offset, int length) newOffsets, newKeys, newValues, - new HashTables(Optional.ofNullable(newRawHashTables), newHashTableEntries), + new HashTables(Optional.ofNullable(newRawHashTables), length, newHashTableEntries), keyType, keyBlockNativeEquals, keyNativeHashCode, @@ -218,6 +218,7 @@ public long getPositionsSizeInBytes(boolean[] positions) Integer.BYTES * HASH_MULTIPLIER * (long) usedEntryCount + getHashTables().getInstanceSizeInBytes(); } + @Override public Block copyRegion(int position, int length) { @@ -250,7 +251,7 @@ public Block copyRegion(int position, int length) newOffsets, newKeys, newValues, - new HashTables(Optional.ofNullable(newRawHashTables), expectedNewHashTableEntries), + new HashTables(Optional.ofNullable(newRawHashTables), length, expectedNewHashTableEntries), keyType, keyBlockNativeEquals, keyNativeHashCode, @@ -258,19 +259,16 @@ public Block copyRegion(int position, int length) } @Override - public T getObject(int position, Class clazz) + public Block getBlock(int position) { - if (clazz != Block.class) { - throw new IllegalArgumentException("clazz must be Block.class"); - } checkReadablePosition(position); int startEntryOffset = getOffset(position); int endEntryOffset = getOffset(position + 1); - return clazz.cast(new SingleMapBlock( + return new SingleMapBlock( startEntryOffset * 2, (endEntryOffset - startEntryOffset) * 2, - this)); + this); } @Override @@ -305,7 +303,7 @@ public Block getSingleValueBlock(int position) new int[] {0, valueLength}, newKeys, newValues, - new HashTables(Optional.ofNullable(newRawHashTables), expectedNewHashTableEntries), + new HashTables(Optional.ofNullable(newRawHashTables), 1, expectedNewHashTableEntries), keyType, keyBlockNativeEquals, keyNativeHashCode, @@ -370,10 +368,13 @@ public static class HashTables @Nullable private volatile int[] hashTables; - // The number of entries of hashTables array as if they are always built. It's used to calculate the retained size. + // The number of hash tables. Each map row corresponds to one hash table if it's built. + private int expectedHashTableCount; + + // The total number of entries of all hashTables as if they are always built. It's used to calculate the retained size. private int expectedEntryCount; - HashTables(Optional hashTables, int expectedEntryCount) + HashTables(Optional hashTables, int expectedHashTableCount, int expectedEntryCount) { if (hashTables.isPresent() && hashTables.get().length != expectedEntryCount) { throw new IllegalArgumentException("hashTables size does not match expectedEntryCount"); @@ -381,6 +382,7 @@ public static class HashTables this.hashTables = hashTables.orElse(null); this.expectedEntryCount = expectedEntryCount; + this.expectedHashTableCount = expectedHashTableCount; } Optional get() @@ -397,6 +399,11 @@ void set(int[] hashTables) this.expectedEntryCount = hashTables.length; } + int getExpectedHashTableCount() + { + return expectedHashTableCount; + } + public long getInstanceSizeInBytes() { return INSTANCE_SIZE; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractRowBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractRowBlock.java index 31f7a02ecc961..6408ad19ca1df 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractRowBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractRowBlock.java @@ -158,14 +158,11 @@ public Block copyRegion(int position, int length) } @Override - public T getObject(int position, Class clazz) + public Block getBlock(int position) { - if (clazz != Block.class) { - throw new IllegalArgumentException("clazz must be Block.class"); - } checkReadablePosition(position); - return clazz.cast(new SingleRowBlock(getFieldBlockOffset(position), getRawFieldBlocks())); + return new SingleRowBlock(getFieldBlockOffset(position), getRawFieldBlocks()); } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleArrayBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleArrayBlock.java index 80581c9ab2e09..92179f0ece16e 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleArrayBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleArrayBlock.java @@ -86,10 +86,10 @@ public Slice getSlice(int position, int offset, int length) } @Override - public T getObject(int position, Class clazz) + public Block getBlock(int position) { checkReadablePosition(position); - return getBlock().getObject(position + start, clazz); + return getBlock().getBlock(position + start); } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleMapBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleMapBlock.java index 6165cb7108bba..779b3b927ea4a 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleMapBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleMapBlock.java @@ -165,14 +165,14 @@ public long hash(int position, int offset, int length) } @Override - public T getObject(int position, Class clazz) + public Block getBlock(int position) { position = getAbsolutePosition(position); if (position % 2 == 0) { - return getRawKeyBlock().getObject(position / 2, clazz); + return getRawKeyBlock().getBlock(position / 2); } else { - return getRawValueBlock().getObject(position / 2, clazz); + return getRawValueBlock().getBlock(position / 2); } } @@ -317,9 +317,9 @@ public Block getBlockUnchecked(int internalPosition) { assert internalPositionInRange(internalPosition, getOffsetBase(), getPositionCount()); if (internalPosition % 2 == 0) { - return getRawKeyBlock().getObject(internalPosition / 2, Block.class); + return getRawKeyBlock().getBlock(internalPosition / 2); } - return getRawValueBlock().getObject(internalPosition / 2, Block.class); + return getRawValueBlock().getBlock(internalPosition / 2); } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleRowBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleRowBlock.java index 0130e5dca4e03..0351e759b7e98 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleRowBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractSingleRowBlock.java @@ -136,10 +136,10 @@ public long hash(int position, int offset, int length) } @Override - public T getObject(int position, Class clazz) + public Block getBlock(int position) { checkFieldIndex(position); - return getRawFieldBlock(position).getObject(rowIndex, clazz); + return getRawFieldBlock(position).getBlock(rowIndex); } @Override @@ -238,7 +238,7 @@ public int getSliceLengthUnchecked(int internalPosition) @Override public Block getBlockUnchecked(int internalPosition) { - return getRawFieldBlock(internalPosition).getObject(rowIndex, Block.class); + return getRawFieldBlock(internalPosition).getBlock(rowIndex); } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractVariableWidthBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractVariableWidthBlock.java index bfb7192eb2fa3..3cae818728b3d 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractVariableWidthBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/AbstractVariableWidthBlock.java @@ -23,9 +23,11 @@ public abstract class AbstractVariableWidthBlock implements Block { - protected abstract Slice getRawSlice(int position); + // TODO: create ColumnarSlice class and expose the rawSlice through it + public abstract Slice getRawSlice(int position); - protected abstract int getPositionOffset(int position); + // TODO: create ColumnarSlice class and expose the offset through it + public abstract int getPositionOffset(int position); protected abstract boolean isEntryNull(int position); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/Block.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/Block.java index 96dcdeec66d02..17503068c35fc 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/Block.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/Block.java @@ -81,9 +81,10 @@ default Slice getSlice(int position, int offset, int length) } /** - * Gets an object in the value at {@code position}. + * Gets a block in the value at {@code position}. + * @return */ - default T getObject(int position, Class clazz) + default Block getBlock(int position) { throw new UnsupportedOperationException(getClass().getName()); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockFlattener.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockFlattener.java index d69c4154a3cc7..c18877cbc1d0b 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockFlattener.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/BlockFlattener.java @@ -53,7 +53,7 @@ private BlockLease flattenDictionaryBlock(DictionaryBlock dictionaryBlock) { Block dictionary = dictionaryBlock.getDictionary(); int positionCount = dictionaryBlock.getPositionCount(); - int[] currentRemappedIds = (int[]) dictionaryBlock.getIds().getBase(); + int[] currentRemappedIds = dictionaryBlock.getRawIds(); // Initially, the below variable is null. After the first pass of the loop, it will be a borrowed array from the allocator, // and it will have reference equality with currentRemappedIds int[] newRemappedIds = null; @@ -61,7 +61,7 @@ private BlockLease flattenDictionaryBlock(DictionaryBlock dictionaryBlock) while (true) { if (dictionary instanceof DictionaryBlock) { dictionaryBlock = (DictionaryBlock) dictionary; - int[] ids = (int[]) dictionaryBlock.getIds().getBase(); + int[] ids = dictionaryBlock.getRawIds(); if (newRemappedIds == null) { newRemappedIds = allocator.borrowIntArray(positionCount); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ColumnarArray.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ColumnarArray.java index a4717ca0817e3..0adeb1a97d13f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ColumnarArray.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ColumnarArray.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.block; +import static io.airlift.slice.SizeOf.sizeOf; import static java.util.Objects.requireNonNull; public class ColumnarArray @@ -70,8 +71,7 @@ private static ColumnarArray toColumnarArray(DictionaryBlock dictionaryBlock) int dictionaryId = dictionaryBlock.getId(position); int length = columnarArray.getLength(dictionaryId); - // adjust to the element block start offset - int startOffset = columnarArray.getOffset(dictionaryId) - columnarArray.getOffset(0); + int startOffset = columnarArray.getOffset(dictionaryId); for (int entryIndex = 0; entryIndex < length; entryIndex++) { dictionaryIds[nextDictionaryIndex] = startOffset + entryIndex; nextDictionaryIndex++; @@ -133,16 +133,26 @@ public boolean isNull(int position) public int getLength(int position) { - return getOffset(position + 1) - getOffset(position); + return (offsets[position + 1 + offsetsOffset] - offsets[position + offsetsOffset]); } - private int getOffset(int position) + public int getOffset(int position) { - return offsets[position + offsetsOffset]; + return (offsets[position + offsetsOffset] - offsets[offsetsOffset]); } public Block getElementsBlock() { return elementsBlock; } + + public Block getNullCheckBlock() + { + return nullCheckBlock; + } + + public long getRetainedSizeInBytes() + { + return nullCheckBlock.getRetainedSizeInBytes() + elementsBlock.getRetainedSizeInBytes() + sizeOf(offsets); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ColumnarMap.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ColumnarMap.java index ede4b07e97d50..e931c90e19067 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ColumnarMap.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ColumnarMap.java @@ -13,6 +13,11 @@ */ package com.facebook.presto.spi.block; +import com.facebook.presto.spi.type.Type; + +import java.util.Optional; + +import static io.airlift.slice.SizeOf.sizeOf; import static java.util.Objects.requireNonNull; public class ColumnarMap @@ -22,6 +27,8 @@ public class ColumnarMap private final int[] offsets; private final Block keysBlock; private final Block valuesBlock; + private final Type keyType; + private final int[] hashTables; public static ColumnarMap toColumnarMap(Block block) { @@ -48,8 +55,9 @@ public static ColumnarMap toColumnarMap(Block block) int totalEntryCount = mapBlock.getOffset(block.getPositionCount()) - firstEntryPosition; Block keysBlock = mapBlock.getRawKeyBlock().getRegion(firstEntryPosition, totalEntryCount); Block valuesBlock = mapBlock.getRawValueBlock().getRegion(firstEntryPosition, totalEntryCount); + Optional hashTables = mapBlock.getHashTables().get(); - return new ColumnarMap(block, offsetBase, offsets, keysBlock, valuesBlock); + return new ColumnarMap(block, offsetBase, offsets, keysBlock, valuesBlock, mapBlock.keyType, hashTables); } private static ColumnarMap toColumnarMap(DictionaryBlock dictionaryBlock) @@ -70,8 +78,7 @@ private static ColumnarMap toColumnarMap(DictionaryBlock dictionaryBlock) int dictionaryId = dictionaryBlock.getId(position); int entryCount = columnarMap.getEntryCount(dictionaryId); - // adjust to the element block start offset - int startOffset = columnarMap.getOffset(dictionaryId) - columnarMap.getOffset(0); + int startOffset = columnarMap.getOffset(dictionaryId); for (int entryIndex = 0; entryIndex < entryCount; entryIndex++) { dictionaryIds[nextDictionaryIndex] = startOffset + entryIndex; nextDictionaryIndex++; @@ -83,7 +90,9 @@ private static ColumnarMap toColumnarMap(DictionaryBlock dictionaryBlock) 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarMap.getKeysBlock(), dictionaryIds), - new DictionaryBlock(dictionaryIds.length, columnarMap.getValuesBlock(), dictionaryIds)); + new DictionaryBlock(dictionaryIds.length, columnarMap.getValuesBlock(), dictionaryIds), + columnarMap.keyType, + columnarMap.getHashTables()); } private static ColumnarMap toColumnarMap(RunLengthEncodedBlock rleBlock) @@ -112,16 +121,20 @@ private static ColumnarMap toColumnarMap(RunLengthEncodedBlock rleBlock) 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarMap.getKeysBlock(), dictionaryIds), - new DictionaryBlock(dictionaryIds.length, columnarMap.getValuesBlock(), dictionaryIds)); + new DictionaryBlock(dictionaryIds.length, columnarMap.getValuesBlock(), dictionaryIds), + columnarMap.keyType, + columnarMap.getHashTables()); } - private ColumnarMap(Block nullCheckBlock, int offsetsOffset, int[] offsets, Block keysBlock, Block valuesBlock) + private ColumnarMap(Block nullCheckBlock, int offsetsOffset, int[] offsets, Block keysBlock, Block valuesBlock, Type keyType, Optional hashTables) { this.nullCheckBlock = nullCheckBlock; this.offsetsOffset = offsetsOffset; this.offsets = offsets; this.keysBlock = keysBlock; this.valuesBlock = valuesBlock; + this.keyType = keyType; + this.hashTables = hashTables.orElse(null); } public int getPositionCount() @@ -139,7 +152,12 @@ public int getEntryCount(int position) return (offsets[position + 1 + offsetsOffset] - offsets[position + offsetsOffset]); } - private int getOffset(int position) + public int getOffset(int position) + { + return (offsets[position + offsetsOffset] - offsets[offsetsOffset]); + } + + public int getAbsoluteOffset(int position) { return offsets[position + offsetsOffset]; } @@ -153,4 +171,24 @@ public Block getValuesBlock() { return valuesBlock; } + + public Block getNullCheckBlock() + { + return nullCheckBlock; + } + + public Type getKeyType() + { + return keyType; + } + + public Optional getHashTables() + { + return Optional.ofNullable(hashTables); + } + + public long getRetainedSizeInBytes() + { + return nullCheckBlock.getRetainedSizeInBytes() + keysBlock.getRetainedSizeInBytes() + valuesBlock.getRetainedSizeInBytes() + sizeOf(offsets) + sizeOf(hashTables); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/ColumnarRow.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/ColumnarRow.java index 36e9fd8c1a1b4..acaa98f442d29 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/ColumnarRow.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/ColumnarRow.java @@ -134,4 +134,23 @@ public Block getField(int index) { return fields[index]; } + + public int getOffset(int position) + { + return ((AbstractRowBlock) nullCheckBlock).getFieldBlockOffset(position); + } + + public Block getNullCheckBlock() + { + return nullCheckBlock; + } + + public long getRetainedSizeInBytes() + { + int fieldsRetainedSize = 0; + for (int i = 0; i < fields.length; i++) { + fieldsRetainedSize += fields[i].getRetainedSizeInBytes(); + } + return nullCheckBlock.getRetainedSizeInBytes() + fieldsRetainedSize; + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlock.java index fe42de16a8851..40f46dff88760 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/DictionaryBlock.java @@ -142,9 +142,9 @@ public Slice getSlice(int position, int offset, int length) } @Override - public T getObject(int position, Class clazz) + public Block getBlock(int position) { - return dictionary.getObject(getId(position), clazz); + return dictionary.getBlock(getId(position)); } @Override @@ -401,6 +401,11 @@ Slice getIds() return Slices.wrappedIntArray(ids, idsOffset, positionCount); } + int[] getRawIds() + { + return ids; + } + public int getId(int position) { checkValidPosition(position, positionCount); @@ -519,7 +524,7 @@ public int getSliceLengthUnchecked(int internalPosition) public Block getBlockUnchecked(int internalPosition) { assert internalPositionInRange(internalPosition, getOffsetBase(), getPositionCount()); - return dictionary.getObject(ids[internalPosition], Block.class); + return dictionary.getBlock(ids[internalPosition]); } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlock.java index 5f5fc9b930bb2..39f110a80b33d 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/LazyBlock.java @@ -92,10 +92,10 @@ public Slice getSlice(int position, int offset, int length) } @Override - public T getObject(int position, Class clazz) + public Block getBlock(int position) { assureLoaded(); - return block.getObject(position, clazz); + return block.getBlock(position); } @Override @@ -343,7 +343,7 @@ public int getSliceLengthUnchecked(int internalPosition) public Block getBlockUnchecked(int internalPosition) { assert block != null : "block is not loaded"; - return block.getObject(internalPosition, Block.class); + return block.getBlock(internalPosition); } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlock.java index f0b1708592b3c..3f60593d762c9 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlock.java @@ -26,6 +26,7 @@ import java.util.function.BiConsumer; import static com.facebook.presto.spi.block.MapBlockBuilder.buildHashTable; +import static com.facebook.presto.spi.block.MapBlockBuilder.verify; import static io.airlift.slice.SizeOf.sizeOf; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -57,6 +58,7 @@ public class MapBlock * @param keyNativeHashCode hash of a key stack type; signature is (K)long */ public static MapBlock fromKeyValueBlock( + int positionCount, Optional mapIsNull, int[] offsets, Block keyBlock, @@ -66,18 +68,16 @@ public static MapBlock fromKeyValueBlock( MethodHandle keyNativeHashCode, MethodHandle keyBlockHashCode) { - validateConstructorArguments(0, offsets.length - 1, mapIsNull.orElse(null), offsets, keyBlock, valueBlock, mapType.getKeyType(), keyBlockNativeEquals, keyNativeHashCode); - - int mapCount = offsets.length - 1; + validateConstructorArguments(0, positionCount, mapIsNull.orElse(null), offsets, keyBlock, valueBlock, mapType.getKeyType(), keyBlockNativeEquals, keyNativeHashCode); return createMapBlockInternal( 0, - mapCount, + positionCount, mapIsNull, offsets, keyBlock, valueBlock, - new HashTables(Optional.empty(), keyBlock.getPositionCount() * HASH_MULTIPLIER), + new HashTables(Optional.empty(), positionCount, keyBlock.getPositionCount() * HASH_MULTIPLIER), mapType.getKeyType(), keyBlockNativeEquals, keyNativeHashCode, @@ -336,7 +336,10 @@ protected void ensureHashTableLoaded() int[] hashTables = new int[getRawKeyBlock().getPositionCount() * HASH_MULTIPLIER]; Arrays.fill(hashTables, -1); - for (int i = 0; i < offsets.length - 1; i++) { + + verify(this.hashTables.getExpectedHashTableCount() <= offsets.length, "incorrect offsets size"); + + for (int i = 0; i < this.hashTables.getExpectedHashTableCount(); i++) { int keyOffset = offsets[i]; int keyCount = offsets[i + 1] - keyOffset; if (keyCount < 0) { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockBuilder.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockBuilder.java index f4e5b943b361a..d2ea021ff12e3 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockBuilder.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockBuilder.java @@ -99,7 +99,7 @@ private MapBlockBuilder( this.mapIsNull = requireNonNull(mapIsNull, "mapIsNull is null"); this.keyBlockBuilder = requireNonNull(keyBlockBuilder, "keyBlockBuilder is null"); this.valueBlockBuilder = requireNonNull(valueBlockBuilder, "valueBlockBuilder is null"); - this.hashTables = new HashTables(Optional.of(requireNonNull(rawHashTables, "hashTables is null")), rawHashTables.length); + this.hashTables = new HashTables(Optional.of(requireNonNull(rawHashTables, "hashTables is null")), 0, rawHashTables.length); } @Override @@ -349,7 +349,7 @@ public Block build() offsets, keyBlockBuilder.build(), valueBlockBuilder.build(), - new HashTables(Optional.of(Arrays.copyOf(rawHashTables.get(), hashTablesEntries)), hashTablesEntries), + new HashTables(Optional.of(Arrays.copyOf(rawHashTables.get(), hashTablesEntries)), positionCount, hashTablesEntries), keyType, keyBlockNativeEquals, keyNativeHashCode, @@ -557,7 +557,7 @@ static int computePosition(long hashcode, int hashTableSize) return (int) ((Integer.toUnsignedLong(Long.hashCode(hashcode)) * hashTableSize) >> 32); } - private static void verify(boolean assertion, String message) + static void verify(boolean assertion, String message) { if (!assertion) { throw new AssertionError(message); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java index f2db717216ff8..b5c567f4c0d22 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/MapBlockEncoding.java @@ -111,6 +111,6 @@ public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceIn int[] offsets = new int[positionCount + 1]; sliceInput.readBytes(wrappedIntArray(offsets)); Optional mapIsNull = EncoderUtil.decodeNullBits(sliceInput, positionCount); - return MapType.createMapBlockInternal(typeManager, keyType, 0, positionCount, mapIsNull, offsets, keyBlock, valueBlock, new HashTables(Optional.ofNullable(hashTable), hashTableLength)); + return MapType.createMapBlockInternal(typeManager, keyType, 0, positionCount, mapIsNull, offsets, keyBlock, valueBlock, new HashTables(Optional.ofNullable(hashTable), positionCount, hashTableLength)); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthEncodedBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthEncodedBlock.java index 3d8615bf51b86..b9081ad8df569 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthEncodedBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/RunLengthEncodedBlock.java @@ -209,10 +209,10 @@ public Slice getSlice(int position, int offset, int length) } @Override - public T getObject(int position, Class clazz) + public Block getBlock(int position) { checkReadablePosition(position); - return value.getObject(0, clazz); + return value.getBlock(0); } @Override @@ -359,7 +359,7 @@ public int getSliceLengthUnchecked(int internalPosition) public Block getBlockUnchecked(int internalPosition) { assert internalPositionInRange(internalPosition, getOffsetBase(), getPositionCount()); - return value.getObject(0, Block.class); + return value.getBlock(0); } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java index e96799bc43391..10349490edf54 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/SingleMapBlockEncoding.java @@ -111,7 +111,7 @@ public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceIn new int[] {0, keyBlock.getPositionCount()}, keyBlock, valueBlock, - new HashTables(Optional.ofNullable(hashTable), hashTableLength), + new HashTables(Optional.ofNullable(hashTable), 1, hashTableLength), keyType, keyBlockNativeEquals, keyNativeHashCode, diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlock.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlock.java index b7f43c4f8dbc4..14cd9d4c762a7 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlock.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlock.java @@ -83,7 +83,7 @@ public VariableWidthBlock(int positionCount, Slice slice, int[] offsets, Optiona } @Override - protected final int getPositionOffset(int position) + public final int getPositionOffset(int position) { return offsets[position + arrayOffset]; } @@ -186,7 +186,7 @@ else if (newValueIsNull != null) { } @Override - protected Slice getRawSlice(int position) + public Slice getRawSlice(int position) { return slice; } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlockBuilder.java b/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlockBuilder.java index 77a68752e3342..3f6a9ad275d83 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlockBuilder.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/block/VariableWidthBlockBuilder.java @@ -78,7 +78,7 @@ public VariableWidthBlockBuilder(@Nullable BlockBuilderStatus blockBuilderStatus } @Override - protected int getPositionOffset(int position) + public int getPositionOffset(int position) { checkValidPosition(position, positions); return getOffset(position); @@ -92,7 +92,7 @@ public int getSliceLength(int position) } @Override - protected Slice getRawSlice(int position) + public Slice getRawSlice(int position) { return sliceOutput.getUnderlyingSlice(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java index b966f6bd09260..46da7d49384a5 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorContext.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.FilterStatsCalculatorService; import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.type.TypeManager; @@ -57,4 +58,9 @@ default RowExpressionService getRowExpressionService() { throw new UnsupportedOperationException(); } + + default FilterStatsCalculatorService getFilterStatsCalculatorService() + { + throw new UnsupportedOperationException(); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java index 556b7beccae62..cc7bc4c2b7f96 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java @@ -51,6 +51,7 @@ import java.util.Optional; import java.util.OptionalLong; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; @@ -257,9 +258,9 @@ default List listTables(ConnectorSession session, Optional> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix); /** - * Get statistics for table for given filtering constraint. + * Get statistics for table for given columns and filtering constraint. */ - default TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint) + default TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Optional tableLayoutHandle, List columnHandles, Constraint constraint) { return TableStatistics.empty(); } @@ -538,7 +539,7 @@ default Map getViews(ConnectorSession /** * @return whether delete without table scan is supported */ - default boolean supportsMetadataDelete(ConnectorSession session, ConnectorTableHandle tableHandle) + default boolean supportsMetadataDelete(ConnectorSession session, ConnectorTableHandle tableHandle, Optional tableLayoutHandle) { throw new PrestoException(NOT_SUPPORTED, "This connector does not support deletes"); } @@ -661,7 +662,7 @@ default List listTablePrivileges(ConnectorSession session, SchemaTabl * This method is unstable and subject to change in the future. */ @Experimental - default void commitPartition(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments) + default CompletableFuture commitPartitionAsync(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments) { throw new PrestoException(NOT_SUPPORTED, "This connector does not support partition commit"); } @@ -672,7 +673,7 @@ default void commitPartition(ConnectorSession session, ConnectorOutputTableHandl * This method is unstable and subject to change in the future. */ @Experimental - default void commitPartition(ConnectorSession session, ConnectorInsertTableHandle tableHandle, Collection fragments) + default CompletableFuture commitPartitionAsync(ConnectorSession session, ConnectorInsertTableHandle tableHandle, Collection fragments) { throw new PrestoException(NOT_SUPPORTED, "This connector does not support partition commit"); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorPageSourceProvider.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorPageSourceProvider.java index aa1c14a700b15..6981f0849ea5f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorPageSourceProvider.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorPageSourceProvider.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; import java.util.List; @@ -25,5 +26,17 @@ public interface ConnectorPageSourceProvider /** * @param columns columns that should show up in the output page, in this order */ - ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, List columns); + @Deprecated + default ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, List columns) + { + throw new UnsupportedOperationException(); + } + + /** + * @param columns columns that should show up in the output page, in this order + */ + default ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, ConnectorTableLayoutHandle layout, List columns) + { + return createPageSource(transactionHandle, session, split, columns); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorPlanOptimizerProvider.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorPlanOptimizerProvider.java index 08b755e350405..c05b248c87a64 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorPlanOptimizerProvider.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorPlanOptimizerProvider.java @@ -19,5 +19,14 @@ public interface ConnectorPlanOptimizerProvider { - Set getConnectorPlanOptimizers(); + /** + * The plan optimizers to be applied before having the notion of distribution. + */ + Set getLogicalPlanOptimizers(); + + /** + * The plan optimizers to be applied after having the notion of distribution. + * The plan will be only executed on a single node. + */ + Set getPhysicalPlanOptimizers(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java index e20500eedbc75..6e30a8c5d695d 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java @@ -54,6 +54,7 @@ import java.util.Optional; import java.util.OptionalLong; import java.util.Set; +import java.util.concurrent.CompletableFuture; import static java.util.Objects.requireNonNull; @@ -282,10 +283,10 @@ public Map> listTableColumns(ConnectorSess } @Override - public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint) + public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Optional tableLayoutHandle, List columnHandles, Constraint constraint) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { - return delegate.getTableStatistics(session, tableHandle, constraint); + return delegate.getTableStatistics(session, tableHandle, tableLayoutHandle, columnHandles, constraint); } } @@ -482,10 +483,10 @@ public void finishDelete(ConnectorSession session, ConnectorTableHandle tableHan } @Override - public boolean supportsMetadataDelete(ConnectorSession session, ConnectorTableHandle tableHandle) + public boolean supportsMetadataDelete(ConnectorSession session, ConnectorTableHandle tableHandle, Optional tableLayoutHandle) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { - return delegate.supportsMetadataDelete(session, tableHandle); + return delegate.supportsMetadataDelete(session, tableHandle, tableLayoutHandle); } } @@ -594,18 +595,18 @@ public List listTablePrivileges(ConnectorSession session, SchemaTable } @Override - public void commitPartition(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments) + public CompletableFuture commitPartitionAsync(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { - delegate.commitPartition(session, tableHandle, fragments); + return delegate.commitPartitionAsync(session, tableHandle, fragments); } } @Override - public void commitPartition(ConnectorSession session, ConnectorInsertTableHandle tableHandle, Collection fragments) + public CompletableFuture commitPartitionAsync(ConnectorSession session, ConnectorInsertTableHandle tableHandle, Collection fragments) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { - delegate.commitPartition(session, tableHandle, fragments); + return delegate.commitPartitionAsync(session, tableHandle, fragments); } } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorPageSourceProvider.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorPageSourceProvider.java index 1583884779efe..a66865ae2a1e7 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorPageSourceProvider.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorPageSourceProvider.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; @@ -39,9 +40,15 @@ public ClassLoaderSafeConnectorPageSourceProvider(ConnectorPageSourceProvider de @Override public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, List columns) + { + throw new UnsupportedOperationException(); + } + + @Override + public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, ConnectorTableLayoutHandle layout, List columns) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { - return delegate.createPageSource(transactionHandle, session, split, columns); + return delegate.createPageSource(transactionHandle, session, split, layout, columns); } } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryContext.java b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryContext.java index 434286d60736a..a7d4a614ecd8c 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryContext.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryContext.java @@ -31,7 +31,6 @@ public class QueryContext private final Optional userAgent; private final Optional clientInfo; private final Set clientTags; - private final Set clientCapabilities; private final Optional source; private final Optional catalog; @@ -53,7 +52,6 @@ public QueryContext( Optional userAgent, Optional clientInfo, Set clientTags, - Set clientCapabilities, Optional source, Optional catalog, Optional schema, @@ -70,7 +68,6 @@ public QueryContext( this.userAgent = requireNonNull(userAgent, "userAgent is null"); this.clientInfo = requireNonNull(clientInfo, "clientInfo is null"); this.clientTags = requireNonNull(clientTags, "clientTags is null"); - this.clientCapabilities = requireNonNull(clientCapabilities, "clientCapabilities is null"); this.source = requireNonNull(source, "source is null"); this.catalog = requireNonNull(catalog, "catalog is null"); this.schema = requireNonNull(schema, "schema is null"); @@ -118,12 +115,6 @@ public Set getClientTags() return clientTags; } - @JsonProperty - public Set getClientCapabilities() - { - return clientCapabilities; - } - @JsonProperty public Optional getSource() { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryStatistics.java b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryStatistics.java index de41835dd9926..8c2e3e1eab57e 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryStatistics.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryStatistics.java @@ -26,6 +26,7 @@ public class QueryStatistics private final Duration queuedTime; private final Optional analysisTime; + private final int peakRunningTasks; private final long peakUserMemoryBytes; // peak of user + system memory private final long peakTotalNonRevocableMemoryBytes; @@ -46,7 +47,8 @@ public class QueryStatistics private final int completedSplits; private final boolean complete; - private final List cpuTimeDistribution; + private final List cpuTimeDistribution; + private final List peakMemoryDistribution; private final List operatorSummaries; @@ -55,6 +57,7 @@ public QueryStatistics( Duration wallTime, Duration queuedTime, Optional analysisTime, + int peakRunningTasks, long peakUserMemoryBytes, long peakTotalNonRevocableMemoryBytes, long peakTaskUserMemory, @@ -70,13 +73,15 @@ public QueryStatistics( List stageGcStatistics, int completedSplits, boolean complete, - List cpuTimeDistribution, + List cpuTimeDistribution, + List peakMemoryDistribution, List operatorSummaries) { this.cpuTime = requireNonNull(cpuTime, "cpuTime is null"); this.wallTime = requireNonNull(wallTime, "wallTime is null"); this.queuedTime = requireNonNull(queuedTime, "queuedTime is null"); this.analysisTime = requireNonNull(analysisTime, "analysisTime is null"); + this.peakRunningTasks = peakRunningTasks; this.peakUserMemoryBytes = peakUserMemoryBytes; this.peakTotalNonRevocableMemoryBytes = peakTotalNonRevocableMemoryBytes; this.peakTaskUserMemory = peakTaskUserMemory; @@ -93,6 +98,7 @@ public QueryStatistics( this.completedSplits = completedSplits; this.complete = complete; this.cpuTimeDistribution = requireNonNull(cpuTimeDistribution, "cpuTimeDistribution is null"); + this.peakMemoryDistribution = requireNonNull(peakMemoryDistribution, "peakMemoryDistribution is null"); this.operatorSummaries = requireNonNull(operatorSummaries, "operatorSummaries is null"); } @@ -116,6 +122,11 @@ public Optional getAnalysisTime() return analysisTime; } + public int getPeakRunningTasks() + { + return peakRunningTasks; + } + public long getPeakUserMemoryBytes() { return peakUserMemoryBytes; @@ -191,11 +202,16 @@ public boolean isComplete() return complete; } - public List getCpuTimeDistribution() + public List getCpuTimeDistribution() { return cpuTimeDistribution; } + public List getPeakMemoryDistribution() + { + return peakMemoryDistribution; + } + public List getOperatorSummaries() { return operatorSummaries; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/StageCpuDistribution.java b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/ResourceDistribution.java similarity index 97% rename from presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/StageCpuDistribution.java rename to presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/ResourceDistribution.java index f3c20a84f7d79..8102de8af8907 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/StageCpuDistribution.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/ResourceDistribution.java @@ -16,7 +16,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -public class StageCpuDistribution +public class ResourceDistribution { private final int stageId; private final int tasks; @@ -32,7 +32,7 @@ public class StageCpuDistribution private final double average; @JsonCreator - public StageCpuDistribution( + public ResourceDistribution( @JsonProperty("stageId") int stageId, @JsonProperty("tasks") int tasks, @JsonProperty("p25") long p25, diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/SplitCompletedEvent.java b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/SplitCompletedEvent.java index 495bebdb34843..95417ce9684ce 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/SplitCompletedEvent.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/SplitCompletedEvent.java @@ -22,6 +22,7 @@ public class SplitCompletedEvent { private final String queryId; private final String stageId; + private final String stageExecutionId; private final String taskId; private final Instant createTime; @@ -36,6 +37,7 @@ public class SplitCompletedEvent public SplitCompletedEvent( String queryId, String stageId, + String stageExecutionId, String taskId, Instant createTime, Optional startTime, @@ -46,6 +48,7 @@ public SplitCompletedEvent( { this.queryId = requireNonNull(queryId, "queryId is null"); this.stageId = requireNonNull(stageId, "stageId is null"); + this.stageExecutionId = requireNonNull(stageExecutionId, "stageExecutionId is null"); this.taskId = requireNonNull(taskId, "taskId is null"); this.createTime = requireNonNull(createTime, "createTime is null"); this.startTime = requireNonNull(startTime, "startTime is null"); @@ -65,6 +68,11 @@ public String getStageId() return stageId; } + public String getStageExecutionId() + { + return stageExecutionId; + } + public String getTaskId() { return taskId; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/StageGcStatistics.java b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/StageGcStatistics.java index 2404bdb91ac66..e939cc5ecaf38 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/StageGcStatistics.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/StageGcStatistics.java @@ -19,6 +19,7 @@ public class StageGcStatistics { private final int stageId; + private final int stageExecutionId; private final int tasks; private final int fullGcTasks; @@ -31,6 +32,7 @@ public class StageGcStatistics @JsonCreator public StageGcStatistics( @JsonProperty("stageId") int stageId, + @JsonProperty("stageExecutionId") int stageExecutionId, @JsonProperty("tasks") int tasks, @JsonProperty("fullGcTasks") int fullGcTasks, @JsonProperty("minFullGcSec") int minFullGcSec, @@ -39,6 +41,7 @@ public StageGcStatistics( @JsonProperty("averageFullGcSec") int averageFullGcSec) { this.stageId = stageId; + this.stageExecutionId = stageExecutionId; this.tasks = tasks; this.fullGcTasks = fullGcTasks; this.minFullGcSec = minFullGcSec; @@ -53,6 +56,12 @@ public int getStageId() return stageId; } + @JsonProperty + public int getStageExecutionId() + { + return stageExecutionId; + } + @JsonProperty public int getTasks() { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/CatalogSchemaPrefix.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/CatalogSchemaPrefix.java new file mode 100644 index 0000000000000..4d9c69b4229a5 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/CatalogSchemaPrefix.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.presto.spi.CatalogSchemaName; + +import java.util.Objects; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class CatalogSchemaPrefix +{ + private final String catalogName; + private final Optional schemaName; + + public CatalogSchemaPrefix(String catalogName, Optional schemaName) + { + this.catalogName = requireNonNull(catalogName, "catalogName is null"); + this.schemaName = requireNonNull(schemaName, "schemaName is null"); + } + + public static CatalogSchemaPrefix of(String prefix) + { + String[] parts = prefix.split("\\."); + if (parts.length != 1 && parts.length != 2) { + throw new IllegalArgumentException("CatalogSchemaPrefix should have 1 or 2 parts"); + } + return parts.length == 1 ? new CatalogSchemaPrefix(parts[0], Optional.empty()) : new CatalogSchemaPrefix(parts[0], Optional.of(parts[1])); + } + + public String getCatalogName() + { + return catalogName; + } + + public Optional getSchemaName() + { + return schemaName; + } + + public boolean includes(CatalogSchemaName catalogSchemaName) + { + return catalogName.equals(catalogSchemaName.getCatalogName()) + && (!schemaName.isPresent() || schemaName.get().equals(catalogSchemaName.getSchemaName())); + } + + public boolean includes(CatalogSchemaPrefix that) + { + return catalogName.equals(that.getCatalogName()) + && (!schemaName.isPresent() || (that.schemaName.isPresent() && schemaName.get().equals(that.schemaName.get()))); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CatalogSchemaPrefix that = (CatalogSchemaPrefix) o; + return Objects.equals(catalogName, that.catalogName) && + Objects.equals(schemaName, that.schemaName); + } + + @Override + public int hashCode() + { + return Objects.hash(catalogName, schemaName); + } + + @Override + public String toString() + { + return catalogName + (schemaName.map(name -> "." + name).orElse("")); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java index c9872dceaeb16..600cc20e08f29 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java @@ -23,5 +23,5 @@ @Experimental public interface FunctionHandle { - CatalogSchemaName getCatalogSchemaName(); + CatalogSchemaName getFunctionNamespace(); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleResolver.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandleResolver.java similarity index 87% rename from presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleResolver.java rename to presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandleResolver.java index b8ed747087447..49474d463eca7 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionHandleResolver.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandleResolver.java @@ -11,9 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.metadata; - -import com.facebook.presto.spi.function.FunctionHandle; +package com.facebook.presto.spi.function; public interface FunctionHandleResolver { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolResolver.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionImplementationType.java similarity index 83% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolResolver.java rename to presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionImplementationType.java index 9b6b8b51fc463..cf7965c033ef3 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolResolver.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionImplementationType.java @@ -11,9 +11,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner; +package com.facebook.presto.spi.function; -public interface SymbolResolver +public enum FunctionImplementationType { - Object getValue(Symbol symbol); + BUILTIN, + SQL } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java index 99aa39d4507c7..a9f7b3460d4ab 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java @@ -25,23 +25,39 @@ public class FunctionMetadata { - private final String name; + private final QualifiedFunctionName name; private final Optional operatorType; private final List argumentTypes; + private final Optional> argumentNames; private final TypeSignature returnType; private final FunctionKind functionKind; + private final FunctionImplementationType implementationType; private final boolean deterministic; private final boolean calledOnNullInput; public FunctionMetadata( - String name, + QualifiedFunctionName name, List argumentTypes, TypeSignature returnType, FunctionKind functionKind, + FunctionImplementationType implementationType, boolean deterministic, boolean calledOnNullInput) { - this(name, Optional.empty(), argumentTypes, returnType, functionKind, deterministic, calledOnNullInput); + this(name, Optional.empty(), argumentTypes, Optional.empty(), returnType, functionKind, implementationType, deterministic, calledOnNullInput); + } + + public FunctionMetadata( + QualifiedFunctionName name, + List argumentTypes, + List argumentNames, + TypeSignature returnType, + FunctionKind functionKind, + FunctionImplementationType implementationType, + boolean deterministic, + boolean calledOnNullInput) + { + this(name, Optional.empty(), argumentTypes, Optional.of(argumentNames), returnType, functionKind, implementationType, deterministic, calledOnNullInput); } public FunctionMetadata( @@ -49,26 +65,31 @@ public FunctionMetadata( List argumentTypes, TypeSignature returnType, FunctionKind functionKind, + FunctionImplementationType implementationType, boolean deterministic, boolean calledOnNullInput) { - this(operatorType.getFunctionName(), Optional.of(operatorType), argumentTypes, returnType, functionKind, deterministic, calledOnNullInput); + this(operatorType.getFunctionName(), Optional.of(operatorType), argumentTypes, Optional.empty(), returnType, functionKind, implementationType, deterministic, calledOnNullInput); } private FunctionMetadata( - String name, + QualifiedFunctionName name, Optional operatorType, List argumentTypes, + Optional> argumentNames, TypeSignature returnType, FunctionKind functionKind, + FunctionImplementationType implementationType, boolean deterministic, boolean calledOnNullInput) { this.name = requireNonNull(name, "name is null"); this.operatorType = requireNonNull(operatorType, "operatorType is null"); this.argumentTypes = unmodifiableList(new ArrayList<>(requireNonNull(argumentTypes, "argumentTypes is null"))); + this.argumentNames = requireNonNull(argumentNames, "argumentNames is null").map(names -> unmodifiableList(new ArrayList<>(names))); this.returnType = requireNonNull(returnType, "returnType is null"); this.functionKind = requireNonNull(functionKind, "functionKind is null"); + this.implementationType = requireNonNull(implementationType, "language is null"); this.deterministic = deterministic; this.calledOnNullInput = calledOnNullInput; } @@ -78,7 +99,7 @@ public FunctionKind getFunctionKind() return functionKind; } - public String getName() + public QualifiedFunctionName getName() { return name; } @@ -88,6 +109,11 @@ public List getArgumentTypes() return argumentTypes; } + public Optional> getArgumentNames() + { + return argumentNames; + } + public TypeSignature getReturnType() { return returnType; @@ -98,6 +124,11 @@ public Optional getOperatorType() return operatorType; } + public FunctionImplementationType getImplementationType() + { + return implementationType; + } + public boolean isDeterministic() { return deterministic; @@ -121,8 +152,10 @@ public boolean equals(Object obj) return Objects.equals(this.name, other.name) && Objects.equals(this.operatorType, other.operatorType) && Objects.equals(this.argumentTypes, other.argumentTypes) && + Objects.equals(this.argumentNames, other.argumentNames) && Objects.equals(this.returnType, other.returnType) && Objects.equals(this.functionKind, other.functionKind) && + Objects.equals(this.implementationType, other.implementationType) && Objects.equals(this.deterministic, other.deterministic) && Objects.equals(this.calledOnNullInput, other.calledOnNullInput); } @@ -130,6 +163,6 @@ public boolean equals(Object obj) @Override public int hashCode() { - return Objects.hash(name, operatorType, argumentTypes, returnType, functionKind, deterministic, calledOnNullInput); + return Objects.hash(name, operatorType, argumentTypes, argumentNames, returnType, functionKind, implementationType, deterministic, calledOnNullInput); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceManager.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceManager.java new file mode 100644 index 0000000000000..035a9e155aa55 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceManager.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.presto.spi.api.Experimental; +import com.facebook.presto.spi.type.TypeSignature; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +@Experimental +public interface FunctionNamespaceManager +{ + /** + * Start a transaction. + */ + FunctionNamespaceTransactionHandle beginTransaction(); + + /** + * Commit the transaction. Will be called at most once and will not be called if + * {@link #abort(FunctionNamespaceTransactionHandle)} is called. + */ + void commit(FunctionNamespaceTransactionHandle transactionHandle); + + /** + * Rollback the transaction. Will be called at most once and will not be called if + * {@link #commit(FunctionNamespaceTransactionHandle)} is called. + */ + void abort(FunctionNamespaceTransactionHandle transactionHandle); + + /** + * Create or replace the specified function. + * TODO: Support transaction + */ + void createFunction(SqlInvokedFunction function, boolean replace); + + /** + * Drop the specified function. + * TODO: Support transaction + */ + void dropFunction(QualifiedFunctionName functionName, Optional> parameterTypes, boolean exists); + + /** + * List all functions managed by the {@link FunctionNamespaceManager}. + * TODO: Support transaction + */ + Collection listFunctions(); + + Collection getFunctions(Optional transactionHandle, QualifiedFunctionName functionName); + + FunctionHandle getFunctionHandle(Optional transactionHandle, Signature signature); + + FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle); + + ScalarFunctionImplementation getScalarFunctionImplementation(FunctionHandle functionHandle); +} diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceManagerFactory.java similarity index 65% rename from presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java rename to presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceManagerFactory.java index d99219339ba9f..fb388ca9264ce 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/FileRewriter.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceManagerFactory.java @@ -11,16 +11,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor.storage; +package com.facebook.presto.spi.function; -import com.facebook.presto.spi.type.Type; - -import java.io.File; -import java.io.IOException; -import java.util.BitSet; import java.util.Map; -public interface FileRewriter +public interface FunctionNamespaceManagerFactory { - OrcFileInfo rewrite(Map columns, File input, File output, BitSet rowsToDelete) throws IOException; + String getName(); + + FunctionHandleResolver getHandleResolver(); + + FunctionNamespaceManager create(Map config); } diff --git a/presto-client/src/main/java/com/facebook/presto/client/ClientCapabilities.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceTransactionHandle.java similarity index 85% rename from presto-client/src/main/java/com/facebook/presto/client/ClientCapabilities.java rename to presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceTransactionHandle.java index 215b5bf9ff5f8..69df8227f8758 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/ClientCapabilities.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceTransactionHandle.java @@ -11,9 +11,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.client; +package com.facebook.presto.spi.function; -public enum ClientCapabilities +public interface FunctionNamespaceTransactionHandle { - PATH; } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/OperatorType.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/OperatorType.java index 63a3ca62c3a35..bf249df9df7ae 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/OperatorType.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/OperatorType.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.spi.function; +import com.facebook.presto.spi.CatalogSchemaName; + import java.util.Arrays; import java.util.Map; import java.util.Optional; @@ -43,16 +45,16 @@ public enum OperatorType XX_HASH_64("XX HASH 64", false), INDETERMINATE("INDETERMINATE", true); - private static final Map OPERATOR_TYPES = Arrays.stream(OperatorType.values()).collect(toMap(OperatorType::getFunctionName, Function.identity())); + private static final Map OPERATOR_TYPES = Arrays.stream(OperatorType.values()).collect(toMap(OperatorType::getFunctionName, Function.identity())); private final String operator; - private final String functionName; + private final QualifiedFunctionName functionName; private final boolean calledOnNullInput; OperatorType(String operator, boolean calledOnNullInput) { this.operator = operator; - this.functionName = "$operator$" + name(); + this.functionName = QualifiedFunctionName.of(new CatalogSchemaName("presto", "default"), "$operator$" + name()); this.calledOnNullInput = calledOnNullInput; } @@ -61,7 +63,7 @@ public String getOperator() return operator; } - public String getFunctionName() + public QualifiedFunctionName getFunctionName() { return functionName; } @@ -71,7 +73,7 @@ public boolean isCalledOnNullInput() return calledOnNullInput; } - public static Optional tryGetOperatorType(String operatorName) + public static Optional tryGetOperatorType(QualifiedFunctionName operatorName) { return Optional.ofNullable(OPERATOR_TYPES.get(operatorName)); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/QualifiedFunctionName.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/QualifiedFunctionName.java new file mode 100644 index 0000000000000..706298c5769cc --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/QualifiedFunctionName.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.presto.spi.CatalogSchemaName; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +import java.util.Objects; + +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public class QualifiedFunctionName +{ + private final CatalogSchemaName functionNamespace; + private final String functionName; + + private QualifiedFunctionName(CatalogSchemaName functionNamespace, String functionName) + { + this.functionNamespace = requireNonNull(functionNamespace, "functionNamespace is null"); + this.functionName = requireNonNull(functionName, "name is null").toLowerCase(ENGLISH); + } + + // This function should only be used for JSON deserialization. Do not use it directly. + @JsonCreator + public static QualifiedFunctionName of(String dottedName) + { + String[] parts = dottedName.split("\\."); + if (parts.length != 3) { + throw new IllegalArgumentException("QualifiedFunctionName should have exactly 3 parts"); + } + return of(new CatalogSchemaName(parts[0], parts[1]), parts[2]); + } + + public static QualifiedFunctionName of(CatalogSchemaName functionNamespace, String name) + { + return new QualifiedFunctionName(functionNamespace, name); + } + + public CatalogSchemaName getFunctionNamespace() + { + return functionNamespace; + } + + // TODO: Examine all callers to limit the usage of the method + public String getFunctionName() + { + return functionName; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + QualifiedFunctionName o = (QualifiedFunctionName) obj; + return Objects.equals(functionNamespace, o.functionNamespace) && + Objects.equals(functionName, o.functionName); + } + + @Override + public int hashCode() + { + return Objects.hash(functionNamespace, functionName); + } + + @JsonValue + @Override + public String toString() + { + return functionNamespace + "." + functionName; + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/RoutineCharacteristics.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/RoutineCharacteristics.java new file mode 100644 index 0000000000000..67f78341d55bb --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/RoutineCharacteristics.java @@ -0,0 +1,97 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import java.util.Objects; + +import static com.facebook.presto.spi.function.RoutineCharacteristics.Determinism.DETERMINISTIC; +import static com.facebook.presto.spi.function.RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class RoutineCharacteristics +{ + public enum Language + { + SQL; + } + + public enum Determinism + { + DETERMINISTIC, + NOT_DETERMINISTIC; + } + + public enum NullCallClause + { + RETURNS_NULL_ON_NULL_INPUT, + CALLED_ON_NULL_INPUT; + } + + private final Language language; + private final Determinism determinism; + private final NullCallClause nullCallClause; + + public RoutineCharacteristics( + Language language, + Determinism determinism, + NullCallClause nullCallClause) + { + this.language = requireNonNull(language, "language is null"); + this.determinism = requireNonNull(determinism, "determinism is null"); + this.nullCallClause = requireNonNull(nullCallClause, "nullCallClause is null"); + } + + public Language getLanguage() + { + return language; + } + + public boolean isDeterministic() + { + return determinism == DETERMINISTIC; + } + + public boolean isCalledOnNullInput() + { + return nullCallClause == CALLED_ON_NULL_INPUT; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RoutineCharacteristics that = (RoutineCharacteristics) o; + return language == that.language + && determinism == that.determinism + && nullCallClause == that.nullCallClause; + } + + @Override + public int hashCode() + { + return Objects.hash(language, determinism, nullCallClause); + } + + @Override + public String toString() + { + return format("(%s, %s, %s)", language, determinism, nullCallClause); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespaceFactory.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/ScalarFunctionImplementation.java similarity index 78% rename from presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespaceFactory.java rename to presto-spi/src/main/java/com/facebook/presto/spi/function/ScalarFunctionImplementation.java index 05b2dd6f1ba67..99c45f251ce99 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionNamespaceFactory.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/ScalarFunctionImplementation.java @@ -11,11 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.metadata; +package com.facebook.presto.spi.function; -public interface FunctionNamespaceFactory -{ - String getName(); +import com.facebook.presto.spi.api.Experimental; - FunctionHandleResolver getHandleResolver(); +@Experimental +public interface ScalarFunctionImplementation +{ } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/Signature.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/Signature.java index 8f2e852bd8edb..8564b2bd44f32 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/Signature.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/Signature.java @@ -30,7 +30,7 @@ public final class Signature { - private final String name; + private final QualifiedFunctionName name; private final FunctionKind kind; private final List typeVariableConstraints; private final List longVariableConstraints; @@ -40,7 +40,7 @@ public final class Signature @JsonCreator public Signature( - @JsonProperty("name") String name, + @JsonProperty("name") QualifiedFunctionName name, @JsonProperty("kind") FunctionKind kind, @JsonProperty("typeVariableConstraints") List typeVariableConstraints, @JsonProperty("longVariableConstraints") List longVariableConstraints, @@ -61,25 +61,25 @@ public Signature( this.variableArity = variableArity; } - public Signature(String name, FunctionKind kind, TypeSignature returnType, TypeSignature... argumentTypes) + public Signature(QualifiedFunctionName name, FunctionKind kind, TypeSignature returnType, TypeSignature... argumentTypes) { this(name, kind, returnType, unmodifiableList(Arrays.asList(argumentTypes))); } - public Signature(String name, FunctionKind kind, TypeSignature returnType, List argumentTypes) + public Signature(QualifiedFunctionName name, FunctionKind kind, TypeSignature returnType, List argumentTypes) { this(name, kind, emptyList(), emptyList(), returnType, argumentTypes, false); } - public Signature withAlias(String name) + @JsonProperty + public QualifiedFunctionName getName() { - return new Signature(name, kind, typeVariableConstraints, longVariableConstraints, getReturnType(), getArgumentTypes(), variableArity); + return name; } - @JsonProperty - public String getName() + public String getNameSuffix() { - return name; + return name.getFunctionName(); } @JsonProperty diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SqlFunction.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunction.java similarity index 81% rename from presto-main/src/main/java/com/facebook/presto/metadata/SqlFunction.java rename to presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunction.java index c9b81e28ec67b..a21c07a122623 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SqlFunction.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunction.java @@ -11,9 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.metadata; - -import com.facebook.presto.spi.function.Signature; +package com.facebook.presto.spi.function; public interface SqlFunction { @@ -23,10 +21,7 @@ public interface SqlFunction boolean isDeterministic(); - default boolean isCalledOnNullInput() - { - return false; - }; + boolean isCalledOnNullInput(); String getDescription(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionHandle.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionHandle.java new file mode 100644 index 0000000000000..e7e26b1627779 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionHandle.java @@ -0,0 +1,94 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.presto.spi.CatalogSchemaName; +import com.facebook.presto.spi.api.Experimental; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +@Experimental +public class SqlFunctionHandle + implements FunctionHandle +{ + private final SqlFunctionId functionId; + private final long version; + + @JsonCreator + public SqlFunctionHandle( + @JsonProperty("functionId") SqlFunctionId functionId, + @JsonProperty("version") long version) + { + this.functionId = requireNonNull(functionId, "functionId is null"); + this.version = version; + } + + @JsonProperty + public SqlFunctionId getFunctionId() + { + return functionId; + } + + @JsonProperty + public long getVersion() + { + return version; + } + + @Override + public CatalogSchemaName getFunctionNamespace() + { + return functionId.getFunctionName().getFunctionNamespace(); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + SqlFunctionHandle o = (SqlFunctionHandle) obj; + return Objects.equals(functionId, o.functionId) + && Objects.equals(version, o.version); + } + + @Override + public int hashCode() + { + return Objects.hash(functionId, version); + } + + @Override + public String toString() + { + return String.format("%s:%s", functionId, version); + } + + public static class Resolver + implements FunctionHandleResolver + { + @Override + public Class getFunctionHandleClass() + { + return SqlFunctionHandle.class; + } + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionId.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionId.java new file mode 100644 index 0000000000000..dd52fc7ab2774 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionId.java @@ -0,0 +1,88 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.presto.spi.api.Experimental; +import com.facebook.presto.spi.type.TypeSignature; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Objects; + +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.joining; + +@Experimental +public class SqlFunctionId +{ + private final QualifiedFunctionName functionName; + private final List argumentTypes; + + @JsonCreator + public SqlFunctionId( + @JsonProperty("functionName") QualifiedFunctionName functionName, + @JsonProperty("argumentTypes") List argumentTypes) + { + this.functionName = requireNonNull(functionName, "functionName is null"); + this.argumentTypes = requireNonNull(argumentTypes, "argumentTypes is null"); + } + + @JsonProperty + public QualifiedFunctionName getFunctionName() + { + return functionName; + } + + @JsonProperty + public List getArgumentTypes() + { + return argumentTypes; + } + + public String getId() + { + return toString(); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + SqlFunctionId o = (SqlFunctionId) obj; + return Objects.equals(functionName, o.functionName) + && Objects.equals(argumentTypes, o.argumentTypes); + } + + @Override + public int hashCode() + { + return Objects.hash(functionName, argumentTypes); + } + + @Override + public String toString() + { + String arguments = argumentTypes.stream() + .map(Object::toString) + .collect(joining(", ")); + return format("%s(%s)", functionName, arguments); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionProperties.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionProperties.java new file mode 100644 index 0000000000000..b95223bed7a0f --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionProperties.java @@ -0,0 +1,127 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.presto.spi.type.TimeZoneKey; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class SqlFunctionProperties +{ + private final boolean parseDecimalLiteralAsDouble; + private final boolean legacyRowFieldOrdinalAccessEnabled; + private final TimeZoneKey timeZoneKey; + private final boolean legacyTimestamp; + + private SqlFunctionProperties( + boolean parseDecimalLiteralAsDouble, + boolean legacyRowFieldOrdinalAccessEnabled, + TimeZoneKey timeZoneKey, + boolean legacyTimestamp) + { + this.parseDecimalLiteralAsDouble = parseDecimalLiteralAsDouble; + this.legacyRowFieldOrdinalAccessEnabled = legacyRowFieldOrdinalAccessEnabled; + this.timeZoneKey = requireNonNull(timeZoneKey, "timeZoneKey is null"); + this.legacyTimestamp = legacyTimestamp; + } + + public boolean isParseDecimalLiteralAsDouble() + { + return parseDecimalLiteralAsDouble; + } + + public boolean isLegacyRowFieldOrdinalAccessEnabled() + { + return legacyRowFieldOrdinalAccessEnabled; + } + + public TimeZoneKey getTimeZoneKey() + { + return timeZoneKey; + } + + @Deprecated + public boolean isLegacyTimestamp() + { + return legacyTimestamp; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof SqlFunctionProperties)) { + return false; + } + SqlFunctionProperties that = (SqlFunctionProperties) o; + return Objects.equals(parseDecimalLiteralAsDouble, that.parseDecimalLiteralAsDouble) && + Objects.equals(legacyRowFieldOrdinalAccessEnabled, that.legacyRowFieldOrdinalAccessEnabled) && + Objects.equals(timeZoneKey, that.timeZoneKey) && + Objects.equals(legacyTimestamp, that.legacyTimestamp); + } + + @Override + public int hashCode() + { + return Objects.hash(parseDecimalLiteralAsDouble, legacyRowFieldOrdinalAccessEnabled, timeZoneKey, legacyTimestamp); + } + + public static Builder builder() + { + return new Builder(); + } + + public static class Builder + { + private boolean parseDecimalLiteralAsDouble; + private boolean legacyRowFieldOrdinalAccessEnabled; + private TimeZoneKey timeZoneKey; + private boolean legacyTimestamp; + + private Builder() {} + + public Builder setParseDecimalLiteralAsDouble(boolean parseDecimalLiteralAsDouble) + { + this.parseDecimalLiteralAsDouble = parseDecimalLiteralAsDouble; + return this; + } + + public Builder setLegacyRowFieldOrdinalAccessEnabled(boolean legacyRowFieldOrdinalAccessEnabled) + { + this.legacyRowFieldOrdinalAccessEnabled = legacyRowFieldOrdinalAccessEnabled; + return this; + } + + public Builder setTimeZoneKey(TimeZoneKey timeZoneKey) + { + this.timeZoneKey = requireNonNull(timeZoneKey, "timeZoneKey is null"); + return this; + } + + public Builder setLegacyTimestamp(boolean legacyTimestamp) + { + this.legacyTimestamp = legacyTimestamp; + return this; + } + + public SqlFunctionProperties build() + { + return new SqlFunctionProperties(parseDecimalLiteralAsDouble, legacyRowFieldOrdinalAccessEnabled, timeZoneKey, legacyTimestamp); + } + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java new file mode 100644 index 0000000000000..68b3f9345f61e --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java @@ -0,0 +1,215 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.presto.spi.api.Experimental; +import com.facebook.presto.spi.type.TypeSignature; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.facebook.presto.spi.function.FunctionKind.SCALAR; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.collectingAndThen; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toList; + +@Experimental +public class SqlInvokedFunction + implements SqlFunction +{ + private final List parameters; + private final String description; + private final RoutineCharacteristics routineCharacteristics; + private final String body; + + private final Signature signature; + private final SqlFunctionId functionId; + private final Optional functionHandle; + + public SqlInvokedFunction( + QualifiedFunctionName functionName, + List parameters, + TypeSignature returnType, + String description, + RoutineCharacteristics routineCharacteristics, + String body, + Optional version) + { + this.parameters = requireNonNull(parameters, "parameters is null"); + this.description = requireNonNull(description, "description is null"); + this.routineCharacteristics = requireNonNull(routineCharacteristics, "routineCharacteristics is null"); + this.body = requireNonNull(body, "body is null"); + + List argumentTypes = parameters.stream() + .map(SqlParameter::getType) + .collect(collectingAndThen(toList(), Collections::unmodifiableList)); + this.signature = new Signature(functionName, SCALAR, returnType, argumentTypes); + this.functionId = new SqlFunctionId(functionName, argumentTypes); + this.functionHandle = version.map(v -> new SqlFunctionHandle(this.functionId, v)); + } + + public SqlInvokedFunction withVersion(long version) + { + if (getVersion().isPresent()) { + throw new IllegalArgumentException(format("function %s is already with version %s", signature.getName(), getVersion().get())); + } + return new SqlInvokedFunction( + signature.getName(), + parameters, + signature.getReturnType(), + description, + routineCharacteristics, + body, + Optional.of(version)); + } + + @Override + public Signature getSignature() + { + return signature; + } + + @Override + public boolean isHidden() + { + return false; + } + + @Override + public boolean isDeterministic() + { + return routineCharacteristics.isDeterministic(); + } + + @Override + public boolean isCalledOnNullInput() + { + return routineCharacteristics.isCalledOnNullInput(); + } + + @Override + public String getDescription() + { + return description; + } + + public List getParameters() + { + return parameters; + } + + public RoutineCharacteristics getRoutineCharacteristics() + { + return routineCharacteristics; + } + + public String getBody() + { + return body; + } + + public SqlFunctionId getFunctionId() + { + return functionId; + } + + public Optional getFunctionHandle() + { + return functionHandle; + } + + public Optional getVersion() + { + return functionHandle.map(SqlFunctionHandle::getVersion); + } + + public FunctionImplementationType getFunctionImplementationType() + { + return FunctionImplementationType.SQL; + } + + public SqlFunctionHandle getRequiredFunctionHandle() + { + Optional functionHandle = getFunctionHandle(); + if (!functionHandle.isPresent()) { + throw new IllegalStateException("missing functionHandle"); + } + return functionHandle.get(); + } + + public long getRequiredVersion() + { + Optional version = getVersion(); + if (!version.isPresent()) { + throw new IllegalStateException("missing version"); + } + return version.get(); + } + + public boolean hasSameDefinitionAs(SqlInvokedFunction function) + { + if (function == null) { + throw new IllegalArgumentException("function is null"); + } + return Objects.equals(parameters, function.parameters) + && Objects.equals(description, function.description) + && Objects.equals(routineCharacteristics, function.routineCharacteristics) + && Objects.equals(body, function.body) + && Objects.equals(signature, function.signature); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + SqlInvokedFunction o = (SqlInvokedFunction) obj; + return Objects.equals(parameters, o.parameters) + && Objects.equals(description, o.description) + && Objects.equals(routineCharacteristics, o.routineCharacteristics) + && Objects.equals(body, o.body) + && Objects.equals(signature, o.signature) + && Objects.equals(functionId, o.functionId) + && Objects.equals(functionHandle, o.functionHandle); + } + + @Override + public int hashCode() + { + return Objects.hash(parameters, description, routineCharacteristics, body, signature, functionId, functionHandle); + } + + @Override + public String toString() + { + return format( + "%s(%s):%s%s {%s} %s", + signature.getName(), + parameters.stream() + .map(Object::toString) + .collect(joining(",")), + signature.getReturnType(), + getVersion().map(version -> ":" + version).orElse(""), + body, + routineCharacteristics); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedScalarFunctionImplementation.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedScalarFunctionImplementation.java new file mode 100644 index 0000000000000..03788a427cf11 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedScalarFunctionImplementation.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.presto.spi.api.Experimental; + +import static java.util.Objects.requireNonNull; + +@Experimental +public class SqlInvokedScalarFunctionImplementation + implements ScalarFunctionImplementation +{ + private final String implementation; + + public SqlInvokedScalarFunctionImplementation(String implementation) + { + this.implementation = requireNonNull(implementation, "implementation is null"); + } + + public String getImplementation() + { + return implementation; + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlParameter.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlParameter.java new file mode 100644 index 0000000000000..452a3e4380858 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlParameter.java @@ -0,0 +1,75 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.presto.spi.type.TypeSignature; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class SqlParameter +{ + private final String name; + private final TypeSignature type; + + @JsonCreator + public SqlParameter( + @JsonProperty("name") String name, + @JsonProperty("type") TypeSignature type) + { + this.name = requireNonNull(name, "name is null"); + this.type = requireNonNull(type, "type is null"); + } + + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public TypeSignature getType() + { + return type; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + SqlParameter o = (SqlParameter) obj; + return Objects.equals(name, o.name) + && Objects.equals(type, o.type); + } + + @Override + public int hashCode() + { + return Objects.hash(name, type); + } + + @Override + public String toString() + { + return String.format("%s %s", name, type); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/ValueWindowFunction.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/ValueWindowFunction.java index 643e860bc426c..8c4d98ad8d841 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/ValueWindowFunction.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/ValueWindowFunction.java @@ -20,6 +20,8 @@ public abstract class ValueWindowFunction { protected WindowIndex windowIndex; + protected boolean ignoreNulls; + private int currentPosition; @Override @@ -56,4 +58,14 @@ public void reset() * @param currentPosition the current position for this row */ public abstract void processRow(BlockBuilder output, int frameStart, int frameEnd, int currentPosition); + + /** + * Set ignore nulls indicator. + * + * @param ignoreNulls true if nulls should be ignored + */ + public void setIgnoreNulls(boolean ignoreNulls) + { + this.ignoreNulls = ignoreNulls; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/AggregationNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/AggregationNode.java similarity index 79% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/plan/AggregationNode.java rename to presto-spi/src/main/java/com/facebook/presto/spi/plan/AggregationNode.java index a4beda25e1c50..e9cee27267a04 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/AggregationNode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/AggregationNode.java @@ -11,39 +11,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.plan; +package com.facebook.presto.spi.plan; -import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.operator.aggregation.InternalAggregationFunction; import com.facebook.presto.spi.function.FunctionHandle; -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; import javax.annotation.concurrent.Immutable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import static com.facebook.presto.sql.planner.plan.AggregationNode.Step.SINGLE; -import static com.google.common.base.Preconditions.checkArgument; +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; +import static java.util.Collections.unmodifiableList; +import static java.util.Collections.unmodifiableMap; +import static java.util.Collections.unmodifiableSet; import static java.util.Objects.requireNonNull; @Immutable -public class AggregationNode - extends InternalPlanNode +public final class AggregationNode + extends PlanNode { private final PlanNode source; private final Map aggregations; @@ -68,7 +67,7 @@ public AggregationNode( super(id); this.source = source; - this.aggregations = ImmutableMap.copyOf(requireNonNull(aggregations, "aggregations is null")); + this.aggregations = unmodifiableMap(new LinkedHashMap<>(requireNonNull(aggregations, "aggregations is null"))); requireNonNull(groupingSets, "groupingSets is null"); groupIdVariable.ifPresent(variable -> checkArgument(groupingSets.getGroupingKeys().contains(variable), "Grouping columns does not contain groupId column")); @@ -86,14 +85,13 @@ public AggregationNode( requireNonNull(preGroupedVariables, "preGroupedVariables is null"); checkArgument(preGroupedVariables.isEmpty() || groupingSets.getGroupingKeys().containsAll(preGroupedVariables), "Pre-grouped variables must be a subset of the grouping keys"); - this.preGroupedVariables = ImmutableList.copyOf(preGroupedVariables); + this.preGroupedVariables = unmodifiableList(new ArrayList<>(preGroupedVariables)); - ImmutableList.Builder outputs = ImmutableList.builder(); - outputs.addAll(groupingSets.getGroupingKeys()); + ArrayList outputs = new ArrayList<>(groupingSets.getGroupingKeys()); hashVariable.ifPresent(outputs::add); - outputs.addAll(aggregations.keySet()); + outputs.addAll(new ArrayList<>(aggregations.keySet())); - this.outputs = outputs.build(); + this.outputs = unmodifiableList(outputs); } public List getGroupingKeys() @@ -133,7 +131,7 @@ public boolean hasNonEmptyGroupingSet() @Override public List getSources() { - return ImmutableList.of(source); + return unmodifiableList(Collections.singletonList(source)); } @Override @@ -196,7 +194,7 @@ public boolean hasOrderings() } @Override - public R accept(InternalPlanVisitor visitor, C context) + public R accept(PlanVisitor visitor, C context) { return visitor.visitAggregation(this, context); } @@ -204,39 +202,8 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new AggregationNode(getId(), Iterables.getOnlyElement(newChildren), aggregations, groupingSets, preGroupedVariables, step, hashVariable, groupIdVariable); - } - - public boolean isDecomposable(FunctionManager functionManager) - { - boolean hasOrderBy = getAggregations().values().stream() - .map(Aggregation::getOrderBy) - .anyMatch(Optional::isPresent); - - boolean hasDistinct = getAggregations().values().stream() - .anyMatch(Aggregation::isDistinct); - - boolean decomposableFunctions = getAggregations().values().stream() - .map(Aggregation::getFunctionHandle) - .map(functionManager::getAggregateFunctionImplementation) - .allMatch(InternalAggregationFunction::isDecomposable); - - return !hasOrderBy && !hasDistinct && decomposableFunctions; - } - - public boolean hasSingleNodeExecutionPreference(FunctionManager functionManager) - { - // There are two kinds of aggregations the have single node execution preference: - // - // 1. aggregations with only empty grouping sets like - // - // SELECT count(*) FROM lineitem; - // - // there is no need for distributed aggregation. Single node FINAL aggregation will suffice, - // since all input have to be aggregated into one line output. - // - // 2. aggregations that must produce default output and are not decomposable, we can not distribute them. - return (hasEmptyGroupingSet() && !hasNonEmptyGroupingSet()) || (hasDefaultOutput() && !isDecomposable(functionManager)); + checkArgument(newChildren.size() == 1, "Unexpected number of elements in list newChildren"); + return new AggregationNode(getId(), newChildren.get(0), aggregations, groupingSets, preGroupedVariables, step, hashVariable, groupIdVariable); } public boolean isStreamable() @@ -246,17 +213,17 @@ public boolean isStreamable() public static GroupingSetDescriptor globalAggregation() { - return singleGroupingSet(ImmutableList.of()); + return singleGroupingSet(unmodifiableList(emptyList())); } public static GroupingSetDescriptor singleGroupingSet(List groupingKeys) { Set globalGroupingSets; if (groupingKeys.isEmpty()) { - globalGroupingSets = ImmutableSet.of(0); + globalGroupingSets = unmodifiableSet(Collections.singleton(0)); } else { - globalGroupingSets = ImmutableSet.of(); + globalGroupingSets = unmodifiableSet(emptySet()); } return new GroupingSetDescriptor(groupingKeys, 1, globalGroupingSets); @@ -286,9 +253,9 @@ public GroupingSetDescriptor( checkArgument(!globalGroupingSets.isEmpty(), "no grouping keys implies at least one global grouping set, but none provided"); } - this.groupingKeys = ImmutableList.copyOf(groupingKeys); + this.groupingKeys = unmodifiableList(new ArrayList<>(groupingKeys)); this.groupingSetCount = groupingSetCount; - this.globalGroupingSets = ImmutableSet.copyOf(globalGroupingSets); + this.globalGroupingSets = unmodifiableSet(new LinkedHashSet<>(globalGroupingSets)); } @JsonProperty @@ -457,4 +424,11 @@ public int hashCode() return Objects.hash(call, filter, orderingScheme, isDistinct, mask); } } + + private static void checkArgument(boolean condition, String message) + { + if (!condition) { + throw new IllegalArgumentException(message); + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Assignments.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/Assignments.java similarity index 88% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Assignments.java rename to presto-spi/src/main/java/com/facebook/presto/spi/plan/Assignments.java index e110450c702ae..b421cfceb4c92 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Assignments.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/Assignments.java @@ -11,15 +11,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.plan; +package com.facebook.presto.spi.plan; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; @@ -30,7 +29,9 @@ import java.util.function.Predicate; import java.util.stream.Collector; -import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.format; +import static java.util.Collections.unmodifiableList; +import static java.util.Collections.unmodifiableMap; import static java.util.Objects.requireNonNull; public class Assignments @@ -68,16 +69,18 @@ public static Assignments of(VariableReferenceExpression variable1, RowExpressio } private final Map assignments; + private final List outputs; @JsonCreator public Assignments(@JsonProperty("assignments") Map assignments) { - this.assignments = ImmutableMap.copyOf(requireNonNull(assignments, "assignments is null")); + this.assignments = unmodifiableMap(new LinkedHashMap<>(requireNonNull(assignments, "assignments is null"))); + this.outputs = unmodifiableList(new ArrayList<>(assignments.keySet())); } public List getOutputs() { - return ImmutableList.copyOf(assignments.keySet()); + return outputs; } @JsonProperty("assignments") @@ -187,12 +190,9 @@ public Builder put(VariableReferenceExpression variable, RowExpression expressio { if (assignments.containsKey(variable)) { RowExpression assignment = assignments.get(variable); - checkState( - assignment.equals(expression), - "Variable %s already has assignment %s, while adding %s", - variable, - assignment, - expression); + if (!assignment.equals(expression)) { + throw new IllegalStateException(format("Variable %s already has assignment %s, while adding %s", variable, assignment, expression)); + } } assignments.put(variable, expression); return this; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExceptNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/ExceptNode.java similarity index 67% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExceptNode.java rename to presto-spi/src/main/java/com/facebook/presto/spi/plan/ExceptNode.java index 3571bf2ad4e95..cd6e3683925e6 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ExceptNode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/ExceptNode.java @@ -11,32 +11,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.plan; +package com.facebook.presto.spi.plan; -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ListMultimap; import javax.annotation.concurrent.Immutable; import java.util.List; +import java.util.Map; @Immutable -public class ExceptNode +public final class ExceptNode extends SetOperationNode { public ExceptNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("sources") List sources, - @JsonProperty("outputToInputs") ListMultimap outputToInputs) + @JsonProperty("outputVariables") List outputVariables, + @JsonProperty("outputToInputs") Map> outputToInputs) { - super(id, sources, outputToInputs); + super(id, sources, outputVariables, outputToInputs); } @Override - public R accept(InternalPlanVisitor visitor, C context) + public R accept(PlanVisitor visitor, C context) { return visitor.visitExcept(this, context); } @@ -44,6 +43,6 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new ExceptNode(getId(), newChildren, getVariableMapping()); + return new ExceptNode(getId(), newChildren, getOutputVariables(), getVariableMapping()); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/FilterStatsCalculatorService.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/FilterStatsCalculatorService.java new file mode 100644 index 0000000000000..a3455bb9652e2 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/FilterStatsCalculatorService.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.spi.plan; + +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.statistics.TableStatistics; +import com.facebook.presto.spi.type.Type; + +import java.util.Map; + +/** + * Estimates the size of the data after applying a predicate using calculations compatible with Cost-Based Optimizer. + */ +public interface FilterStatsCalculatorService +{ + /** + * @param tableStatistics Table-level and column-level statistics. Columns are identified using ColumnHandles. + * @param predicate Filter expression referring to columns by name + * @param columnNames Mapping from ColumnHandles used in tableStatistics to column names used in the predicate + * @param columnTypes Mapping from column names used in the predicate to column types + */ + TableStatistics filterStats( + TableStatistics tableStatistics, + RowExpression predicate, + ConnectorSession session, + Map columnNames, + Map columnTypes); +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/IntersectNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/IntersectNode.java similarity index 68% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/plan/IntersectNode.java rename to presto-spi/src/main/java/com/facebook/presto/spi/plan/IntersectNode.java index 66742b090bdc9..78c7e71bb9f7e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/IntersectNode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/IntersectNode.java @@ -11,34 +11,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.plan; +package com.facebook.presto.spi.plan; -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ListMultimap; import javax.annotation.concurrent.Immutable; import java.util.List; +import java.util.Map; @Immutable -public class IntersectNode +public final class IntersectNode extends SetOperationNode { @JsonCreator public IntersectNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("sources") List sources, - @JsonProperty("outputToInputs") ListMultimap outputToInputs) + @JsonProperty("outputVariables") List outputVariables, + @JsonProperty("outputToInputs") Map> outputToInputs) { - super(id, sources, outputToInputs); + super(id, sources, outputVariables, outputToInputs); } @Override - public R accept(InternalPlanVisitor visitor, C context) + public R accept(PlanVisitor visitor, C context) { return visitor.visitIntersect(this, context); } @@ -46,6 +45,6 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new IntersectNode(getId(), newChildren, getVariableMapping()); + return new IntersectNode(getId(), newChildren, getOutputVariables(), getVariableMapping()); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/LimitNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/LimitNode.java new file mode 100644 index 0000000000000..77cfe476b3b97 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/LimitNode.java @@ -0,0 +1,126 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.plan; + +import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.concurrent.Immutable; + +import java.util.List; + +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static java.util.Collections.singletonList; +import static java.util.Objects.requireNonNull; + +@Immutable +public final class LimitNode + extends PlanNode +{ + /** + * Stages of `LimitNode`: + * + * PARTIAL: `LimitNode` is in the distributed plan and generates partial results on local workers. + * FINAL: `LimitNode` is in the distributed plan and finalizes the partial results from `PARTIAL` nodes. + */ + public enum Step + { + PARTIAL, + FINAL + } + + private final PlanNode source; + private final long count; + private final Step step; + + @JsonCreator + public LimitNode( + @JsonProperty("id") PlanNodeId id, + @JsonProperty("source") PlanNode source, + @JsonProperty("count") long count, + @JsonProperty("step") Step step) + { + super(id); + checkCondition(count >= 0, INVALID_FUNCTION_ARGUMENT, "count must be greater than or equal to zero"); + + this.source = requireNonNull(source, "source is null"); + this.count = count; + this.step = requireNonNull(step, "step is null"); + } + + @Override + public List getSources() + { + return singletonList(source); + } + + /** + * LimitNode only expects a single upstream PlanNode. + */ + @JsonProperty + public PlanNode getSource() + { + return source; + } + + /** + * Get the limit `N` number of results to return. + */ + @JsonProperty + public long getCount() + { + return count; + } + + @JsonProperty + public Step getStep() + { + return step; + } + + public boolean isPartial() + { + return step == Step.PARTIAL; + } + + @Override + public List getOutputVariables() + { + return source.getOutputVariables(); + } + + @Override + public R accept(PlanVisitor visitor, C context) + { + return visitor.visitLimit(this, context); + } + + @Override + public PlanNode replaceChildren(List newChildren) + { + checkCondition(newChildren != null && newChildren.size() == 1, GENERIC_INTERNAL_ERROR, "Expect exactly 1 child PlanNode"); + return new LimitNode(getId(), newChildren.get(0), count, getStep()); + } + + private static void checkCondition(boolean condition, ErrorCodeSupplier errorCode, String message) + { + if (!condition) { + throw new PrestoException(errorCode, message); + } + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/Ordering.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/Ordering.java new file mode 100644 index 0000000000000..89888bce9b68d --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/Ordering.java @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.plan; + +import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.concurrent.Immutable; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +@Immutable +public class Ordering +{ + private final VariableReferenceExpression variable; + private final SortOrder sortOrder; + + @JsonCreator + public Ordering( + @JsonProperty("variable") VariableReferenceExpression variable, + @JsonProperty("sortOrder") SortOrder sortOrder) + { + this.variable = requireNonNull(variable, "variable is null"); + this.sortOrder = requireNonNull(sortOrder, "sortOrder is null"); + } + + @JsonProperty + public VariableReferenceExpression getVariable() + { + return variable; + } + + @JsonProperty + public SortOrder getSortOrder() + { + return sortOrder; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Ordering that = (Ordering) o; + return Objects.equals(variable, that.variable) && + Objects.equals(sortOrder, that.sortOrder); + } + + @Override + public int hashCode() + { + return Objects.hash(variable, sortOrder); + } + + @Override + public String toString() + { + StringBuilder stringBuilder = new StringBuilder(this.getClass().getSimpleName()); + stringBuilder.append(" {"); + stringBuilder.append("variable='").append(variable).append('\''); + stringBuilder.append(", sortOrder='").append(sortOrder).append('\''); + stringBuilder.append('}'); + return stringBuilder.toString(); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/OrderingScheme.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/OrderingScheme.java new file mode 100644 index 0000000000000..e04d7e07fc79f --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/OrderingScheme.java @@ -0,0 +1,129 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.plan; + +import com.facebook.presto.spi.block.SortOrder; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import static java.lang.String.format; +import static java.util.Collections.unmodifiableList; +import static java.util.Collections.unmodifiableMap; +import static java.util.Collections.unmodifiableSet; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; + +public class OrderingScheme +{ + private final List orderBy; + private final Map orderings; + + @JsonCreator + public OrderingScheme(@JsonProperty("orderBy") List orderBy) + { + requireNonNull(orderBy, "orderBy is null"); + checkArgument(!orderBy.isEmpty(), "orderBy is empty"); + this.orderBy = immutableListCopyOf(orderBy); + this.orderings = immutableMapCopyOf(orderBy.stream().collect(toMap(Ordering::getVariable, Ordering::getSortOrder))); + } + + @JsonProperty + public List getOrderBy() + { + return orderBy; + } + + public List getOrderByVariables() + { + return orderBy.stream().map(Ordering::getVariable).collect(toList()); + } + + /** + * Sort order of ORDER BY variables. + */ + public Map getOrderingsMap() + { + return orderings; + } + + public SortOrder getOrdering(VariableReferenceExpression variable) + { + checkArgument(orderings.containsKey(variable), "No ordering for variable: %s", variable); + return orderings.get(variable); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + OrderingScheme that = (OrderingScheme) o; + return Objects.equals(orderBy, that.orderBy) && + Objects.equals(orderings, that.orderings); + } + + @Override + public int hashCode() + { + return Objects.hash(orderBy, orderings); + } + + @Override + public String toString() + { + StringBuilder stringBuilder = new StringBuilder(this.getClass().getSimpleName()); + stringBuilder.append(" {"); + stringBuilder.append("orderBy='").append(orderBy).append('\''); + stringBuilder.append(", orderings='").append(orderings).append('\''); + stringBuilder.append('}'); + return stringBuilder.toString(); + } + + private static void checkArgument(boolean condition, String messageFormat, Object... args) + { + if (!condition) { + throw new IllegalArgumentException(format(messageFormat, args)); + } + } + + private static Set immutableSetCopyOf(Collection collection) + { + return unmodifiableSet(new TreeSet<>(requireNonNull(collection, "collection is null"))); + } + + private static List immutableListCopyOf(Collection collection) + { + return unmodifiableList(new ArrayList<>(requireNonNull(collection, "collection is null"))); + } + + private static Map immutableMapCopyOf(Map map) + { + return unmodifiableMap(new TreeMap<>(requireNonNull(map, "map is null"))); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java index 8dc59c192fd3a..50fab2c80996d 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java @@ -20,6 +20,16 @@ public abstract class PlanVisitor */ public abstract R visitPlan(PlanNode node, C context); + public R visitAggregation(AggregationNode node, C context) + { + return visitPlan(node, context); + } + + public R visitProject(ProjectNode node, C context) + { + return visitPlan(node, context); + } + public R visitFilter(FilterNode node, C context) { return visitPlan(node, context); @@ -30,8 +40,33 @@ public R visitTableScan(TableScanNode node, C context) return visitPlan(node, context); } + public R visitUnion(UnionNode node, C context) + { + return visitPlan(node, context); + } + + public R visitIntersect(IntersectNode node, C context) + { + return visitPlan(node, context); + } + + public R visitExcept(ExceptNode node, C context) + { + return visitPlan(node, context); + } + public R visitValues(ValuesNode node, C context) { return visitPlan(node, context); } + + public R visitLimit(LimitNode node, C context) + { + return visitPlan(node, context); + } + + public R visitTopN(TopNNode node, C context) + { + return visitPlan(node, context); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ProjectNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/ProjectNode.java similarity index 79% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ProjectNode.java rename to presto-spi/src/main/java/com/facebook/presto/spi/plan/ProjectNode.java index 218abfef22615..c485b9570e278 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/ProjectNode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/ProjectNode.java @@ -11,25 +11,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.plan; +package com.facebook.presto.spi.plan; -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import javax.annotation.concurrent.Immutable; import java.util.List; +import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; @Immutable -public class ProjectNode - extends InternalPlanNode +public final class ProjectNode + extends PlanNode { private final PlanNode source; private final Assignments assignments; @@ -64,7 +61,7 @@ public Assignments getAssignments() @Override public List getSources() { - return ImmutableList.of(source); + return singletonList(source); } @JsonProperty @@ -74,7 +71,7 @@ public PlanNode getSource() } @Override - public R accept(InternalPlanVisitor visitor, C context) + public R accept(PlanVisitor visitor, C context) { return visitor.visitProject(this, context); } @@ -82,6 +79,10 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new ProjectNode(getId(), Iterables.getOnlyElement(newChildren), assignments); + requireNonNull(newChildren, "newChildren list is null"); + if (newChildren.size() != 1) { + throw new IllegalArgumentException("newChildren list has multiple items"); + } + return new ProjectNode(getId(), newChildren.get(0), assignments); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/SetOperationNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/SetOperationNode.java new file mode 100644 index 0000000000000..5d8880c788c8a --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/SetOperationNode.java @@ -0,0 +1,120 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.plan; + +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.concurrent.Immutable; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static java.lang.String.format; +import static java.util.Collections.unmodifiableList; +import static java.util.Collections.unmodifiableMap; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toList; + +@Immutable +public abstract class SetOperationNode + extends PlanNode +{ + private final List sources; + private final Map> outputToInputs; + private final List outputVariables; + + @JsonCreator + protected SetOperationNode( + @JsonProperty("id") PlanNodeId id, + @JsonProperty("sources") List sources, + @JsonProperty("outputVariables") List outputVariables, + @JsonProperty("outputToInputs") Map> outputToInputs) + { + super(id); + + requireNonNull(sources, "sources is null"); + checkArgument(!sources.isEmpty(), "Must have at least one source"); + requireNonNull(outputToInputs, "outputToInputs is null"); + + this.sources = unmodifiableList(new ArrayList<>(sources)); + Map> copiedMap = new LinkedHashMap<>(); + outputToInputs.forEach((key, value) -> copiedMap.put(key, unmodifiableList(new ArrayList<>(value)))); + this.outputToInputs = unmodifiableMap(copiedMap); + this.outputVariables = unmodifiableList(new ArrayList<>(outputVariables)); + + for (Collection inputs : this.outputToInputs.values()) { + checkArgument( + inputs.size() == this.sources.size(), + format("Every source needs to map its variables to an output %s operation variables", this.getClass().getSimpleName())); + } + + // Make sure each source positionally corresponds to their variable values in the Multimap + for (int i = 0; i < sources.size(); i++) { + for (List expectedInputs : this.outputToInputs.values()) { + checkArgument(sources.get(i).getOutputVariables().contains(expectedInputs.get(i)), "Source does not provide required variables"); + } + } + } + + @Override + @JsonProperty + public List getSources() + { + return sources; + } + + @JsonProperty + public Map> getVariableMapping() + { + return outputToInputs; + } + + @Override + @JsonProperty + public List getOutputVariables() + { + return outputVariables; + } + + public List sourceOutputLayout(int sourceIndex) + { + // Make sure the sourceOutputLayout variables are listed in the same order as the corresponding output variables + return unmodifiableList(getOutputVariables().stream().map(variable -> outputToInputs.get(variable).get(sourceIndex)).collect(toList())); + } + + /** + * Returns the output to input variable mapping for the given source channel + */ + public Map sourceVariableMap(int sourceIndex) + { + Map result = new LinkedHashMap<>(); + for (Map.Entry> entry : outputToInputs.entrySet()) { + result.put(entry.getKey(), entry.getValue().get(sourceIndex)); + } + + return unmodifiableMap(result); + } + + private static void checkArgument(boolean condition, String message) + { + if (!condition) { + throw new IllegalArgumentException(message); + } + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/TableScanNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/TableScanNode.java index 7420b708b1ef9..7c9348f6aabe3 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/plan/TableScanNode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/TableScanNode.java @@ -46,6 +46,9 @@ public final class TableScanNode private final TupleDomain enforcedConstraint; + /** + * This constructor is for JSON deserialization only. Do not use! + */ @JsonCreator public TableScanNode( @JsonProperty("id") PlanNodeId id, @@ -53,7 +56,6 @@ public TableScanNode( @JsonProperty("outputVariables") List outputVariables, @JsonProperty("assignments") Map assignments) { - // This constructor is for JSON deserialization only. Do not use. super(id); this.table = requireNonNull(table, "table is null"); this.outputVariables = unmodifiableList(requireNonNull(outputVariables, "outputVariables is null")); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TopNNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/TopNNode.java similarity index 63% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TopNNode.java rename to presto-spi/src/main/java/com/facebook/presto/spi/plan/TopNNode.java index d99db37731658..e80057007252a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TopNNode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/TopNNode.java @@ -11,30 +11,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.plan; +package com.facebook.presto.spi.plan; -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.facebook.presto.sql.planner.OrderingScheme; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import javax.annotation.concurrent.Immutable; import java.util.List; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; -import static com.facebook.presto.util.Failures.checkCondition; -import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; +import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; @Immutable -public class TopNNode - extends InternalPlanNode +public final class TopNNode + extends PlanNode { + /** + * Stages of `TopNNode`: + * + * SINGLE: `TopNNode` is in the logical plan. + * PARTIAL: `TopNNode` is in the distributed plan, and generates partial results of `TopN` on local workers. + * FINAL: `TopNNode` is in the distributed plan, and finalizes the partial results from `PARTIAL` nodes. + */ public enum Step { SINGLE, @@ -48,7 +53,8 @@ public enum Step private final Step step; @JsonCreator - public TopNNode(@JsonProperty("id") PlanNodeId id, + public TopNNode( + @JsonProperty("id") PlanNodeId id, @JsonProperty("source") PlanNode source, @JsonProperty("count") long count, @JsonProperty("orderingScheme") OrderingScheme orderingScheme, @@ -70,7 +76,7 @@ public TopNNode(@JsonProperty("id") PlanNodeId id, @Override public List getSources() { - return ImmutableList.of(source); + return singletonList(source); } @JsonProperty("source") @@ -104,7 +110,7 @@ public Step getStep() } @Override - public R accept(InternalPlanVisitor visitor, C context) + public R accept(PlanVisitor visitor, C context) { return visitor.visitTopN(this, context); } @@ -112,6 +118,21 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new TopNNode(getId(), Iterables.getOnlyElement(newChildren), count, orderingScheme, step); + checkCondition(newChildren != null && newChildren.size() == 1, GENERIC_INTERNAL_ERROR, "Expect exactly 1 child PlanNode"); + return new TopNNode(getId(), newChildren.get(0), count, orderingScheme, step); + } + + private static void checkArgument(boolean condition, String message) + { + if (!condition) { + throw new IllegalArgumentException(message); + } + } + + private static void checkCondition(boolean condition, ErrorCodeSupplier errorCode, String formatString, Object... args) + { + if (!condition) { + throw new PrestoException(errorCode, format(formatString, args)); + } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/UnionNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/UnionNode.java similarity index 68% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/plan/UnionNode.java rename to presto-spi/src/main/java/com/facebook/presto/spi/plan/UnionNode.java index 8b105a87a256a..f392306e35d26 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/UnionNode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/UnionNode.java @@ -11,34 +11,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.plan; +package com.facebook.presto.spi.plan; -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ListMultimap; import javax.annotation.concurrent.Immutable; import java.util.List; +import java.util.Map; @Immutable -public class UnionNode +public final class UnionNode extends SetOperationNode { @JsonCreator public UnionNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("sources") List sources, - @JsonProperty("outputToInputs") ListMultimap outputToInputs) + @JsonProperty("outputVariables") List outputVariables, + @JsonProperty("outputToInputs") Map> outputToInputs) { - super(id, sources, outputToInputs); + super(id, sources, outputVariables, outputToInputs); } @Override - public R accept(InternalPlanVisitor visitor, C context) + public R accept(PlanVisitor visitor, C context) { return visitor.visitUnion(this, context); } @@ -46,6 +45,6 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new UnionNode(getId(), newChildren, getVariableMapping()); + return new UnionNode(getId(), newChildren, getOutputVariables(), getVariableMapping()); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/predicate/Domain.java b/presto-spi/src/main/java/com/facebook/presto/spi/predicate/Domain.java index c595e72ef38f6..6582580a53e06 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/predicate/Domain.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/predicate/Domain.java @@ -82,7 +82,7 @@ public static Domain singleValue(Type type, Object value) return new Domain(ValueSet.of(type, value), false); } - public static Domain multipleValues(Type type, List values) + public static Domain multipleValues(Type type, List values) { if (values.isEmpty()) { throw new IllegalArgumentException("values cannot be empty"); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/predicate/Primitives.java b/presto-spi/src/main/java/com/facebook/presto/spi/predicate/Primitives.java index 074b19a1d2d4a..3b3efc63d985f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/predicate/Primitives.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/predicate/Primitives.java @@ -30,7 +30,7 @@ * @author Kevin Bourrillion * @since 1.0 */ -final class Primitives +public final class Primitives { private Primitives() {} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/predicate/SortedRangeSet.java b/presto-spi/src/main/java/com/facebook/presto/spi/predicate/SortedRangeSet.java index d158841a37024..1d67afbc10882 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/predicate/SortedRangeSet.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/predicate/SortedRangeSet.java @@ -32,6 +32,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static java.util.Objects.requireNonNull; /** @@ -419,6 +420,26 @@ SortedRangeSet build() result.put(current.getLow(), current); } + // TODO find a more generic way to do this + if (type == BOOLEAN) { + boolean trueAllowed = false; + boolean falseAllowed = false; + for (Map.Entry entry : result.entrySet()) { + if (entry.getValue().includes(Marker.exactly(BOOLEAN, true))) { + trueAllowed = true; + } + if (entry.getValue().includes(Marker.exactly(BOOLEAN, false))) { + falseAllowed = true; + } + } + + if (trueAllowed && falseAllowed) { + result = new TreeMap<>(); + result.put(Range.all(BOOLEAN).getLow(), Range.all(BOOLEAN)); + return new SortedRangeSet(BOOLEAN, result); + } + } + return new SortedRangeSet(type, result); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/predicate/TupleDomain.java b/presto-spi/src/main/java/com/facebook/presto/spi/predicate/TupleDomain.java index af1565c0c40d9..1bdf83b53e571 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/predicate/TupleDomain.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/predicate/TupleDomain.java @@ -34,6 +34,7 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collector; +import java.util.stream.Collectors; import static java.util.Collections.unmodifiableMap; import static java.util.Objects.requireNonNull; @@ -119,6 +120,46 @@ public static Optional> extractFixedValuesToConst .collect(toLinkedMap(Map.Entry::getKey, entry -> new ConstantExpression(entry.getValue().getNullableSingleValue(), entry.getValue().getType())))); } + /** + * Extract all column constraints that require a set of one (or more) values in their respective Domains. All the + * ranges (see {@link SortedRangeSet}) must have fixed values. + * Returns an empty Optional if the Domain is none. + */ + public static Optional>> extractFixedValueSets(TupleDomain tupleDomain) + { + if (!tupleDomain.getDomains().isPresent()) { + return Optional.empty(); + } + + Map> fixedValues = new HashMap<>(); + for (Map.Entry entry : tupleDomain.getDomains().get().entrySet()) { + Optional> values = extractFixedValueSet(entry.getValue()); + if (values.isPresent()) { + fixedValues.put(entry.getKey(), values.get()); + } + } + + return Optional.of(fixedValues); + } + + private static Optional> extractFixedValueSet(Domain domain) + { + ValueSet values = domain.getValues(); + if (!values.getType().isOrderable()) { + return Optional.empty(); + } + + List ranges = values.getRanges().getOrderedRanges(); + boolean allSingleValue = ranges.stream().allMatch(Range::isSingleValue); + if (!allSingleValue) { + return Optional.empty(); + } + + return Optional.of(ranges.stream() + .map(range -> new NullableValue(domain.getType(), range.getSingleValue())) + .collect(Collectors.toSet())); + } + /** * Convert a map of columns to values into the TupleDomain which requires * those columns to be fixed to those values. Null is allowed as a fixed value. diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/ConstantExpression.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/ConstantExpression.java index f808e0afdd9f9..96e835c552706 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/relation/ConstantExpression.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/ConstantExpression.java @@ -14,6 +14,7 @@ package com.facebook.presto.spi.relation; import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.predicate.Primitives; import com.facebook.presto.spi.predicate.Utils; import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.annotation.JsonCreator; @@ -35,7 +36,9 @@ public final class ConstantExpression public ConstantExpression(Object value, Type type) { requireNonNull(type, "type is null"); - + if (value != null && !Primitives.wrap(type.getJavaType()).isInstance(value)) { + throw new IllegalArgumentException(String.format("Object '%s' does not match type %s", value, type.getJavaType())); + } this.value = value; this.type = type; } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/DomainTranslator.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/DomainTranslator.java index af7e2491db778..bcd31b64fcfaa 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/relation/DomainTranslator.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/DomainTranslator.java @@ -14,6 +14,7 @@ package com.facebook.presto.spi.relation; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.spi.predicate.TupleDomain; import java.util.Optional; @@ -24,10 +25,16 @@ public interface DomainTranslator { interface ColumnExtractor { - Optional extract(RowExpression expression); + /** + * Given an expression and values domain, determine whether the expression qualifies as a + * "column" and return its desired representation. + * + * Return Optional.empty() if expression doesn't qualify. + */ + Optional extract(RowExpression expression, Domain domain); } - RowExpression toPredicate(TupleDomain tupleDomain); + RowExpression toPredicate(TupleDomain tupleDomain); /** * Convert a RowExpression predicate into an ExtractionResult consisting of: @@ -59,7 +66,7 @@ public RowExpression getRemainingExpression() } } - ColumnExtractor BASIC_COLUMN_EXTRACTOR = expression -> { + ColumnExtractor BASIC_COLUMN_EXTRACTOR = (expression, domain) -> { if (expression instanceof VariableReferenceExpression) { return Optional.of((VariableReferenceExpression) expression); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/ExpressionOptimizer.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/ExpressionOptimizer.java index c00453b4e63b1..70f72a555058c 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/relation/ExpressionOptimizer.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/ExpressionOptimizer.java @@ -15,6 +15,8 @@ import com.facebook.presto.spi.ConnectorSession; +import java.util.function.Function; + public interface ExpressionOptimizer { /** @@ -22,15 +24,21 @@ public interface ExpressionOptimizer */ RowExpression optimize(RowExpression rowExpression, Level level, ConnectorSession session); + Object optimize(RowExpression expression, Level level, ConnectorSession session, Function variableResolver); + enum Level { /** - * SERIALIZABLE guarantees the optimized RowExpression can be serialized and deserialized + * SERIALIZABLE guarantees the optimized RowExpression can be serialized and deserialized. */ SERIALIZABLE, /** - * MOST_OPTIMIZED removes all redundancy in a RowExpression but can end up with non-serializable objects (e.g., Regex). + * OPTIMIZED removes all redundancy in a RowExpression but can end up with non-serializable objects (e.g., Regex). + */ + OPTIMIZED, + /** + * EVALUATE attempt to evaluate the RowExpression into a constant, even though it can be non-deterministic. */ - MOST_OPTIMIZED + EVALUATED } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/LogicalRowExpressions.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/LogicalRowExpressions.java deleted file mode 100644 index e76e9820ed475..0000000000000 --- a/presto-spi/src/main/java/com/facebook/presto/spi/relation/LogicalRowExpressions.java +++ /dev/null @@ -1,514 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.spi.relation; - -import com.facebook.presto.spi.function.StandardFunctionResolution; -import com.facebook.presto.spi.relation.SpecialFormExpression.Form; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Queue; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND; -import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.OR; -import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; -import static java.util.Collections.singletonList; -import static java.util.Collections.unmodifiableList; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toList; - -public final class LogicalRowExpressions -{ - public static final ConstantExpression TRUE_CONSTANT = new ConstantExpression(true, BOOLEAN); - public static final ConstantExpression FALSE_CONSTANT = new ConstantExpression(false, BOOLEAN); - - private final DeterminismEvaluator determinismEvaluator; - private final StandardFunctionResolution functionResolution; - - public LogicalRowExpressions(DeterminismEvaluator determinismEvaluator, StandardFunctionResolution functionResolution) - { - this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); - this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); - } - - public static List extractConjuncts(RowExpression expression) - { - return extractPredicates(AND, expression); - } - - public static List extractDisjuncts(RowExpression expression) - { - return extractPredicates(OR, expression); - } - - public static List extractPredicates(RowExpression expression) - { - if (expression instanceof SpecialFormExpression) { - Form form = ((SpecialFormExpression) expression).getForm(); - if (form == AND || form == OR) { - return extractPredicates(form, expression); - } - } - return singletonList(expression); - } - - public static List extractPredicates(Form form, RowExpression expression) - { - if (expression instanceof SpecialFormExpression && ((SpecialFormExpression) expression).getForm() == form) { - SpecialFormExpression specialFormExpression = (SpecialFormExpression) expression; - if (specialFormExpression.getArguments().size() != 2) { - throw new IllegalStateException("logical binary expression requires exactly 2 operands"); - } - - List predicates = new ArrayList<>(); - predicates.addAll(extractPredicates(form, specialFormExpression.getArguments().get(0))); - predicates.addAll(extractPredicates(form, specialFormExpression.getArguments().get(1))); - return unmodifiableList(predicates); - } - - return singletonList(expression); - } - - public static RowExpression and(RowExpression... expressions) - { - return and(Arrays.asList(expressions)); - } - - public static RowExpression and(Collection expressions) - { - return binaryExpression(AND, expressions); - } - - public static RowExpression or(RowExpression... expressions) - { - return or(Arrays.asList(expressions)); - } - - public static RowExpression or(Collection expressions) - { - return binaryExpression(OR, expressions); - } - - public static RowExpression binaryExpression(Form form, Collection expressions) - { - requireNonNull(form, "operator is null"); - requireNonNull(expressions, "expressions is null"); - - if (expressions.isEmpty()) { - switch (form) { - case AND: - return TRUE_CONSTANT; - case OR: - return FALSE_CONSTANT; - default: - throw new IllegalArgumentException("Unsupported binary expression operator"); - } - } - - // Build balanced tree for efficient recursive processing that - // preserves the evaluation order of the input expressions. - // - // The tree is built bottom up by combining pairs of elements into - // binary AND expressions. - // - // Example: - // - // Initial state: - // a b c d e - // - // First iteration: - // - // /\ /\ e - // a b c d - // - // Second iteration: - // - // / \ e - // /\ /\ - // a b c d - // - // - // Last iteration: - // - // / \ - // / \ e - // /\ /\ - // a b c d - - Queue queue = new ArrayDeque<>(expressions); - while (queue.size() > 1) { - Queue buffer = new ArrayDeque<>(); - - // combine pairs of elements - while (queue.size() >= 2) { - List arguments = Arrays.asList(queue.remove(), queue.remove()); - buffer.add(new SpecialFormExpression(form, BOOLEAN, arguments)); - } - - // if there's and odd number of elements, just append the last one - if (!queue.isEmpty()) { - buffer.add(queue.remove()); - } - - // continue processing the pairs that were just built - queue = buffer; - } - - return queue.remove(); - } - - public RowExpression combinePredicates(Form form, RowExpression... expressions) - { - return combinePredicates(form, Arrays.asList(expressions)); - } - - public RowExpression combinePredicates(Form form, Collection expressions) - { - if (form == AND) { - return combineConjuncts(expressions); - } - return combineDisjuncts(expressions); - } - - public RowExpression combineConjuncts(RowExpression... expressions) - { - return combineConjuncts(Arrays.asList(expressions)); - } - - public RowExpression combineConjuncts(Collection expressions) - { - requireNonNull(expressions, "expressions is null"); - - List conjuncts = expressions.stream() - .flatMap(e -> extractConjuncts(e).stream()) - .filter(e -> !e.equals(TRUE_CONSTANT)) - .collect(toList()); - - conjuncts = removeDuplicates(conjuncts); - - if (conjuncts.contains(FALSE_CONSTANT)) { - return FALSE_CONSTANT; - } - - return and(conjuncts); - } - - public RowExpression combineDisjuncts(RowExpression... expressions) - { - return combineDisjuncts(Arrays.asList(expressions)); - } - - public RowExpression combineDisjuncts(Collection expressions) - { - return combineDisjunctsWithDefault(expressions, FALSE_CONSTANT); - } - - public RowExpression combineDisjunctsWithDefault(Collection expressions, RowExpression emptyDefault) - { - requireNonNull(expressions, "expressions is null"); - - List disjuncts = expressions.stream() - .flatMap(e -> extractDisjuncts(e).stream()) - .filter(e -> !e.equals(FALSE_CONSTANT)) - .collect(toList()); - - disjuncts = removeDuplicates(disjuncts); - - if (disjuncts.contains(TRUE_CONSTANT)) { - return TRUE_CONSTANT; - } - - return disjuncts.isEmpty() ? emptyDefault : or(disjuncts); - } - - /** - * Given a logical expression, the goal is to push negation to the leaf nodes. - * This only applies to propositional logic. this utility cannot be applied to high-order logic. - * Examples of non-applicable cases could be f(a AND b) > 5 - * - * An applicable example: - * - * NOT - * | - * ___OR_ AND - * / \ / \ - * NOT OR ==> AND AND - * | / \ / \ / \ - * AND c NOT a b NOT d - * / \ | | - * a b d c - */ - public RowExpression pushNegationToLeaves(RowExpression expression) - { - return expression.accept(new PushNegationVisitor(), null); - } - - /** - * Given a logical expression, the goal is to convert to conjuctive normal form (CNF). - * This requires making a call to `pushNegationToLeaves`. There is no guarantee as to - * the balance of the resulting expression tree. - * - * This only applies to propositional logic. this utility cannot be applied to high-order logic. - * Examples of non-applicable cases could be f(a AND b) > 5 - * - * NOTE: This may exponentially increase the number of RowExpressions in the expression. - * - * An applicable example: - * - * NOT - * | - * ___OR_ AND - * / \ / \ - * NOT OR ==> OR AND - * | / \ / \ / \ - * OR c NOT a b NOT d - * / \ | | - * a b d c - */ - public RowExpression convertToConjunctiveNormalForm(RowExpression expression) - { - return convertToNormalForm(expression, AND); - } - - /** - * Given a logical expression, the goal is to convert to disjunctive normal form (DNF). - * The same limitations, format, and risks apply as for converting to conjunctive normal form (CNF). - * - * An applicable example: - * - * NOT OR - * | / \ - * ___OR_ AND AND - * / \ / \ / \ - * NOT OR ==> a AND b AND - * | / \ / \ / \ - * OR c NOT NOT d NOT d - * / \ | | | - * a b d c c - */ - public RowExpression convertToDisjunctiveNormalForm(RowExpression expression) - { - return convertToNormalForm(expression, OR); - } - - public RowExpression convertToNormalForm(RowExpression expression, Form clauseJoiner) - { - return pushNegationToLeaves(expression).accept(new ConvertNormalFormVisitor(), clauseJoiner); - } - - public RowExpression filterDeterministicConjuncts(RowExpression expression) - { - return filterConjuncts(expression, this.determinismEvaluator::isDeterministic); - } - - public RowExpression filterNonDeterministicConjuncts(RowExpression expression) - { - return filterConjuncts(expression, predicate -> !this.determinismEvaluator.isDeterministic(predicate)); - } - - public RowExpression filterConjuncts(RowExpression expression, Predicate predicate) - { - List conjuncts = extractConjuncts(expression).stream() - .filter(predicate) - .collect(Collectors.toList()); - - return combineConjuncts(conjuncts); - } - - /** - * Removes duplicate deterministic expressions. Preserves the relative order - * of the expressions in the list. - */ - private List removeDuplicates(List expressions) - { - Set seen = new HashSet<>(); - - List result = new ArrayList<>(); - for (RowExpression expression : expressions) { - if (!determinismEvaluator.isDeterministic(expression)) { - result.add(expression); - } - else if (!seen.contains(expression)) { - result.add(expression); - seen.add(expression); - } - } - - return unmodifiableList(result); - } - - private boolean isConjunctionOrDisjunction(RowExpression expression) - { - if (expression instanceof SpecialFormExpression) { - Form form = ((SpecialFormExpression) expression).getForm(); - return form == AND || form == OR; - } - return false; - } - - private final class PushNegationVisitor - implements RowExpressionVisitor - { - @Override - public RowExpression visitCall(CallExpression call, Void context) - { - if (!isNegationExpression(call)) { - return call; - } - - RowExpression argument = call.getArguments().get(0); - - // eliminate two consecutive negations - if (isNegationExpression(argument)) { - return ((CallExpression) argument).getArguments().get(0).accept(new PushNegationVisitor(), null); - } - - if (!isConjunctionOrDisjunction(argument)) { - return call; - } - - // push negation through conjunction or disjunction - SpecialFormExpression specialForm = ((SpecialFormExpression) argument); - RowExpression left = specialForm.getArguments().get(0); - RowExpression right = specialForm.getArguments().get(1); - if (specialForm.getForm() == AND) { - // !(a AND b) ==> !a OR !b - return or(notCallExpression(left).accept(new PushNegationVisitor(), null), notCallExpression(right).accept(new PushNegationVisitor(), null)); - } - // !(a OR b) ==> !a AND !b - return and(notCallExpression(left).accept(new PushNegationVisitor(), null), notCallExpression(right).accept(new PushNegationVisitor(), null)); - } - - @Override - public RowExpression visitSpecialForm(SpecialFormExpression specialForm, Void context) - { - if (!isConjunctionOrDisjunction(specialForm)) { - return specialForm; - } - - RowExpression left = specialForm.getArguments().get(0); - RowExpression right = specialForm.getArguments().get(1); - - if (specialForm.getForm() == AND) { - return and(left.accept(new PushNegationVisitor(), null), right.accept(new PushNegationVisitor(), null)); - } - return or(left.accept(new PushNegationVisitor(), null), right.accept(new PushNegationVisitor(), null)); - } - - @Override - public RowExpression visitInputReference(InputReferenceExpression reference, Void context) - { - return reference; - } - - @Override - public RowExpression visitConstant(ConstantExpression literal, Void context) - { - return literal; - } - - @Override - public RowExpression visitLambda(LambdaDefinitionExpression lambda, Void context) - { - return lambda; - } - - @Override - public RowExpression visitVariableReference(VariableReferenceExpression reference, Void context) - { - return reference; - } - - private boolean isNegationExpression(RowExpression expression) - { - return expression instanceof CallExpression && ((CallExpression) expression).getFunctionHandle().equals(functionResolution.notFunction()); - } - - private RowExpression notCallExpression(RowExpression argument) - { - return new CallExpression("not", functionResolution.notFunction(), BOOLEAN, singletonList(argument)); - } - } - - private class ConvertNormalFormVisitor - implements RowExpressionVisitor - { - @Override - public RowExpression visitSpecialForm(SpecialFormExpression specialForm, Form clauseJoiner) - { - if (!isConjunctionOrDisjunction(specialForm)) { - return specialForm; - } - - // normalize arguments - RowExpression left = specialForm.getArguments().get(0).accept(new ConvertNormalFormVisitor(), clauseJoiner); - RowExpression right = specialForm.getArguments().get(1).accept(new ConvertNormalFormVisitor(), clauseJoiner); - - // check if already in correct form - if (specialForm.getForm() == clauseJoiner) { - return clauseJoiner == AND ? and(left, right) : or(left, right); - } - - // else, we expand and rewrite based on distributive property of Boolean algebra, for example - // (l1 OR l2) AND (r1 OR r2) <=> (l1 AND r1) OR (l1 AND r2) OR (l2 AND r1) OR (l2 AND r2) - Form termJoiner = clauseJoiner == AND ? OR : AND; - List leftClauses = extractPredicates(clauseJoiner, left); - List rightClauses = extractPredicates(clauseJoiner, right); - List permutationClauses = new ArrayList<>(); - for (RowExpression leftClause : leftClauses) { - for (RowExpression rightClause : rightClauses) { - permutationClauses.add(termJoiner == AND ? and(leftClause, rightClause) : or(leftClause, rightClause)); - } - } - return combinePredicates(clauseJoiner, unmodifiableList(permutationClauses)); - } - - @Override - public RowExpression visitCall(CallExpression call, Form clauseJoiner) - { - return call; - } - - @Override - public RowExpression visitInputReference(InputReferenceExpression reference, SpecialFormExpression.Form clauseJoiner) - { - return reference; - } - - @Override - public RowExpression visitConstant(ConstantExpression literal, SpecialFormExpression.Form clauseJoiner) - { - return literal; - } - - @Override - public RowExpression visitLambda(LambdaDefinitionExpression lambda, SpecialFormExpression.Form clauseJoiner) - { - return lambda; - } - - @Override - public RowExpression visitVariableReference(VariableReferenceExpression reference, SpecialFormExpression.Form clauseJoiner) - { - return reference; - } - } -} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/PredicateCompiler.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/PredicateCompiler.java index 49a323b1a2363..bc02c08568188 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/relation/PredicateCompiler.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/PredicateCompiler.java @@ -14,6 +14,7 @@ package com.facebook.presto.spi.relation; import com.facebook.presto.spi.api.Experimental; +import com.facebook.presto.spi.function.SqlFunctionProperties; import java.util.function.Supplier; @@ -27,5 +28,5 @@ public interface PredicateCompiler /** * Predicate expression may not contain any variable references, only input references. */ - Supplier compilePredicate(RowExpression predicate); + Supplier compilePredicate(SqlFunctionProperties sqlFunctionProperties, RowExpression predicate); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java index 836fbf2859eab..12710f71b0f76 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.spi.relation; +import com.facebook.presto.spi.ConnectorSession; + /** * A set of services/utilities that are helpful for connectors to operate on row expressions */ @@ -25,4 +27,9 @@ public interface RowExpressionService PredicateCompiler getPredicateCompiler(); DeterminismEvaluator getDeterminismEvaluator(); + + /** + * @return user-friendly representation of the expression similar to original SQL + */ + String formatRowExpression(ConnectorSession session, RowExpression expression); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/security/Identity.java b/presto-spi/src/main/java/com/facebook/presto/spi/security/Identity.java index 1cdc3b5006c59..3b72895ce0da5 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/security/Identity.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/security/Identity.java @@ -32,12 +32,7 @@ public class Identity public Identity(String user, Optional principal) { - this(user, principal, emptyMap()); - } - - public Identity(String user, Optional principal, Map roles) - { - this(user, principal, roles, emptyMap()); + this(user, principal, emptyMap(), emptyMap()); } public Identity(String user, Optional principal, Map roles, Map extraCredentials) diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/ArrayType.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/ArrayType.java index 68b3874246baf..1e8e434bbe5b8 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/ArrayType.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/ArrayType.java @@ -63,8 +63,8 @@ public boolean isOrderable() @Override public boolean equalTo(Block leftBlock, int leftPosition, Block rightBlock, int rightPosition) { - Block leftArray = leftBlock.getObject(leftPosition, Block.class); - Block rightArray = rightBlock.getObject(rightPosition, Block.class); + Block leftArray = leftBlock.getBlock(leftPosition); + Block rightArray = rightBlock.getBlock(rightPosition); if (leftArray.getPositionCount() != rightArray.getPositionCount()) { return false; @@ -99,8 +99,8 @@ public int compareTo(Block leftBlock, int leftPosition, Block rightBlock, int ri throw new UnsupportedOperationException(getTypeSignature() + " type is not orderable"); } - Block leftArray = leftBlock.getObject(leftPosition, Block.class); - Block rightArray = rightBlock.getObject(rightPosition, Block.class); + Block leftArray = leftBlock.getBlock(leftPosition); + Block rightArray = rightBlock.getBlock(rightPosition); int len = Math.min(leftArray.getPositionCount(), rightArray.getPositionCount()); int index = 0; @@ -132,7 +132,7 @@ public Object getObjectValue(ConnectorSession session, Block block, int position return ((AbstractArrayBlock) block).apply((valuesBlock, start, length) -> arrayBlockToObjectValues(session, valuesBlock, start, length), position); } else { - Block arrayBlock = block.getObject(position, Block.class); + Block arrayBlock = block.getBlock(position); return arrayBlockToObjectValues(session, arrayBlock, 0, arrayBlock.getPositionCount()); } } @@ -180,7 +180,7 @@ public void writeSlice(BlockBuilder blockBuilder, Slice value, int offset, int l @Override public Block getObject(Block block, int position) { - return block.getObject(position, Block.class); + return block.getBlock(position); } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/MapType.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/MapType.java index 6f8b6b9bc5a91..ee5d69b0aa272 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/MapType.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/MapType.java @@ -126,8 +126,8 @@ public long hash(Block block, int position) @Override public boolean equalTo(Block leftBlock, int leftPosition, Block rightBlock, int rightPosition) { - Block leftMapBlock = leftBlock.getObject(leftPosition, Block.class); - Block rightMapBlock = rightBlock.getObject(rightPosition, Block.class); + Block leftMapBlock = leftBlock.getBlock(leftPosition); + Block rightMapBlock = rightBlock.getBlock(rightPosition); if (leftMapBlock.getPositionCount() != rightMapBlock.getPositionCount()) { return false; @@ -202,7 +202,7 @@ public Object getObjectValue(ConnectorSession session, Block block, int position return null; } - Block singleMapBlock = block.getObject(position, Block.class); + Block singleMapBlock = block.getBlock(position); if (!(singleMapBlock instanceof SingleMapBlock)) { throw new UnsupportedOperationException("Map is encoded with legacy block representation"); } @@ -228,7 +228,7 @@ public void appendTo(Block block, int position, BlockBuilder blockBuilder) @Override public Block getObject(Block block, int position) { - return block.getObject(position, Block.class); + return block.getBlock(position); } @Override @@ -258,9 +258,10 @@ public String getDisplayName() return "map(" + keyType.getDisplayName() + ", " + valueType.getDisplayName() + ")"; } - public Block createBlockFromKeyValue(Optional mapIsNull, int[] offsets, Block keyBlock, Block valueBlock) + public Block createBlockFromKeyValue(int positionCount, Optional mapIsNull, int[] offsets, Block keyBlock, Block valueBlock) { return MapBlock.fromKeyValueBlock( + positionCount, mapIsNull, offsets, keyBlock, diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/RowType.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/RowType.java index f492f8db9a9dc..763d5ae6c7741 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/RowType.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/RowType.java @@ -166,7 +166,7 @@ public void appendTo(Block block, int position, BlockBuilder blockBuilder) @Override public Block getObject(Block block, int position) { - return block.getObject(position, Block.class); + return block.getBlock(position); } @Override @@ -229,8 +229,8 @@ public boolean isOrderable() @Override public boolean equalTo(Block leftBlock, int leftPosition, Block rightBlock, int rightPosition) { - Block leftRow = leftBlock.getObject(leftPosition, Block.class); - Block rightRow = rightBlock.getObject(rightPosition, Block.class); + Block leftRow = leftBlock.getBlock(leftPosition); + Block rightRow = rightBlock.getBlock(rightPosition); for (int i = 0; i < leftRow.getPositionCount(); i++) { checkElementNotNull(leftRow.isNull(i)); @@ -247,8 +247,8 @@ public boolean equalTo(Block leftBlock, int leftPosition, Block rightBlock, int @Override public int compareTo(Block leftBlock, int leftPosition, Block rightBlock, int rightPosition) { - Block leftRow = leftBlock.getObject(leftPosition, Block.class); - Block rightRow = rightBlock.getObject(rightPosition, Block.class); + Block leftRow = leftBlock.getBlock(leftPosition); + Block rightRow = rightBlock.getBlock(rightPosition); for (int i = 0; i < leftRow.getPositionCount(); i++) { checkElementNotNull(leftRow.isNull(i)); @@ -269,7 +269,7 @@ public int compareTo(Block leftBlock, int leftPosition, Block rightBlock, int ri @Override public long hash(Block block, int position) { - Block arrayBlock = block.getObject(position, Block.class); + Block arrayBlock = block.getBlock(position); long result = 1; for (int i = 0; i < arrayBlock.getPositionCount(); i++) { Type elementType = fields.get(i).getType(); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/StandardTypes.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/StandardTypes.java index bcdd37624c0b8..253d5bb2612af 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/StandardTypes.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/StandardTypes.java @@ -41,6 +41,7 @@ public final class StandardTypes public static final String MAP = "map"; public static final String JSON = "json"; public static final String IPADDRESS = "ipaddress"; + public static final String IPPREFIX = "ipprefix"; public static final String GEOMETRY = "Geometry"; public static final String BING_TILE = "BingTile"; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/TestStandardErrorCode.java b/presto-spi/src/test/java/com/facebook/presto/spi/TestStandardErrorCode.java index 3ad8398d3ce4e..f0403b080398a 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/TestStandardErrorCode.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/TestStandardErrorCode.java @@ -19,10 +19,10 @@ import java.util.Iterator; import java.util.Set; +import static com.facebook.airlift.testing.Assertions.assertGreaterThan; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INSUFFICIENT_RESOURCES; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static io.airlift.testing.Assertions.assertGreaterThan; -import static io.airlift.testing.Assertions.assertLessThan; import static java.util.Arrays.asList; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/TestSubfieldTokenizer.java b/presto-spi/src/test/java/com/facebook/presto/spi/TestSubfieldTokenizer.java index 7c3096320204d..aa0b327fc8d7e 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/TestSubfieldTokenizer.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/TestSubfieldTokenizer.java @@ -33,7 +33,6 @@ public void test() { List elements = ImmutableList.of( new NestedField("b"), - new NestedField("$bucket"), new Subfield.LongSubscript(2), new Subfield.StringSubscript("z"), Subfield.allSubscripts(), @@ -70,6 +69,14 @@ private static void assertPath(Subfield path) assertEquals(new Subfield(((NestedField) tokenizer.next()).getName(), Streams.stream(tokenizer).collect(toImmutableList())), path); } + @Test + public void testColumnNames() + { + assertPath(new Subfield("$bucket", ImmutableList.of())); + assertPath(new Subfield("apollo-11", ImmutableList.of())); + assertPath(new Subfield("a/b/c:12", ImmutableList.of())); + } + @Test public void testInvalidPaths() { diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/block/CountingArrayAllocator.java b/presto-spi/src/test/java/com/facebook/presto/spi/block/CountingArrayAllocator.java index e42d1584c52cf..ba83c69071c9b 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/block/CountingArrayAllocator.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/block/CountingArrayAllocator.java @@ -17,7 +17,7 @@ * Test class which always allocates a new array but keeps track of the number * of borrowed arrays. */ -class CountingArrayAllocator +public class CountingArrayAllocator implements ArrayAllocator { private int borrowedArrays; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingSession.java b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingSession.java index 140e7c84cb3ff..49e1ec57529c5 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingSession.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/block/TestingSession.java @@ -15,6 +15,7 @@ import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.SqlFunctionProperties; import com.facebook.presto.spi.security.ConnectorIdentity; import com.facebook.presto.spi.type.TimeZoneKey; @@ -53,6 +54,12 @@ public TimeZoneKey getTimeZoneKey() return UTC_KEY; } + @Override + public Optional getClientInfo() + { + return Optional.of("TestClientInfo"); + } + @Override public Locale getLocale() { @@ -77,6 +84,12 @@ public boolean isLegacyTimestamp() return true; } + @Override + public SqlFunctionProperties getSqlFunctionProperties() + { + return SqlFunctionProperties.builder().setTimeZoneKey(UTC_KEY).build(); + } + @Override public T getProperty(String name, Class type) { diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestAllOrNoneValueSet.java b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestAllOrNoneValueSet.java index 4cb6bcd05522d..77ccf3915d0a3 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestAllOrNoneValueSet.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestAllOrNoneValueSet.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.spi.predicate; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.type.TestingTypeDeserializer; import com.facebook.presto.spi.type.TestingTypeManager; import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; -import io.airlift.json.ObjectMapperProvider; import io.airlift.slice.Slices; import org.testng.annotations.Test; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestDomain.java b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestDomain.java index 4ce9d00f25f85..418f8bd3e7cdb 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestDomain.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestDomain.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.predicate; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.TestingBlockEncodingSerde; import com.facebook.presto.spi.block.TestingBlockJsonSerde; @@ -22,7 +23,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableList; -import io.airlift.json.ObjectMapperProvider; import io.airlift.slice.Slices; import org.testng.annotations.Test; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestEquatableValueSet.java b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestEquatableValueSet.java index 428cab87d5906..836adeae50d91 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestEquatableValueSet.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestEquatableValueSet.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.predicate; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.TestingBlockEncodingSerde; import com.facebook.presto.spi.block.TestingBlockJsonSerde; @@ -23,7 +24,6 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import io.airlift.json.ObjectMapperProvider; import org.testng.annotations.Test; import java.util.Iterator; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestMarker.java b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestMarker.java index 7095e25b8ac1d..ae824642ba21d 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestMarker.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestMarker.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.predicate; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.TestingBlockEncodingSerde; import com.facebook.presto.spi.block.TestingBlockJsonSerde; @@ -24,7 +25,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; -import io.airlift.json.ObjectMapperProvider; import org.testng.annotations.Test; import java.util.Map; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestRange.java b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestRange.java index 4e7a7b233b2c6..f4ce45a29f1f2 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestRange.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestRange.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.predicate; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.TestingBlockEncodingSerde; import com.facebook.presto.spi.block.TestingBlockJsonSerde; @@ -21,7 +22,6 @@ import com.facebook.presto.spi.type.Type; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; -import io.airlift.json.ObjectMapperProvider; import org.testng.annotations.Test; import static com.facebook.presto.spi.type.BigintType.BIGINT; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestSortedRangeSet.java b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestSortedRangeSet.java index 958b40f141e1c..a7fc8725308cd 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestSortedRangeSet.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestSortedRangeSet.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.predicate; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.TestingBlockEncodingSerde; import com.facebook.presto.spi.block.TestingBlockJsonSerde; @@ -23,7 +24,6 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import io.airlift.json.ObjectMapperProvider; import org.testng.annotations.Test; import static com.facebook.presto.spi.type.BigintType.BIGINT; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestTupleDomain.java b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestTupleDomain.java index 069156d3868d7..08d8ac1bb2e25 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestTupleDomain.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/predicate/TestTupleDomain.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.spi.predicate; +import com.facebook.airlift.json.ObjectMapperProvider; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.TestingColumnHandle; import com.facebook.presto.spi.block.Block; @@ -28,7 +29,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableMap; -import io.airlift.json.ObjectMapperProvider; import org.testng.annotations.Test; import java.io.IOException; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/resourceGroups/TestResourceGroupId.java b/presto-spi/src/test/java/com/facebook/presto/spi/resourceGroups/TestResourceGroupId.java index 5f007876fd033..d7a39f8ed169d 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/resourceGroups/TestResourceGroupId.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/resourceGroups/TestResourceGroupId.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.spi.resourceGroups; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/security/TestSelectedRole.java b/presto-spi/src/test/java/com/facebook/presto/spi/security/TestSelectedRole.java index 245e8131e6e1e..c463f110d3e69 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/security/TestSelectedRole.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/security/TestSelectedRole.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.spi.security; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import org.testng.annotations.Test; import java.util.Optional; -import static io.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static org.testng.Assert.assertEquals; public class TestSelectedRole diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/type/TestParameterKind.java b/presto-spi/src/test/java/com/facebook/presto/spi/type/TestParameterKind.java index 5c37e2b8cd549..ae3cb079fc694 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/type/TestParameterKind.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/type/TestParameterKind.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.spi.type; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.ObjectMapperProvider; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; diff --git a/presto-sql-function/pom.xml b/presto-sql-function/pom.xml new file mode 100644 index 0000000000000..ec06da64e1b49 --- /dev/null +++ b/presto-sql-function/pom.xml @@ -0,0 +1,59 @@ + + + + presto-root + com.facebook.presto + 0.231-SNAPSHOT + + 4.0.0 + + presto-sql-function + presto-sql-function + + + ${project.parent.basedir} + + + + + com.facebook.presto + presto-spi + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + com.google.code.findbugs + jsr305 + + + + com.google.guava + guava + + + + com.facebook.airlift + configuration + + + + io.airlift + units + + + + + org.testng + testng + test + + + diff --git a/presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/AbstractSqlInvokedFunctionNamespaceManager.java b/presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/AbstractSqlInvokedFunctionNamespaceManager.java new file mode 100644 index 0000000000000..b28839bb4ada2 --- /dev/null +++ b/presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/AbstractSqlInvokedFunctionNamespaceManager.java @@ -0,0 +1,206 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sqlfunction; + +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionNamespaceManager; +import com.facebook.presto.spi.function.FunctionNamespaceTransactionHandle; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.function.ScalarFunctionImplementation; +import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionHandle; +import com.facebook.presto.spi.function.SqlFunctionId; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.facebook.presto.spi.function.SqlInvokedScalarFunctionImplementation; +import com.facebook.presto.spi.function.SqlParameter; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + +import javax.annotation.ParametersAreNonnullByDefault; +import javax.annotation.concurrent.GuardedBy; + +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static com.facebook.presto.spi.function.FunctionImplementationType.SQL; +import static com.facebook.presto.spi.function.FunctionKind.SCALAR; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +public abstract class AbstractSqlInvokedFunctionNamespaceManager + implements FunctionNamespaceManager +{ + private final ConcurrentMap transactions = new ConcurrentHashMap<>(); + + private final LoadingCache> functions; + private final LoadingCache metadataByHandle; + private final LoadingCache implementationByHandle; + + public AbstractSqlInvokedFunctionNamespaceManager(SqlInvokedFunctionNamespaceManagerConfig config) + { + this.functions = CacheBuilder.newBuilder() + .expireAfterWrite(config.getFunctionCacheExpiration().toMillis(), MILLISECONDS) + .build(new CacheLoader>() + { + @Override + @ParametersAreNonnullByDefault + public Collection load(QualifiedFunctionName functionName) + { + Collection functions = fetchFunctionsDirect(functionName); + for (SqlInvokedFunction function : functions) { + metadataByHandle.put(function.getRequiredFunctionHandle(), sqlInvokedFunctionToMetadata(function)); + } + return functions; + } + }); + this.metadataByHandle = CacheBuilder.newBuilder() + .expireAfterWrite(config.getFunctionInstanceCacheExpiration().toMillis(), MILLISECONDS) + .build(new CacheLoader() + { + @Override + @ParametersAreNonnullByDefault + public FunctionMetadata load(SqlFunctionHandle functionHandle) + { + return fetchFunctionMetadataDirect(functionHandle); + } + }); + this.implementationByHandle = CacheBuilder.newBuilder() + .expireAfterWrite(config.getFunctionInstanceCacheExpiration().toMillis(), MILLISECONDS) + .build(new CacheLoader() + { + @Override + public ScalarFunctionImplementation load(SqlFunctionHandle functionHandle) + { + return fetchFunctionImplementationDirect(functionHandle); + } + }); + } + + protected abstract Collection fetchFunctionsDirect(QualifiedFunctionName functionName); + + protected abstract FunctionMetadata fetchFunctionMetadataDirect(SqlFunctionHandle functionHandle); + + protected abstract ScalarFunctionImplementation fetchFunctionImplementationDirect(SqlFunctionHandle functionHandle); + + @Override + public final FunctionNamespaceTransactionHandle beginTransaction() + { + UuidFunctionNamespaceTransactionHandle transactionHandle = UuidFunctionNamespaceTransactionHandle.create(); + transactions.put(transactionHandle, new FunctionCollection()); + return transactionHandle; + } + + @Override + public final void commit(FunctionNamespaceTransactionHandle transactionHandle) + { + // Transactional commit is not supported yet. + transactions.remove(transactionHandle); + } + + @Override + public final void abort(FunctionNamespaceTransactionHandle transactionHandle) + { + // Transactional rollback is not supported yet. + transactions.remove(transactionHandle); + } + + @Override + public final Collection getFunctions(Optional transactionHandle, QualifiedFunctionName functionName) + { + checkArgument(transactionHandle.isPresent(), "missing transactionHandle"); + return transactions.get(transactionHandle.get()).loadAndGetFunctionsTransactional(functionName); + } + + @Override + public final FunctionHandle getFunctionHandle(Optional transactionHandle, Signature signature) + { + checkArgument(transactionHandle.isPresent(), "missing transactionHandle"); + // This is the only assumption in this class that we're dealing with sql-invoked regular function. + SqlFunctionId functionId = new SqlFunctionId(signature.getName(), signature.getArgumentTypes()); + return transactions.get(transactionHandle.get()).getFunctionHandle(functionId); + } + + @Override + public final FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) + { + checkArgument(functionHandle instanceof SqlFunctionHandle, "Unsupported FunctionHandle type '%s'", functionHandle.getClass().getSimpleName()); + return metadataByHandle.getUnchecked((SqlFunctionHandle) functionHandle); + } + + @Override + public final ScalarFunctionImplementation getScalarFunctionImplementation(FunctionHandle functionHandle) + { + checkArgument(functionHandle instanceof SqlFunctionHandle, "Unsupported FunctionHandle type '%s'", functionHandle.getClass().getSimpleName()); + return implementationByHandle.getUnchecked((SqlFunctionHandle) functionHandle); + } + + protected void refreshFunctionsCache(QualifiedFunctionName functionName) + { + functions.refresh(functionName); + } + + protected static FunctionMetadata sqlInvokedFunctionToMetadata(SqlInvokedFunction function) + { + return new FunctionMetadata( + function.getSignature().getName(), + function.getSignature().getArgumentTypes(), + function.getParameters().stream() + .map(SqlParameter::getName) + .collect(toImmutableList()), + function.getSignature().getReturnType(), + SCALAR, + function.getFunctionImplementationType(), + function.isDeterministic(), + function.isCalledOnNullInput()); + } + + protected static ScalarFunctionImplementation sqlInvokedFunctionToImplementation(SqlInvokedFunction function) + { + checkArgument(function.getFunctionImplementationType().equals(SQL)); + return new SqlInvokedScalarFunctionImplementation(function.getBody()); + } + + private Collection fetchFunctions(QualifiedFunctionName functionName) + { + return functions.getUnchecked(functionName); + } + + private class FunctionCollection + { + @GuardedBy("this") + private final Map> functions = new ConcurrentHashMap<>(); + + @GuardedBy("this") + private final Map functionHandles = new ConcurrentHashMap<>(); + + public synchronized Collection loadAndGetFunctionsTransactional(QualifiedFunctionName functionName) + { + Collection functions = this.functions.computeIfAbsent(functionName, AbstractSqlInvokedFunctionNamespaceManager.this::fetchFunctions); + functionHandles.putAll(functions.stream().collect(toImmutableMap(SqlInvokedFunction::getFunctionId, SqlInvokedFunction::getRequiredFunctionHandle))); + return functions; + } + + public synchronized FunctionHandle getFunctionHandle(SqlFunctionId functionId) + { + return functionHandles.get(functionId); + } + } +} diff --git a/presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/SqlInvokedFunctionNamespaceManagerConfig.java b/presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/SqlInvokedFunctionNamespaceManagerConfig.java new file mode 100644 index 0000000000000..194c64ef46ec6 --- /dev/null +++ b/presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/SqlInvokedFunctionNamespaceManagerConfig.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sqlfunction; + +import com.facebook.airlift.configuration.Config; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; + +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.MINUTES; + +public class SqlInvokedFunctionNamespaceManagerConfig +{ + private Duration functionCacheExpiration = new Duration(5, MINUTES); + private Duration functionInstanceCacheExpiration = new Duration(8, HOURS); + + @MinDuration("0ns") + public Duration getFunctionCacheExpiration() + { + return functionCacheExpiration; + } + + @Config("function-cache-expiration") + public SqlInvokedFunctionNamespaceManagerConfig setFunctionCacheExpiration(Duration functionCacheExpiration) + { + this.functionCacheExpiration = functionCacheExpiration; + return this; + } + + @MinDuration("0ns") + public Duration getFunctionInstanceCacheExpiration() + { + return functionInstanceCacheExpiration; + } + + @Config("function-instance-cache-expiration") + public SqlInvokedFunctionNamespaceManagerConfig setFunctionInstanceCacheExpiration(Duration functionInstanceCacheExpiration) + { + this.functionInstanceCacheExpiration = functionInstanceCacheExpiration; + return this; + } +} diff --git a/presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/UuidFunctionNamespaceTransactionHandle.java b/presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/UuidFunctionNamespaceTransactionHandle.java new file mode 100644 index 0000000000000..609b338b45ed6 --- /dev/null +++ b/presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/UuidFunctionNamespaceTransactionHandle.java @@ -0,0 +1,71 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sqlfunction; + +import com.facebook.presto.spi.function.FunctionNamespaceTransactionHandle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +import java.util.Objects; +import java.util.UUID; + +import static java.util.Objects.requireNonNull; + +public class UuidFunctionNamespaceTransactionHandle + implements FunctionNamespaceTransactionHandle +{ + private final UUID uuid; + + private UuidFunctionNamespaceTransactionHandle(UUID uuid) + { + this.uuid = requireNonNull(uuid, "uuid is null"); + } + + public static UuidFunctionNamespaceTransactionHandle create() + { + return new UuidFunctionNamespaceTransactionHandle(UUID.randomUUID()); + } + + @JsonCreator + public static UuidFunctionNamespaceTransactionHandle valueOf(String value) + { + return new UuidFunctionNamespaceTransactionHandle(UUID.fromString(value)); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final UuidFunctionNamespaceTransactionHandle o = (UuidFunctionNamespaceTransactionHandle) obj; + return Objects.equals(uuid, o.uuid); + } + + @Override + public int hashCode() + { + return Objects.hash(uuid); + } + + @Override + @JsonValue + public String toString() + { + return uuid.toString(); + } +} diff --git a/presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/testing/SqlInvokedFunctionTestUtils.java b/presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/testing/SqlInvokedFunctionTestUtils.java new file mode 100644 index 0000000000000..36e3a565a6d0a --- /dev/null +++ b/presto-sql-function/src/main/java/com/facebook/presto/sqlfunction/testing/SqlInvokedFunctionTestUtils.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sqlfunction.testing; + +import com.facebook.presto.spi.CatalogSchemaName; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.function.RoutineCharacteristics; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.facebook.presto.spi.function.SqlParameter; +import com.google.common.collect.ImmutableList; + +import java.util.Optional; + +import static com.facebook.presto.spi.function.RoutineCharacteristics.Determinism.DETERMINISTIC; +import static com.facebook.presto.spi.function.RoutineCharacteristics.Language.SQL; +import static com.facebook.presto.spi.function.RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT; +import static com.facebook.presto.spi.function.RoutineCharacteristics.NullCallClause.RETURNS_NULL_ON_NULL_INPUT; +import static com.facebook.presto.spi.type.StandardTypes.DOUBLE; +import static com.facebook.presto.spi.type.StandardTypes.INTEGER; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; + +public class SqlInvokedFunctionTestUtils +{ + private SqlInvokedFunctionTestUtils() + { + } + + public static final QualifiedFunctionName POWER_TOWER = QualifiedFunctionName.of(new CatalogSchemaName("unittest", "memory"), "power_tower"); + + public static final SqlInvokedFunction FUNCTION_POWER_TOWER_DOUBLE = new SqlInvokedFunction( + POWER_TOWER, + ImmutableList.of(new SqlParameter("x", parseTypeSignature(DOUBLE))), + parseTypeSignature(DOUBLE), + "power tower", + new RoutineCharacteristics(SQL, DETERMINISTIC, CALLED_ON_NULL_INPUT), + "pow(x, x)", + Optional.empty()); + + public static final SqlInvokedFunction FUNCTION_POWER_TOWER_DOUBLE_UPDATED = new SqlInvokedFunction( + POWER_TOWER, + ImmutableList.of(new SqlParameter("x", parseTypeSignature(DOUBLE))), + parseTypeSignature(DOUBLE), + "power tower", + new RoutineCharacteristics(SQL, DETERMINISTIC, RETURNS_NULL_ON_NULL_INPUT), + "pow(x, x)", + Optional.empty()); + + public static final SqlInvokedFunction FUNCTION_POWER_TOWER_INT = new SqlInvokedFunction( + POWER_TOWER, + ImmutableList.of(new SqlParameter("x", parseTypeSignature(INTEGER))), + parseTypeSignature(INTEGER), + "power tower", + new RoutineCharacteristics(SQL, DETERMINISTIC, RETURNS_NULL_ON_NULL_INPUT), + "pow(x, x)", + Optional.empty()); +} diff --git a/presto-sql-function/src/test/java/com/facebook/presto/sqlfunction/TestSqlInvokedFunctionNamespaceManagerConfig.java b/presto-sql-function/src/test/java/com/facebook/presto/sqlfunction/TestSqlInvokedFunctionNamespaceManagerConfig.java new file mode 100644 index 0000000000000..46d4510886645 --- /dev/null +++ b/presto-sql-function/src/test/java/com/facebook/presto/sqlfunction/TestSqlInvokedFunctionNamespaceManagerConfig.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sqlfunction; + +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.MINUTES; + +public class TestSqlInvokedFunctionNamespaceManagerConfig +{ + @Test + public void testDefault() + { + assertRecordedDefaults(recordDefaults(SqlInvokedFunctionNamespaceManagerConfig.class) + .setFunctionCacheExpiration(new Duration(5, MINUTES)) + .setFunctionInstanceCacheExpiration(new Duration(8, HOURS))); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("function-cache-expiration", "10m") + .put("function-instance-cache-expiration", "4h") + .build(); + SqlInvokedFunctionNamespaceManagerConfig expected = new SqlInvokedFunctionNamespaceManagerConfig() + .setFunctionCacheExpiration(new Duration(10, MINUTES)) + .setFunctionInstanceCacheExpiration(new Duration(4, HOURS)); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-sqlserver/pom.xml b/presto-sqlserver/pom.xml index 4e13df6b97dbc..d6362a2e63f48 100644 --- a/presto-sqlserver/pom.xml +++ b/presto-sqlserver/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.225-SNAPSHOT + 0.231-SNAPSHOT 4.0.0 @@ -27,7 +27,7 @@ - io.airlift + com.facebook.airlift configuration diff --git a/presto-sqlserver/src/main/java/com/facebook/presto/plugin/sqlserver/SqlServerClient.java b/presto-sqlserver/src/main/java/com/facebook/presto/plugin/sqlserver/SqlServerClient.java index daf3926230481..f507246716da1 100644 --- a/presto-sqlserver/src/main/java/com/facebook/presto/plugin/sqlserver/SqlServerClient.java +++ b/presto-sqlserver/src/main/java/com/facebook/presto/plugin/sqlserver/SqlServerClient.java @@ -18,6 +18,7 @@ import com.facebook.presto.plugin.jdbc.DriverConnectionFactory; import com.facebook.presto.plugin.jdbc.JdbcColumnHandle; import com.facebook.presto.plugin.jdbc.JdbcConnectorId; +import com.facebook.presto.plugin.jdbc.JdbcIdentity; import com.facebook.presto.plugin.jdbc.JdbcTableHandle; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; @@ -44,9 +45,9 @@ public SqlServerClient(JdbcConnectorId connectorId, BaseJdbcConfig config) } @Override - protected void renameTable(String catalogName, SchemaTableName oldTable, SchemaTableName newTable) + protected void renameTable(JdbcIdentity identity, String catalogName, SchemaTableName oldTable, SchemaTableName newTable) { - try (Connection connection = connectionFactory.openConnection()) { + try (Connection connection = connectionFactory.openConnection(identity)) { String sql = format( "sp_rename %s, %s", singleQuote(catalogName, oldTable.getSchemaName(), oldTable.getTableName()), @@ -59,9 +60,9 @@ protected void renameTable(String catalogName, SchemaTableName oldTable, SchemaT } @Override - public void renameColumn(JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName) + public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName) { - try (Connection connection = connectionFactory.openConnection()) { + try (Connection connection = connectionFactory.openConnection(identity)) { String sql = format( "sp_rename %s, %s, 'COLUMN'", singleQuote(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName(), jdbcColumn.getColumnName()), diff --git a/presto-sqlserver/src/main/java/com/facebook/presto/plugin/sqlserver/SqlServerClientModule.java b/presto-sqlserver/src/main/java/com/facebook/presto/plugin/sqlserver/SqlServerClientModule.java index 455a42378553c..e68ed578a95c9 100644 --- a/presto-sqlserver/src/main/java/com/facebook/presto/plugin/sqlserver/SqlServerClientModule.java +++ b/presto-sqlserver/src/main/java/com/facebook/presto/plugin/sqlserver/SqlServerClientModule.java @@ -19,7 +19,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; public class SqlServerClientModule implements Module diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index c6e97fa2758c4..06cb42f1b8aa8 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-teradata-functions @@ -17,7 +17,7 @@ - io.airlift + com.facebook.airlift concurrent diff --git a/presto-teradata-functions/src/main/java/com/facebook/presto/teradata/functions/TeradataDateFunctions.java b/presto-teradata-functions/src/main/java/com/facebook/presto/teradata/functions/TeradataDateFunctions.java index 9b6e844d71a9d..670e906a5f3b6 100644 --- a/presto-teradata-functions/src/main/java/com/facebook/presto/teradata/functions/TeradataDateFunctions.java +++ b/presto-teradata-functions/src/main/java/com/facebook/presto/teradata/functions/TeradataDateFunctions.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.teradata.functions; +import com.facebook.airlift.concurrent.ThreadLocalCache; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.Description; @@ -20,7 +21,6 @@ import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.TimeZoneKey; -import io.airlift.concurrent.ThreadLocalCache; import io.airlift.slice.Slice; import org.joda.time.DateTimeZone; import org.joda.time.chrono.ISOChronology; diff --git a/presto-testing-docker/pom.xml b/presto-testing-docker/pom.xml new file mode 100644 index 0000000000000..1e3e615abd379 --- /dev/null +++ b/presto-testing-docker/pom.xml @@ -0,0 +1,79 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.231-SNAPSHOT + + + presto-testing-docker + presto-testing-docker + + + ${project.parent.basedir} + + + + + com.facebook.airlift + log + + + + com.facebook.airlift + testing + + + + com.google.guava + guava + + + + com.starburstdata + starburst-spotify-docker-client + + + + net.jodah + failsafe + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + ci + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + ci-only + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + diff --git a/presto-testing-docker/src/main/java/com/facebook/presto/testing/docker/DockerContainer.java b/presto-testing-docker/src/main/java/com/facebook/presto/testing/docker/DockerContainer.java new file mode 100644 index 0000000000000..0bc85156c5f0c --- /dev/null +++ b/presto-testing-docker/src/main/java/com/facebook/presto/testing/docker/DockerContainer.java @@ -0,0 +1,251 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.testing.docker; + +import com.facebook.airlift.log.Logger; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.spotify.docker.client.DefaultDockerClient; +import com.spotify.docker.client.DockerClient; +import com.spotify.docker.client.exceptions.ContainerNotFoundException; +import com.spotify.docker.client.messages.Container; +import com.spotify.docker.client.messages.ContainerConfig; +import com.spotify.docker.client.messages.HostConfig; +import com.spotify.docker.client.messages.PortBinding; +import net.jodah.failsafe.Failsafe; +import net.jodah.failsafe.RetryPolicy; +import net.jodah.failsafe.function.CheckedConsumer; + +import java.io.Closeable; +import java.net.Socket; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.airlift.testing.Closeables.closeAllSuppress; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.common.collect.MoreCollectors.toOptional; +import static java.lang.String.format; +import static java.time.temporal.ChronoUnit.MINUTES; +import static java.time.temporal.ChronoUnit.SECONDS; +import static java.util.Objects.requireNonNull; + +public final class DockerContainer + implements Closeable +{ + private static final Logger LOG = Logger.get(DockerContainer.class); + + private static final boolean DEBUG = false; + + private static final String HOST_IP = "127.0.0.1"; + private final String image; + private final Map environment; + private DockerClient dockerClient; + private String containerId; + + private Map hostPorts; + + public DockerContainer(String image, List ports, Map environment, CheckedConsumer healthCheck) + { + this.image = requireNonNull(image, "image is null"); + this.environment = ImmutableMap.copyOf(requireNonNull(environment, "environment is null")); + try { + startContainer(ports, healthCheck); + } + catch (Exception e) { + closeAllSuppress(e, this); + throw new RuntimeException(e); + } + } + + private void startContainer(List ports, CheckedConsumer healthCheck) + throws Exception + { + dockerClient = DefaultDockerClient.fromEnv().build(); + if (dockerClient.listImages(DockerClient.ListImagesParam.byName(image)).isEmpty()) { + checkState(!image.endsWith("-SNAPSHOT"), "Unavailable snapshot image %s, please build before running tests", image); + LOG.info("Pulling image %s...", image); + dockerClient.pull(image); + } + if (DEBUG) { + Optional testingContainer = dockerClient.listContainers().stream() + .filter(container -> container.image().equals(image)) + .collect(toOptional()); + if (testingContainer.isPresent()) { + containerId = testingContainer.get().id(); + LOG.info("Container for %s already exists with id: %s", image, containerId); + calculateHostPorts(ports); + } + else { + createContainer(ports); + } + } + else { + createContainer(ports); + } + + checkState(isContainerUp(), "Container was not started properly"); + + LOG.info("Auto-assigned host ports are %s", hostPorts); + + waitForContainer(healthCheck); + } + + private boolean isContainerUp() + { + try { + return dockerClient.inspectContainer(containerId).state().running(); + } + catch (ContainerNotFoundException e) { + return false; + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void createContainer(List ports) + throws Exception + { + LOG.info("Starting docker container from image %s", image); + + Map> portBindings = ports.stream() + .collect(toImmutableMap(Object::toString, port -> ImmutableList.of(PortBinding.create(HOST_IP, "0")))); + Set exposedPorts = ports.stream() + .map(Object::toString) + .collect(toImmutableSet()); + + containerId = dockerClient.createContainer(ContainerConfig.builder() + .hostConfig(HostConfig.builder() + .portBindings(portBindings) + .build()) + .exposedPorts(exposedPorts) + .env(environment.entrySet().stream() + .map(entry -> format("%s=%s", entry.getKey(), entry.getValue())) + .collect(toImmutableList())) + .image(image) + .build()) + .id(); + + LOG.info("Started docker container with id: %s", containerId); + + dockerClient.startContainer(containerId); + + calculateHostPorts(ports); + + waitForContainerPorts(ports); + } + + private void waitForContainer(CheckedConsumer healthCheck) + { + RetryPolicy retryPolicy = new RetryPolicy<>() + .withMaxDuration(Duration.of(10, MINUTES)) + .withMaxAttempts(Integer.MAX_VALUE) // limited by MaxDuration + .abortOn(error -> !isContainerUp()) + .onRetry(event -> LOG.info(format("Waiting for container for %s [%s]...", image, event.getLastFailure()))) + .withDelay(Duration.of(10, SECONDS)); + Failsafe.with(retryPolicy).run(() -> healthCheck.accept(this::getHostPort)); + } + + private void waitForContainerPorts(List ports) + { + List hostPorts = ports.stream() + .map(this::getHostPort) + .collect(toImmutableList()); + + RetryPolicy retryPolicy = new RetryPolicy<>() + .withMaxDuration(Duration.of(10, MINUTES)) + .withMaxAttempts(Integer.MAX_VALUE) // limited by MaxDuration + .abortOn(error -> !isContainerUp()) + .withDelay(Duration.of(5, SECONDS)) + .onRetry(event -> LOG.info("Waiting for ports %s that are exposed on %s on %s ...", ports, HOST_IP, hostPorts)); + + Failsafe.with(retryPolicy).run(() -> { + for (int port : ports) { + try (Socket socket = new Socket(HOST_IP, getHostPort(port))) { + checkState(socket.isConnected()); + } + } + }); + } + + private void calculateHostPorts(List ports) + throws Exception + { + hostPorts = dockerClient.inspectContainer(containerId).networkSettings() + .ports() + .entrySet() + .stream() + .filter(entry -> ports.contains(extractPort(entry))) + .collect(toImmutableMap( + entry -> extractPort(entry), + entry -> entry.getValue().stream() + .peek(portBinding -> { + checkState(portBinding.hostIp().equals(HOST_IP), "Unexpected port binding found: %s", portBinding); + }) + .map(PortBinding::hostPort) + .collect(toOptional()) + .map(Integer::parseInt) + .orElseThrow(() -> new IllegalStateException("Could not extract port mapping from: " + entry)))); + } + + public int getHostPort(int port) + { + checkArgument(hostPorts.keySet().contains(port), "Port %s is not bound", port); + return hostPorts.get(port); + } + + private static int extractPort(Entry> entry) + { + checkArgument(!entry.getKey().contains("/udp"), "UDP port binding is not supported"); + return Integer.parseInt(entry.getKey().replace("/tcp", "")); + } + + private void removeContainer(String containerId) + { + try { + LOG.info("Killing container %s", containerId); + dockerClient.killContainer(containerId); + dockerClient.removeContainer(containerId); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() + { + if (dockerClient == null) { + return; + } + if (!DEBUG && containerId != null) { + removeContainer(containerId); + } + dockerClient.close(); + dockerClient = null; + } + + public interface HostPortProvider + { + int getHostPort(int containerPort); + } +} diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 4a4c3196c8dbb..b6aeeb1b4221f 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 44fc4d69c9cc5..bc9de425fe8e4 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-tests @@ -48,37 +48,37 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift configuration - io.airlift.discovery + com.facebook.airlift.discovery discovery-server - io.airlift + com.facebook.airlift json - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift node @@ -88,7 +88,7 @@ - io.airlift + com.facebook.airlift stats @@ -103,7 +103,7 @@ - io.airlift + com.facebook.airlift testing @@ -176,7 +176,7 @@ - io.airlift + com.facebook.airlift log-manager runtime diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java index 79dc42d2b7ce8..9ad7a2181f5db 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java @@ -1338,4 +1338,17 @@ public void testQuantileDigestMergeGroupBy() "GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); } + + @Test + public void testGroupedRow() + { + assertQuery( + "SELECT count(r[1]), count(r[2]) " + + "FROM (" + + " SELECT orderkey, max_by(ROW(orderstatus, shippriority), orderstatus) AS r " + + " FROM orders " + + " GROUP BY orderkey" + + ")", + "SELECT 15000, 15000"); + } } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestDistributedQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestDistributedQueries.java index 0e9754143f12d..c82f7850aa330 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestDistributedQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestDistributedQueries.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.tests; +import com.facebook.airlift.testing.Assertions; import com.facebook.presto.Session; import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.execution.QueryInfo; @@ -28,7 +29,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.UncheckedTimeoutException; -import io.airlift.testing.Assertions; import io.airlift.units.Duration; import org.intellij.lang.annotations.Language; import org.testng.annotations.Test; diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java index 19844fc3c0d42..6255fe17d6de7 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java @@ -15,8 +15,8 @@ import com.facebook.presto.Session; import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.metadata.BuiltInFunction; import com.facebook.presto.metadata.FunctionListBuilder; -import com.facebook.presto.metadata.SqlFunction; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spi.type.Decimals; @@ -41,6 +41,7 @@ import io.airlift.tpch.TpchTable; import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; import org.intellij.lang.annotations.Language; +import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -103,7 +104,7 @@ public abstract class AbstractTestQueries extends AbstractTestQueryFramework { // We can just use the default type registry, since we don't use any parametric types - protected static final List CUSTOM_FUNCTIONS = new FunctionListBuilder() + protected static final List CUSTOM_FUNCTIONS = new FunctionListBuilder() .aggregates(CustomSum.class) .window(CustomRank.class) .scalars(CustomAdd.class) @@ -445,6 +446,29 @@ public void testRowFieldAccessorInAggregate() "SELECT * FROM VALUES (31, 3), (41, 1)"); } + @Test + public void testTryMapTransformValueFunction() + { + // MaterializedResult#Builder doesn't support null row. Coalesce null value to empty map for comparison. + MaterializedResult actual = computeActual("" + + "SELECT COALESCE( TRY( TRANSFORM_VALUES( id, (k, v) -> k / v ) ) , MAP() )" + + "FROM ( VALUES " + + "(MAP(ARRAY[1, 2], ARRAY[0, 0])), " + + "(MAP(ARRAY[1, 2], ARRAY[1, 2])), " + + "(MAP(ARRAY[28, 56], ARRAY[2, 4])), " + + "(MAP(ARRAY[4, 5], ARRAY[0, 0])), " + + "(MAP(ARRAY[12, 72], ARRAY[3, 6]))) AS t (id)"); + + MaterializedResult expected = resultBuilder(getSession(), mapType(INTEGER, INTEGER)) + .row(ImmutableMap.of()) + .row(ImmutableMap.of(1, 1, 2, 1)) + .row(ImmutableMap.of(28, 14, 56, 14)) + .row(ImmutableMap.of()) + .row(ImmutableMap.of(12, 4, 72, 12)) + .build(); + assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows()); + } + @Test public void testRowFieldAccessorInWindowFunction() { @@ -643,6 +667,14 @@ public void testUnnest() "SELECT * FROM (SELECT custkey FROM orders ORDER BY orderkey LIMIT 1) CROSS JOIN (VALUES (10, 1), (20, 2), (30, 3))"); assertQuery("SELECT * FROM orders, UNNEST(ARRAY[1])", "SELECT orders.*, 1 FROM orders"); + assertQuery("SELECT a FROM (" + + " SELECT l.arr AS arr FROM (" + + " SELECT orderkey, ARRAY[1,2,3] AS arr FROM orders ORDER BY orderkey LIMIT 1) l" + + " FULL OUTER JOIN (" + + " SELECT orderkey, ARRAY[1,2,3] AS arr FROM orders ORDER BY orderkey LIMIT 1) o" + + " ON l.orderkey = o.orderkey) " + + "CROSS JOIN UNNEST(arr) AS t (a)", + "SELECT * FROM (VALUES (1), (2), (3))"); assertQueryFails( "SELECT * FROM (VALUES array[2, 2]) a(x) LEFT OUTER JOIN UNNEST(x) ON true", @@ -4076,7 +4108,7 @@ public void testWindowNoChannels() public void testInvalidWindowFunction() { assertQueryFails("SELECT abs(x) OVER ()\n" + - "FROM (VALUES (1), (2), (3)) t(x)", + "FROM (VALUES (1), (2), (3)) t(x)", "line 1:1: Not a window function: abs"); } @@ -4621,6 +4653,8 @@ public void testExplainDdl() { assertExplainDdl("CREATE TABLE foo (pk bigint)", "CREATE TABLE foo"); assertExplainDdl("CREATE VIEW foo AS SELECT * FROM orders", "CREATE VIEW foo"); + assertExplainDdl("CREATE OR REPLACE FUNCTION testing.default.tan (x int) RETURNS double COMMENT 'tangent trigonometric function' LANGUAGE SQL DETERMINISTIC CALLED ON NULL INPUT RETURN sin(x) / cos(x)", "CREATE FUNCTION testing.default.tan"); + assertExplainDdl("DROP FUNCTION IF EXISTS testing.default.tan (int)", "DROP FUNCTION testing.default.tan"); assertExplainDdl("DROP TABLE orders"); assertExplainDdl("DROP VIEW view"); assertExplainDdl("ALTER TABLE orders RENAME TO new_name"); @@ -4939,7 +4973,6 @@ public void testShowSession() getSession().getSource(), getSession().getCatalog(), getSession().getSchema(), - getSession().getPath(), getSession().getTraceToken(), getSession().getTimeZoneKey(), getSession().getLocale(), @@ -4947,7 +4980,6 @@ public void testShowSession() getSession().getUserAgent(), getSession().getClientInfo(), getSession().getClientTags(), - getSession().getClientCapabilities(), getSession().getResourceEstimates(), getSession().getStartTime(), ImmutableMap.builder() @@ -7950,6 +7982,9 @@ public void testDescribeOutputNonSelect() assertDescribeOutputEmpty("ALTER TABLE foo RENAME TO bar"); assertDescribeOutputEmpty("DROP TABLE foo"); assertDescribeOutputEmpty("CREATE VIEW foo AS SELECT * FROM nation"); + assertDescribeOutputEmpty("CREATE FUNCTION testing.default.tan (x int) RETURNS double COMMENT 'tangent trigonometric function' LANGUAGE SQL DETERMINISTIC CALLED ON NULL INPUT RETURN sin(x) / cos(x)"); + assertDescribeOutputEmpty("DROP FUNCTION IF EXISTS testing.default.tan (int)"); + assertDescribeOutputEmpty("DROP VIEW foo"); assertDescribeOutputEmpty("PREPARE test FROM SELECT * FROM orders"); assertDescribeOutputEmpty("EXECUTE test"); @@ -8215,6 +8250,53 @@ public void testRightJoinWithEmptyProbeSide() "WITH small_part AS (SELECT * FROM part WHERE name = 'a') SELECT lineitem.orderkey FROM small_part RIGHT JOIN lineitem ON small_part.partkey = lineitem.partkey"); } + @Test + public void CardinalityOfFilterSimplification() + { + MaterializedResult result = computeActual("" + + "SELECT cardinality(filter(array_agg(orderkey), x -> x % 2 = 0)) " + + "FROM orders " + + "WHERE orderkey < 20"); + + assertEquals(result.getRowCount(), 1); + Assert.assertEquals(result.getMaterializedRows().get(0).getField(0), 3L); + } + + @Test + public void CardinalityOfFilterComparisonSimplification() + { + MaterializedResult result = computeActual("" + + "SELECT cardinality(filter(array_agg(orderkey), x -> x % 3 = 0)) = 1 " + + "FROM orders " + + "WHERE orderkey < 20"); + + assertEquals(result.getRowCount(), 1); + Assert.assertEquals(result.getMaterializedRows().get(0).getField(0), true); + } + + @Test + public void CardinalityOfFilteGreaterThanZeroSimplification() + { + MaterializedResult result = computeActual("" + + "SELECT cardinality(filter(array_agg(orderkey), x -> x % 2 = 0)) > 0 " + + "FROM orders " + + "WHERE orderkey < 20"); + + assertEquals(result.getRowCount(), 1); + Assert.assertEquals(result.getMaterializedRows().get(0).getField(0), true); + } + + @Test + public void CardinalityOfFilteEqualsZeroSimplification() + { + MaterializedResult result = computeActual("" + + "SELECT cardinality(filter(array_agg(orderkey), x -> x > 2)) = 0 " + + "FROM orders " + + "WHERE orderkey < 20"); + + assertEquals(result.getRowCount(), 1); + Assert.assertEquals(result.getMaterializedRows().get(0).getField(0), false); + } protected Session noJoinReordering() { return Session.builder(getSession()) diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java index d3c77a1298080..f0caf36db5aae 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java @@ -52,6 +52,7 @@ import java.util.OptionalLong; import java.util.function.Consumer; +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; import static com.facebook.presto.sql.ParsingUtil.createParsingOptions; import static com.facebook.presto.sql.SqlFormatter.formatSql; import static com.facebook.presto.transaction.TransactionBuilder.transaction; @@ -59,7 +60,6 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.airlift.testing.Closeables.closeAllRuntimeException; import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.Objects.requireNonNull; @@ -215,6 +215,11 @@ protected void assertQuerySucceeds(Session session, @Language("SQL") String sql) QueryAssertions.assertQuerySucceeds(queryRunner, session, sql); } + protected void assertQuerySucceeds(@Language("SQL") String sql) + { + QueryAssertions.assertQuerySucceeds(queryRunner, getSession(), sql); + } + protected void assertQueryFailsEventually(@Language("SQL") String sql, @Language("RegExp") String expectedMessageRegExp, Duration timeout) { QueryAssertions.assertQueryFailsEventually(queryRunner, getSession(), sql, expectedMessageRegExp, timeout); @@ -371,7 +376,6 @@ private QueryExplainer getQueryExplainer() List optimizers = new PlanOptimizers( metadata, sqlParser, - featuresConfig, forceSingleNode, new MBeanExporter(new TestingMBeanServer()), queryRunner.getSplitManager(), diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestRepartitionQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestRepartitionQueries.java new file mode 100644 index 0000000000000..ad9b658c625e6 --- /dev/null +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestRepartitionQueries.java @@ -0,0 +1,330 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests; + +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + +public abstract class AbstractTestRepartitionQueries + extends AbstractTestQueryFramework +{ + protected AbstractTestRepartitionQueries(QueryRunnerSupplier supplier) + { + super(supplier); + } + + @Test + public void testBoolean() + { + assertQuery("WITH lineitem_ex AS (\n" + + " SELECT\n" + + " partkey,\n" + + " suppkey,\n" + + " IF (mod(orderkey + linenumber, 11) = 0, FALSE, TRUE) AS flag\n" + + " FROM lineitem\n" + + ")\n" + + "SELECT\n" + + " CHECKSUM(l.flag)\n" + + "FROM lineitem_ex l,\n" + + " partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey\n" + + " AND l.suppkey = p.suppkey\n" + + " AND p.availqty < 1000", + new byte[] {126, 85, 59, 72, -71, -42, 49, 21}); + } + + @Test + public void testSmallInt() + { + assertQuery("WITH lineitem_ex AS (\n" + + " SELECT\n" + + " partkey,\n" + + " suppkey,\n" + + " CAST(linenumber AS SMALLINT) AS l_linenumber\n" + + " FROM lineitem\n" + + ")\n" + + "SELECT\n" + + " CHECKSUM(l.l_linenumber)\n" + + "FROM lineitem_ex l,\n" + + " partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey\n" + + " AND l.suppkey = p.suppkey\n" + + " AND p.availqty < 1000", + new byte[] {-87, -84, -37, 75, -52, 22, 87, -29}); + } + + @Test + public void testInteger() + { + assertQuery("SELECT\n" + + " CHECKSUM(l.linenumber)\n" + + " FROM lineitem l, partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey\n" + + " AND l.suppkey = p.suppkey\n" + + " AND p.availqty < 1000", + new byte[] {-87, -84, -37, 75, -52, 22, 87, -29}); + } + + @Test + public void testBigInt() + { + assertQuery("SELECT\n" + + " CHECKSUM(l.partkey)\n" + + " FROM lineitem l, partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey\n" + + " AND l.suppkey = p.suppkey\n" + + " AND p.availqty < 1000", + new byte[] {122, -36, -24, 126, -7, 61, -78, 5}); + } + + @Test + public void testIpAddress() + { + assertQuery("WITH lineitem_ex AS \n" + + "(\n" + + "SELECT\n" + + " partkey,\n" + + " CAST(\n" + + " CONCAT(\n" + + " CONCAT(\n" + + " CONCAT(\n" + + " CONCAT(\n" + + " CONCAT(\n" + + " CONCAT(CAST((orderkey % 255) AS VARCHAR), '.'),\n" + + " CAST((partkey % 255) AS VARCHAR)\n" + + " ),\n" + + " '.'\n" + + " ),\n" + + " CAST(suppkey AS VARCHAR)\n" + + " ),\n" + + " '.'\n" + + " ),\n" + + " CAST(linenumber AS VARCHAR)\n" + + " ) AS ipaddress\n" + + " ) AS ip\n" + + " FROM lineitem\n" + + " )\n" + + "SELECT\n" + + " CHECKSUM(l.ip) \n" + + "FROM lineitem_ex l,\n" + + " partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey", + new byte[] {92, -57, -31, -119, -122, 9, 118, -31}); + } + + @Test + public void testBigintWithNulls() + { + assertQuery("WITH lineitem_ex AS (\n" + + " SELECT\n" + + " IF (mod(orderkey + linenumber, 11) = 0, NULL, partkey) AS l_partkey\n" + + " FROM lineitem\n" + + ")\n" + + "SELECT\n" + + " CHECKSUM(l.l_partkey)\n" + + "FROM lineitem_ex l,\n" + + " partsupp p\n" + + "WHERE\n" + + " l.l_partkey = p.partkey", + new byte[] {-4, -54, 21, 27, 121, 66, 3, 35}); + } + + @Test + public void testVarchar() + { + assertQuery("SELECT\n" + + " CHECKSUM(l.comment)\n" + + " FROM lineitem l, partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey\n" + + " AND l.suppkey = p.suppkey\n" + + " AND p.availqty < 1000", + new byte[] {14, -4, 62, 87, 52, 53, -101, -100}); + } + + @Test + public void testDictionaryOfVarcharWithNulls() + { + assertQuery("SELECT\n" + + " CHECKSUM(l.shipmode)\n" + + " FROM lineitem l, partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey\n" + + " AND l.suppkey = p.suppkey\n" + + " AND p.availqty < 1000", + new byte[] {127, 108, -69, -115, -43, 44, -54, 88}); + } + + @Test + public void testArrayOfBigInt() + { + assertQuery("WITH lineitem_ex AS (\n" + + " SELECT\n" + + " partkey,\n" + + " suppkey,\n" + + " ARRAY[orderkey, partkey, suppkey] AS l_array\n" + + " FROM lineitem\n" + + ")\n" + + "SELECT\n" + + " CHECKSUM(l.l_array)\n" + + "FROM lineitem_ex l,\n" + + " partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey\n" + + " AND l.suppkey = p.suppkey\n" + + " AND p.availqty < 1000", + new byte[] {10, -59, 46, 87, 22, -93, 58, -16}); + } + + @Test + public void testArrayOfArray() + { + assertQuery("WITH lineitem_ex AS (\n" + + " SELECT\n" + + " partkey,\n" + + " suppkey,\n" + + " ARRAY[ARRAY[orderkey, partkey], ARRAY[suppkey]] AS l_array\n" + + " FROM lineitem\n" + + ")\n" + + "SELECT\n" + + " CHECKSUM(l.l_array)\n" + + "FROM lineitem_ex l,\n" + + " partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey\n" + + " AND l.suppkey = p.suppkey\n" + + " AND p.availqty < 1000", + new byte[] {10, -59, 46, 87, 22, -93, 58, -16}); + } + + @Test + public void testStruct() + { + assertQuery("WITH lineitem_ex AS (\n" + + " SELECT\n" + + " orderkey,\n" + + " CAST(\n" + + " ROW (\n" + + " partkey,\n" + + " suppkey,\n" + + " extendedprice,\n" + + " discount,\n" + + " quantity,\n" + + " shipdate,\n" + + " receiptdate,\n" + + " commitdate,\n" + + " comment\n" + + " ) AS ROW(\n" + + " l_partkey BIGINT,\n" + + " l_suppkey BIGINT,\n" + + " l_extendedprice DOUBLE,\n" + + " l_discount DOUBLE,\n" + + " l_quantity DOUBLE,\n" + + " l_shipdate DATE,\n" + + " l_receiptdate DATE,\n" + + " l_commitdate DATE,\n" + + " l_comment VARCHAR(44)\n" + + " )\n" + + " ) AS l_shipment\n" + + " FROM lineitem\n" + + ")\n" + + "SELECT\n" + + " CHECKSUM(l.l_shipment)\n" + + "FROM lineitem_ex l,\n" + + " orders o\n" + + "WHERE\n" + + " l.orderkey = o.orderkey", + new byte[] {-56, 110, 18, -107, -123, 121, 87, 79}); + } + + @Test + public void testStructWithNulls() + { + assertQuery("WITH lineitem_ex AS (\n" + + " SELECT\n" + + " orderkey,\n" + + " CAST(\n" + + " IF (\n" + + " mod(partkey, 5) = 0,\n" + + " NULL,\n" + + " ROW(\n" + + " COMMENT,\n" + + " IF (\n" + + " mod(partkey, 13) = 0,\n" + + " NULL,\n" + + " CONCAT(CAST(partkey AS VARCHAR), COMMENT)\n" + + " )\n" + + " )\n" + + " ) AS ROW(s1 VARCHAR, s2 VARCHAR)\n" + + " ) AS l_partkey_struct\n" + + " FROM lineitem\n" + + ")\n" + + "SELECT\n" + + " CHECKSUM(l.l_partkey_struct)\n" + + "FROM lineitem_ex l,\n" + + " orders o\n" + + "WHERE\n" + + " l.orderkey = o.orderkey", + new byte[] {67, 108, 83, 92, 16, -5, 66, 65}); + } + + @Test + public void testMaps() + { + assertQuery("WITH lineitem_ex AS (\n" + + " SELECT\n" + + " partkey,\n" + + " suppkey,\n" + + " MAP(ARRAY[1, 2, 3], ARRAY[orderkey, partkey, suppkey]) AS l_map\n" + + " FROM lineitem\n" + + ")\n" + + "SELECT\n" + + " CHECKSUM(l.l_map)\n" + + "FROM lineitem_ex l,\n" + + " partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey\n" + + " AND l.suppkey = p.suppkey\n" + + " AND p.availqty < 1000", + new byte[] {-28, 76, -12, -42, 116, -118, -9, 46}); + + assertQuery("WITH lineitem_ex AS (\n" + + " SELECT\n" + + " partkey,\n" + + " suppkey,\n" + + " MAP(ARRAY[1, 2, 3], ARRAY[orderkey, partkey, suppkey]) AS l_map\n" + + " FROM lineitem\n" + + ")\n" + + "SELECT\n" + + " CHECKSUM(l.l_map[1])\n" + + "FROM lineitem_ex l,\n" + + " partsupp p\n" + + "WHERE\n" + + " l.partkey = p.partkey\n" + + " AND l.suppkey = p.suppkey\n" + + " AND p.availqty < 1000\n", + new byte[] {121, 123, -114, -120, -18, 98, -124, 105}); + } + + private void assertQuery(String query, byte[] checksum) + { + assertEquals(computeActual(query).getOnlyValue(), checksum); + } +} diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestingPrestoClient.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestingPrestoClient.java index b613c1f520473..6fc0daefef572 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestingPrestoClient.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestingPrestoClient.java @@ -147,7 +147,6 @@ private static ClientSession toClientSession(Session session, URI server, Durati session.getClientInfo().orElse(null), session.getCatalog().orElse(null), session.getSchema().orElse(null), - session.getPath().toString(), session.getTimeZoneKey().getId(), session.getLocale(), resourceEstimates.build(), diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/CreateHll.java b/presto-tests/src/main/java/com/facebook/presto/tests/CreateHll.java index 7feabe5db22ba..92d798173a488 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/CreateHll.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/CreateHll.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.tests; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; import io.airlift.slice.Slice; -import io.airlift.stats.cardinality.HyperLogLog; import static com.facebook.presto.operator.aggregation.ApproximateSetAggregation.DEFAULT_STANDARD_ERROR; import static com.facebook.presto.operator.aggregation.HyperLogLogUtils.standardErrorToBuckets; diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java index 1f9bc39929669..d6ca19f6cff43 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.tests; +import com.facebook.airlift.discovery.server.testing.TestingDiscoveryServer; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.testing.Assertions; import com.facebook.presto.Session; import com.facebook.presto.Session.SessionBuilder; import com.facebook.presto.cost.StatsCalculator; @@ -43,9 +46,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.Closer; -import io.airlift.discovery.server.testing.TestingDiscoveryServer; -import io.airlift.log.Logger; -import io.airlift.testing.Assertions; import io.airlift.units.Duration; import org.intellij.lang.annotations.Language; @@ -167,7 +167,7 @@ private DistributedQueryRunner( start = System.nanoTime(); for (TestingPrestoServer server : servers) { - server.getMetadata().addFunctions(AbstractTestQueries.CUSTOM_FUNCTIONS); + server.getMetadata().registerBuiltInFunctions(AbstractTestQueries.CUSTOM_FUNCTIONS); } log.info("Added functions in %s", nanosSince(start).convertToMostSuccinctTimeUnit()); diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/H2QueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/H2QueryRunner.java index d3416d507f7c8..f18180a16a891 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/H2QueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/H2QueryRunner.java @@ -22,6 +22,8 @@ import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.CharType; import com.facebook.presto.spi.type.DecimalType; +import com.facebook.presto.spi.type.RowType; +import com.facebook.presto.spi.type.TimestampType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.VarcharType; import com.facebook.presto.testing.MaterializedResult; @@ -47,10 +49,12 @@ import java.sql.Date; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Timestamp; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -68,6 +72,7 @@ import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; import static com.facebook.presto.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.spi.type.Varchars.isVarcharType; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static com.facebook.presto.tpch.TpchRecordSet.createTpchRecordSet; @@ -195,6 +200,11 @@ public MaterializedResult execute(Session session, @Language("SQL") String sql, return materializedRows; } + public Handle getHandle() + { + return handle; + } + private static RowMapper rowMapper(List types) { return new RowMapper() @@ -289,6 +299,15 @@ else if (isCharType(type)) { row.add(padEnd(stringValue, ((CharType) type).getLength(), ' ')); } } + else if (VARBINARY.equals(type)) { + byte[] binary = resultSet.getBytes(i); + if (resultSet.wasNull()) { + row.add(null); + } + else { + row.add(binary); + } + } else if (DATE.equals(type)) { // resultSet.getDate(i) doesn't work if JVM's zone skipped day being retrieved (e.g. 2011-12-30 and Pacific/Apia zone) LocalDate dateValue = resultSet.getObject(i, LocalDate.class); @@ -363,7 +382,16 @@ else if (type instanceof ArrayType) { row.add(null); } else { - row.add(newArrayList((Object[]) array.getArray())); + row.add(newArrayList(mapArrayValues(((ArrayType) type), (Object[]) array.getArray()))); + } + } + else if (type instanceof RowType) { + Array array = resultSet.getArray(i); + if (resultSet.wasNull()) { + row.add(null); + } + else { + row.add(newArrayList(mapRowValues((RowType) type, (Object[]) array.getArray()))); } } else { @@ -375,6 +403,55 @@ else if (type instanceof ArrayType) { }; } + private static Object[] mapArrayValues(ArrayType arrayType, Object[] values) + { + Type elementType = arrayType.getElementType(); + if (elementType instanceof ArrayType) { + return Arrays.stream(values) + .map(v -> v == null ? null : newArrayList((Object[]) v)) + .toArray(); + } + + if (elementType instanceof RowType) { + RowType rowType = (RowType) elementType; + return Arrays.stream(values) + .map(v -> v == null ? null : newArrayList(mapRowValues(rowType, (Object[]) v))) + .toArray(); + } + + if (elementType instanceof CharType) { + int length = ((CharType) elementType).getLength(); + return Arrays.stream(values) + .map(String.class::cast) + .map(v -> v == null ? null : padEnd(v, length, ' ')) + .toArray(); + } + + if (elementType instanceof TimestampType) { + return Arrays.stream(values) + .map(v -> v == null ? null : ((Timestamp) v).toLocalDateTime()) + .toArray(); + } + + return values; + } + + private static Object[] mapRowValues(RowType rowType, Object[] values) + { + int fieldCount = rowType.getFields().size(); + Object[] fields = new Object[fieldCount]; + for (int j = 0; j < fieldCount; j++) { + Type fieldType = rowType.getTypeParameters().get(j); + if (fieldType instanceof RowType) { + fields[j] = newArrayList(mapRowValues((RowType) fieldType, (Object[]) values[j])); + } + else { + fields[j] = values[j]; + } + } + return fields; + } + private static void insertRows(ConnectorTableMetadata tableMetadata, Handle handle, RecordSet data) { List columns = tableMetadata.getColumns().stream() diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/QueryAssertions.java b/presto-tests/src/main/java/com/facebook/presto/tests/QueryAssertions.java index 3fea48b4587e4..aea7f2ddb359b 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/QueryAssertions.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/QueryAssertions.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.tests; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollector; import com.facebook.presto.metadata.QualifiedObjectName; @@ -26,7 +27,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Multiset; import com.google.common.collect.Multisets; -import io.airlift.log.Logger; import io.airlift.tpch.TpchTable; import io.airlift.units.Duration; import org.intellij.lang.annotations.Language; diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java index 876de3376e56a..e49c3f77f89c7 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.tests; +import com.facebook.airlift.testing.Closeables; import com.facebook.presto.Session; import com.facebook.presto.cost.StatsCalculator; import com.facebook.presto.metadata.AllNodes; @@ -34,7 +35,6 @@ import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.testing.Closeables; import org.intellij.lang.annotations.Language; import java.util.List; @@ -75,7 +75,7 @@ public StandaloneQueryRunner(Session defaultSession) refreshNodes(); - server.getMetadata().addFunctions(AbstractTestQueries.CUSTOM_FUNCTIONS); + server.getMetadata().registerBuiltInFunctions(AbstractTestQueries.CUSTOM_FUNCTIONS); SessionPropertyManager sessionPropertyManager = server.getMetadata().getSessionPropertyManager(); sessionPropertyManager.addSystemSessionProperties(TEST_SYSTEM_PROPERTIES); diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/StatefulSleepingSum.java b/presto-tests/src/main/java/com/facebook/presto/tests/StatefulSleepingSum.java index 719a0ce6fba20..35abb77c33670 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/StatefulSleepingSum.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/StatefulSleepingSum.java @@ -16,8 +16,9 @@ import com.facebook.presto.metadata.BoundVariables; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.metadata.SqlScalarFunction; -import com.facebook.presto.operator.scalar.ScalarFunctionImplementation; +import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -26,8 +27,9 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicInteger; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; -import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; +import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.constructorMethodHandle; @@ -43,7 +45,7 @@ public class StatefulSleepingSum private StatefulSleepingSum() { super(new Signature( - "stateful_sleeping_sum", + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "stateful_sleeping_sum"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("bigint")), ImmutableList.of(), @@ -71,10 +73,10 @@ public String getDescription() } @Override - public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) { int args = 4; - return new ScalarFunctionImplementation( + return new BuiltInScalarFunctionImplementation( false, nCopies(args, valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), methodHandle(StatefulSleepingSum.class, "statefulSleepingSum", State.class, double.class, long.class, long.class, long.class), diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/TestingPrestoClient.java b/presto-tests/src/main/java/com/facebook/presto/tests/TestingPrestoClient.java index 67c9154c336e1..839159cf1beb1 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/TestingPrestoClient.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/TestingPrestoClient.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.DecimalType; import com.facebook.presto.spi.type.MapType; +import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.SqlTimestamp; import com.facebook.presto.spi.type.SqlTimestampWithTimeZone; import com.facebook.presto.spi.type.Type; @@ -238,6 +239,14 @@ else if (type instanceof MapType) { e -> convertToRowValue(((MapType) type).getKeyType(), e.getKey()), e -> convertToRowValue(((MapType) type).getValueType(), e.getValue()))); } + else if (type instanceof RowType) { + Map data = (Map) value; + RowType rowType = (RowType) type; + + return rowType.getFields().stream() + .map(field -> convertToRowValue(field.getType(), data.get(field.getName().get()))) + .collect(toList()); + } else if (type instanceof DecimalType) { return new BigDecimal((String) value); } diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestBeginQuery.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestBeginQuery.java index 70bb7d2a541ab..5ba6d09e69e36 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestBeginQuery.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestBeginQuery.java @@ -14,8 +14,12 @@ package com.facebook.presto.execution; import com.facebook.presto.Session; +import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorHandleResolver; +import com.facebook.presto.spi.ConnectorPageSource; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; import com.facebook.presto.spi.FixedPageSource; import com.facebook.presto.spi.Plugin; import com.facebook.presto.spi.connector.Connector; @@ -43,6 +47,7 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -214,7 +219,13 @@ public ConnectorSplitManager getSplitManager() @Override public ConnectorPageSourceProvider getPageSourceProvider() { - return (transactionHandle, session, split, columns) -> new FixedPageSource(ImmutableList.of()); + return new ConnectorPageSourceProvider() { + @Override + public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, ConnectorTableLayoutHandle layout, List columns) + { + return new FixedPageSource(ImmutableList.of()); + } + }; } @Override diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestCompletedEventWarnings.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestCompletedEventWarnings.java index 5fa22f728b1a9..63a28bf55ed91 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestCompletedEventWarnings.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestCompletedEventWarnings.java @@ -26,7 +26,7 @@ import com.google.common.collect.ImmutableMap; import org.intellij.lang.annotations.Language; import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.util.List; @@ -39,6 +39,7 @@ import static com.google.common.collect.Iterables.getOnlyElement; import static org.testng.Assert.fail; +@Test(singleThreaded = true) public class TestCompletedEventWarnings { private static final int EXPECTED_EVENTS = 3; @@ -46,7 +47,7 @@ public class TestCompletedEventWarnings private QueryRunner queryRunner; private EventsBuilder generatedEvents; - @BeforeMethod + @BeforeClass public void setUp() throws Exception { diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestEventListener.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestEventListener.java index 496870e97fed0..4b761b34cb09a 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestEventListener.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestEventListener.java @@ -157,7 +157,7 @@ public void testNormalQuery() // Sum of row count processed by all leaf stages is equal to the number of rows in the table long actualCompletedPositions = splitCompletedEvents.stream() - .filter(e -> !e.getStageId().endsWith(".0")) // filter out the root stage + .filter(e -> !e.getStageExecutionId().endsWith(".0.0")) // filter out the root stage .mapToLong(e -> e.getStatistics().getCompletedPositions()) .sum(); diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestFinalQueryInfo.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestFinalQueryInfo.java index f4e5e842ac2fb..9f364201f6065 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestFinalQueryInfo.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestFinalQueryInfo.java @@ -29,9 +29,9 @@ import java.util.Locale; import java.util.Optional; +import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.client.StatementClientFactory.newStatementClient; -import static io.airlift.concurrent.MoreFutures.tryGetFutureValue; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static org.testng.Assert.assertTrue; @@ -71,7 +71,6 @@ private static QueryId startQuery(String sql, DistributedQueryRunner queryRunner null, null, null, - null, "America/Los_Angeles", Locale.ENGLISH, ImmutableMap.of(), diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestQueryRunnerUtil.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestQueryRunnerUtil.java index aae63ed6b80e1..516c3cb086fec 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestQueryRunnerUtil.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestQueryRunnerUtil.java @@ -24,9 +24,9 @@ import java.util.Map; import java.util.Set; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.execution.QueryState.RUNNING; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.concurrent.MoreFutures.getFutureValue; import static java.util.concurrent.TimeUnit.MILLISECONDS; public final class TestQueryRunnerUtil diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestWarnings.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestWarnings.java index 99fef30943bc8..1357c407eecc3 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestWarnings.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestWarnings.java @@ -27,10 +27,10 @@ import java.util.List; import java.util.Set; +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.execution.TestQueryRunnerUtil.createQueryRunner; import static com.facebook.presto.spi.StandardWarningCode.PARSER_WARNING; import static com.facebook.presto.spi.StandardWarningCode.TOO_MANY_STAGES; -import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static org.testng.Assert.fail; @@ -64,23 +64,29 @@ public void testStageCountWarningThreshold() .append(stageIndex); } String query = queryBuilder.toString(); - Session session = testSessionBuilder() - .setCatalog("tpch") - .setSchema("tiny") - .build(); - assertWarnings(queryRunner, session, query, ImmutableList.of(TOO_MANY_STAGES.toWarningCode())); - assertWarnings(queryRunner, session, noWarningsQuery, ImmutableList.of()); + assertWarnings(queryRunner, TEST_SESSION, query, ImmutableList.of(TOO_MANY_STAGES.toWarningCode())); + assertWarnings(queryRunner, TEST_SESSION, noWarningsQuery, ImmutableList.of()); } @Test public void testNonReservedWordWarning() { String query = "SELECT CURRENT_ROLE, t.current_role FROM (VALUES (3)) t(current_role)"; - Session session = testSessionBuilder() - .setCatalog("tpch") - .setSchema("tiny") - .build(); - assertWarnings(queryRunner, session, query, ImmutableList.of(PARSER_WARNING.toWarningCode())); + assertWarnings(queryRunner, TEST_SESSION, query, ImmutableList.of(PARSER_WARNING.toWarningCode())); + } + + @Test + public void testNewReservedWordsWarning() + { + String query = "SELECT CALLED, t.called FROM (VALUES (3)) t(called)"; + assertWarnings(queryRunner, TEST_SESSION, query, ImmutableList.of(PARSER_WARNING.toWarningCode())); + } + + @Test + public void testQuotedIdentifiersDoNotTriggerWarning() + { + String query = "SELECT \"CALLED\" FROM (VALUES (3)) t(\"called\")"; + assertWarnings(queryRunner, TEST_SESSION, query, ImmutableList.of()); } private static void assertWarnings(QueryRunner queryRunner, Session session, @Language("SQL") String sql, List expectedWarnings) diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestingSessionContext.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestingSessionContext.java index e48a174726dce..a7c1054d3c3b7 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestingSessionContext.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestingSessionContext.java @@ -56,12 +56,6 @@ public String getSchema() return session.getSchema().orElse(null); } - @Override - public String getPath() - { - return session.getPath().toString(); - } - @Override public String getSource() { @@ -98,12 +92,6 @@ public Set getClientTags() return session.getClientTags(); } - @Override - public Set getClientCapabilities() - { - return session.getClientCapabilities(); - } - @Override public ResourceEstimates getResourceEstimates() { diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/TestResourceGroupIntegration.java b/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/TestResourceGroupIntegration.java index 5eafe5db9620c..28ae0502fb91a 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/TestResourceGroupIntegration.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/TestResourceGroupIntegration.java @@ -23,8 +23,8 @@ import java.util.List; +import static com.facebook.airlift.testing.Assertions.assertLessThan; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.testing.Assertions.assertLessThan; import static io.airlift.units.Duration.nanosSince; import static java.util.concurrent.TimeUnit.SECONDS; import static org.testng.Assert.assertEquals; diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/H2ResourceGroupConfigurationManagerFactory.java b/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/H2ResourceGroupConfigurationManagerFactory.java index 6f5e88dae8a60..167d80fbec71e 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/H2ResourceGroupConfigurationManagerFactory.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/H2ResourceGroupConfigurationManagerFactory.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.execution.resourceGroups.db; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; +import com.facebook.airlift.node.NodeModule; import com.facebook.presto.resourceGroups.VariableMap; import com.facebook.presto.resourceGroups.db.DbResourceGroupConfigurationManager; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; @@ -21,9 +24,6 @@ import com.facebook.presto.spi.resourceGroups.ResourceGroupConfigurationManagerContext; import com.facebook.presto.spi.resourceGroups.ResourceGroupConfigurationManagerFactory; import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; -import io.airlift.node.NodeModule; import java.util.Map; diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/H2ResourceGroupsModule.java b/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/H2ResourceGroupsModule.java index 207e26460db77..a06cf6c18cc77 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/H2ResourceGroupsModule.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/H2ResourceGroupsModule.java @@ -27,7 +27,7 @@ import javax.inject.Singleton; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; public class H2ResourceGroupsModule implements Module diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/H2TestUtil.java b/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/H2TestUtil.java index d279fd975276d..c4f3a290913ff 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/H2TestUtil.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/H2TestUtil.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution.resourceGroups.db; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.Session; import com.facebook.presto.execution.QueryManager; import com.facebook.presto.execution.QueryState; @@ -28,18 +29,17 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.json.JsonCodec; import java.util.List; import java.util.Random; import java.util.Set; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; import static com.facebook.presto.execution.QueryState.RUNNING; import static com.facebook.presto.execution.QueryState.TERMINAL_QUERY_STATES; import static com.facebook.presto.spi.StandardErrorCode.CONFIGURATION_INVALID; import static com.facebook.presto.spi.resourceGroups.QueryType.EXPLAIN; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.json.JsonCodec.listJsonCodec; import static java.util.concurrent.TimeUnit.MILLISECONDS; class H2TestUtil diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/TestQueuesDb.java b/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/TestQueuesDb.java index 031b26aba2f3b..3b5417bcd8312 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/TestQueuesDb.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/resourceGroups/db/TestQueuesDb.java @@ -33,6 +33,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; +import static com.facebook.airlift.testing.Assertions.assertContains; import static com.facebook.presto.SystemSessionProperties.QUERY_MAX_EXECUTION_TIME; import static com.facebook.presto.execution.QueryState.FAILED; import static com.facebook.presto.execution.QueryState.FINISHED; @@ -56,7 +57,6 @@ import static com.facebook.presto.spi.StandardErrorCode.INVALID_RESOURCE_GROUP; import static com.facebook.presto.spi.StandardErrorCode.QUERY_REJECTED; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.testing.Assertions.assertContains; import static java.lang.String.format; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.testng.Assert.assertEquals; diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedSpilledQueries.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedSpilledQueries.java index d3fe51ad93849..50e017841c9bd 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedSpilledQueries.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedSpilledQueries.java @@ -68,4 +68,10 @@ public void testAssignUniqueId() // TODO: disabled until https://github.com/prestodb/presto/issues/8926 is resolved // due to long running query test created many spill files on disk. } + + @Override + public void testLimitWithJoin() + { + // TODO: disable until https://github.com/prestodb/presto/issues/13859 is resolved. + } } diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestLocalQueries.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestLocalQueries.java index 6a0bb87757902..a3dd09ce6e1dd 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestLocalQueries.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestLocalQueries.java @@ -32,6 +32,7 @@ import java.util.Optional; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.SystemSessionProperties.PUSH_PARTIAL_AGGREGATION_THROUGH_JOIN; import static com.facebook.presto.spi.predicate.Marker.Bound.EXACTLY; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; @@ -43,7 +44,6 @@ import static com.facebook.presto.testing.assertions.Assert.assertEquals; import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.json.JsonCodec.jsonCodec; public class TestLocalQueries extends AbstractTestQueries @@ -70,7 +70,7 @@ public static LocalQueryRunner createLocalQueryRunner() new TpchConnectorFactory(1), ImmutableMap.of()); - localQueryRunner.getMetadata().addFunctions(CUSTOM_FUNCTIONS); + localQueryRunner.getMetadata().registerBuiltInFunctions(CUSTOM_FUNCTIONS); SessionPropertyManager sessionPropertyManager = localQueryRunner.getMetadata().getSessionPropertyManager(); sessionPropertyManager.addSystemSessionProperties(TEST_SYSTEM_PROPERTIES); diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryPlanDeterminism.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryPlanDeterminism.java index 05fcd17b83d55..3c72675416d1a 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryPlanDeterminism.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryPlanDeterminism.java @@ -71,7 +71,7 @@ private static LocalQueryRunner createLocalQueryRunner() new TpchConnectorFactory(1), ImmutableMap.of()); - localQueryRunner.getMetadata().addFunctions(CUSTOM_FUNCTIONS); + localQueryRunner.getMetadata().registerBuiltInFunctions(CUSTOM_FUNCTIONS); SessionPropertyManager sessionPropertyManager = localQueryRunner.getMetadata().getSessionPropertyManager(); sessionPropertyManager.addSystemSessionProperties(TEST_SYSTEM_PROPERTIES); diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryTaskLimit.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryTaskLimit.java new file mode 100644 index 0000000000000..9ab62af845df9 --- /dev/null +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryTaskLimit.java @@ -0,0 +1,107 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests; + +import com.facebook.presto.Session; +import com.facebook.presto.server.BasicQueryInfo; +import com.facebook.presto.tpch.TpchPlugin; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import static com.facebook.presto.spi.StandardErrorCode.QUERY_HAS_TOO_MANY_STAGES; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static java.util.concurrent.Executors.newCachedThreadPool; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + +public class TestQueryTaskLimit +{ + private ExecutorService executor; + + @BeforeClass + public void setUp() + { + executor = newCachedThreadPool(); + } + + @AfterClass(alwaysRun = true) + public void shutdown() + { + executor.shutdownNow(); + executor = null; + } + + @Test(timeOut = 30_000, expectedExceptions = ExecutionException.class, expectedExceptionsMessageRegExp = ".*Query killed because the cluster is overloaded with too many tasks.*") + public void testExceedTaskLimit() + throws Exception + { + Session defaultSession = testSessionBuilder() + .setCatalog("tpch") + .setSchema("sf1000") + .build(); + + ImmutableMap extraProperties = ImmutableMap.builder() + .put("experimental.max-total-running-task-count", "4") + .put("experimental.max-query-running-task-count", "4") + .build(); + + try (DistributedQueryRunner queryRunner = createQueryRunner(defaultSession, extraProperties)) { + Future query = executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")); + + waitForQueryToBeKilled(queryRunner); + + query.get(); + } + } + + public static DistributedQueryRunner createQueryRunner(Session session, Map properties) + throws Exception + { + DistributedQueryRunner queryRunner = new DistributedQueryRunner(session, 2, properties); + + try { + queryRunner.installPlugin(new TpchPlugin()); + queryRunner.createCatalog("tpch", "tpch"); + return queryRunner; + } + catch (Exception e) { + queryRunner.close(); + throw e; + } + } + + private void waitForQueryToBeKilled(DistributedQueryRunner queryRunner) + throws InterruptedException + { + while (true) { + for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) { + if (info.getState().isDone()) { + assertNotNull(info.getErrorCode()); + assertEquals(info.getErrorCode().getCode(), QUERY_HAS_TOO_MANY_STAGES.toErrorCode().getCode()); + MILLISECONDS.sleep(100); + return; + } + } + MILLISECONDS.sleep(10); + } + } +} diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestRepartitionQueries.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestRepartitionQueries.java new file mode 100644 index 0000000000000..c3a7c2fb4a685 --- /dev/null +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestRepartitionQueries.java @@ -0,0 +1,27 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests; + +import com.facebook.presto.tests.tpch.TpchQueryRunnerBuilder; + +public class TestRepartitionQueries + extends AbstractTestRepartitionQueries +{ + public TestRepartitionQueries() + { + super(() -> TpchQueryRunnerBuilder.builder() + .setSingleExtraProperty("experimental.optimized-repartitioning", "true") + .build()); + } +} diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestWithOptimizedWriter.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestRepartitionQueriesWithSmallPages.java similarity index 50% rename from presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestWithOptimizedWriter.java rename to presto-tests/src/test/java/com/facebook/presto/tests/TestRepartitionQueriesWithSmallPages.java index 28eff3d093f29..25b40ad966078 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/integration/TestRaptorIntegrationSmokeTestWithOptimizedWriter.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestRepartitionQueriesWithSmallPages.java @@ -11,23 +11,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.raptor.integration; +package com.facebook.presto.tests; +import com.facebook.presto.tests.tpch.TpchQueryRunnerBuilder; import com.google.common.collect.ImmutableMap; -import static com.facebook.presto.raptor.RaptorQueryRunner.createRaptorQueryRunner; - -public class TestRaptorIntegrationSmokeTestWithOptimizedWriter - extends TestRaptorIntegrationSmokeTest +public class TestRepartitionQueriesWithSmallPages + extends AbstractTestRepartitionQueries { - @SuppressWarnings("unused") - public TestRaptorIntegrationSmokeTestWithOptimizedWriter() - { - this(() -> createRaptorQueryRunner(ImmutableMap.of(), true, false, ImmutableMap.of("storage.orc.optimized-writer-stage", "ENABLED_AND_VALIDATED"))); - } - - protected TestRaptorIntegrationSmokeTestWithOptimizedWriter(QueryRunnerSupplier supplier) + protected TestRepartitionQueriesWithSmallPages() { - super(supplier); + super(() -> TpchQueryRunnerBuilder.builder() + .setExtraProperties( + ImmutableMap.of("experimental.optimized-repartitioning", "true", + // Use small SerializedPages to force flushing + "driver.max-page-partitioning-buffer-size", "200B")) + .build()); } } diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestSqlFunctions.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestSqlFunctions.java new file mode 100644 index 0000000000000..9a14d7f5c4265 --- /dev/null +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestSqlFunctions.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests; + +import com.facebook.presto.tests.tpch.TpchQueryRunnerBuilder; +import org.testng.annotations.Test; + +public class TestSqlFunctions + extends AbstractTestQueryFramework +{ + protected TestSqlFunctions() + { + super(() -> TpchQueryRunnerBuilder.builder().build()); + } + + @Test + public void testCreateFunctionInvalidFunctionName() + { + assertQueryFails( + "CREATE FUNCTION testing.tan (x int) RETURNS double COMMENT 'tangent trigonometric function' RETURN sin(x) / cos(x)", + ".*Function name should be in the form of catalog\\.schema\\.function_name, found: testing\\.tan"); + assertQueryFails( + "CREATE FUNCTION presto.default.tan (x int) RETURNS double COMMENT 'tangent trigonometric function' RETURN sin(x) / cos(x)", + "Cannot create function in built-in function namespace: presto\\.default\\.tan"); + } + + @Test + public void testCreateFunctionInvalidSemantics() + { + assertQueryFails( + "CREATE FUNCTION testing.default.tan (x int) RETURNS varchar COMMENT 'tangent trigonometric function' RETURN sin(x) / cos(x)", + "Function implementation type 'double' does not match declared return type 'varchar'"); + assertQueryFails( + "CREATE FUNCTION testing.default.tan (x int) RETURNS varchar COMMENT 'tangent trigonometric function' RETURN sin(y) / cos(y)", + ".*Column 'y' cannot be resolved"); + assertQueryFails( + "CREATE FUNCTION testing.default.tan (x double) RETURNS double COMMENT 'tangent trigonometric function' RETURN sum(x)", + ".*CREATE FUNCTION body cannot contain aggregations, window functions or grouping operations:.*"); + } + + @Test + public void testDropFunctionInvalidFunctionName() + { + assertQueryFails( + "DROP FUNCTION IF EXISTS testing.tan", + ".*Function name should be in the form of catalog\\.schema\\.function_name, found: testing\\.tan"); + assertQueryFails( + "DROP FUNCTION presto.default.tan (double)", + "Cannot drop function in built-in function namespace: presto\\.default\\.tan"); + } +} diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/tpch/TpchQueryRunner.java b/presto-tests/src/test/java/com/facebook/presto/tests/tpch/TpchQueryRunner.java index c1aff11e4dd44..7dd7e7f13545c 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/tpch/TpchQueryRunner.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/tpch/TpchQueryRunner.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.tests.tpch; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.log.Logging; import com.facebook.presto.tests.DistributedQueryRunner; -import io.airlift.log.Logger; -import io.airlift.log.Logging; public final class TpchQueryRunner { diff --git a/presto-thrift-connector-api/pom.xml b/presto-thrift-connector-api/pom.xml index a3eb9d6c1454d..1afb8337c4a25 100644 --- a/presto-thrift-connector-api/pom.xml +++ b/presto-thrift-connector-api/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-thrift-connector-api @@ -62,7 +62,7 @@ - io.airlift + com.facebook.airlift stats test diff --git a/presto-thrift-connector-api/src/test/java/com/facebook/presto/connector/thrift/api/TestReadWrite.java b/presto-thrift-connector-api/src/test/java/com/facebook/presto/connector/thrift/api/TestReadWrite.java index f2ffde83cac5a..d8003768fe5a6 100644 --- a/presto-thrift-connector-api/src/test/java/com/facebook/presto/connector/thrift/api/TestReadWrite.java +++ b/presto-thrift-connector-api/src/test/java/com/facebook/presto/connector/thrift/api/TestReadWrite.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.thrift.api; +import com.facebook.airlift.stats.cardinality.HyperLogLog; import com.facebook.presto.operator.index.PageRecordSet; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; @@ -22,7 +23,6 @@ import com.facebook.presto.spi.type.VarcharType; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; -import io.airlift.stats.cardinality.HyperLogLog; import org.testng.annotations.Test; import java.util.ArrayList; diff --git a/presto-thrift-connector/pom.xml b/presto-thrift-connector/pom.xml index e91d05d8836e7..da3494ca63331 100644 --- a/presto-thrift-connector/pom.xml +++ b/presto-thrift-connector/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-thrift-connector @@ -59,12 +59,12 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift log @@ -74,7 +74,7 @@ - io.airlift + com.facebook.airlift stats @@ -89,7 +89,7 @@ - io.airlift + com.facebook.airlift configuration @@ -99,13 +99,13 @@ - io.airlift + com.facebook.airlift concurrent - io.airlift + com.facebook.airlift log-manager runtime @@ -161,7 +161,7 @@ - io.airlift + com.facebook.airlift testing test diff --git a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnector.java b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnector.java index 08706cc74ecc1..a038917341ef0 100644 --- a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnector.java +++ b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnector.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.connector.thrift; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorIndexProvider; import com.facebook.presto.spi.connector.ConnectorMetadata; @@ -21,8 +23,6 @@ import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spi.transaction.IsolationLevel; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import javax.inject.Inject; @@ -101,10 +101,6 @@ public final void shutdown() try { lifeCycleManager.stop(); } - catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - log.error(ie, "Interrupted while shutting down connector"); - } catch (Exception e) { log.error(e, "Error shutting down connector"); } diff --git a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorConfig.java b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorConfig.java index ce482cec3b840..89befcb247501 100644 --- a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorConfig.java +++ b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.connector.thrift; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import io.airlift.units.DataSize; import io.airlift.units.MaxDataSize; import io.airlift.units.MinDataSize; diff --git a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorFactory.java b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorFactory.java index a24a2d8d7d107..0de774d246207 100644 --- a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorFactory.java +++ b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorFactory.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.thrift; +import com.facebook.airlift.bootstrap.Bootstrap; import com.facebook.drift.transport.netty.client.DriftNettyClientModule; import com.facebook.presto.connector.thrift.util.RebindSafeMBeanServer; import com.facebook.presto.spi.ConnectorHandleResolver; @@ -22,7 +23,6 @@ import com.facebook.presto.spi.type.TypeManager; import com.google.inject.Injector; import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; import org.weakref.jmx.guice.MBeanModule; import javax.management.MBeanServer; @@ -79,10 +79,6 @@ public Connector create(String catalogName, Map config, Connecto return injector.getInstance(ThriftConnector.class); } - catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted while creating connector", ie); - } catch (Exception e) { throwIfUnchecked(e); throw new RuntimeException(e); diff --git a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorStats.java b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorStats.java index b292bf9f0fcba..5a0cd537731cc 100644 --- a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorStats.java +++ b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorStats.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.connector.thrift; -import io.airlift.stats.DistributionStat; +import com.facebook.airlift.stats.DistributionStat; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; diff --git a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftIndexPageSource.java b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftIndexPageSource.java index d4abc22a374fd..fa3a7c58dc86d 100644 --- a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftIndexPageSource.java +++ b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftIndexPageSource.java @@ -44,6 +44,9 @@ import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicLong; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.MoreFutures.toCompletableFuture; +import static com.facebook.airlift.concurrent.MoreFutures.whenAnyComplete; import static com.facebook.presto.connector.thrift.api.PrestoThriftPageResult.fromRecordSet; import static com.facebook.presto.connector.thrift.util.ThriftExceptions.catchingThriftException; import static com.facebook.presto.connector.thrift.util.TupleDomainConversion.tupleDomainToThriftTupleDomain; @@ -52,9 +55,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Futures.nonCancellationPropagating; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.MoreFutures.toCompletableFuture; -import static io.airlift.concurrent.MoreFutures.whenAnyComplete; import static java.lang.Math.min; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; @@ -77,6 +77,7 @@ public class ThriftIndexPageSource private final AtomicLong readTimeNanos = new AtomicLong(0); private long completedBytes; + private long completedPositions; private CompletableFuture statusFuture; private ListenableFuture splitFuture; @@ -143,6 +144,12 @@ public long getCompletedBytes() return completedBytes; } + @Override + public long getCompletedPositions() + { + return completedPositions; + } + @Override public long getReadTimeNanos() { @@ -206,6 +213,7 @@ public Page getNextPage() if (page != null) { long pageSize = page.getSizeInBytes(); completedBytes += pageSize; + completedPositions += page.getPositionCount(); stats.addIndexPageSize(pageSize); } else { diff --git a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftModule.java b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftModule.java index 501108c90e620..f221fb65c0486 100644 --- a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftModule.java +++ b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftModule.java @@ -28,10 +28,10 @@ import java.util.Optional; import java.util.concurrent.Executor; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static com.facebook.drift.client.ExceptionClassification.NORMAL_EXCEPTION; import static com.facebook.drift.client.guice.DriftClientBinder.driftClientBinder; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.airlift.configuration.ConfigBinder.configBinder; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newFixedThreadPool; import static org.weakref.jmx.ObjectNames.generatedNameOf; diff --git a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftPageSource.java b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftPageSource.java index 8b10422d1354e..472d3f7718ef9 100644 --- a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftPageSource.java +++ b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftPageSource.java @@ -32,12 +32,12 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; +import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; +import static com.facebook.airlift.concurrent.MoreFutures.toCompletableFuture; import static com.facebook.presto.connector.thrift.util.ThriftExceptions.catchingThriftException; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.util.concurrent.Futures.nonCancellationPropagating; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.airlift.concurrent.MoreFutures.toCompletableFuture; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; @@ -56,6 +56,7 @@ public class ThriftPageSource private CompletableFuture future; private final ThriftConnectorStats stats; private long completedBytes; + private long completedPositions; public ThriftPageSource( DriftClient client, @@ -106,6 +107,12 @@ public long getCompletedBytes() return completedBytes; } + @Override + public long getCompletedPositions() + { + return completedPositions; + } + @Override public long getReadTimeNanos() { @@ -181,6 +188,7 @@ private Page processBatch(PrestoThriftPageResult rowsBatch) if (page != null) { long pageSize = page.getSizeInBytes(); completedBytes += pageSize; + completedPositions += page.getPositionCount(); stats.addScanPageSize(pageSize); } else { diff --git a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftSplitManager.java b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftSplitManager.java index 14fedd29a444d..22e98aa204f66 100644 --- a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftSplitManager.java +++ b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftSplitManager.java @@ -46,6 +46,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import static com.facebook.airlift.concurrent.MoreFutures.toCompletableFuture; import static com.facebook.presto.connector.thrift.util.ThriftExceptions.catchingThriftException; import static com.facebook.presto.connector.thrift.util.TupleDomainConversion.tupleDomainToThriftTupleDomain; import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; @@ -54,7 +55,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static io.airlift.concurrent.MoreFutures.toCompletableFuture; import static java.util.Objects.requireNonNull; public class ThriftSplitManager diff --git a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/location/ExtendedSimpleAddressSelectorBinder.java b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/location/ExtendedSimpleAddressSelectorBinder.java index 86523e7a6998d..6073b79444491 100644 --- a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/location/ExtendedSimpleAddressSelectorBinder.java +++ b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/location/ExtendedSimpleAddressSelectorBinder.java @@ -24,7 +24,7 @@ import java.lang.annotation.Annotation; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; public class ExtendedSimpleAddressSelectorBinder implements AddressSelectorBinder diff --git a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/util/RebindSafeMBeanServer.java b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/util/RebindSafeMBeanServer.java index f8fec2de99f88..1708349f9b35d 100644 --- a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/util/RebindSafeMBeanServer.java +++ b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/util/RebindSafeMBeanServer.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.connector.thrift.util; -import io.airlift.log.Logger; +import com.facebook.airlift.log.Logger; import javax.annotation.concurrent.ThreadSafe; import javax.management.Attribute; diff --git a/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/TestThriftConnectorConfig.java b/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/TestThriftConnectorConfig.java index 37a45535bf67b..d2f804b063f1b 100644 --- a/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/TestThriftConnectorConfig.java +++ b/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/TestThriftConnectorConfig.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.connector.thrift; +import com.facebook.airlift.configuration.testing.ConfigAssertions; import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; import io.airlift.units.DataSize; import org.testng.annotations.Test; diff --git a/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/TestThriftPlugin.java b/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/TestThriftPlugin.java index 92e546bfec14e..ec53a43f6a2f2 100644 --- a/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/TestThriftPlugin.java +++ b/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/TestThriftPlugin.java @@ -23,8 +23,8 @@ import java.util.Map; import java.util.ServiceLoader; +import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.google.common.collect.Iterables.getOnlyElement; -import static io.airlift.testing.Assertions.assertInstanceOf; import static org.testng.Assert.assertNotNull; public class TestThriftPlugin diff --git a/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/integration/ThriftQueryRunner.java b/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/integration/ThriftQueryRunner.java index 3e9a755d405dd..5051a4ff92df4 100644 --- a/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/integration/ThriftQueryRunner.java +++ b/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/integration/ThriftQueryRunner.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.connector.thrift.integration; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.log.Logging; import com.facebook.drift.codec.ThriftCodecManager; import com.facebook.drift.server.DriftServer; import com.facebook.drift.server.DriftService; @@ -41,16 +43,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.airlift.log.Logger; -import io.airlift.log.Logging; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.locks.Lock; +import static com.facebook.airlift.testing.Closeables.closeQuietly; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static io.airlift.testing.Closeables.closeQuietly; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; diff --git a/presto-thrift-testing-server/pom.xml b/presto-thrift-testing-server/pom.xml index 0ebdda709d58e..ce596957516a9 100644 --- a/presto-thrift-testing-server/pom.xml +++ b/presto-thrift-testing-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-thrift-testing-server @@ -55,12 +55,12 @@ - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift bootstrap @@ -75,7 +75,7 @@ - io.airlift + com.facebook.airlift concurrent @@ -90,7 +90,7 @@ - io.airlift + com.facebook.airlift json diff --git a/presto-thrift-testing-server/src/main/java/com/facebook/presto/connector/thrift/server/ThriftTpchServer.java b/presto-thrift-testing-server/src/main/java/com/facebook/presto/connector/thrift/server/ThriftTpchServer.java index 181fa4609202e..c2179e754af97 100644 --- a/presto-thrift-testing-server/src/main/java/com/facebook/presto/connector/thrift/server/ThriftTpchServer.java +++ b/presto-thrift-testing-server/src/main/java/com/facebook/presto/connector/thrift/server/ThriftTpchServer.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.connector.thrift.server; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.log.Logger; import com.facebook.drift.transport.netty.server.DriftNettyServerModule; import com.google.common.collect.ImmutableList; import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.log.Logger; import java.util.List; diff --git a/presto-thrift-testing-server/src/main/java/com/facebook/presto/connector/thrift/server/ThriftTpchService.java b/presto-thrift-testing-server/src/main/java/com/facebook/presto/connector/thrift/server/ThriftTpchService.java index 061698ec39a67..fa987657331ff 100644 --- a/presto-thrift-testing-server/src/main/java/com/facebook/presto/connector/thrift/server/ThriftTpchService.java +++ b/presto-thrift-testing-server/src/main/java/com/facebook/presto/connector/thrift/server/ThriftTpchService.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.connector.thrift.server; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.connector.thrift.api.PrestoThriftBlock; import com.facebook.presto.connector.thrift.api.PrestoThriftColumnMetadata; import com.facebook.presto.connector.thrift.api.PrestoThriftId; @@ -37,7 +38,6 @@ import com.google.common.primitives.Ints; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import io.airlift.json.JsonCodec; import io.airlift.tpch.TpchColumn; import io.airlift.tpch.TpchEntity; import io.airlift.tpch.TpchTable; @@ -50,6 +50,8 @@ import java.util.List; import java.util.Set; +import static com.facebook.airlift.concurrent.Threads.threadsNamed; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.connector.thrift.api.PrestoThriftBlock.fromBlock; import static com.facebook.presto.connector.thrift.server.SplitInfo.normalSplit; import static com.facebook.presto.spi.block.PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES; @@ -58,8 +60,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; -import static io.airlift.concurrent.Threads.threadsNamed; -import static io.airlift.json.JsonCodec.jsonCodec; import static java.lang.Math.min; import static java.util.concurrent.Executors.newFixedThreadPool; import static java.util.stream.Collectors.toList; diff --git a/presto-tpcds/pom.xml b/presto-tpcds/pom.xml index 95a723b066e8d..7c0673c6db121 100644 --- a/presto-tpcds/pom.xml +++ b/presto-tpcds/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-tpcds @@ -32,7 +32,7 @@ - io.airlift + com.facebook.airlift json @@ -104,13 +104,13 @@ - io.airlift + com.facebook.airlift log test - io.airlift + com.facebook.airlift log-manager test diff --git a/presto-tpcds/src/main/java/com/facebook/presto/tpcds/TpcdsMetadata.java b/presto-tpcds/src/main/java/com/facebook/presto/tpcds/TpcdsMetadata.java index 3a30a50b1c38d..9d5ca71916d5f 100644 --- a/presto-tpcds/src/main/java/com/facebook/presto/tpcds/TpcdsMetadata.java +++ b/presto-tpcds/src/main/java/com/facebook/presto/tpcds/TpcdsMetadata.java @@ -148,14 +148,14 @@ private static ConnectorTableMetadata getTableMetadata(String schemaName, Table } @Override - public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint) + public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Optional tableLayoutHandle, List columnHandles, Constraint constraint) { TpcdsTableHandle tpcdsTableHandle = (TpcdsTableHandle) tableHandle; Table table = Table.getTable(tpcdsTableHandle.getTableName()); String schemaName = scaleFactorSchemaName(tpcdsTableHandle.getScaleFactor()); - return tpcdsTableStatisticsFactory.create(schemaName, table, getColumnHandles(session, tableHandle)); + return tpcdsTableStatisticsFactory.create(schemaName, table, columnHandles); } @Override diff --git a/presto-tpcds/src/main/java/com/facebook/presto/tpcds/statistics/TpcdsTableStatisticsFactory.java b/presto-tpcds/src/main/java/com/facebook/presto/tpcds/statistics/TpcdsTableStatisticsFactory.java index 3b40f1243f970..d6019e2fdcbae 100644 --- a/presto-tpcds/src/main/java/com/facebook/presto/tpcds/statistics/TpcdsTableStatisticsFactory.java +++ b/presto-tpcds/src/main/java/com/facebook/presto/tpcds/statistics/TpcdsTableStatisticsFactory.java @@ -29,6 +29,7 @@ import io.airlift.slice.Slice; import java.time.LocalDate; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -45,14 +46,14 @@ public class TpcdsTableStatisticsFactory { private final TableStatisticsDataRepository statisticsDataRepository = new TableStatisticsDataRepository(); - public TableStatistics create(String schemaName, Table table, Map columnHandles) + public TableStatistics create(String schemaName, Table table, List columnHandles) { Optional statisticsDataOptional = statisticsDataRepository.load(schemaName, table); return statisticsDataOptional.map(statisticsData -> toTableStatistics(columnHandles, statisticsData)) .orElse(TableStatistics.empty()); } - private TableStatistics toTableStatistics(Map columnHandles, TableStatisticsData statisticsData) + private TableStatistics toTableStatistics(List columnHandles, TableStatisticsData statisticsData) { long rowCount = statisticsData.getRowCount(); TableStatistics.Builder tableStatistics = TableStatistics.builder() @@ -60,9 +61,9 @@ private TableStatistics toTableStatistics(Map columnHandle if (rowCount > 0) { Map columnsData = statisticsData.getColumns(); - for (Map.Entry entry : columnHandles.entrySet()) { - TpcdsColumnHandle columnHandle = (TpcdsColumnHandle) entry.getValue(); - tableStatistics.setColumnStatistics(entry.getValue(), toColumnStatistics(columnsData.get(entry.getKey()), columnHandle.getType(), rowCount)); + for (ColumnHandle handle : columnHandles) { + TpcdsColumnHandle columnHandle = (TpcdsColumnHandle) handle; + tableStatistics.setColumnStatistics(columnHandle, toColumnStatistics(columnsData.get(columnHandle.getColumnName()), columnHandle.getType(), rowCount)); } } diff --git a/presto-tpcds/src/test/java/com/facebook/presto/tpcds/TestTpcds.java b/presto-tpcds/src/test/java/com/facebook/presto/tpcds/TestTpcds.java index 93e220e24c514..fd161db3e760f 100644 --- a/presto-tpcds/src/test/java/com/facebook/presto/tpcds/TestTpcds.java +++ b/presto-tpcds/src/test/java/com/facebook/presto/tpcds/TestTpcds.java @@ -71,9 +71,4 @@ public void testLargeInWithShortDecimal() assertQuerySucceeds("SELECT i_current_price FROM item WHERE i_current_price IN (i_wholesale_cost, " + longValues + ")"); assertQuerySucceeds("SELECT i_current_price FROM item WHERE i_current_price NOT IN (i_wholesale_cost, " + longValues + ")"); } - - private void assertQuerySucceeds(String sql) - { - computeActual(sql); - } } diff --git a/presto-tpcds/src/test/java/com/facebook/presto/tpcds/TestTpcdsMetadataStatistics.java b/presto-tpcds/src/test/java/com/facebook/presto/tpcds/TestTpcdsMetadataStatistics.java index bcbffe6160476..fcaad7134a688 100644 --- a/presto-tpcds/src/test/java/com/facebook/presto/tpcds/TestTpcdsMetadataStatistics.java +++ b/presto-tpcds/src/test/java/com/facebook/presto/tpcds/TestTpcdsMetadataStatistics.java @@ -14,6 +14,7 @@ package com.facebook.presto.tpcds; +import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorTableHandle; @@ -22,13 +23,15 @@ import com.facebook.presto.spi.statistics.DoubleRange; import com.facebook.presto.spi.statistics.Estimate; import com.facebook.presto.spi.statistics.TableStatistics; +import com.google.common.collect.ImmutableList; import com.teradata.tpcds.Table; import com.teradata.tpcds.column.CallCenterColumn; import com.teradata.tpcds.column.WebSiteColumn; -import io.airlift.json.JsonCodec; import org.testng.annotations.Test; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Stream; import static com.facebook.presto.spi.Constraint.alwaysTrue; @@ -52,7 +55,8 @@ public void testNoTableStatsForNotSupportedSchema() .forEach(table -> { SchemaTableName schemaTableName = new SchemaTableName(schemaName, table.getName()); ConnectorTableHandle tableHandle = metadata.getTableHandle(session, schemaTableName); - TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, alwaysTrue()); + List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); + TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, Optional.empty(), columnHandles, alwaysTrue()); assertTrue(tableStatistics.getRowCount().isUnknown()); assertTrue(tableStatistics.getColumnStatistics().isEmpty()); })); @@ -66,7 +70,8 @@ public void testTableStatsExistenceSupportedSchema() .forEach(table -> { SchemaTableName schemaTableName = new SchemaTableName(schemaName, table.getName()); ConnectorTableHandle tableHandle = metadata.getTableHandle(session, schemaTableName); - TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, alwaysTrue()); + List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); + TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, Optional.empty(), columnHandles, alwaysTrue()); assertFalse(tableStatistics.getRowCount().isUnknown()); for (ColumnHandle column : metadata.getColumnHandles(session, tableHandle).values()) { assertTrue(tableStatistics.getColumnStatistics().containsKey(column)); @@ -80,12 +85,12 @@ public void testTableStatsDetails() { SchemaTableName schemaTableName = new SchemaTableName("sf1", Table.CALL_CENTER.getName()); ConnectorTableHandle tableHandle = metadata.getTableHandle(session, schemaTableName); - TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, alwaysTrue()); + Map columnHandles = metadata.getColumnHandles(session, tableHandle); + TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, Optional.empty(), ImmutableList.copyOf(columnHandles.values()), alwaysTrue()); estimateAssertion.assertClose(tableStatistics.getRowCount(), Estimate.of(6), "Row count does not match"); // all columns have stats - Map columnHandles = metadata.getColumnHandles(session, tableHandle); for (ColumnHandle column : columnHandles.values()) { assertTrue(tableStatistics.getColumnStatistics().containsKey(column)); assertNotNull(tableStatistics.getColumnStatistics().get(column)); @@ -150,9 +155,8 @@ public void testNullFraction() { SchemaTableName schemaTableName = new SchemaTableName("sf1", Table.WEB_SITE.getName()); ConnectorTableHandle tableHandle = metadata.getTableHandle(session, schemaTableName); - TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, alwaysTrue()); - Map columnHandles = metadata.getColumnHandles(session, tableHandle); + TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, Optional.empty(), ImmutableList.copyOf(columnHandles.values()), alwaysTrue()); // some null values assertColumnStatistics( @@ -169,7 +173,8 @@ public void testTableStatisticsSerialization() { SchemaTableName schemaTableName = new SchemaTableName("sf1", Table.WEB_SITE.getName()); ConnectorTableHandle tableHandle = metadata.getTableHandle(session, schemaTableName); - TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, alwaysTrue()); + List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values()); + TableStatistics tableStatistics = metadata.getTableStatistics(session, tableHandle, Optional.empty(), columnHandles, alwaysTrue()); Entry entry = tableStatistics.getColumnStatistics().entrySet().iterator().next(); diff --git a/presto-tpcds/src/test/java/com/facebook/presto/tpcds/TpcdsQueryRunner.java b/presto-tpcds/src/test/java/com/facebook/presto/tpcds/TpcdsQueryRunner.java index a72342e2b0bb3..a506e5a7c6e5c 100644 --- a/presto-tpcds/src/test/java/com/facebook/presto/tpcds/TpcdsQueryRunner.java +++ b/presto-tpcds/src/test/java/com/facebook/presto/tpcds/TpcdsQueryRunner.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.tpcds; +import com.facebook.airlift.log.Logger; +import com.facebook.airlift.log.Logging; import com.facebook.presto.Session; import com.facebook.presto.tests.DistributedQueryRunner; import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import io.airlift.log.Logging; import java.util.Map; diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 97b1fbc6cdec5..3a312e3fd979c 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-tpch diff --git a/presto-tpch/src/main/java/com/facebook/presto/tpch/TpchMetadata.java b/presto-tpch/src/main/java/com/facebook/presto/tpch/TpchMetadata.java index 9a64cb58f38ba..5330054e6d907 100644 --- a/presto-tpch/src/main/java/com/facebook/presto/tpch/TpchMetadata.java +++ b/presto-tpch/src/main/java/com/facebook/presto/tpch/TpchMetadata.java @@ -68,6 +68,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import static com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT; @@ -78,6 +79,7 @@ import static com.facebook.presto.spi.type.VarcharType.createVarcharType; import static com.facebook.presto.tpch.util.PredicateUtils.convertToPredicate; import static com.facebook.presto.tpch.util.PredicateUtils.filterOutColumnFromPredicate; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Maps.asMap; import static io.airlift.tpch.OrderColumn.ORDER_STATUS; @@ -320,7 +322,7 @@ public Map> listTableColumns(ConnectorSess } @Override - public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint) + public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Optional tableLayoutHandle, List columnHandles, Constraint constraint) { TpchTableHandle tpchTableHandle = (TpchTableHandle) tableHandle; String tableName = tpchTableHandle.getTableName(); @@ -331,7 +333,6 @@ public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTab } Optional optionalTableStatisticsData = statisticsEstimator.estimateStats(tpchTable, columnValuesRestrictions, tpchTableHandle.getScaleFactor()); - Map columnHandles = getColumnHandles(session, tpchTableHandle); return optionalTableStatisticsData .map(tableStatisticsData -> toTableStatistics(optionalTableStatisticsData.get(), tpchTableHandle, columnHandles)) .orElse(TableStatistics.empty()); @@ -370,23 +371,25 @@ private Map, List> avoidTrivialOrderStatusRestriction(List } } - private TableStatistics toTableStatistics(TableStatisticsData tableStatisticsData, TpchTableHandle tpchTableHandle, Map columnHandles) + private TableStatistics toTableStatistics(TableStatisticsData tableStatisticsData, TpchTableHandle tpchTableHandle, List columnHandles) { + TpchTable table = TpchTable.getTable(tpchTableHandle.getTableName()); + + Map columnHandleByName = columnHandles.stream() + .map(TpchColumnHandle.class::cast) + .collect(toImmutableMap(TpchColumnHandle::getColumnName, Function.identity())); + TableStatistics.Builder builder = TableStatistics.builder() .setRowCount(Estimate.of(tableStatisticsData.getRowCount())); tableStatisticsData.getColumns().forEach((columnName, stats) -> { - TpchColumnHandle columnHandle = (TpchColumnHandle) getColumnHandle(tpchTableHandle, columnHandles, columnName); - builder.setColumnStatistics(columnHandle, toColumnStatistics(stats, columnHandle.getType())); + TpchColumnHandle columnHandle = columnHandleByName.get(columnNaming.getName(table.getColumn(columnName))); + if (columnHandle != null) { + builder.setColumnStatistics(columnHandle, toColumnStatistics(stats, columnHandle.getType())); + } }); return builder.build(); } - private ColumnHandle getColumnHandle(TpchTableHandle tpchTableHandle, Map columnHandles, String columnName) - { - TpchTable table = TpchTable.getTable(tpchTableHandle.getTableName()); - return columnHandles.get(columnNaming.getName(table.getColumn(columnName))); - } - private ColumnStatistics toColumnStatistics(ColumnStatisticsData stats, Type columnType) { return ColumnStatistics.builder() diff --git a/presto-tpch/src/test/java/com/facebook/presto/tpch/TestTpchMetadata.java b/presto-tpch/src/test/java/com/facebook/presto/tpch/TestTpchMetadata.java index fea4307cb5ea0..16987b5fa5410 100644 --- a/presto-tpch/src/test/java/com/facebook/presto/tpch/TestTpchMetadata.java +++ b/presto-tpch/src/test/java/com/facebook/presto/tpch/TestTpchMetadata.java @@ -136,7 +136,8 @@ private void testTableStats(String schema, TpchTable table, double expectedRo private void testTableStats(String schema, TpchTable table, Constraint constraint, double expectedRowCount) { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName(schema, table.getTableName())); - TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, constraint); + List columnHandles = ImmutableList.copyOf(tpchMetadata.getColumnHandles(session, tableHandle).values()); + TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, Optional.empty(), columnHandles, constraint); double actualRowCountValue = tableStatistics.getRowCount().getValue(); assertEquals(tableStatistics.getRowCount(), Estimate.of(actualRowCountValue)); @@ -146,7 +147,8 @@ private void testTableStats(String schema, TpchTable table, Constraint table) { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName(schema, table.getTableName())); - TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, alwaysTrue()); + List columnHandles = ImmutableList.copyOf(tpchMetadata.getColumnHandles(session, tableHandle).values()); + TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, Optional.empty(), columnHandles, alwaysTrue()); assertTrue(tableStatistics.getRowCount().isUnknown()); } @@ -237,7 +239,8 @@ private void testColumnStats(String schema, TpchTable table, TpchColumn co private void testColumnStats(String schema, TpchTable table, TpchColumn column, Constraint constraint, ColumnStatistics expected) { TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName(schema, table.getTableName())); - TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, constraint); + List columnHandles = ImmutableList.copyOf(tpchMetadata.getColumnHandles(session, tableHandle).values()); + TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, Optional.empty(), columnHandles, constraint); ColumnHandle columnHandle = tpchMetadata.getColumnHandles(session, tableHandle).get(column.getSimplifiedColumnName()); ColumnStatistics actual = tableStatistics.getColumnStatistics().get(columnHandle); diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index c67f2c26e24a0..1f0a67605c83b 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.225-SNAPSHOT + 0.231-SNAPSHOT presto-verifier @@ -33,6 +33,17 @@ + + com.facebook.presto + presto-hive-metastore + + + com.facebook.presto.hive + hive-apache + + + + com.facebook.presto presto-jdbc @@ -58,11 +69,21 @@ presto-thrift-connector + + com.fasterxml.jackson.core + jackson-annotations + + com.fasterxml.jackson.core jackson-core + + com.fasterxml.jackson.core + jackson-databind + + com.google.code.findbugs jsr305 @@ -85,32 +106,42 @@ - io.airlift + com.facebook.airlift bootstrap - io.airlift + com.facebook.airlift + concurrent + + + + com.facebook.airlift configuration - io.airlift + com.facebook.airlift event - io.airlift + com.facebook.airlift + http-client + + + + com.facebook.airlift json - io.airlift + com.facebook.airlift log - io.airlift + com.facebook.airlift log-manager @@ -134,6 +165,11 @@ validation-api + + javax.ws.rs + javax.ws.rs-api + + mysql mysql-connector-java @@ -170,14 +206,20 @@ - io.airlift + com.facebook.airlift testing test - io.airlift - testing-mysql-server + com.facebook.presto + testing-mysql-server-5 + test + + + + com.facebook.presto + testing-mysql-server-base test diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/PrestoVerifyCommand.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/PrestoVerifyCommand.java index dca3814b41e52..032e0bb7210ce 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/PrestoVerifyCommand.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/PrestoVerifyCommand.java @@ -16,12 +16,13 @@ import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.tree.Property; import com.facebook.presto.verifier.framework.AbstractVerifyCommand; -import com.facebook.presto.verifier.framework.PrestoExceptionClassifier; import com.facebook.presto.verifier.framework.SourceQuery; -import com.facebook.presto.verifier.framework.SqlExceptionClassifier; +import com.facebook.presto.verifier.prestoaction.PrestoExceptionClassifier; +import com.facebook.presto.verifier.prestoaction.SqlExceptionClassifier; import com.facebook.presto.verifier.resolver.ExceededGlobalMemoryLimitFailureResolver; import com.facebook.presto.verifier.resolver.ExceededTimeLimitFailureResolver; -import com.facebook.presto.verifier.resolver.FailureResolver; +import com.facebook.presto.verifier.resolver.FailureResolverFactory; +import com.facebook.presto.verifier.resolver.TooManyOpenPartitionsFailureResolver; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.inject.Module; @@ -72,11 +73,12 @@ public SqlExceptionClassifier getSqlExceptionClassifier() } @Override - public List getFailureResolvers() + public List getFailureResolverFactories() { return ImmutableList.of( - new ExceededGlobalMemoryLimitFailureResolver(), - new ExceededTimeLimitFailureResolver()); + new ExceededGlobalMemoryLimitFailureResolver.Factory(), + new ExceededTimeLimitFailureResolver.Factory(), + new TooManyOpenPartitionsFailureResolver.Factory()); } @Override diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForControl.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForControl.java new file mode 100644 index 0000000000000..ab3321a97af55 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForControl.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.annotation; + +import javax.inject.Qualifier; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@Qualifier +public @interface ForControl +{ +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForTest.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForTest.java new file mode 100644 index 0000000000000..9a82aa905cc8f --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForTest.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.annotation; + +import javax.inject.Qualifier; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@Qualifier +public @interface ForTest +{ +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisDetails.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisDetails.java new file mode 100644 index 0000000000000..77b4f3fcf6731 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisDetails.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.event; + +import com.facebook.airlift.event.client.EventField; +import com.facebook.airlift.event.client.EventType; +import com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.google.common.collect.ImmutableList; + +import javax.annotation.concurrent.Immutable; + +import java.util.List; +import java.util.Optional; + +@Immutable +@EventType("DeterminismAnalysisDetails") +public class DeterminismAnalysisDetails +{ + private final List runs; + private final String limitQueryAnalysis; + private final String limitQueryAnalysisQueryId; + + @JsonCreator + public DeterminismAnalysisDetails( + List runs, + LimitQueryDeterminismAnalysis limitQueryAnalysis, + Optional limitQueryAnalysisQueryId) + { + this.runs = ImmutableList.copyOf(runs); + this.limitQueryAnalysis = limitQueryAnalysis.name(); + this.limitQueryAnalysisQueryId = limitQueryAnalysisQueryId.orElse(null); + } + + @EventField + public List getRuns() + { + return runs; + } + + @EventField + public String getLimitQueryAnalysis() + { + return limitQueryAnalysis; + } + + @EventField + public String getLimitQueryAnalysisQueryId() + { + return limitQueryAnalysisQueryId; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisRun.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisRun.java new file mode 100644 index 0000000000000..9f60cfa6e51ca --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/DeterminismAnalysisRun.java @@ -0,0 +1,103 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.event; + +import com.facebook.airlift.event.client.EventField; +import com.facebook.airlift.event.client.EventType; + +import javax.annotation.concurrent.Immutable; + +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +@Immutable +@EventType("DeterminismAnalysisRun") +public class DeterminismAnalysisRun +{ + private final String tableName; + private final String queryId; + private final String checksumQueryId; + + private DeterminismAnalysisRun( + Optional tableName, + Optional queryId, + Optional checksumQueryId) + { + this.tableName = tableName.orElse(null); + this.queryId = queryId.orElse(null); + this.checksumQueryId = checksumQueryId.orElse(null); + } + + @EventField + public String getTableName() + { + return tableName; + } + + @EventField + public String getQueryId() + { + return queryId; + } + + @EventField + public String getChecksumQueryId() + { + return checksumQueryId; + } + + public static Builder builder() + { + return new Builder(); + } + + public static class Builder + { + private String tableName; + private String queryId; + private String checksumQueryId; + + private Builder() + { + } + + public Builder setTableName(String tableName) + { + checkState(this.tableName == null, "tableName is already set"); + this.tableName = requireNonNull(tableName, "tableName is null"); + return this; + } + + public Builder setQueryId(String queryId) + { + checkState(this.queryId == null, "queryId is already set"); + this.queryId = requireNonNull(queryId, "queryId is null"); + return this; + } + + public Builder setChecksumQueryId(String checksumQueryId) + { + checkState(this.checksumQueryId == null, "checksumQueryId is already set"); + this.checksumQueryId = requireNonNull(checksumQueryId, "checksumQueryId is null"); + return this; + } + + public DeterminismAnalysisRun build() + { + return new DeterminismAnalysisRun(Optional.ofNullable(tableName), Optional.ofNullable(queryId), Optional.ofNullable(checksumQueryId)); + } + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/EventClientModule.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/EventClientModule.java index b26713106e98c..d516e03bc3931 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/EventClientModule.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/EventClientModule.java @@ -13,14 +13,14 @@ */ package com.facebook.presto.verifier.event; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.airlift.event.client.EventClient; import com.facebook.presto.verifier.framework.VerifierConfig; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets.SetView; import com.google.inject.Binder; import com.google.inject.multibindings.Multibinder; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.airlift.event.client.EventClient; import java.util.Map; import java.util.Set; diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/HumanReadableEventClient.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/HumanReadableEventClient.java index f3de0d5e90a54..143784556b8a0 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/HumanReadableEventClient.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/HumanReadableEventClient.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.verifier.event; +import com.facebook.airlift.event.client.AbstractEventClient; import com.facebook.presto.verifier.framework.VerifierConfig; -import io.airlift.event.client.AbstractEventClient; import javax.inject.Inject; diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/JsonEventClient.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/JsonEventClient.java index 83f77749fa9d8..ec0d69bb3b9f8 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/JsonEventClient.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/JsonEventClient.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.verifier.event; +import com.facebook.airlift.event.client.AbstractEventClient; +import com.facebook.airlift.event.client.JsonEventSerializer; import com.facebook.presto.verifier.framework.VerifierConfig; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; -import io.airlift.event.client.AbstractEventClient; -import io.airlift.event.client.JsonEventSerializer; import javax.inject.Inject; diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/FailureInfo.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryFailure.java similarity index 62% rename from presto-verifier/src/main/java/com/facebook/presto/verifier/event/FailureInfo.java rename to presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryFailure.java index e66a4fe8c42df..72d19ae5d588d 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/FailureInfo.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryFailure.java @@ -13,9 +13,9 @@ */ package com.facebook.presto.verifier.event; -import com.facebook.presto.verifier.framework.QueryOrigin.QueryStage; -import io.airlift.event.client.EventField; -import io.airlift.event.client.EventType; +import com.facebook.airlift.event.client.EventField; +import com.facebook.airlift.event.client.EventType; +import com.facebook.presto.verifier.framework.QueryStage; import javax.annotation.concurrent.Immutable; @@ -24,22 +24,37 @@ import static java.util.Objects.requireNonNull; @Immutable -@EventType("QueryInfo") -public class FailureInfo +@EventType("QueryFailure") +public class QueryFailure { + private final String clusterType; private final String queryStage; private final String errorCode; + private final boolean retryable; private final String prestoQueryId; private final String stacktrace; - public FailureInfo(QueryStage queryStage, String errorCode, Optional prestoQueryId, String stacktrace) + public QueryFailure( + QueryStage queryStage, + String errorCode, + boolean retryable, + Optional prestoQueryId, + String stacktrace) { - this.queryStage = queryStage.name(); + this.queryStage = requireNonNull(queryStage.name(), "queryStage is null"); + this.clusterType = requireNonNull(queryStage.getTargetCluster().name(), "cluster is null"); this.errorCode = requireNonNull(errorCode, "errorCode is null"); + this.retryable = retryable; this.prestoQueryId = prestoQueryId.orElse(null); this.stacktrace = requireNonNull(stacktrace, "stacktrace is null"); } + @EventField + public String getClusterType() + { + return clusterType; + } + @EventField public String getQueryStage() { @@ -52,6 +67,12 @@ public String getErrorCode() return errorCode; } + @EventField + public boolean isRetryable() + { + return retryable; + } + @EventField public String getPrestoQueryId() { diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryInfo.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryInfo.java index d09048f9a4c13..dbba1047a1cba 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryInfo.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/QueryInfo.java @@ -13,8 +13,8 @@ */ package com.facebook.presto.verifier.event; -import io.airlift.event.client.EventField; -import io.airlift.event.client.EventType; +import com.facebook.airlift.event.client.EventField; +import com.facebook.airlift.event.client.EventType; import javax.annotation.concurrent.Immutable; @@ -38,7 +38,14 @@ public class QueryInfo private final String checksumQuery; private final Double cpuTimeSecs; private final Double wallTimeSecs; - private final List allFailures; + + public QueryInfo( + String catalog, + String schema, + String originalQuery) + { + this(catalog, schema, originalQuery, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + } public QueryInfo( String catalog, @@ -51,8 +58,7 @@ public QueryInfo( Optional> teardownQueries, Optional checksumQuery, Optional cpuTimeSecs, - Optional wallTimeSecs, - List allFailures) + Optional wallTimeSecs) { this.catalog = requireNonNull(catalog, "catalog is null"); this.schema = requireNonNull(schema, "schema is null"); @@ -65,7 +71,6 @@ public QueryInfo( this.checksumQuery = checksumQuery.orElse(null); this.cpuTimeSecs = cpuTimeSecs.orElse(null); this.wallTimeSecs = wallTimeSecs.orElse(null); - this.allFailures = requireNonNull(allFailures, "allFailures is null"); } @EventField @@ -133,10 +138,4 @@ public Double getWallTimeSecs() { return wallTimeSecs; } - - @EventField - public List getAllFailures() - { - return allFailures; - } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java index fadd69b7c269a..8897b1b2ade3b 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/event/VerifierQueryEvent.java @@ -13,14 +13,19 @@ */ package com.facebook.presto.verifier.event; +import com.facebook.airlift.event.client.EventField; +import com.facebook.airlift.event.client.EventType; +import com.facebook.presto.verifier.framework.DeterminismAnalysis; import com.facebook.presto.verifier.framework.SkippedReason; -import io.airlift.event.client.EventField; -import io.airlift.event.client.EventType; +import com.facebook.presto.verifier.framework.SourceQuery; +import com.google.common.collect.ImmutableList; import javax.annotation.concurrent.Immutable; +import java.util.List; import java.util.Optional; +import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SKIPPED; import static java.util.Objects.requireNonNull; @Immutable @@ -43,6 +48,8 @@ public enum EventStatus private final String skippedReason; private final Boolean deterministic; + private final String determinismAnalysis; + private final DeterminismAnalysisDetails determinismAnalysisDetails; private final String resolveMessage; private final QueryInfo controlQueryInfo; @@ -51,30 +58,69 @@ public enum EventStatus private final String errorCode; private final String errorMessage; + private final QueryFailure finalQueryFailure; + private final List queryFailures; + public VerifierQueryEvent( String suite, String testId, String name, EventStatus status, Optional skippedReason, - Optional deterministic, + Optional determinismAnalysis, + Optional determinismAnalysisDetails, Optional resolveMessage, QueryInfo controlQueryInfo, QueryInfo testQueryInfo, Optional errorCode, - Optional errorMessage) + Optional errorMessage, + Optional finalQueryFailure, + List queryFailures) { this.suite = requireNonNull(suite, "suite is null"); this.testId = requireNonNull(testId, "testId is null"); this.name = requireNonNull(name, "name is null"); this.status = status.name(); this.skippedReason = skippedReason.map(SkippedReason::name).orElse(null); - this.deterministic = deterministic.orElse(null); + this.deterministic = determinismAnalysis.filter(d -> !d.isUnknown()).map(DeterminismAnalysis::isDeterministic).orElse(null); + this.determinismAnalysis = determinismAnalysis.map(DeterminismAnalysis::name).orElse(null); + this.determinismAnalysisDetails = determinismAnalysisDetails.orElse(null); this.resolveMessage = resolveMessage.orElse(null); this.controlQueryInfo = requireNonNull(controlQueryInfo, "controlQueryInfo is null"); this.testQueryInfo = requireNonNull(testQueryInfo, "testQueryInfo is null"); this.errorCode = errorCode.orElse(null); this.errorMessage = errorMessage.orElse(null); + this.finalQueryFailure = finalQueryFailure.orElse(null); + this.queryFailures = ImmutableList.copyOf(queryFailures); + } + + public static VerifierQueryEvent skipped( + String suite, + String testId, + SourceQuery sourceQuery, + SkippedReason skippedReason) + { + return new VerifierQueryEvent( + suite, + testId, + sourceQuery.getName(), + SKIPPED, + Optional.of(skippedReason), + Optional.empty(), + Optional.empty(), + Optional.empty(), + new QueryInfo( + sourceQuery.getControlConfiguration().getCatalog(), + sourceQuery.getControlConfiguration().getSchema(), + sourceQuery.getControlQuery()), + new QueryInfo( + sourceQuery.getTestConfiguration().getCatalog(), + sourceQuery.getTestConfiguration().getSchema(), + sourceQuery.getTestQuery()), + Optional.empty(), + Optional.empty(), + Optional.empty(), + ImmutableList.of()); } @EventField @@ -108,11 +154,24 @@ public String getSkippedReason() } @EventField + @Deprecated public Boolean getDeterministic() { return deterministic; } + @EventField + public String getDeterminismAnalysis() + { + return determinismAnalysis; + } + + @EventField + public DeterminismAnalysisDetails getDeterminismAnalysisDetails() + { + return determinismAnalysisDetails; + } + @EventField public String getResolveMessage() { @@ -142,4 +201,16 @@ public String getErrorMessage() { return errorMessage; } + + @EventField + public QueryFailure getFinalQueryFailure() + { + return finalQueryFailure; + } + + @EventField + public List getQueryFailures() + { + return queryFailures; + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java index ee5e5a82d8d52..3133e88301537 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java @@ -13,41 +13,47 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.airlift.log.Logger; import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.SqlFormatter; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.verifier.checksum.ChecksumResult; -import com.facebook.presto.verifier.event.FailureInfo; +import com.facebook.presto.verifier.event.DeterminismAnalysisDetails; import com.facebook.presto.verifier.event.QueryInfo; import com.facebook.presto.verifier.event.VerifierQueryEvent; import com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus; import com.facebook.presto.verifier.framework.MatchResult.MatchType; -import com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster; -import com.facebook.presto.verifier.resolver.FailureResolver; -import io.airlift.log.Logger; +import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.facebook.presto.verifier.resolver.FailureResolverManager; +import com.facebook.presto.verifier.rewrite.QueryRewriter; import io.airlift.units.Duration; +import javax.annotation.Nullable; + import java.util.List; import java.util.Optional; +import static com.facebook.presto.spi.StandardErrorCode.COMPILER_ERROR; import static com.facebook.presto.spi.StandardErrorCode.EXCEEDED_TIME_LIMIT; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.FAILED; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.FAILED_RESOLVED; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SKIPPED; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SUCCEEDED; -import static com.facebook.presto.verifier.framework.QueryOrigin.QueryStage.MAIN; -import static com.facebook.presto.verifier.framework.QueryOrigin.QueryStage.SETUP; -import static com.facebook.presto.verifier.framework.QueryOrigin.QueryStage.TEARDOWN; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.CONTROL; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.TEST; -import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; -import static com.facebook.presto.verifier.framework.QueryOrigin.forSetup; -import static com.facebook.presto.verifier.framework.QueryOrigin.forTeardown; +import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; +import static com.facebook.presto.verifier.framework.ClusterType.TEST; +import static com.facebook.presto.verifier.framework.QueryStage.CHECKSUM; +import static com.facebook.presto.verifier.framework.QueryStage.CONTROL_MAIN; +import static com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS; +import static com.facebook.presto.verifier.framework.QueryStage.TEST_MAIN; +import static com.facebook.presto.verifier.framework.QueryStage.forMain; +import static com.facebook.presto.verifier.framework.QueryStage.forSetup; +import static com.facebook.presto.verifier.framework.QueryStage.forTeardown; import static com.facebook.presto.verifier.framework.SkippedReason.CONTROL_QUERY_FAILED; import static com.facebook.presto.verifier.framework.SkippedReason.CONTROL_QUERY_TIMED_OUT; import static com.facebook.presto.verifier.framework.SkippedReason.CONTROL_SETUP_QUERY_FAILED; import static com.facebook.presto.verifier.framework.SkippedReason.FAILED_BEFORE_CONTROL_QUERY; import static com.facebook.presto.verifier.framework.SkippedReason.NON_DETERMINISTIC; +import static com.facebook.presto.verifier.framework.SkippedReason.VERIFIER_LIMITATION; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Throwables.getStackTraceAsString; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -66,41 +72,35 @@ public abstract class AbstractVerification private final PrestoAction prestoAction; private final SourceQuery sourceQuery; private final QueryRewriter queryRewriter; - private final List failureResolvers; + private final FailureResolverManager failureResolverManager; + private final VerificationContext verificationContext; private final String testId; private final boolean runTearDownOnResultMismatch; - private final boolean failureResolverEnabled; - - private final VerificationContext verificationContext = new VerificationContext(); public AbstractVerification( VerificationResubmitter verificationResubmitter, PrestoAction prestoAction, SourceQuery sourceQuery, QueryRewriter queryRewriter, - List failureResolvers, - VerifierConfig config) + FailureResolverManager failureResolverManager, + VerificationContext verificationContext, + VerifierConfig verifierConfig) { this.verificationResubmitter = requireNonNull(verificationResubmitter, "verificationResubmitter is null"); this.prestoAction = requireNonNull(prestoAction, "prestoAction is null"); this.sourceQuery = requireNonNull(sourceQuery, "sourceQuery is null"); this.queryRewriter = requireNonNull(queryRewriter, "queryRewriter is null"); - this.failureResolvers = requireNonNull(failureResolvers, "failureResolvers is null"); + this.failureResolverManager = requireNonNull(failureResolverManager, "failureResolverManager is null"); + this.verificationContext = requireNonNull(verificationContext, "verificationContext is null"); - this.testId = requireNonNull(config.getTestId(), "testId is null"); - this.runTearDownOnResultMismatch = config.isRunTearDownOnResultMismatch(); - this.failureResolverEnabled = config.isFailureResolverEnabled(); + this.testId = requireNonNull(verifierConfig.getTestId(), "testId is null"); + this.runTearDownOnResultMismatch = verifierConfig.isRunTeardownOnResultMismatch(); } - protected abstract VerificationResult verify(QueryBundle control, QueryBundle test); + protected abstract MatchResult verify(QueryBundle control, QueryBundle test); - protected abstract Optional isDeterministic(QueryBundle control, ChecksumResult firstChecksum); - - protected VerificationContext getVerificationContext() - { - return verificationContext; - } + protected abstract DeterminismAnalysis analyzeDeterminism(QueryBundle control, ChecksumResult firstChecksum); @Override public SourceQuery getSourceQuery() @@ -114,32 +114,35 @@ public Optional run() boolean resultMismatched = false; QueryBundle control = null; QueryBundle test = null; - VerificationResult verificationResult = null; - Optional deterministic = Optional.empty(); + MatchResult matchResult = null; + Optional determinismAnalysis = Optional.empty(); - Optional controlQueryStats = Optional.empty(); - Optional testQueryStats = Optional.empty(); + QueryStats controlQueryStats = null; + QueryStats testQueryStats = null; try { - control = queryRewriter.rewriteQuery(sourceQuery.getControlQuery(), CONTROL, getConfiguration(CONTROL), getVerificationContext()); - test = queryRewriter.rewriteQuery(sourceQuery.getTestQuery(), TEST, getConfiguration(TEST), getVerificationContext()); - controlQueryStats = Optional.of(setupAndRun(control, CONTROL)); - testQueryStats = Optional.of(setupAndRun(test, TEST)); - verificationResult = verify(control, test); - - deterministic = verificationResult.getMatchResult().isMismatchPossiblyCausedByNonDeterminism() ? - isDeterministic(control, verificationResult.getMatchResult().getControlChecksum()) : - Optional.empty(); - resultMismatched = deterministic.orElse(true) && !verificationResult.getMatchResult().isMatched(); + control = queryRewriter.rewriteQuery(sourceQuery.getControlQuery(), CONTROL); + test = queryRewriter.rewriteQuery(sourceQuery.getTestQuery(), TEST); + controlQueryStats = setupAndRun(control, false); + testQueryStats = setupAndRun(test, false); + matchResult = verify(control, test); + + if (matchResult.isMismatchPossiblyCausedByNonDeterminism()) { + determinismAnalysis = Optional.of(analyzeDeterminism(control, matchResult.getControlChecksum())); + } + boolean maybeDeterministic = !determinismAnalysis.isPresent() || + determinismAnalysis.get().isDeterministic() || + determinismAnalysis.get().isUnknown(); + resultMismatched = maybeDeterministic && !matchResult.isMatched(); return Optional.of(buildEvent( Optional.of(control), Optional.of(test), - controlQueryStats, - testQueryStats, + Optional.ofNullable(controlQueryStats), + Optional.ofNullable(testQueryStats), Optional.empty(), - Optional.of(verificationResult), - deterministic)); + Optional.of(matchResult), + determinismAnalysis)); } catch (QueryException e) { if (verificationResubmitter.resubmit(this, e)) { @@ -148,11 +151,11 @@ public Optional run() return Optional.of(buildEvent( Optional.ofNullable(control), Optional.ofNullable(test), - controlQueryStats, - testQueryStats, + Optional.ofNullable(controlQueryStats), + Optional.ofNullable(testQueryStats), Optional.of(e), - Optional.ofNullable(verificationResult), - deterministic)); + Optional.ofNullable(matchResult), + determinismAnalysis)); } catch (Throwable t) { log.error(t); @@ -160,12 +163,8 @@ public Optional run() } finally { if (!resultMismatched || runTearDownOnResultMismatch) { - if (control != null) { - teardownSafely(control, CONTROL); - } - if (test != null) { - teardownSafely(test, TEST); - } + teardownSafely(control); + teardownSafely(test); } } } @@ -180,64 +179,70 @@ protected QueryRewriter getQueryRewriter() return queryRewriter; } - protected QueryConfiguration getConfiguration(TargetCluster cluster) + protected VerificationContext getVerificationContext() { - checkState(cluster == CONTROL || cluster == TEST, "Unexpected TargetCluster %s", cluster); - return cluster == CONTROL ? sourceQuery.getControlConfiguration() : sourceQuery.getTestConfiguration(); + return verificationContext; } - protected void setup(QueryBundle control, TargetCluster cluster) + protected QueryStats setupAndRun(QueryBundle bundle, boolean determinismAnalysis) { - for (Statement setupQuery : control.getSetupQueries()) { - prestoAction.execute(setupQuery, getConfiguration(cluster), forSetup(cluster), getVerificationContext()); + checkState(!determinismAnalysis || bundle.getCluster() == CONTROL, "Determinism analysis can only be run on control cluster"); + QueryStage setupStage = determinismAnalysis ? DETERMINISM_ANALYSIS : forSetup(bundle.getCluster()); + QueryStage mainStage = determinismAnalysis ? DETERMINISM_ANALYSIS : forMain(bundle.getCluster()); + + for (Statement setupQuery : bundle.getSetupQueries()) { + prestoAction.execute(setupQuery, setupStage); } + return getPrestoAction().execute(bundle.getQuery(), mainStage); } - protected void teardownSafely(QueryBundle control, TargetCluster cluster) + protected void teardownSafely(@Nullable QueryBundle bundle) { - for (Statement teardownQuery : control.getTeardownQueries()) { + if (bundle == null) { + return; + } + + for (Statement teardownQuery : bundle.getTeardownQueries()) { try { - prestoAction.execute(teardownQuery, getConfiguration(cluster), forTeardown(cluster), getVerificationContext()); + prestoAction.execute(teardownQuery, forTeardown(bundle.getCluster())); } catch (Throwable t) { - log.warn("Failed to teardown %s: %s", cluster.name().toLowerCase(ENGLISH), formatSql(teardownQuery)); + log.warn("Failed to teardown %s: %s", bundle.getCluster().name().toLowerCase(ENGLISH), formatSql(teardownQuery)); } } } - protected QueryStats setupAndRun(QueryBundle control, TargetCluster cluster) - { - setup(control, cluster); - return getPrestoAction().execute(control.getQuery(), getConfiguration(cluster), forMain(cluster), getVerificationContext()); - } - private VerifierQueryEvent buildEvent( Optional control, Optional test, Optional controlStats, Optional testStats, Optional queryException, - Optional verificationResult, - Optional deterministic) + Optional matchResult, + Optional determinismAnalysis) { - boolean succeeded = verificationResult.isPresent() && verificationResult.get().getMatchResult().isMatched(); + boolean succeeded = matchResult.isPresent() && matchResult.get().isMatched(); QueryState controlState = getQueryState(controlStats, queryException, CONTROL); QueryState testState = getQueryState(testStats, queryException, TEST); String errorMessage = null; if (!succeeded) { - errorMessage = format("Test state %s, Control state %s\n", testState.name(), controlState.name()); + errorMessage = format("Test state %s, Control state %s.\n\n", testState.name(), controlState.name()); if (queryException.isPresent()) { - errorMessage += getStackTraceAsString(queryException.get().getCause()); + errorMessage += format( + "%s query failed on %s cluster:\n%s", + queryException.get().getQueryStage().name().replace("_", " "), + queryException.get().getQueryStage().getTargetCluster(), + getStackTraceAsString(queryException.get().getCause())); } - if (verificationResult.isPresent()) { - errorMessage += verificationResult.get().getMatchResult().getResultsComparison(); + if (matchResult.isPresent()) { + errorMessage += matchResult.get().getResultsComparison(); } } EventStatus status; - Optional skippedReason = getSkippedReason(controlState, deterministic); + Optional skippedReason = getSkippedReason(controlState, determinismAnalysis, queryException); Optional resolveMessage = Optional.empty(); if (succeeded) { status = SUCCEEDED; @@ -248,22 +253,22 @@ else if (skippedReason.isPresent()) { else { if (controlState == QueryState.SUCCEEDED && queryException.isPresent()) { checkState(controlStats.isPresent(), "control succeeded but control stats is missing"); - resolveMessage = resolveFailure(controlStats.get(), queryException.get()); + resolveMessage = failureResolverManager.resolve(controlStats.get(), queryException.get(), test); } status = resolveMessage.isPresent() ? FAILED_RESOLVED : FAILED; } - controlStats = queryException.isPresent() && queryException.get().getQueryOrigin().equals(forMain(CONTROL)) ? + controlStats = queryException.isPresent() && queryException.get().getQueryStage() == CONTROL_MAIN ? queryException.get().getQueryStats() : controlStats; - testStats = queryException.isPresent() && queryException.get().getQueryOrigin().equals(forMain(TEST)) ? + testStats = queryException.isPresent() && queryException.get().getQueryStage() == TEST_MAIN ? queryException.get().getQueryStats() : testStats; Optional errorCode = Optional.empty(); if (!succeeded) { errorCode = Optional.ofNullable(queryException.map(QueryException::getErrorCode).orElse( - verificationResult.map(VerificationResult::getMatchResult).map(MatchResult::getMatchType).map(MatchType::name).orElse(null))); + matchResult.map(MatchResult::getMatchType).map(MatchType::name).orElse(null))); } return new VerifierQueryEvent( @@ -272,40 +277,32 @@ else if (skippedReason.isPresent()) { sourceQuery.getName(), status, skippedReason, - deterministic, + determinismAnalysis, + determinismAnalysis.isPresent() ? + Optional.of(new DeterminismAnalysisDetails( + verificationContext.getDeterminismAnalysisRuns(), + verificationContext.getLimitQueryAnalysis(), + verificationContext.getLimitQueryAnalysisQueryId())) : + Optional.empty(), resolveMessage, buildQueryInfo( sourceQuery.getControlConfiguration(), sourceQuery.getControlQuery(), - verificationResult.map(VerificationResult::getControlChecksumQueryId), - verificationResult.map(VerificationResult::getControlChecksumQuery), + verificationContext.getControlChecksumQueryId(), + verificationContext.getControlChecksumQuery(), control, - controlStats, - verificationContext.getAllFailures(CONTROL)), + controlStats), buildQueryInfo( sourceQuery.getTestConfiguration(), sourceQuery.getTestQuery(), - verificationResult.map(VerificationResult::getTestChecksumQueryId), - verificationResult.map(VerificationResult::getTestChecksumQuery), + verificationContext.getTestChecksumQueryId(), + verificationContext.getTestChecksumQuery(), test, - testStats, - verificationContext.getAllFailures(TEST)), + testStats), errorCode, - Optional.ofNullable(errorMessage)); - } - - private Optional resolveFailure(QueryStats controlStats, QueryException queryException) - { - if (!failureResolverEnabled) { - return Optional.empty(); - } - for (FailureResolver failureResolver : failureResolvers) { - Optional resolveMessage = failureResolver.resolve(controlStats, queryException); - if (resolveMessage.isPresent()) { - return resolveMessage; - } - } - return Optional.empty(); + Optional.ofNullable(errorMessage), + queryException.map(QueryException::toQueryFailure), + verificationContext.getQueryFailures()); } private static QueryInfo buildQueryInfo( @@ -314,8 +311,7 @@ private static QueryInfo buildQueryInfo( Optional checksumQueryId, Optional checksumQuery, Optional queryBundle, - Optional queryStats, - List allFailures) + Optional queryStats) { return new QueryInfo( configuration.getCatalog(), @@ -328,8 +324,7 @@ private static QueryInfo buildQueryInfo( queryBundle.map(QueryBundle::getTeardownQueries).map(AbstractVerification::formatSqls), checksumQuery, millisToSeconds(queryStats.map(QueryStats::getCpuTimeMillis)), - millisToSeconds(queryStats.map(QueryStats::getWallTimeMillis)), - allFailures); + millisToSeconds(queryStats.map(QueryStats::getWallTimeMillis))); } protected static String formatSql(Statement statement) @@ -344,7 +339,10 @@ protected static List formatSqls(List statements) .collect(toImmutableList()); } - private static Optional getSkippedReason(QueryState controlState, Optional deterministic) + private static Optional getSkippedReason( + QueryState controlState, + Optional determinismAnalysis, + Optional queryException) { switch (controlState) { case FAILED: @@ -356,9 +354,15 @@ private static Optional getSkippedReason(QueryState controlState, case NOT_RUN: return Optional.of(FAILED_BEFORE_CONTROL_QUERY); } - if (!deterministic.orElse(true)) { + if (determinismAnalysis.isPresent() && determinismAnalysis.get().isNonDeterministic()) { return Optional.of(NON_DETERMINISTIC); } + if (queryException.isPresent() && + queryException.get().getQueryStage().equals(CHECKSUM) && + queryException.get().getPrestoErrorCode().isPresent() && + queryException.get().getPrestoErrorCode().get().equals(COMPILER_ERROR)) { + return Optional.of(VERIFIER_LIMITATION); + } return Optional.empty(); } @@ -367,23 +371,23 @@ private static Optional millisToSeconds(Optional millis) return millis.map(value -> new Duration(value, MILLISECONDS).getValue(SECONDS)); } - private static QueryState getQueryState(Optional statsFromResult, Optional queryException, TargetCluster cluster) + private static QueryState getQueryState(Optional statsFromResult, Optional queryException, ClusterType cluster) { if (statsFromResult.isPresent()) { return QueryState.SUCCEEDED; } - if (!queryException.isPresent() || queryException.get().getQueryOrigin().getCluster() != cluster) { + if (!queryException.isPresent() || queryException.get().getQueryStage().getTargetCluster() != cluster) { return QueryState.NOT_RUN; } - if (queryException.get().getQueryOrigin().getStage() == SETUP) { + if (queryException.get().getQueryStage().isSetup()) { return QueryState.FAILED_TO_SETUP; } - if (queryException.get().getQueryOrigin().getStage() == MAIN) { + if (queryException.get().getQueryStage().isMain()) { return queryException.get().getPrestoErrorCode().map(errorCode -> errorCode == EXCEEDED_TIME_LIMIT).orElse(false) ? QueryState.TIMED_OUT : QueryState.FAILED; } - if (queryException.get().getQueryOrigin().getStage() == TEARDOWN) { + if (queryException.get().getQueryStage().isTeardown()) { return QueryState.FAILED_TO_TEARDOWN; } return QueryState.NOT_RUN; diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerifyCommand.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerifyCommand.java index 01f4d72456d35..fabf707d60c00 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerifyCommand.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerifyCommand.java @@ -13,15 +13,15 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; import com.facebook.presto.verifier.event.EventClientModule; import com.facebook.presto.verifier.source.SourceQueryModule; import com.google.common.collect.ImmutableList; import com.google.inject.Injector; import com.google.inject.Module; import io.airlift.airline.Arguments; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.bootstrap.LifeCycleManager; -import io.airlift.log.Logger; import static com.google.common.base.Throwables.throwIfUnchecked; @@ -45,7 +45,7 @@ public void run() getSqlParserOptions(), getCustomQueryFilterClasses(), getSqlExceptionClassifier(), - getFailureResolvers(), + getFailureResolverFactories(), getTablePropertyOverrides())) .add(new SourceQueryModule(getCustomSourceQuerySupplierTypes())) .add(new EventClientModule(getCustomEventClientTypes())) diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ClusterType.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ClusterType.java new file mode 100644 index 0000000000000..2038d8a08b97f --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ClusterType.java @@ -0,0 +1,20 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +public enum ClusterType +{ + CONTROL, + TEST, +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/Column.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/Column.java index fbb442d7b6d59..c00dd65f5540e 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/Column.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/Column.java @@ -13,13 +13,10 @@ */ package com.facebook.presto.verifier.framework; -import com.facebook.presto.block.BlockEncodingManager; -import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.tree.Identifier; -import com.facebook.presto.type.TypeRegistry; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; @@ -47,11 +44,6 @@ public enum Category } private static final Set FLOATING_POINT_TYPES = ImmutableSet.of(DOUBLE, REAL); - private static final TypeRegistry typeRegistry = new TypeRegistry(); - - static { - new FunctionManager(typeRegistry, new BlockEncodingManager(typeRegistry), new FeaturesConfig()); - } private final String name; private final Category category; @@ -85,10 +77,10 @@ public Type getType() return type; } - public static Column fromResultSet(ResultSet resultSet) + public static Column fromResultSet(TypeManager typeManager, ResultSet resultSet) throws SQLException { - Type type = typeRegistry.getType(parseTypeSignature(resultSet.getString("Type"))); + Type type = typeManager.getType(parseTypeSignature(resultSet.getString("Type"))); Category category; if (FLOATING_POINT_TYPES.contains(type)) { category = FLOATING_POINT; diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java index 75b6b5d8c3b6b..5aea0f1d289c8 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java @@ -13,101 +13,148 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Query; import com.facebook.presto.sql.tree.ShowColumns; import com.facebook.presto.verifier.checksum.ChecksumResult; import com.facebook.presto.verifier.checksum.ChecksumValidator; import com.facebook.presto.verifier.checksum.ColumnMatchResult; +import com.facebook.presto.verifier.event.DeterminismAnalysisRun; import com.facebook.presto.verifier.framework.MatchResult.MatchType; -import com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster; -import com.facebook.presto.verifier.resolver.FailureResolver; +import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.facebook.presto.verifier.resolver.FailureResolverManager; +import com.facebook.presto.verifier.rewrite.QueryRewriter; import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalLong; +import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; +import static com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED; +import static com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED_DATA_CHANGED; +import static com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED_INCONSISTENT_SCHEMA; +import static com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED_QUERY_FAILURE; +import static com.facebook.presto.verifier.framework.DeterminismAnalysis.DETERMINISTIC; +import static com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_COLUMNS; +import static com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_LIMIT_CLAUSE; +import static com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_ROW_COUNT; import static com.facebook.presto.verifier.framework.MatchResult.MatchType.COLUMN_MISMATCH; import static com.facebook.presto.verifier.framework.MatchResult.MatchType.MATCH; import static com.facebook.presto.verifier.framework.MatchResult.MatchType.ROW_COUNT_MISMATCH; import static com.facebook.presto.verifier.framework.MatchResult.MatchType.SCHEMA_MISMATCH; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.CONTROL; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.TEST; -import static com.facebook.presto.verifier.framework.QueryOrigin.forChecksum; -import static com.facebook.presto.verifier.framework.QueryOrigin.forDescribe; +import static com.facebook.presto.verifier.framework.QueryStage.CHECKSUM; +import static com.facebook.presto.verifier.framework.QueryStage.DESCRIBE; +import static com.facebook.presto.verifier.framework.VerifierUtil.callWithQueryStatsConsumer; +import static com.facebook.presto.verifier.framework.VerifierUtil.runWithQueryStatsConsumer; import static com.google.common.collect.Iterables.getOnlyElement; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; public class DataVerification extends AbstractVerification { + private final TypeManager typeManager; private final ChecksumValidator checksumValidator; + private final LimitQueryDeterminismAnalyzer limitQueryDeterminismAnalyzer; + private final boolean runTeardownForDeterminismAnalysis; + + private final int maxDeterminismAnalysisRuns; public DataVerification( VerificationResubmitter verificationResubmitter, PrestoAction prestoAction, SourceQuery sourceQuery, QueryRewriter queryRewriter, - List failureResolvers, - VerifierConfig config, - ChecksumValidator checksumValidator) + FailureResolverManager failureResolverManager, + VerificationContext verificationContext, + VerifierConfig verifierConfig, + TypeManager typeManager, + ChecksumValidator checksumValidator, + LimitQueryDeterminismAnalyzer limitQueryDeterminismAnalyzer) { - super(verificationResubmitter, prestoAction, sourceQuery, queryRewriter, failureResolvers, config); + super(verificationResubmitter, prestoAction, sourceQuery, queryRewriter, failureResolverManager, verificationContext, verifierConfig); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.checksumValidator = requireNonNull(checksumValidator, "checksumValidator is null"); + this.limitQueryDeterminismAnalyzer = requireNonNull(limitQueryDeterminismAnalyzer, "limitQueryDeterminismAnalyzer is null"); + this.runTeardownForDeterminismAnalysis = verifierConfig.isRunTeardownForDeterminismAnalysis(); + this.maxDeterminismAnalysisRuns = verifierConfig.getMaxDeterminismAnalysisRuns(); } @Override - public VerificationResult verify(QueryBundle control, QueryBundle test) + public MatchResult verify(QueryBundle control, QueryBundle test) { - List controlColumns = getColumns(control.getTableName(), CONTROL); - List testColumns = getColumns(test.getTableName(), TEST); - ChecksumQueryAndResult controlChecksum = computeChecksum(control, controlColumns, CONTROL); - ChecksumQueryAndResult testChecksum = computeChecksum(test, testColumns, TEST); - return new VerificationResult( - controlChecksum.getQueryId(), - testChecksum.getQueryId(), - formatSql(controlChecksum.getQuery()), - formatSql(testChecksum.getQuery()), - match( - controlColumns, - testColumns, - controlChecksum.getResult(), - testChecksum.getResult())); + List controlColumns = getColumns(control.getTableName()); + List testColumns = getColumns(test.getTableName()); + + Query controlChecksumQuery = checksumValidator.generateChecksumQuery(control.getTableName(), controlColumns); + Query testChecksumQuery = checksumValidator.generateChecksumQuery(test.getTableName(), testColumns); + + getVerificationContext().setControlChecksumQuery(formatSql(controlChecksumQuery)); + getVerificationContext().setTestChecksumQuery(formatSql(testChecksumQuery)); + + QueryResult controlChecksum = callWithQueryStatsConsumer( + () -> executeChecksumQuery(controlChecksumQuery), + stats -> getVerificationContext().setControlChecksumQueryId(stats.getQueryId())); + QueryResult testChecksum = callWithQueryStatsConsumer( + () -> executeChecksumQuery(testChecksumQuery), + stats -> getVerificationContext().setTestChecksumQueryId(stats.getQueryId())); + + return match(controlColumns, testColumns, getOnlyElement(controlChecksum.getResults()), getOnlyElement(testChecksum.getResults())); } @Override - protected Optional isDeterministic(QueryBundle control, ChecksumResult firstChecksum) + protected DeterminismAnalysis analyzeDeterminism(QueryBundle control, ChecksumResult controlChecksum) { - List columns = getColumns(control.getTableName(), CONTROL); + List columns = getColumns(control.getTableName()); + List queryBundles = new ArrayList<>(); - QueryBundle secondRun = null; - QueryBundle thirdRun = null; try { - secondRun = getQueryRewriter().rewriteQuery(getSourceQuery().getControlQuery(), CONTROL, getConfiguration(CONTROL), getVerificationContext()); - setupAndRun(secondRun, CONTROL); - if (!match(columns, columns, firstChecksum, computeChecksum(secondRun, columns, CONTROL).getResult()).isMatched()) { - return Optional.of(false); + for (int i = 0; i < maxDeterminismAnalysisRuns; i++) { + QueryBundle queryBundle = getQueryRewriter().rewriteQuery(getSourceQuery().getControlQuery(), CONTROL); + queryBundles.add(queryBundle); + DeterminismAnalysisRun.Builder run = getVerificationContext().startDeterminismAnalysisRun().setTableName(queryBundle.getTableName().toString()); + + runWithQueryStatsConsumer(() -> setupAndRun(queryBundle, true), stats -> run.setQueryId(stats.getQueryId())); + + Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getTableName(), columns); + ChecksumResult testChecksum = getOnlyElement(callWithQueryStatsConsumer( + () -> executeChecksumQuery(checksumQuery), + stats -> run.setChecksumQueryId(stats.getQueryId())).getResults()); + + DeterminismAnalysis determinismAnalysis = matchResultToDeterminism(match(columns, columns, controlChecksum, testChecksum)); + if (determinismAnalysis != DETERMINISTIC) { + return determinismAnalysis; + } } - thirdRun = getQueryRewriter().rewriteQuery(getSourceQuery().getControlQuery(), CONTROL, getConfiguration(CONTROL), getVerificationContext()); - setupAndRun(thirdRun, CONTROL); - if (!match(columns, columns, firstChecksum, computeChecksum(thirdRun, columns, CONTROL).getResult()).isMatched()) { - return Optional.of(false); + LimitQueryDeterminismAnalysis analysis = limitQueryDeterminismAnalyzer.analyze(control, controlChecksum.getRowCount(), getVerificationContext()); + getVerificationContext().setLimitQueryAnalysis(analysis); + + switch (analysis) { + case NON_DETERMINISTIC: + return NON_DETERMINISTIC_LIMIT_CLAUSE; + case NOT_RUN: + case DETERMINISTIC: + return DETERMINISTIC; + case FAILED_DATA_CHANGED: + return ANALYSIS_FAILED_DATA_CHANGED; + default: + throw new IllegalArgumentException(format("Invalid analysis: %s", analysis)); } - - return Optional.of(true); + } + catch (QueryException qe) { + return ANALYSIS_FAILED_QUERY_FAILURE; } catch (Throwable t) { - return Optional.empty(); + return ANALYSIS_FAILED; } finally { - if (secondRun != null) { - teardownSafely(secondRun, CONTROL); - } - if (thirdRun != null) { - teardownSafely(thirdRun, CONTROL); + if (runTeardownForDeterminismAnalysis) { + queryBundles.forEach(this::teardownSafely); } } } @@ -148,59 +195,31 @@ private MatchResult match( mismatchedColumns); } - private List getColumns(QualifiedName tableName, TargetCluster cluster) + private DeterminismAnalysis matchResultToDeterminism(MatchResult matchResult) { - return getPrestoAction() - .execute( - new ShowColumns(tableName), - getConfiguration(cluster), - forDescribe(), - getVerificationContext(), - Column::fromResultSet) - .getResults(); + switch (matchResult.getMatchType()) { + case MATCH: + return DETERMINISTIC; + case SCHEMA_MISMATCH: + return ANALYSIS_FAILED_INCONSISTENT_SCHEMA; + case ROW_COUNT_MISMATCH: + return NON_DETERMINISTIC_ROW_COUNT; + case COLUMN_MISMATCH: + return NON_DETERMINISTIC_COLUMNS; + default: + throw new IllegalArgumentException(format("Invalid MatchResult: %s", matchResult)); + } } - private ChecksumQueryAndResult computeChecksum(QueryBundle bundle, List columns, TargetCluster cluster) + private List getColumns(QualifiedName tableName) { - Query checksumQuery = checksumValidator.generateChecksumQuery(bundle.getTableName(), columns); - QueryResult queryResult = getPrestoAction().execute( - checksumQuery, - getConfiguration(cluster), - forChecksum(), - getVerificationContext(), - ChecksumResult::fromResultSet); - return new ChecksumQueryAndResult( - queryResult.getQueryStats().getQueryId(), - checksumQuery, - getOnlyElement(queryResult.getResults())); + return getPrestoAction() + .execute(new ShowColumns(tableName), DESCRIBE, resultSet -> Column.fromResultSet(typeManager, resultSet)) + .getResults(); } - private class ChecksumQueryAndResult + private QueryResult executeChecksumQuery(Query query) { - private final String queryId; - private final Query query; - private final ChecksumResult result; - - public ChecksumQueryAndResult(String queryId, Query query, ChecksumResult result) - { - this.queryId = requireNonNull(queryId, "queryId is null"); - this.query = requireNonNull(query, "query is null"); - this.result = requireNonNull(result, "result is null"); - } - - public String getQueryId() - { - return queryId; - } - - public Query getQuery() - { - return query; - } - - public ChecksumResult getResult() - { - return result; - } + return getPrestoAction().execute(query, CHECKSUM, ChecksumResult::fromResultSet); } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalysis.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalysis.java new file mode 100644 index 0000000000000..564f6f7f4fd8b --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DeterminismAnalysis.java @@ -0,0 +1,57 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +import static java.util.Objects.requireNonNull; + +public enum DeterminismAnalysis +{ + DETERMINISTIC(Category.DETERMINISTIC), + NON_DETERMINISTIC_COLUMNS(Category.NON_DETERMINISTIC), + NON_DETERMINISTIC_ROW_COUNT(Category.NON_DETERMINISTIC), + NON_DETERMINISTIC_LIMIT_CLAUSE(Category.NON_DETERMINISTIC), + ANALYSIS_FAILED_INCONSISTENT_SCHEMA(Category.UNKNOWN), + ANALYSIS_FAILED_QUERY_FAILURE(Category.UNKNOWN), + ANALYSIS_FAILED_DATA_CHANGED(Category.UNKNOWN), + ANALYSIS_FAILED(Category.UNKNOWN); + + private enum Category + { + DETERMINISTIC, + NON_DETERMINISTIC, + UNKNOWN + } + + private final Category category; + + DeterminismAnalysis(Category category) + { + this.category = requireNonNull(category, "category is null"); + } + + public boolean isDeterministic() + { + return category == Category.DETERMINISTIC; + } + + public boolean isNonDeterministic() + { + return category == Category.NON_DETERMINISTIC; + } + + public boolean isUnknown() + { + return category == Category.UNKNOWN; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ForwardingDriver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ForwardingDriver.java deleted file mode 100644 index 5839ae567c0bc..0000000000000 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/ForwardingDriver.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.verifier.framework; - -import java.sql.Connection; -import java.sql.Driver; -import java.sql.DriverPropertyInfo; -import java.sql.SQLException; -import java.sql.SQLFeatureNotSupportedException; -import java.util.Properties; -import java.util.logging.Logger; - -import static java.util.Objects.requireNonNull; - -class ForwardingDriver - implements Driver -{ - private final Driver driver; - - ForwardingDriver(Driver driver) - { - this.driver = requireNonNull(driver); - } - - @Override - public boolean acceptsURL(String url) - throws SQLException - { - return driver.acceptsURL(url); - } - - @Override - public Connection connect(String url, Properties info) - throws SQLException - { - return driver.connect(url, info); - } - - @Override - public int getMajorVersion() - { - return driver.getMajorVersion(); - } - - @Override - public int getMinorVersion() - { - return driver.getMinorVersion(); - } - - @Override - public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) - throws SQLException - { - return driver.getPropertyInfo(url, info); - } - - @Override - public boolean jdbcCompliant() - { - return driver.jdbcCompliant(); - } - - @Override - public Logger getParentLogger() - throws SQLFeatureNotSupportedException - { - return driver.getParentLogger(); - } -} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/JdbcDriverUtil.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/JdbcDriverUtil.java deleted file mode 100644 index 0789b374c6b5b..0000000000000 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/JdbcDriverUtil.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.verifier.framework; - -import com.google.common.collect.ImmutableList; - -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.file.Paths; -import java.sql.Driver; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.List; -import java.util.Optional; - -public class JdbcDriverUtil -{ - private JdbcDriverUtil() - { - } - - public static void initializeDrivers( - Optional additionalJdbcDriverPath, - Optional controlJdbcDriverClass, - Optional testJdbcDriverClass) - { - if (additionalJdbcDriverPath.isPresent()) { - List urlList = getUrls(additionalJdbcDriverPath.get()); - URL[] urls = new URL[urlList.size()]; - urlList.toArray(urls); - controlJdbcDriverClass.ifPresent(driver -> loadJdbcDriver(urls, driver)); - testJdbcDriverClass.ifPresent(driver -> loadJdbcDriver(urls, driver)); - } - } - - private static void loadJdbcDriver(URL[] urls, String jdbcClassName) - { - try (URLClassLoader classLoader = new URLClassLoader(urls)) { - Driver driver = (Driver) Class.forName(jdbcClassName, true, classLoader).getConstructor().newInstance(); - // The code calling the DriverManager to load the driver needs to be in the same class loader as the driver - // In order to bypass this we create a shim that wraps the specified jdbc driver class. - // TODO: Change the implementation to be DataSource based instead of DriverManager based. - DriverManager.registerDriver(new ForwardingDriver(driver)); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - catch (SQLException | ReflectiveOperationException e) { - throw new RuntimeException(e); - } - } - - private static List getUrls(String path) - { - try { - ImmutableList.Builder urlList = ImmutableList.builder(); - File driverPath = new File(path); - if (!driverPath.isDirectory()) { - return urlList.add(Paths.get(path).toUri().toURL()).build(); - } - File[] files = driverPath.listFiles((dir, name) -> name.endsWith(".jar")); - if (files == null) { - return urlList.build(); - } - for (File file : files) { - // Does not handle nested directories - if (file.isDirectory()) { - continue; - } - urlList.add(Paths.get(file.getAbsolutePath()).toUri().toURL()); - } - return urlList.build(); - } - catch (MalformedURLException e) { - throw new RuntimeException(e); - } - } -} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalysis.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalysis.java new file mode 100644 index 0000000000000..c017ce5d5fe09 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalysis.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +public enum LimitQueryDeterminismAnalysis +{ + NOT_RUN, + NON_DETERMINISTIC, + DETERMINISTIC, + FAILED_DATA_CHANGED, +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalyzer.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalyzer.java new file mode 100644 index 0000000000000..74fdae65a164c --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/LimitQueryDeterminismAnalyzer.java @@ -0,0 +1,139 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +import com.facebook.presto.sql.tree.CreateTableAsSelect; +import com.facebook.presto.sql.tree.FunctionCall; +import com.facebook.presto.sql.tree.Insert; +import com.facebook.presto.sql.tree.LongLiteral; +import com.facebook.presto.sql.tree.QualifiedName; +import com.facebook.presto.sql.tree.Query; +import com.facebook.presto.sql.tree.QuerySpecification; +import com.facebook.presto.sql.tree.Select; +import com.facebook.presto.sql.tree.SingleColumn; +import com.facebook.presto.sql.tree.Statement; +import com.facebook.presto.sql.tree.TableSubquery; +import com.facebook.presto.sql.tree.With; +import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.google.common.collect.ImmutableList; + +import java.util.Optional; + +import static com.facebook.presto.sql.QueryUtil.simpleQuery; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.DETERMINISTIC; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.FAILED_DATA_CHANGED; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.NON_DETERMINISTIC; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.NOT_RUN; +import static com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS; +import static com.facebook.presto.verifier.framework.VerifierUtil.callWithQueryStatsConsumer; +import static com.google.common.collect.Iterables.getOnlyElement; +import static java.util.Objects.requireNonNull; + +public class LimitQueryDeterminismAnalyzer +{ + private final PrestoAction prestoAction; + private final boolean enabled; + + public LimitQueryDeterminismAnalyzer(PrestoAction prestoAction, VerifierConfig verifierConfig) + { + this.prestoAction = requireNonNull(prestoAction, "prestoAction is null"); + this.enabled = verifierConfig.isEnableLimitQueryDeterminismAnalyzer(); + } + + public LimitQueryDeterminismAnalysis analyze(QueryBundle control, long rowCount, VerificationContext verificationContext) + { + if (!enabled) { + return NOT_RUN; + } + + Statement statement = control.getQuery(); + Query query; + + // A query is rewritten to either an Insert or a CreateTableAsSelect + if (statement instanceof Insert) { + query = ((Insert) statement).getQuery(); + } + else if (statement instanceof CreateTableAsSelect) { + query = ((CreateTableAsSelect) statement).getQuery(); + } + else { + return NOT_RUN; + } + + // Flatten TableSubquery + if (query.getQueryBody() instanceof TableSubquery) { + Optional with = query.getWith(); + while (query.getQueryBody() instanceof TableSubquery) { + // ORDER BY and LIMIT must be empty according to syntax + if (query.getOrderBy().isPresent() || query.getLimit().isPresent()) { + return NOT_RUN; + } + query = ((TableSubquery) query.getQueryBody()).getQuery(); + // WITH must be empty according to syntax + if (query.getWith().isPresent()) { + return NOT_RUN; + } + } + query = new Query(with, query.getQueryBody(), query.getOrderBy(), query.getLimit()); + } + + // If Query contains ORDER_BY, we won't be able to make conclusion even if there is LIMIT clause. + if (query.getOrderBy().isPresent()) { + return NOT_RUN; + } + + Query queryNoLimit; + if (query.getLimit().isPresent()) { + queryNoLimit = new Query(query.getWith(), query.getQueryBody(), Optional.empty(), Optional.empty()); + } + else if (query.getQueryBody() instanceof QuerySpecification) { + QuerySpecification querySpecification = (QuerySpecification) query.getQueryBody(); + if (querySpecification.getOrderBy().isPresent() || !querySpecification.getLimit().isPresent()) { + return NOT_RUN; + } + queryNoLimit = new Query( + query.getWith(), + new QuerySpecification( + querySpecification.getSelect(), + querySpecification.getFrom(), + querySpecification.getWhere(), + querySpecification.getGroupBy(), + querySpecification.getHaving(), + Optional.empty(), + Optional.empty()), + Optional.empty(), + Optional.empty()); + } + else { + return NOT_RUN; + } + + Query rowCountQuery = simpleQuery( + new Select(false, ImmutableList.of(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of(new LongLiteral("1")))))), + new TableSubquery(queryNoLimit)); + + QueryResult result = callWithQueryStatsConsumer( + () -> prestoAction.execute(rowCountQuery, DETERMINISM_ANALYSIS, resultSet -> resultSet.getLong(1)), + stats -> verificationContext.setLimitQueryAnalysisQueryId(stats.getQueryId())); + + long rowCountNoLimit = getOnlyElement(result.getResults()); + if (rowCountNoLimit > rowCount) { + return NON_DETERMINISTIC; + } + if (rowCountNoLimit == rowCount) { + return DETERMINISTIC; + } + return FAILED_DATA_CHANGED; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryBundle.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryBundle.java index 2fb895592f8cc..228f299bb56c7 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryBundle.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryBundle.java @@ -18,34 +18,34 @@ import com.google.common.collect.ImmutableList; import java.util.List; -import java.util.Optional; -import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; public class QueryBundle { - private final Optional tableName; + private final QualifiedName tableName; private final List setupQueries; private final Statement query; private final List teardownQueries; + private final ClusterType cluster; public QueryBundle( - Optional tableName, + QualifiedName tableName, List setupQueries, Statement query, - List teardownQueries) + List teardownQueries, + ClusterType cluster) { this.tableName = requireNonNull(tableName, "tableName is null"); this.setupQueries = ImmutableList.copyOf(setupQueries); - this.query = requireNonNull(query, "mainQueries is null"); + this.query = requireNonNull(query, "query is null"); this.teardownQueries = ImmutableList.copyOf(teardownQueries); + this.cluster = requireNonNull(cluster, "cluster is null"); } public QualifiedName getTableName() { - checkState(tableName.isPresent(), "tableName is missing"); - return tableName.get(); + return tableName; } public List getSetupQueries() @@ -62,4 +62,9 @@ public List getTeardownQueries() { return teardownQueries; } + + public ClusterType getCluster() + { + return cluster; + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java index 7b08fc2f47c46..373b1d40977da 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfiguration.java @@ -17,16 +17,21 @@ import org.jdbi.v3.core.mapper.reflect.ColumnName; import org.jdbi.v3.core.mapper.reflect.JdbiConstructor; +import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; import java.util.Optional; +import static com.facebook.presto.verifier.framework.QueryConfigurationOverrides.SessionPropertiesOverrideStrategy.OVERRIDE; +import static com.facebook.presto.verifier.framework.QueryConfigurationOverrides.SessionPropertiesOverrideStrategy.SUBSTITUTE; import static java.util.Objects.requireNonNull; public class QueryConfiguration { private final String catalog; private final String schema; - private final String username; + private final Optional username; private final Optional password; private final Map sessionProperties; @@ -34,25 +39,36 @@ public class QueryConfiguration public QueryConfiguration( @ColumnName("catalog") String catalog, @ColumnName("schema") String schema, - @ColumnName("username") String username, + @ColumnName("username") Optional username, @ColumnName("password") Optional password, @ColumnName("session_properties") Optional> sessionProperties) - { - this(catalog, schema, username, password, sessionProperties.orElse(ImmutableMap.of())); - } - - public QueryConfiguration( - String catalog, - String schema, - String username, - Optional password, - Map sessionProperties) { this.catalog = requireNonNull(catalog, "catalog is null"); this.schema = requireNonNull(schema, "schema is null"); this.username = requireNonNull(username, "username is null"); this.password = requireNonNull(password, "password is null"); - this.sessionProperties = ImmutableMap.copyOf(sessionProperties); + this.sessionProperties = ImmutableMap.copyOf(sessionProperties.orElse(ImmutableMap.of())); + } + + public QueryConfiguration applyOverrides(QueryConfigurationOverrides overrides) + { + Map sessionProperties = this.sessionProperties; + if (overrides.getSessionPropertiesOverrideStrategy() == OVERRIDE) { + sessionProperties = overrides.getSessionPropertiesOverride(); + } + else if (overrides.getSessionPropertiesOverrideStrategy() == SUBSTITUTE) { + sessionProperties = new HashMap<>(sessionProperties); + for (Entry entry : overrides.getSessionPropertiesOverride().entrySet()) { + sessionProperties.put(entry.getKey(), entry.getValue()); + } + } + + return new QueryConfiguration( + overrides.getCatalogOverride().orElse(catalog), + overrides.getSchemaOverride().orElse(schema), + Optional.ofNullable(overrides.getUsernameOverride().orElse(username.orElse(null))), + Optional.ofNullable(overrides.getPasswordOverride().orElse(password.orElse(null))), + Optional.of(sessionProperties)); } public String getCatalog() @@ -65,7 +81,7 @@ public String getSchema() return schema; } - public String getUsername() + public Optional getUsername() { return username; } @@ -79,4 +95,27 @@ public Map getSessionProperties() { return sessionProperties; } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + QueryConfiguration o = (QueryConfiguration) obj; + return Objects.equals(catalog, o.catalog) && + Objects.equals(schema, o.schema) && + Objects.equals(username, o.username) && + Objects.equals(password, o.password) && + Objects.equals(sessionProperties, o.sessionProperties); + } + + @Override + public int hashCode() + { + return Objects.hash(catalog, schema, username, password, sessionProperties); + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfigurationOverrides.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfigurationOverrides.java new file mode 100644 index 0000000000000..44ab37a716ad1 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfigurationOverrides.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +import java.util.Map; +import java.util.Optional; + +public interface QueryConfigurationOverrides +{ + enum SessionPropertiesOverrideStrategy + { + NO_ACTION, + OVERRIDE, + SUBSTITUTE, + } + + Optional getCatalogOverride(); + + Optional getSchemaOverride(); + + Optional getUsernameOverride(); + + Optional getPasswordOverride(); + + SessionPropertiesOverrideStrategy getSessionPropertiesOverrideStrategy(); + + Map getSessionPropertiesOverride(); +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfigurationOverridesConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfigurationOverridesConfig.java new file mode 100644 index 0000000000000..c442c8c33ef3b --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryConfigurationOverridesConfig.java @@ -0,0 +1,131 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; + +import javax.validation.constraints.NotNull; + +import java.io.IOException; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.verifier.framework.QueryConfigurationOverrides.SessionPropertiesOverrideStrategy.NO_ACTION; + +public class QueryConfigurationOverridesConfig + implements QueryConfigurationOverrides +{ + private Optional catalogOverride = Optional.empty(); + private Optional schemaOverride = Optional.empty(); + private Optional usernameOverride = Optional.empty(); + private Optional passwordOverride = Optional.empty(); + private SessionPropertiesOverrideStrategy sessionPropertiesOverrideStrategy = NO_ACTION; + private Map sessionPropertiesOverride = ImmutableMap.of(); + + @Override + public Optional getCatalogOverride() + { + return catalogOverride; + } + + @ConfigDescription("Overrides the catalog for all the queries") + @Config("catalog-override") + public QueryConfigurationOverridesConfig setCatalogOverride(String catalogOverride) + { + this.catalogOverride = Optional.ofNullable(catalogOverride); + return this; + } + + @Override + public Optional getSchemaOverride() + { + return schemaOverride; + } + + @ConfigDescription("Overrides the schema for all the queries") + @Config("schema-override") + public QueryConfigurationOverridesConfig setSchemaOverride(String schemaOverride) + { + this.schemaOverride = Optional.ofNullable(schemaOverride); + return this; + } + + @Override + public Optional getUsernameOverride() + { + return usernameOverride; + } + + @ConfigDescription("Overrides the username for all the queries") + @Config("username-override") + public QueryConfigurationOverridesConfig setUsernameOverride(String usernameOverride) + { + this.usernameOverride = Optional.ofNullable(usernameOverride); + return this; + } + + @Override + public Optional getPasswordOverride() + { + return passwordOverride; + } + + @ConfigDescription("Overrides the password for all the queries") + @Config("password-override") + public QueryConfigurationOverridesConfig setPasswordOverride(String passwordOverride) + { + this.passwordOverride = Optional.ofNullable(passwordOverride); + return this; + } + + @Override + @NotNull + public SessionPropertiesOverrideStrategy getSessionPropertiesOverrideStrategy() + { + return sessionPropertiesOverrideStrategy; + } + + @Config("session-properties-override-strategy") + public QueryConfigurationOverridesConfig setSessionPropertiesOverrideStrategy(SessionPropertiesOverrideStrategy sessionPropertiesOverrideStrategy) + { + this.sessionPropertiesOverrideStrategy = sessionPropertiesOverrideStrategy; + return this; + } + + @Override + public Map getSessionPropertiesOverride() + { + return sessionPropertiesOverride; + } + + @Config("session-properties-override") + public QueryConfigurationOverridesConfig setSessionPropertiesOverride(String sessionPropertyOverride) + { + if (sessionPropertyOverride == null) { + return this; + } + + try { + this.sessionPropertiesOverride = new ObjectMapper().readValue(sessionPropertyOverride, new TypeReference>() {}); + } + catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryException.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryException.java index 4b899dcac7c4b..7dd03ffc26efd 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryException.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryException.java @@ -16,12 +16,14 @@ import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.spi.ErrorCode; import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.verifier.event.QueryFailure; import com.google.common.base.Function; import java.util.Optional; import static com.facebook.presto.verifier.framework.QueryException.Type.CLUSTER_CONNECTION; import static com.facebook.presto.verifier.framework.QueryException.Type.PRESTO; +import static com.google.common.base.Throwables.getStackTraceAsString; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -52,7 +54,7 @@ public enum Type private final Optional prestoErrorCode; private final boolean retryable; private final Optional queryStats; - private final QueryOrigin queryOrigin; + private final QueryStage queryStage; private QueryException( Throwable cause, @@ -60,24 +62,24 @@ private QueryException( Optional prestoErrorCode, boolean retryable, Optional queryStats, - QueryOrigin queryOrigin) + QueryStage queryStage) { super(cause); this.type = requireNonNull(type, "type is null"); this.prestoErrorCode = requireNonNull(prestoErrorCode, "errorCode is null"); this.retryable = retryable; this.queryStats = requireNonNull(queryStats, "queryStats is null"); - this.queryOrigin = requireNonNull(queryOrigin, "queryOrigin is null"); + this.queryStage = requireNonNull(queryStage, "queryStage is null"); } - public static QueryException forClusterConnection(Throwable cause, QueryOrigin queryOrigin) + public static QueryException forClusterConnection(Throwable cause, QueryStage queryStage) { - return new QueryException(cause, CLUSTER_CONNECTION, Optional.empty(), true, Optional.empty(), queryOrigin); + return new QueryException(cause, CLUSTER_CONNECTION, Optional.empty(), true, Optional.empty(), queryStage); } - public static QueryException forPresto(Throwable cause, Optional prestoErrorCode, boolean retryable, Optional queryStats, QueryOrigin queryOrigin) + public static QueryException forPresto(Throwable cause, Optional prestoErrorCode, boolean retryable, Optional queryStats, QueryStage queryStage) { - return new QueryException(cause, PRESTO, prestoErrorCode, retryable, queryStats, queryOrigin); + return new QueryException(cause, PRESTO, prestoErrorCode, retryable, queryStats, queryStage); } public Type getType() @@ -100,13 +102,23 @@ public Optional getQueryStats() return queryStats; } - public QueryOrigin getQueryOrigin() + public QueryStage getQueryStage() { - return queryOrigin; + return queryStage; } public String getErrorCode() { return format("%s(%s)", type.name(), type.descriptionGenerator.apply(this)); } + + public QueryFailure toQueryFailure() + { + return new QueryFailure( + queryStage, + getErrorCode(), + retryable, + queryStats.map(QueryStats::getQueryId), + getStackTraceAsString(this)); + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryOrigin.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryOrigin.java deleted file mode 100644 index 50c6690579ce4..0000000000000 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryOrigin.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.verifier.framework; - -import java.util.Objects; - -import static com.facebook.presto.verifier.framework.QueryOrigin.QueryStage.CHECKSUM; -import static com.facebook.presto.verifier.framework.QueryOrigin.QueryStage.DESCRIBE; -import static com.facebook.presto.verifier.framework.QueryOrigin.QueryStage.MAIN; -import static com.facebook.presto.verifier.framework.QueryOrigin.QueryStage.REWRITE; -import static com.facebook.presto.verifier.framework.QueryOrigin.QueryStage.SETUP; -import static com.facebook.presto.verifier.framework.QueryOrigin.QueryStage.TEARDOWN; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.CONTROL; -import static java.util.Objects.requireNonNull; - -public class QueryOrigin -{ - public enum TargetCluster - { - CONTROL, - TEST, - } - - public enum QueryStage - { - SETUP, - MAIN, - TEARDOWN, - REWRITE, - DESCRIBE, - CHECKSUM, - } - - private final TargetCluster cluster; - private final QueryStage stage; - - private QueryOrigin(TargetCluster cluster, QueryStage stage) - { - this.cluster = requireNonNull(cluster, "cluster is null"); - this.stage = requireNonNull(stage, "stage is null"); - } - - public static QueryOrigin forSetup(TargetCluster group) - { - return new QueryOrigin(group, SETUP); - } - - public static QueryOrigin forMain(TargetCluster group) - { - return new QueryOrigin(group, MAIN); - } - - public static QueryOrigin forTeardown(TargetCluster group) - { - return new QueryOrigin(group, TEARDOWN); - } - - public static QueryOrigin forRewrite() - { - return new QueryOrigin(CONTROL, REWRITE); - } - - public static QueryOrigin forDescribe() - { - return new QueryOrigin(CONTROL, DESCRIBE); - } - - public static QueryOrigin forChecksum() - { - return new QueryOrigin(CONTROL, CHECKSUM); - } - - public TargetCluster getCluster() - { - return cluster; - } - - public QueryStage getStage() - { - return stage; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - QueryOrigin o = (QueryOrigin) obj; - return Objects.equals(cluster, o.cluster) && - Objects.equals(stage, o.stage); - } - - @Override - public int hashCode() - { - return Objects.hash(cluster, stage); - } -} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryStage.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryStage.java new file mode 100644 index 0000000000000..72226c617831d --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryStage.java @@ -0,0 +1,80 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; +import static com.facebook.presto.verifier.framework.ClusterType.TEST; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public enum QueryStage +{ + CONTROL_SETUP(CONTROL), + CONTROL_MAIN(CONTROL), + CONTROL_TEARDOWN(CONTROL), + + TEST_SETUP(TEST), + TEST_MAIN(TEST), + TEST_TEARDOWN(TEST), + + REWRITE(CONTROL), + DESCRIBE(CONTROL), + CHECKSUM(CONTROL), + DETERMINISM_ANALYSIS(CONTROL); + + private final ClusterType targetCluster; + + QueryStage(ClusterType targetCluster) + { + this.targetCluster = requireNonNull(targetCluster, "targetCluster is null"); + } + + public ClusterType getTargetCluster() + { + return targetCluster; + } + + public boolean isSetup() + { + return this == CONTROL_SETUP || this == TEST_SETUP; + } + + public boolean isMain() + { + return this == CONTROL_MAIN || this == TEST_MAIN; + } + + public boolean isTeardown() + { + return this == CONTROL_TEARDOWN || this == TEST_TEARDOWN; + } + + public static QueryStage forSetup(ClusterType cluster) + { + checkState(cluster == CONTROL || cluster == TEST, "Invalid cluster: %s", cluster); + return cluster == CONTROL ? CONTROL_SETUP : TEST_SETUP; + } + + public static QueryStage forMain(ClusterType cluster) + { + checkState(cluster == CONTROL || cluster == TEST, "Invalid cluster: %s", cluster); + return cluster == CONTROL ? CONTROL_MAIN : TEST_MAIN; + } + + public static QueryStage forTeardown(ClusterType cluster) + { + checkState(cluster == CONTROL || cluster == TEST, "Invalid cluster: %s", cluster); + return cluster == CONTROL ? CONTROL_TEARDOWN : TEST_TEARDOWN; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/SkippedReason.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/SkippedReason.java index f3f7d9b148ac5..02ba16c87eb4c 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/SkippedReason.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/SkippedReason.java @@ -15,9 +15,16 @@ public enum SkippedReason { + SYNTAX_ERROR, + UNSUPPORTED_QUERY_TYPE, + MISMATCHED_QUERY_TYPE, + CUSTOM_FILTER, + CONTROL_QUERY_FAILED, CONTROL_SETUP_QUERY_FAILED, CONTROL_QUERY_TIMED_OUT, FAILED_BEFORE_CONTROL_QUERY, - NON_DETERMINISTIC + NON_DETERMINISTIC, + + VERIFIER_LIMITATION, } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationContext.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationContext.java index 1c7080e37e8f8..a32d817cf002f 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationContext.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationContext.java @@ -13,44 +13,121 @@ */ package com.facebook.presto.verifier.framework; -import com.facebook.presto.jdbc.QueryStats; -import com.facebook.presto.verifier.event.FailureInfo; -import com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster; +import com.facebook.presto.verifier.event.DeterminismAnalysisRun; +import com.facebook.presto.verifier.event.QueryFailure; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; -import java.util.EnumMap; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.Optional; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.CONTROL; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.TEST; -import static com.google.common.base.Throwables.getStackTraceAsString; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.NOT_RUN; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; public class VerificationContext { - private Map> failures = new EnumMap<>(TargetCluster.class); + private String controlChecksumQueryId; + private String controlChecksumQuery; + private String testChecksumQueryId; + private String testChecksumQuery; - public VerificationContext() + private ImmutableList.Builder determinismAnalysisRuns = ImmutableList.builder(); + private LimitQueryDeterminismAnalysis limitQueryAnalysis; + private String limitQueryAnalysisQueryId; + + private ImmutableSet.Builder queryExceptions = ImmutableSet.builder(); + + public Optional getControlChecksumQueryId() + { + return Optional.ofNullable(controlChecksumQueryId); + } + + public void setControlChecksumQueryId(String controlChecksumQueryId) + { + checkState(this.controlChecksumQueryId == null, "controlChecksumQueryId is already set"); + this.controlChecksumQueryId = requireNonNull(controlChecksumQueryId, "controlChecksumQueryId is null"); + } + + public Optional getControlChecksumQuery() + { + return Optional.ofNullable(controlChecksumQuery); + } + + public void setControlChecksumQuery(String controlChecksumQuery) + { + checkState(this.controlChecksumQuery == null, "controlChecksumQuery is already set"); + this.controlChecksumQuery = requireNonNull(controlChecksumQuery, "controlChecksumQuery is null"); + } + + public Optional getTestChecksumQueryId() + { + return Optional.ofNullable(testChecksumQueryId); + } + + public void setTestChecksumQueryId(String testChecksumQueryId) + { + checkState(this.testChecksumQueryId == null, "testChecksumQueryId is already set"); + this.testChecksumQueryId = requireNonNull(testChecksumQueryId, "testChecksumQueryId is null"); + } + + public Optional getTestChecksumQuery() + { + return Optional.ofNullable(testChecksumQuery); + } + + public void setTestChecksumQuery(String testChecksumQuery) + { + checkState(this.testChecksumQuery == null, "testChecksumQuery is already set"); + this.testChecksumQuery = requireNonNull(testChecksumQuery, "testChecksumQuery is null"); + } + + public List getDeterminismAnalysisRuns() + { + return determinismAnalysisRuns.build().stream() + .map(DeterminismAnalysisRun.Builder::build) + .collect(toImmutableList()); + } + + public DeterminismAnalysisRun.Builder startDeterminismAnalysisRun() + { + DeterminismAnalysisRun.Builder run = DeterminismAnalysisRun.builder(); + determinismAnalysisRuns.add(run); + return run; + } + + public LimitQueryDeterminismAnalysis getLimitQueryAnalysis() + { + return limitQueryAnalysis == null ? NOT_RUN : limitQueryAnalysis; + } + + public void setLimitQueryAnalysis(LimitQueryDeterminismAnalysis limitQueryAnalysis) + { + checkState(this.limitQueryAnalysis == null, "limitQueryAnalysis is already set"); + this.limitQueryAnalysis = requireNonNull(limitQueryAnalysis, "limitQueryAnalysis is null"); + } + + public Optional getLimitQueryAnalysisQueryId() + { + return Optional.ofNullable(limitQueryAnalysisQueryId); + } + + public void setLimitQueryAnalysisQueryId(String limitQueryAnalysisQueryId) { - failures.put(CONTROL, new LinkedHashSet<>()); - failures.put(TEST, new LinkedHashSet<>()); + checkState(this.limitQueryAnalysisQueryId == null, "limitQueryAnalysisQueryId is already set"); + this.limitQueryAnalysisQueryId = requireNonNull(limitQueryAnalysisQueryId, "limitQueryAnalysisQueryId is null"); } - public void recordFailure(QueryException exception) + public void addException(QueryException exception) { - failures.get(exception.getQueryOrigin().getCluster()).add(exception); + queryExceptions.add(exception); } - public List getAllFailures(TargetCluster cluster) + public List getQueryFailures() { - return failures.get(cluster).stream() - .map(exception -> new FailureInfo( - exception.getQueryOrigin().getStage(), - exception.getErrorCode(), - exception.getQueryStats().map(QueryStats::getQueryId), - getStackTraceAsString(exception))) + return queryExceptions.build().stream() + .map(QueryException::toQueryFailure) .collect(toImmutableList()); } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationFactory.java index 42c80777e5fc0..6f623e66547f0 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationFactory.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationFactory.java @@ -13,14 +13,22 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.verifier.annotation.ForTest; import com.facebook.presto.verifier.checksum.ChecksumValidator; -import com.facebook.presto.verifier.resolver.FailureResolver; +import com.facebook.presto.verifier.prestoaction.NodeResourceClient; +import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.facebook.presto.verifier.prestoaction.PrestoActionFactory; +import com.facebook.presto.verifier.resolver.FailureResolverConfig; +import com.facebook.presto.verifier.resolver.FailureResolverFactoryContext; +import com.facebook.presto.verifier.resolver.FailureResolverManager; +import com.facebook.presto.verifier.resolver.FailureResolverManagerFactory; +import com.facebook.presto.verifier.rewrite.QueryRewriter; +import com.facebook.presto.verifier.rewrite.QueryRewriterFactory; import javax.inject.Inject; -import java.util.List; - import static com.facebook.presto.verifier.framework.VerifierUtil.PARSING_OPTIONS; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -28,27 +36,36 @@ public class VerificationFactory { private final SqlParser sqlParser; - private final PrestoAction prestoAction; - private final QueryRewriter queryRewriter; + private final PrestoActionFactory prestoActionFactory; + private final QueryRewriterFactory queryRewriterFactory; + private final FailureResolverManagerFactory failureResolverManagerFactory; + private final NodeResourceClient testResourceClient; private final ChecksumValidator checksumValidator; - private final List failureResolvers; - private final VerifierConfig config; + private final VerifierConfig verifierConfig; + private final TypeManager typeManager; + private final FailureResolverConfig failureResolverConfig; @Inject public VerificationFactory( SqlParser sqlParser, - PrestoAction prestoAction, - QueryRewriter queryRewriter, + PrestoActionFactory prestoActionFactory, + QueryRewriterFactory queryRewriterFactory, + FailureResolverManagerFactory failureResolverManagerFactory, + @ForTest NodeResourceClient testResourceClient, ChecksumValidator checksumValidator, - List failureResolvers, - VerifierConfig config) + VerifierConfig verifierConfig, + TypeManager typeManager, + FailureResolverConfig failureResolverConfig) { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); - this.prestoAction = requireNonNull(prestoAction, "prestoAction is null"); - this.queryRewriter = requireNonNull(queryRewriter, "queryRewriter is null"); + this.prestoActionFactory = requireNonNull(prestoActionFactory, "prestoActionFactory is null"); + this.queryRewriterFactory = requireNonNull(queryRewriterFactory, "queryRewriterFactory is null"); + this.failureResolverManagerFactory = requireNonNull(failureResolverManagerFactory, "failureResolverManagerFactory is null"); + this.testResourceClient = requireNonNull(testResourceClient, "testResourceClient is null"); this.checksumValidator = requireNonNull(checksumValidator, "checksumValidator is null"); - this.failureResolvers = requireNonNull(failureResolvers, "failureResolvers is null"); - this.config = requireNonNull(config, "config is null"); + this.verifierConfig = requireNonNull(verifierConfig, "config is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.failureResolverConfig = requireNonNull(failureResolverConfig, "failureResolverConfig is null"); } public Verification get(VerificationResubmitter verificationResubmitter, SourceQuery sourceQuery) @@ -56,7 +73,26 @@ public Verification get(VerificationResubmitter verificationResubmitter, SourceQ QueryType queryType = QueryType.of(sqlParser.createStatement(sourceQuery.getControlQuery(), PARSING_OPTIONS)); switch (queryType.getCategory()) { case DATA_PRODUCING: - return new DataVerification(verificationResubmitter, prestoAction, sourceQuery, queryRewriter, failureResolvers, config, checksumValidator); + VerificationContext verificationContext = new VerificationContext(); + PrestoAction prestoAction = prestoActionFactory.create(sourceQuery, verificationContext); + QueryRewriter queryRewriter = queryRewriterFactory.create(prestoAction); + FailureResolverManager failureResolverManager = failureResolverManagerFactory.create(new FailureResolverFactoryContext( + sqlParser, + prestoAction, + testResourceClient, + failureResolverConfig)); + LimitQueryDeterminismAnalyzer limitQueryDeterminismAnalyzer = new LimitQueryDeterminismAnalyzer(prestoAction, verifierConfig); + return new DataVerification( + verificationResubmitter, + prestoAction, + sourceQuery, + queryRewriter, + failureResolverManager, + verificationContext, + verifierConfig, + typeManager, + checksumValidator, + limitQueryDeterminismAnalyzer); default: throw new IllegalStateException(format("Unsupported query type: %s", queryType)); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationManager.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationManager.java index 60f06409fe7de..08ae4fd4a6ea6 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationManager.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationManager.java @@ -13,16 +13,18 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.airlift.event.client.EventClient; +import com.facebook.airlift.log.Logger; import com.facebook.presto.sql.parser.ParsingException; import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.verifier.annotation.ForControl; +import com.facebook.presto.verifier.annotation.ForTest; import com.facebook.presto.verifier.event.VerifierQueryEvent; import com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus; import com.facebook.presto.verifier.source.SourceQuerySupplier; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import io.airlift.event.client.EventClient; -import io.airlift.log.Logger; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -30,7 +32,9 @@ import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; import java.util.EnumMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -47,10 +51,13 @@ import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.FAILED_RESOLVED; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SKIPPED; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SUCCEEDED; -import static com.facebook.presto.verifier.framework.JdbcDriverUtil.initializeDrivers; -import static com.facebook.presto.verifier.framework.PrestoExceptionClassifier.shouldResubmit; import static com.facebook.presto.verifier.framework.QueryType.Category.DATA_PRODUCING; +import static com.facebook.presto.verifier.framework.SkippedReason.CUSTOM_FILTER; +import static com.facebook.presto.verifier.framework.SkippedReason.MISMATCHED_QUERY_TYPE; +import static com.facebook.presto.verifier.framework.SkippedReason.SYNTAX_ERROR; +import static com.facebook.presto.verifier.framework.SkippedReason.UNSUPPORTED_QUERY_TYPE; import static com.facebook.presto.verifier.framework.VerifierUtil.PARSING_OPTIONS; +import static com.facebook.presto.verifier.prestoaction.PrestoExceptionClassifier.shouldResubmit; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.Thread.currentThread; import static java.util.Objects.requireNonNull; @@ -67,18 +74,10 @@ public class VerificationManager private final Set eventClients; private final List> customQueryFilters; - private final Optional additionalJdbcDriverPath; - private final Optional controlJdbcDriverClass; - private final Optional testJdbcDriverClass; + private final QueryConfigurationOverrides controlOverrides; + private final QueryConfigurationOverrides testOverrides; - private final Optional controlCatalogOverride; - private final Optional controlSchemaOverride; - private final Optional controlUsernameOverride; - private final Optional controlPasswordOverride; - private final Optional testCatalogOverride; - private final Optional testSchemaOverride; - private final Optional testUsernameOverride; - private final Optional testPasswordOverride; + private final String testId; private final Optional> whitelist; private final Optional> blacklist; @@ -100,6 +99,8 @@ public VerificationManager( SqlParser sqlParser, Set eventClients, List> customQueryFilters, + @ForControl QueryConfigurationOverrides controlOverrides, + @ForTest QueryConfigurationOverrides testOverrides, VerifierConfig config) { this.sourceQuerySupplier = requireNonNull(sourceQuerySupplier, "sourceQuerySupplier is null"); @@ -108,18 +109,10 @@ public VerificationManager( this.eventClients = ImmutableSet.copyOf(eventClients); this.customQueryFilters = requireNonNull(customQueryFilters, "customQueryFilters is null"); - this.additionalJdbcDriverPath = requireNonNull(config.getAdditionalJdbcDriverPath(), "additionalJdbcDriverPath is null"); - this.controlJdbcDriverClass = requireNonNull(config.getControlJdbcDriverClass(), "controlJdbcDriverClass is null"); - this.testJdbcDriverClass = requireNonNull(config.getTestJdbcDriverClass(), "testJdbcDriverClass is null"); + this.controlOverrides = requireNonNull(controlOverrides, "controlOverrides is null"); + this.testOverrides = requireNonNull(testOverrides, "testOverride is null"); - this.controlCatalogOverride = requireNonNull(config.getControlCatalogOverride(), "controlCatalogOverride is null"); - this.controlSchemaOverride = requireNonNull(config.getControlSchemaOverride(), "controlSchemaOverride is null"); - this.controlUsernameOverride = requireNonNull(config.getControlUsernameOverride(), "controlUsernameOverride is null"); - this.controlPasswordOverride = requireNonNull(config.getControlPasswordOverride(), "controlPasswordOverride is null"); - this.testCatalogOverride = requireNonNull(config.getTestCatalogOverride(), "testCatalogOverride is null"); - this.testSchemaOverride = requireNonNull(config.getTestSchemaOverride(), "testSchemaOverride is null"); - this.testUsernameOverride = requireNonNull(config.getTestUsernameOverride(), "testUsernameOverride is null"); - this.testPasswordOverride = requireNonNull(config.getTestPasswordOverride(), "testPasswordOverride is null"); + this.testId = requireNonNull(config.getTestId(), "testId is null"); this.whitelist = requireNonNull(config.getWhitelist(), "whitelist is null"); this.blacklist = requireNonNull(config.getBlacklist(), "blacklist is null"); @@ -133,7 +126,6 @@ public VerificationManager( @PostConstruct public void start() { - initializeDrivers(additionalJdbcDriverPath, controlJdbcDriverClass, testJdbcDriverClass); this.executor = newFixedThreadPool(maxConcurrency); this.completionService = new ExecutorCompletionService<>(executor); @@ -201,18 +193,8 @@ private List applyOverrides(List sourceQueries) sourceQuery.getName(), sourceQuery.getControlQuery(), sourceQuery.getTestQuery(), - new QueryConfiguration( - testCatalogOverride.orElse(sourceQuery.getTestConfiguration().getCatalog()), - testSchemaOverride.orElse(sourceQuery.getTestConfiguration().getSchema()), - testUsernameOverride.orElse(sourceQuery.getTestConfiguration().getUsername()), - Optional.ofNullable(testPasswordOverride.orElse(sourceQuery.getTestConfiguration().getPassword().orElse(null))), - sourceQuery.getTestConfiguration().getSessionProperties()), - new QueryConfiguration( - controlCatalogOverride.orElse(sourceQuery.getControlConfiguration().getCatalog()), - controlSchemaOverride.orElse(sourceQuery.getControlConfiguration().getSchema()), - controlUsernameOverride.orElse(sourceQuery.getControlConfiguration().getUsername()), - Optional.ofNullable(controlPasswordOverride.orElse(sourceQuery.getControlConfiguration().getPassword().orElse(null))), - sourceQuery.getControlConfiguration().getSessionProperties()))) + sourceQuery.getControlConfiguration().applyOverrides(controlOverrides), + sourceQuery.getTestConfiguration().applyOverrides(testOverrides))) .collect(toImmutableList()); } @@ -247,14 +229,23 @@ private List filterQueryType(List sourceQueries) try { QueryType controlQueryType = QueryType.of(sqlParser.createStatement(sourceQuery.getControlQuery(), PARSING_OPTIONS)); QueryType testQueryType = QueryType.of(sqlParser.createStatement(sourceQuery.getTestQuery(), PARSING_OPTIONS)); - if (controlQueryType == testQueryType && controlQueryType.getCategory() == DATA_PRODUCING) { + + if (controlQueryType != testQueryType) { + postEvent(VerifierQueryEvent.skipped(sourceQuery.getSuite(), testId, sourceQuery, MISMATCHED_QUERY_TYPE)); + } + else if (controlQueryType.getCategory() != DATA_PRODUCING) { + postEvent(VerifierQueryEvent.skipped(sourceQuery.getSuite(), testId, sourceQuery, UNSUPPORTED_QUERY_TYPE)); + } + else { selected.add(sourceQuery); } } catch (ParsingException e) { log.warn("Failed to parse query: %s", sourceQuery.getName()); + postEvent(VerifierQueryEvent.skipped(sourceQuery.getSuite(), testId, sourceQuery, SYNTAX_ERROR)); } catch (UnsupportedQueryTypeException ignored) { + postEvent(VerifierQueryEvent.skipped(sourceQuery.getSuite(), testId, sourceQuery, UNSUPPORTED_QUERY_TYPE)); } } List selectQueries = selected.build(); @@ -269,10 +260,16 @@ private List applyCustomFilters(List sourceQueries) } log.info("Applying custom query filters"); + sourceQueries = new ArrayList<>(sourceQueries); for (Predicate filter : customQueryFilters) { - sourceQueries = sourceQueries.stream() - .filter(filter::test) - .collect(toImmutableList()); + Iterator iterator = sourceQueries.iterator(); + while (iterator.hasNext()) { + SourceQuery sourceQuery = iterator.next(); + if (!filter.test(sourceQuery)) { + iterator.remove(); + postEvent(VerifierQueryEvent.skipped(sourceQuery.getSuite(), testId, sourceQuery, CUSTOM_FILTER)); + } + } log.info("Applying custom filter %s... Remaining queries: %s", filter.getClass().getSimpleName(), sourceQueries.size()); } return sourceQueries; @@ -308,9 +305,7 @@ private void reportProgressUntilFinished() } else { statusCount.compute(EventStatus.valueOf(event.get().getStatus()), (status, count) -> count == null ? 1 : count + 1); - for (EventClient eventClient : eventClients) { - eventClient.post(event.get()); - } + postEvent(event.get()); } double progress = ((double) completed) / queriesSubmitted.get() * 100; @@ -334,4 +329,11 @@ private void reportProgressUntilFinished() } } } + + private void postEvent(VerifierQueryEvent event) + { + for (EventClient eventClient : eventClients) { + eventClient.post(event); + } + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationResult.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationResult.java deleted file mode 100644 index a8f1ec0402843..0000000000000 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerificationResult.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.verifier.framework; - -import static java.util.Objects.requireNonNull; - -public class VerificationResult -{ - private final String controlChecksumQueryId; - private final String testChecksumQueryId; - private final String controlChecksumQuery; - private final String testChecksumQuery; - private final MatchResult matchResult; - - public VerificationResult( - String controlChecksumQueryId, - String testChecksumQueryId, - String controlChecksumQuery, - String testChecksumQuery, - MatchResult matchResult) - { - this.controlChecksumQueryId = requireNonNull(controlChecksumQueryId, "controlChecksumQueryId is null"); - this.testChecksumQueryId = requireNonNull(testChecksumQueryId, "testChecksumQueryId is null"); - this.controlChecksumQuery = requireNonNull(controlChecksumQuery, "controlChecksumQuery is null"); - this.testChecksumQuery = requireNonNull(testChecksumQuery, "testChecksumQuery is null"); - this.matchResult = requireNonNull(matchResult, "matchResult is null"); - } - - public String getControlChecksumQueryId() - { - return controlChecksumQueryId; - } - - public String getControlChecksumQuery() - { - return controlChecksumQuery; - } - - public String getTestChecksumQueryId() - { - return testChecksumQueryId; - } - - public String getTestChecksumQuery() - { - return testChecksumQuery; - } - - public MatchResult getMatchResult() - { - return matchResult; - } -} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierConfig.java index ae6109546bc02..dbdf05ec60d14 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierConfig.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierConfig.java @@ -13,13 +13,10 @@ */ package com.facebook.presto.verifier.framework; -import com.facebook.presto.sql.tree.QualifiedName; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableSet; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.units.Duration; -import io.airlift.units.MinDuration; import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; @@ -28,34 +25,9 @@ import java.util.Set; import static com.facebook.presto.verifier.source.MySqlSourceQuerySupplier.MYSQL_SOURCE_QUERY_SUPPLIER; -import static java.util.concurrent.TimeUnit.MINUTES; public class VerifierConfig { - private Optional additionalJdbcDriverPath = Optional.empty(); - private Optional controlJdbcDriverClass = Optional.empty(); - private Optional testJdbcDriverClass = Optional.empty(); - - private String controlJdbcUrl; - private String testJdbcUrl; - - private Duration controlTimeout = new Duration(10, MINUTES); - private Duration testTimeout = new Duration(30, MINUTES); - private Duration metadataTimeout = new Duration(3, MINUTES); - private Duration checksumTimeout = new Duration(20, MINUTES); - - private QualifiedName controlTablePrefix = QualifiedName.of("tmp_verifier_control"); - private QualifiedName testTablePrefix = QualifiedName.of("tmp_verifier_test"); - - private Optional controlCatalogOverride = Optional.empty(); - private Optional controlSchemaOverride = Optional.empty(); - private Optional controlUsernameOverride = Optional.empty(); - private Optional controlPasswordOverride = Optional.empty(); - private Optional testCatalogOverride = Optional.empty(); - private Optional testSchemaOverride = Optional.empty(); - private Optional testUsernameOverride = Optional.empty(); - private Optional testPasswordOverride = Optional.empty(); - private Optional> whitelist = Optional.empty(); private Optional> blacklist = Optional.empty(); @@ -72,273 +44,12 @@ public class VerifierConfig private double relativeErrorMargin = 1e-4; private double absoluteErrorMargin = 1e-12; - private boolean runTearDownOnResultMismatch; - private boolean failureResolverEnabled = true; - private int verificationResubmissionLimit = 2; - - @NotNull - public Optional getAdditionalJdbcDriverPath() - { - return additionalJdbcDriverPath; - } + private boolean runTeardownOnResultMismatch; + private boolean runTeardownForDeterminismAnalysis; - @ConfigDescription("Path for test jdbc driver") - @Config("additional-jdbc-driver-path") - public VerifierConfig setAdditionalJdbcDriverPath(String additionalJdbcDriverPath) - { - this.additionalJdbcDriverPath = Optional.ofNullable(additionalJdbcDriverPath); - return this; - } - - @NotNull - public Optional getControlJdbcDriverClass() - { - return controlJdbcDriverClass; - } - - @ConfigDescription("Fully qualified control JDBC driver name") - @Config("control.jdbc-driver-class") - public VerifierConfig setControlJdbcDriverClass(String controlJdbcDriverClass) - { - this.controlJdbcDriverClass = Optional.ofNullable(controlJdbcDriverClass); - return this; - } - - @NotNull - public Optional getTestJdbcDriverClass() - { - return testJdbcDriverClass; - } - - @ConfigDescription("Fully qualified test JDBC driver name") - @Config("test.jdbc-driver-class") - public VerifierConfig setTestJdbcDriverClass(String testJdbcDriverClass) - { - this.testJdbcDriverClass = Optional.ofNullable(testJdbcDriverClass); - return this; - } - - @NotNull - public String getControlJdbcUrl() - { - return controlJdbcUrl; - } - - @ConfigDescription("URL for the control cluster") - @Config("control.jdbc-url") - public VerifierConfig setControlJdbcUrl(String controlJdbcUrl) - { - this.controlJdbcUrl = controlJdbcUrl; - return this; - } - - @NotNull - public String getTestJdbcUrl() - { - return testJdbcUrl; - } - - @ConfigDescription("URL for the test cluster") - @Config("test.jdbc-url") - public VerifierConfig setTestJdbcUrl(String testJdbcUrl) - { - this.testJdbcUrl = testJdbcUrl; - return this; - } - - @MinDuration("1s") - public Duration getControlTimeout() - { - return controlTimeout; - } - - @ConfigDescription("Timeout for queries to the control cluster") - @Config("control.timeout") - public VerifierConfig setControlTimeout(Duration controlTimeout) - { - this.controlTimeout = controlTimeout; - return this; - } - - @MinDuration("1s") - public Duration getTestTimeout() - { - return testTimeout; - } - - @ConfigDescription("Timeout for queries to the test cluster") - @Config("test.timeout") - public VerifierConfig setTestTimeout(Duration testTimeout) - { - this.testTimeout = testTimeout; - return this; - } - - @MinDuration("1s") - public Duration getMetadataTimeout() - { - return metadataTimeout; - } - - @Config("metadata.timeout") - public VerifierConfig setMetadataTimeout(Duration metadataTimeout) - { - this.metadataTimeout = metadataTimeout; - return this; - } - - @MinDuration("1s") - public Duration getChecksumTimeout() - { - return checksumTimeout; - } - - @Config("checksum.timeout") - public VerifierConfig setChecksumTimeout(Duration checksumTimeout) - { - this.checksumTimeout = checksumTimeout; - return this; - } - - @NotNull - public QualifiedName getControlTablePrefix() - { - return controlTablePrefix; - } - - @ConfigDescription("The prefix to use for temporary control shadow tables. May be fully qualified like 'tmp_catalog.tmp_schema.tmp_'") - @Config("control.table-prefix") - public VerifierConfig setControlTablePrefix(String controlTablePrefix) - { - this.controlTablePrefix = controlTablePrefix == null ? - null : - QualifiedName.of(Splitter.on(".").splitToList(controlTablePrefix)); - return this; - } - - @NotNull - public QualifiedName getTestTablePrefix() - { - return testTablePrefix; - } - - @ConfigDescription("The prefix to use for temporary test shadow tables. May be fully qualified like 'tmp_catalog.tmp_schema.tmp_'") - @Config("test.table-prefix") - public VerifierConfig setTestTablePrefix(String testTablePrefix) - { - this.testTablePrefix = testTablePrefix == null ? - null : - QualifiedName.of(Splitter.on(".").splitToList(testTablePrefix)); - return this; - } - - public Optional getControlCatalogOverride() - { - return controlCatalogOverride; - } - - @ConfigDescription("Overrides the control_catalog field in all queries in the suites") - @Config("control.catalog-override") - public VerifierConfig setControlCatalogOverride(String controlCatalogOverride) - { - this.controlCatalogOverride = Optional.ofNullable(controlCatalogOverride); - return this; - } - - public Optional getControlSchemaOverride() - { - return controlSchemaOverride; - } - - @ConfigDescription("Overrides the control_schema field in all queries in the suites") - @Config("control.schema-override") - public VerifierConfig setControlSchemaOverride(String controlSchemaOverride) - { - this.controlSchemaOverride = Optional.ofNullable(controlSchemaOverride); - return this; - } - - @NotNull - public Optional getControlUsernameOverride() - { - return controlUsernameOverride; - } - - @ConfigDescription("Username for control cluster") - @Config("control.username-override") - public VerifierConfig setControlUsernameOverride(String controlUsernameOverride) - { - this.controlUsernameOverride = Optional.ofNullable(controlUsernameOverride); - return this; - } - - @NotNull - public Optional getControlPasswordOverride() - { - return controlPasswordOverride; - } - - @ConfigDescription("Password for control cluster") - @Config("control.password-override") - public VerifierConfig setControlPasswordOverride(String controlPasswordOverride) - { - this.controlPasswordOverride = Optional.ofNullable(controlPasswordOverride); - return this; - } - - public Optional getTestCatalogOverride() - { - return testCatalogOverride; - } - - @ConfigDescription("Overrides the test_catalog field in all queries in the suites") - @Config("test.catalog-override") - public VerifierConfig setTestCatalogOverride(String testCatalogOverride) - { - this.testCatalogOverride = Optional.ofNullable(testCatalogOverride); - return this; - } - - public Optional getTestSchemaOverride() - { - return testSchemaOverride; - } - - @ConfigDescription("Overrides the test_schema field in all queries in the suites") - @Config("test.schema-override") - public VerifierConfig setTestSchemaOverride(String testSchemaOverride) - { - this.testSchemaOverride = Optional.ofNullable(testSchemaOverride); - return this; - } - - @NotNull - public Optional getTestUsernameOverride() - { - return testUsernameOverride; - } - - @ConfigDescription("Username for test cluster") - @Config("test.username-override") - public VerifierConfig setTestUsernameOverride(String testUsernameOverride) - { - this.testUsernameOverride = Optional.ofNullable(testUsernameOverride); - return this; - } - - @NotNull - public Optional getTestPasswordOverride() - { - return testPasswordOverride; - } - - @ConfigDescription("Password for test cluster") - @Config("test.password-override") - public VerifierConfig setTestPasswordOverride(String testPasswordOverride) - { - this.testPasswordOverride = Optional.ofNullable(testPasswordOverride); - return this; - } + private int maxDeterminismAnalysisRuns = 2; + private boolean enableLimitQueryDeterminismAnalyzer = true; + private int verificationResubmissionLimit = 2; @NotNull public Optional> getWhitelist() @@ -511,28 +222,54 @@ public VerifierConfig setAbsoluteErrorMargin(double absoluteErrorMargin) return this; } - public boolean isRunTearDownOnResultMismatch() + public boolean isRunTeardownOnResultMismatch() { - return runTearDownOnResultMismatch; + return runTeardownOnResultMismatch; } @ConfigDescription("When set to false, temporary tables are not dropped in case of checksum failure") @Config("run-teardown-on-result-mismatch") - public VerifierConfig setRunTearDownOnResultMismatch(boolean runTearDownOnResultMismatch) + public VerifierConfig setRunTeardownOnResultMismatch(boolean runTeardownOnResultMismatch) + { + this.runTeardownOnResultMismatch = runTeardownOnResultMismatch; + return this; + } + + public boolean isRunTeardownForDeterminismAnalysis() + { + return runTeardownForDeterminismAnalysis; + } + + @ConfigDescription("When set to false, temporary tables are not dropped for determinism analysis runs") + @Config("run-teardown-for-determinism-analysis") + public VerifierConfig setRunTeardownForDeterminismAnalysis(boolean runTeardownForDeterminismAnalysis) + { + this.runTeardownForDeterminismAnalysis = runTeardownForDeterminismAnalysis; + return this; + } + + @Min(0) + public int getMaxDeterminismAnalysisRuns() + { + return maxDeterminismAnalysisRuns; + } + + @Config("max-determinism-analysis-runs") + public VerifierConfig setMaxDeterminismAnalysisRuns(int maxDeterminismAnalysisRuns) { - this.runTearDownOnResultMismatch = runTearDownOnResultMismatch; + this.maxDeterminismAnalysisRuns = maxDeterminismAnalysisRuns; return this; } - public boolean isFailureResolverEnabled() + public boolean isEnableLimitQueryDeterminismAnalyzer() { - return failureResolverEnabled; + return enableLimitQueryDeterminismAnalyzer; } - @Config("failure-resolver.enabled") - public VerifierConfig setFailureResolverEnabled(boolean failureResolverEnabled) + @Config("enable-limit-query-determinism-analyzer") + public VerifierConfig setEnableLimitQueryDeterminismAnalyzer(boolean enableLimitQueryDeterminismAnalyzer) { - this.failureResolverEnabled = failureResolverEnabled; + this.enableLimitQueryDeterminismAnalyzer = enableLimitQueryDeterminismAnalyzer; return this; } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierModule.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierModule.java index 6de96df40beb9..f1807f40b6872 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierModule.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierModule.java @@ -13,32 +13,62 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.presto.block.BlockEncodingManager; +import com.facebook.presto.metadata.CatalogManager; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.HandleJsonModule; +import com.facebook.presto.spi.block.BlockEncoding; +import com.facebook.presto.spi.block.BlockEncodingSerde; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.tree.Property; +import com.facebook.presto.transaction.ForTransactionManager; +import com.facebook.presto.transaction.InMemoryTransactionManager; +import com.facebook.presto.transaction.TransactionManager; +import com.facebook.presto.transaction.TransactionManagerConfig; +import com.facebook.presto.type.TypeRegistry; +import com.facebook.presto.verifier.annotation.ForControl; +import com.facebook.presto.verifier.annotation.ForTest; import com.facebook.presto.verifier.checksum.ChecksumValidator; import com.facebook.presto.verifier.checksum.FloatingPointColumnValidator; import com.facebook.presto.verifier.checksum.OrderableArrayColumnValidator; import com.facebook.presto.verifier.checksum.SimpleColumnValidator; -import com.facebook.presto.verifier.resolver.FailureResolver; +import com.facebook.presto.verifier.prestoaction.SqlExceptionClassifier; +import com.facebook.presto.verifier.prestoaction.VerificationPrestoActionModule; +import com.facebook.presto.verifier.resolver.FailureResolverFactory; +import com.facebook.presto.verifier.resolver.FailureResolverModule; import com.facebook.presto.verifier.retry.ForClusterConnection; import com.facebook.presto.verifier.retry.ForPresto; import com.facebook.presto.verifier.retry.RetryConfig; +import com.facebook.presto.verifier.rewrite.VerificationQueryRewriterModule; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; import com.google.inject.Inject; import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Provides; +import com.google.inject.Scopes; import com.google.inject.TypeLiteral; -import io.airlift.configuration.AbstractConfigurationAwareModule; import javax.inject.Provider; +import javax.inject.Singleton; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; import java.util.function.Predicate; +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static com.google.inject.Scopes.SINGLETON; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static com.google.inject.multibindings.Multibinder.newSetBinder; import static java.util.Objects.requireNonNull; +import static java.util.concurrent.Executors.newCachedThreadPool; +import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; public class VerifierModule extends AbstractConfigurationAwareModule @@ -46,26 +76,31 @@ public class VerifierModule private final SqlParserOptions sqlParserOptions; private final List>> customQueryFilterClasses; private final SqlExceptionClassifier exceptionClassifier; - private final List failureResolvers; + private final List failureResolverFactories; private final List tablePropertyOverrides; public VerifierModule( SqlParserOptions sqlParserOptions, List>> customQueryFilterClasses, SqlExceptionClassifier exceptionClassifier, - List failureResolvers, + List failureResolverFactories, List tablePropertyOverrides) { this.sqlParserOptions = requireNonNull(sqlParserOptions, "sqlParserOptions is null"); this.customQueryFilterClasses = ImmutableList.copyOf(customQueryFilterClasses); this.exceptionClassifier = requireNonNull(exceptionClassifier, "exceptionClassifier is null"); - this.failureResolvers = requireNonNull(failureResolvers, "failureResolvers is null"); + this.failureResolverFactories = requireNonNull(failureResolverFactories, "failureResolverFactories is null"); this.tablePropertyOverrides = requireNonNull(tablePropertyOverrides, "tablePropertyOverrides is null"); } protected final void setup(Binder binder) { configBinder(binder).bindConfig(VerifierConfig.class); + configBinder(binder).bindConfig(QueryConfigurationOverridesConfig.class, ForControl.class, "control"); + configBinder(binder).bindConfig(QueryConfigurationOverridesConfig.class, ForTest.class, "test"); + binder.bind(QueryConfigurationOverrides.class).annotatedWith(ForControl.class).to(Key.get(QueryConfigurationOverridesConfig.class, ForControl.class)).in(SINGLETON); + binder.bind(QueryConfigurationOverrides.class).annotatedWith(ForTest.class).to(Key.get(QueryConfigurationOverridesConfig.class, ForTest.class)).in(SINGLETON); + configBinder(binder).bindConfig(RetryConfig.class, ForClusterConnection.class, "cluster-connection"); configBinder(binder).bindConfig(RetryConfig.class, ForPresto.class, "presto"); @@ -73,10 +108,35 @@ protected final void setup(Binder binder) binder.bind(customQueryFilterClass).in(SINGLETON); } + // block encoding + binder.bind(BlockEncodingSerde.class).to(BlockEncodingManager.class).in(Scopes.SINGLETON); + newSetBinder(binder, BlockEncoding.class); + + // catalog + binder.bind(CatalogManager.class).in(Scopes.SINGLETON); + + // function + binder.bind(FunctionManager.class).in(SINGLETON); + + // handle resolver + binder.install(new HandleJsonModule()); + + // parser binder.bind(SqlParserOptions.class).toInstance(sqlParserOptions); binder.bind(SqlParser.class).in(SINGLETON); - binder.bind(QueryRewriter.class).in(SINGLETON); - binder.bind(PrestoAction.class).to(JdbcPrestoAction.class).in(SINGLETON); + + // transaction + configBinder(binder).bindConfig(TransactionManagerConfig.class); + + // type + configBinder(binder).bindConfig(FeaturesConfig.class); + binder.bind(TypeManager.class).to(TypeRegistry.class).in(SINGLETON); + newSetBinder(binder, Type.class); + + // verifier + install(new VerificationPrestoActionModule(exceptionClassifier)); + install(new VerificationQueryRewriterModule()); + install(new FailureResolverModule(failureResolverFactories)); binder.bind(VerificationManager.class).in(SINGLETON); binder.bind(VerificationFactory.class).in(SINGLETON); binder.bind(ChecksumValidator.class).in(SINGLETON); @@ -84,8 +144,6 @@ protected final void setup(Binder binder) binder.bind(FloatingPointColumnValidator.class).in(SINGLETON); binder.bind(OrderableArrayColumnValidator.class).in(SINGLETON); binder.bind(new TypeLiteral>>() {}).toProvider(new CustomQueryFilterProvider(customQueryFilterClasses)); - binder.bind(SqlExceptionClassifier.class).toInstance(exceptionClassifier); - binder.bind(new TypeLiteral>() {}).toInstance(failureResolvers); binder.bind(new TypeLiteral>() {}).toInstance(tablePropertyOverrides); } @@ -116,4 +174,31 @@ public List> get() return customVerificationFilters.build(); } } + + @Provides + @Singleton + @ForTransactionManager + public static ScheduledExecutorService createTransactionIdleCheckExecutor() + { + return newSingleThreadScheduledExecutor(daemonThreadsNamed("transaction-idle-check")); + } + + @Provides + @Singleton + @ForTransactionManager + public static ExecutorService createTransactionFinishingExecutor() + { + return newCachedThreadPool(daemonThreadsNamed("transaction-finishing-%s")); + } + + @Provides + @Singleton + public static TransactionManager createTransactionManager( + TransactionManagerConfig config, + CatalogManager catalogManager, + @ForTransactionManager ScheduledExecutorService idleCheckExecutor, + @ForTransactionManager ExecutorService finishingExecutor) + { + return InMemoryTransactionManager.create(config, idleCheckExecutor, catalogManager, finishingExecutor); + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java index 7b2a39904cf86..543063217cbc3 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifierUtil.java @@ -13,10 +13,15 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.tree.Identifier; +import java.util.function.Consumer; +import java.util.function.Function; + import static com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DOUBLE; +import static com.google.common.base.Functions.identity; public class VerifierUtil { @@ -30,4 +35,32 @@ public static Identifier delimitedIdentifier(String name) { return new Identifier(name, true); } + + public static void runWithQueryStatsConsumer(Callable callable, Consumer queryStatsConsumer) + { + callWithQueryStatsConsumer(callable, identity(), queryStatsConsumer); + } + + public static QueryResult callWithQueryStatsConsumer(Callable> callable, Consumer queryStatsConsumer) + { + return callWithQueryStatsConsumer(callable, QueryResult::getQueryStats, queryStatsConsumer); + } + + private static V callWithQueryStatsConsumer(Callable callable, Function queryStatsTransformer, Consumer queryStatsConsumer) + { + try { + V result = callable.call(); + queryStatsConsumer.accept(queryStatsTransformer.apply(result)); + return result; + } + catch (QueryException e) { + e.getQueryStats().ifPresent(queryStatsConsumer); + throw e; + } + } + + public interface Callable + { + V call(); + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifyCommand.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifyCommand.java index a668996232990..38828094a9898 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifyCommand.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/VerifyCommand.java @@ -15,7 +15,8 @@ import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.tree.Property; -import com.facebook.presto.verifier.resolver.FailureResolver; +import com.facebook.presto.verifier.prestoaction.SqlExceptionClassifier; +import com.facebook.presto.verifier.resolver.FailureResolverFactory; import com.google.inject.Module; import java.util.List; @@ -37,7 +38,7 @@ public interface VerifyCommand SqlExceptionClassifier getSqlExceptionClassifier(); - List getFailureResolvers(); + List getFailureResolverFactories(); List getTablePropertyOverrides(); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/HttpNodeResourceClient.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/HttpNodeResourceClient.java new file mode 100644 index 0000000000000..5fe405209bf88 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/HttpNodeResourceClient.java @@ -0,0 +1,87 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.json.ObjectMapperProvider; +import com.facebook.presto.verifier.retry.RetryConfig; +import com.facebook.presto.verifier.retry.RetryDriver; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.inject.Inject; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; + +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.airlift.http.client.StringResponseHandler.StringResponse; +import static com.facebook.airlift.http.client.StringResponseHandler.createStringResponseHandler; +import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static javax.ws.rs.core.HttpHeaders.CONTENT_TYPE; +import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static javax.ws.rs.core.Response.Status.OK; + +public class HttpNodeResourceClient + implements NodeResourceClient +{ + private final HttpClient httpClient; + private final PrestoAddress prestoAddress; + private final RetryDriver networkRetry; + + @Inject + public HttpNodeResourceClient( + HttpClient httpClient, + PrestoClusterConfig prestoAddress, + RetryConfig networkRetryConfig) + { + this.httpClient = requireNonNull(httpClient, "httpClient is null"); + this.prestoAddress = requireNonNull(prestoAddress, "prestoAddress is null"); + this.networkRetry = new RetryDriver<>(networkRetryConfig, PrestoExceptionClassifier::isClusterConnectionException, RuntimeException.class, e -> {}); + } + + @Override + public int getClusterSize(String path) + { + return networkRetry.run(format("getJsonResponse()", path), () -> getClusterSizeOnce(path)); + } + + private int getClusterSizeOnce(String path) + { + Request request = prepareGet() + .setUri(prestoAddress.getHttpUri(path)) + .setHeader(CONTENT_TYPE, APPLICATION_JSON) + .build(); + + StringResponse response = httpClient.execute(request, createStringResponseHandler()); + checkState( + response.getStatusCode() == OK.getStatusCode(), + "Invalid response: %s %s", + response.getStatusCode(), + response.getStatusMessage()); + + List> values; + try { + values = new ObjectMapperProvider().get().readValue(response.getBody(), new TypeReference>>() {}); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + + return values.size(); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/JdbcPrestoAction.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/JdbcPrestoAction.java similarity index 60% rename from presto-verifier/src/main/java/com/facebook/presto/verifier/framework/JdbcPrestoAction.java rename to presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/JdbcPrestoAction.java index 1ba47de08c06c..eb187b7c16221 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/JdbcPrestoAction.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/JdbcPrestoAction.java @@ -11,13 +11,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.verifier.framework; +package com.facebook.presto.verifier.prestoaction; import com.facebook.presto.jdbc.PrestoConnection; import com.facebook.presto.jdbc.PrestoStatement; import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.tree.Statement; -import com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster; +import com.facebook.presto.verifier.framework.QueryConfiguration; +import com.facebook.presto.verifier.framework.QueryException; +import com.facebook.presto.verifier.framework.QueryResult; +import com.facebook.presto.verifier.framework.QueryStage; +import com.facebook.presto.verifier.framework.VerificationContext; import com.facebook.presto.verifier.retry.ForClusterConnection; import com.facebook.presto.verifier.retry.ForPresto; import com.facebook.presto.verifier.retry.RetryConfig; @@ -26,8 +30,6 @@ import com.google.common.collect.ImmutableMap; import io.airlift.units.Duration; -import javax.inject.Inject; - import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLClientInfoException; @@ -40,10 +42,7 @@ import static com.facebook.presto.sql.SqlFormatter.formatSql; import static com.facebook.presto.verifier.framework.QueryException.Type.CLUSTER_CONNECTION; import static com.facebook.presto.verifier.framework.QueryException.Type.PRESTO; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.CONTROL; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.TEST; import static com.google.common.base.Preconditions.checkState; -import static java.lang.String.format; import static java.util.Objects.requireNonNull; public class JdbcPrestoAction @@ -52,83 +51,71 @@ public class JdbcPrestoAction private static final String QUERY_MAX_EXECUTION_TIME = "query_max_execution_time"; private final SqlExceptionClassifier exceptionClassifier; + private final QueryConfiguration queryConfiguration; - private final Map clusterUrls; - private final Duration controlTimeout; - private final Duration testTimeout; + private final String jdbcUrl; + private final Duration queryTimeout; private final Duration metadataTimeout; private final Duration checksumTimeout; - private final RetryDriver networkRetry; - private final RetryDriver prestoRetry; + private final RetryDriver networkRetry; + private final RetryDriver prestoRetry; - @Inject public JdbcPrestoAction( SqlExceptionClassifier exceptionClassifier, - VerifierConfig config, + QueryConfiguration queryConfiguration, + VerificationContext verificationContext, + PrestoClusterConfig prestoClusterConfig, @ForClusterConnection RetryConfig networkRetryConfig, @ForPresto RetryConfig prestoRetryConfig) { this.exceptionClassifier = requireNonNull(exceptionClassifier, "exceptionClassifier is null"); - this.clusterUrls = ImmutableMap.of( - CONTROL, config.getControlJdbcUrl(), - TEST, config.getTestJdbcUrl()); - this.controlTimeout = requireNonNull(config.getControlTimeout(), "controlTimeout is null"); - this.testTimeout = requireNonNull(config.getTestTimeout(), "testTimeout is null"); - this.metadataTimeout = requireNonNull(config.getMetadataTimeout(), "metadataTimeout is null"); - this.checksumTimeout = requireNonNull(config.getChecksumTimeout(), "checksumTimeout is null"); - - this.networkRetry = new RetryDriver(networkRetryConfig, queryException -> queryException.getType() == CLUSTER_CONNECTION); - this.prestoRetry = new RetryDriver(prestoRetryConfig, queryException -> queryException.getType() == PRESTO && queryException.isRetryable()); + this.queryConfiguration = requireNonNull(queryConfiguration, "queryConfiguration is null"); + + this.jdbcUrl = requireNonNull(prestoClusterConfig.getJdbcUrl(), "jdbcUrl is null"); + this.queryTimeout = requireNonNull(prestoClusterConfig.getQueryTimeout(), "queryTimeout is null"); + this.metadataTimeout = requireNonNull(prestoClusterConfig.getMetadataTimeout(), "metadataTimeout is null"); + this.checksumTimeout = requireNonNull(prestoClusterConfig.getChecksumTimeout(), "checksumTimeout is null"); + + this.networkRetry = new RetryDriver<>( + networkRetryConfig, + queryException -> queryException.getType() == CLUSTER_CONNECTION, + QueryException.class, + verificationContext::addException); + this.prestoRetry = new RetryDriver<>( + prestoRetryConfig, + queryException -> queryException.getType() == PRESTO && queryException.isRetryable(), + QueryException.class, + verificationContext::addException); } @Override - public QueryStats execute( - Statement statement, - QueryConfiguration configuration, - QueryOrigin queryOrigin, - VerificationContext context) + public QueryStats execute(Statement statement, QueryStage queryStage) { - return execute(statement, configuration, queryOrigin, context, Optional.empty()).getQueryStats(); + return execute(statement, queryStage, Optional.empty()).getQueryStats(); } @Override - public QueryResult execute( - Statement statement, - QueryConfiguration configuration, - QueryOrigin queryOrigin, - VerificationContext context, - ResultSetConverter converter) + public QueryResult execute(Statement statement, QueryStage queryStage, ResultSetConverter converter) { - return execute(statement, configuration, queryOrigin, context, Optional.of(converter)); + return execute(statement, queryStage, Optional.of(converter)); } - private QueryResult execute( - Statement statement, - QueryConfiguration configuration, - QueryOrigin queryOrigin, - VerificationContext context, - Optional> converter) + private QueryResult execute(Statement statement, QueryStage queryStage, Optional> converter) { return prestoRetry.run( "presto", - context, () -> networkRetry.run( "presto-cluster-connection", - context, - () -> executeOnce(statement, configuration, queryOrigin, converter))); + () -> executeOnce(statement, queryStage, converter))); } - private QueryResult executeOnce( - Statement statement, - QueryConfiguration configuration, - QueryOrigin queryOrigin, - Optional> converter) + private QueryResult executeOnce(Statement statement, QueryStage queryStage, Optional> converter) { String query = formatSql(statement, Optional.empty()); ProgressMonitor progressMonitor = new ProgressMonitor(); - try (PrestoConnection connection = getConnection(configuration, queryOrigin)) { + try (PrestoConnection connection = getConnection(queryStage)) { try (java.sql.Statement jdbcStatement = connection.createStatement()) { PrestoStatement prestoStatement = jdbcStatement.unwrap(PrestoStatement.class); prestoStatement.setProgressMonitor(progressMonitor); @@ -164,7 +151,7 @@ private QueryResult executeOnce( } } catch (SQLException e) { - throw exceptionClassifier.createException(queryOrigin, progressMonitor.getLastQueryStats(), e); + throw exceptionClassifier.createException(queryStage, progressMonitor.getLastQueryStats(), e); } } @@ -176,29 +163,27 @@ private void consumeResultSet(ResultSet resultSet) } } - private PrestoConnection getConnection( - QueryConfiguration configuration, - QueryOrigin queryOrigin) + private PrestoConnection getConnection(QueryStage queryStage) throws SQLException { PrestoConnection connection = DriverManager.getConnection( - clusterUrls.get(queryOrigin.getCluster()), - configuration.getUsername(), - configuration.getPassword().orElse(null)) + jdbcUrl, + queryConfiguration.getUsername().orElse(null), + queryConfiguration.getPassword().orElse(null)) .unwrap(PrestoConnection.class); try { connection.setClientInfo("ApplicationName", "verifier-test"); - connection.setCatalog(configuration.getCatalog()); - connection.setSchema(configuration.getSchema()); + connection.setCatalog(queryConfiguration.getCatalog()); + connection.setSchema(queryConfiguration.getSchema()); } catch (SQLClientInfoException ignored) { // Do nothing } Map sessionProperties = ImmutableMap.builder() - .putAll(configuration.getSessionProperties()) - .put(QUERY_MAX_EXECUTION_TIME, getTimeout(queryOrigin).toString()) + .putAll(queryConfiguration.getSessionProperties()) + .put(QUERY_MAX_EXECUTION_TIME, getTimeout(queryStage).toString()) .build(); for (Entry entry : sessionProperties.entrySet()) { connection.setSessionProperty(entry.getKey(), entry.getValue()); @@ -206,23 +191,16 @@ private PrestoConnection getConnection( return connection; } - private Duration getTimeout(QueryOrigin queryOrigin) + private Duration getTimeout(QueryStage queryStage) { - TargetCluster cluster = queryOrigin.getCluster(); - checkState(cluster == CONTROL || cluster == TEST, "Invalid TargetCluster: %s", cluster); - - switch (queryOrigin.getStage()) { - case SETUP: - case MAIN: - case TEARDOWN: - return cluster == CONTROL ? controlTimeout : testTimeout; + switch (queryStage) { case REWRITE: case DESCRIBE: return metadataTimeout; case CHECKSUM: return checksumTimeout; default: - throw new IllegalArgumentException(format("Invalid QueryStage: %s", queryOrigin.getStage())); + return queryTimeout; } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/NodeResourceClient.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/NodeResourceClient.java new file mode 100644 index 0000000000000..599a7f6c81279 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/NodeResourceClient.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +public interface NodeResourceClient +{ + int getClusterSize(String path); +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoAction.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAction.java similarity index 73% rename from presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoAction.java rename to presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAction.java index 6c4876c0cad7d..2e7dd33b6ed5f 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoAction.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAction.java @@ -11,10 +11,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.verifier.framework; +package com.facebook.presto.verifier.prestoaction; import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.sql.tree.Statement; +import com.facebook.presto.verifier.framework.QueryResult; +import com.facebook.presto.verifier.framework.QueryStage; import com.google.common.collect.ImmutableList; import java.sql.ResultSet; @@ -38,16 +40,7 @@ R apply(ResultSet resultSet) }; } - QueryStats execute( - Statement statement, - QueryConfiguration configuration, - QueryOrigin queryOrigin, - VerificationContext context); + QueryStats execute(Statement statement, QueryStage queryStage); - QueryResult execute( - Statement statement, - QueryConfiguration configuration, - QueryOrigin queryOrigin, - VerificationContext context, - ResultSetConverter converter); + QueryResult execute(Statement statement, QueryStage queryStage, ResultSetConverter converter); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoActionFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoActionFactory.java new file mode 100644 index 0000000000000..1f29a4941ca4d --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoActionFactory.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.presto.verifier.framework.SourceQuery; +import com.facebook.presto.verifier.framework.VerificationContext; + +public interface PrestoActionFactory +{ + PrestoAction create(SourceQuery sourceQuery, VerificationContext verificationContext); +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAddress.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAddress.java new file mode 100644 index 0000000000000..d3b93c7184cc5 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoAddress.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import java.net.URI; +import java.util.Map; + +import static java.lang.String.format; +import static java.util.stream.Collectors.joining; + +public interface PrestoAddress +{ + String getHost(); + + int getJdbcPort(); + + int getHttpPort(); + + Map getJdbcUrlParameters(); + + default String getJdbcUrl() + { + String query = getJdbcUrlParameters().entrySet().stream() + .map(entry -> format("%s=%s", entry.getKey(), entry.getValue())) + .collect(joining("&")); + return format("jdbc:presto://%s:%s?%s", getHost(), getJdbcPort(), query); + } + + default URI getHttpUri(String path) + { + return URI.create(format("http://%s:%s", getHost(), getHttpPort())).resolve(path); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoClusterConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoClusterConfig.java new file mode 100644 index 0000000000000..01f061ca54e32 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoClusterConfig.java @@ -0,0 +1,149 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.airlift.configuration.Config; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +import java.io.IOException; +import java.util.Map; + +import static java.util.concurrent.TimeUnit.MINUTES; + +public class PrestoClusterConfig + implements PrestoAddress +{ + private String host; + private int jdbcPort; + private int httpPort; + private Map jdbcUrlParameters = ImmutableMap.of(); + + private Duration queryTimeout = new Duration(60, MINUTES); + private Duration metadataTimeout = new Duration(3, MINUTES); + private Duration checksumTimeout = new Duration(30, MINUTES); + + @Override + @NotNull + public String getHost() + { + return host; + } + + @Config("host") + public PrestoClusterConfig setHost(String host) + { + this.host = host; + return this; + } + + @Override + @Min(0) + @Max(65535) + public int getJdbcPort() + { + return jdbcPort; + } + + @Config("jdbc-port") + public PrestoClusterConfig setJdbcPort(int jdbcPort) + { + this.jdbcPort = jdbcPort; + return this; + } + + @Override + @Min(0) + @Max(65535) + public int getHttpPort() + { + return httpPort; + } + + @Config("http-port") + public PrestoClusterConfig setHttpPort(int httpPort) + { + this.httpPort = httpPort; + return this; + } + + @Override + @NotNull + public Map getJdbcUrlParameters() + { + return jdbcUrlParameters; + } + + @Config("jdbc-url-parameters") + public PrestoClusterConfig setJdbcUrlParameters(String jdbcUrlParameters) + { + if (jdbcUrlParameters == null) { + return this; + } + + try { + this.jdbcUrlParameters = new ObjectMapper().readValue(jdbcUrlParameters, new TypeReference>() {}); + } + catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @MinDuration("1s") + public Duration getQueryTimeout() + { + return queryTimeout; + } + + @Config("query-timeout") + public PrestoClusterConfig setQueryTimeout(Duration queryTimeout) + { + this.queryTimeout = queryTimeout; + return this; + } + + @MinDuration("1s") + public Duration getMetadataTimeout() + { + return metadataTimeout; + } + + @Config("metadata-timeout") + public PrestoClusterConfig setMetadataTimeout(Duration metadataTimeout) + { + this.metadataTimeout = metadataTimeout; + return this; + } + + @MinDuration("1s") + public Duration getChecksumTimeout() + { + return checksumTimeout; + } + + @Config("checksum-timeout") + public PrestoClusterConfig setChecksumTimeout(Duration checksumTimeout) + { + this.checksumTimeout = checksumTimeout; + return this; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoExceptionClassifier.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java similarity index 87% rename from presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoExceptionClassifier.java rename to presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java index 27eb42b85c978..8fa5ae20114cf 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/PrestoExceptionClassifier.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java @@ -11,14 +11,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.verifier.framework; +package com.facebook.presto.verifier.prestoaction; import com.facebook.presto.connector.thrift.ThriftErrorCode; import com.facebook.presto.hive.HiveErrorCode; +import com.facebook.presto.hive.MetastoreErrorCode; import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.plugin.jdbc.JdbcErrorCode; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.StandardErrorCode; +import com.facebook.presto.verifier.framework.QueryException; +import com.facebook.presto.verifier.framework.QueryStage; import com.google.common.collect.ImmutableSet; import java.io.EOFException; @@ -34,15 +37,15 @@ import static com.facebook.presto.connector.thrift.ThriftErrorCode.THRIFT_SERVICE_CONNECTION_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CURSOR_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILE_NOT_FOUND; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_TABLE_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.HiveErrorCode.HIVE_TOO_MANY_OPEN_PARTITIONS; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_CLOSE_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_DATA_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_OPEN_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_TABLE_DROPPED_DURING_QUERY; import static com.facebook.presto.plugin.jdbc.JdbcErrorCode.JDBC_ERROR; import static com.facebook.presto.spi.StandardErrorCode.ABANDONED_TASK; import static com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE; @@ -62,6 +65,7 @@ public class PrestoExceptionClassifier { private static final Set DEFAULT_ERRORS = ImmutableSet.builder() .addAll(asList(StandardErrorCode.values())) + .addAll(asList(MetastoreErrorCode.values())) .addAll(asList(HiveErrorCode.values())) .addAll(asList(JdbcErrorCode.values())) .addAll(asList(ThriftErrorCode.values())) @@ -116,15 +120,15 @@ public PrestoExceptionClassifier( .build(); } - public QueryException createException(QueryOrigin queryOrigin, Optional queryStats, SQLException cause) + public QueryException createException(QueryStage queryStage, Optional queryStats, SQLException cause) { Optional clusterConnectionExceptionCause = getClusterConnectionExceptionCause(cause); if (clusterConnectionExceptionCause.isPresent()) { - return QueryException.forClusterConnection(clusterConnectionExceptionCause.get(), queryOrigin); + return QueryException.forClusterConnection(clusterConnectionExceptionCause.get(), queryStage); } Optional errorCode = Optional.ofNullable(errorByCode.get(cause.getErrorCode())); - return QueryException.forPresto(cause, errorCode, errorCode.isPresent() && retryableErrors.contains(errorCode.get()), queryStats, queryOrigin); + return QueryException.forPresto(cause, errorCode, errorCode.isPresent() && retryableErrors.contains(errorCode.get()), queryStats, queryStage); } public static boolean shouldResubmit(QueryException queryException) @@ -133,6 +137,11 @@ public static boolean shouldResubmit(QueryException queryException) && DEFAULT_REQUEUABLE_ERRORS.contains(queryException.getPrestoErrorCode().get()); } + public static boolean isClusterConnectionException(Throwable t) + { + return getClusterConnectionExceptionCause(t).isPresent(); + } + private static Optional getClusterConnectionExceptionCause(Throwable t) { while (t != null) { diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoAction.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoAction.java new file mode 100644 index 0000000000000..6dd68e28ff9b8 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoAction.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.sql.tree.Statement; +import com.facebook.presto.verifier.framework.ClusterType; +import com.facebook.presto.verifier.framework.QueryResult; +import com.facebook.presto.verifier.framework.QueryStage; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; +import static com.facebook.presto.verifier.framework.ClusterType.TEST; + +public class RoutingPrestoAction + implements PrestoAction +{ + private final Map prestoActions; + + public RoutingPrestoAction(PrestoAction controlAction, PrestoAction testAction) + { + this.prestoActions = ImmutableMap.of( + CONTROL, controlAction, + TEST, testAction); + } + + @Override + public QueryStats execute(Statement statement, QueryStage queryStage) + { + return prestoActions.get(queryStage.getTargetCluster()).execute(statement, queryStage); + } + + @Override + public QueryResult execute(Statement statement, QueryStage queryStage, ResultSetConverter converter) + { + return prestoActions.get(queryStage.getTargetCluster()).execute(statement, queryStage, converter); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoActionFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoActionFactory.java new file mode 100644 index 0000000000000..59b678b2d582e --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/RoutingPrestoActionFactory.java @@ -0,0 +1,71 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.presto.verifier.annotation.ForControl; +import com.facebook.presto.verifier.annotation.ForTest; +import com.facebook.presto.verifier.framework.SourceQuery; +import com.facebook.presto.verifier.framework.VerificationContext; +import com.facebook.presto.verifier.retry.ForClusterConnection; +import com.facebook.presto.verifier.retry.ForPresto; +import com.facebook.presto.verifier.retry.RetryConfig; + +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; + +public class RoutingPrestoActionFactory + implements PrestoActionFactory +{ + private final SqlExceptionClassifier exceptionClassifier; + private final PrestoClusterConfig controlClusterConfig; + private final PrestoClusterConfig testClusterConfig; + private final RetryConfig networkRetryConfig; + private final RetryConfig prestoRetryConfig; + + @Inject + public RoutingPrestoActionFactory( + SqlExceptionClassifier exceptionClassifier, + @ForControl PrestoClusterConfig controlClusterConfig, + @ForTest PrestoClusterConfig testClusterConfig, + @ForClusterConnection RetryConfig networkRetryConfig, + @ForPresto RetryConfig prestoRetryConfig) + { + this.exceptionClassifier = requireNonNull(exceptionClassifier, "exceptionClassifier is null"); + this.controlClusterConfig = requireNonNull(controlClusterConfig, "controlClusterConfig is null"); + this.testClusterConfig = requireNonNull(testClusterConfig, "testClusterConfig is null"); + this.networkRetryConfig = requireNonNull(networkRetryConfig, "networkRetryConfig is null"); + this.prestoRetryConfig = requireNonNull(prestoRetryConfig, "verifierConfig is null"); + } + + @Override + public RoutingPrestoAction create(SourceQuery sourceQuery, VerificationContext verificationContext) + { + return new RoutingPrestoAction( + new JdbcPrestoAction( + exceptionClassifier, + sourceQuery.getControlConfiguration(), + verificationContext, + controlClusterConfig, + networkRetryConfig, + prestoRetryConfig), + new JdbcPrestoAction( + exceptionClassifier, + sourceQuery.getTestConfiguration(), + verificationContext, + testClusterConfig, + networkRetryConfig, + prestoRetryConfig)); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/SqlExceptionClassifier.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/SqlExceptionClassifier.java similarity index 71% rename from presto-verifier/src/main/java/com/facebook/presto/verifier/framework/SqlExceptionClassifier.java rename to presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/SqlExceptionClassifier.java index c5918404ac148..094ff5ccde716 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/SqlExceptionClassifier.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/SqlExceptionClassifier.java @@ -11,14 +11,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.verifier.framework; +package com.facebook.presto.verifier.prestoaction; import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.verifier.framework.QueryException; +import com.facebook.presto.verifier.framework.QueryStage; import java.sql.SQLException; import java.util.Optional; public interface SqlExceptionClassifier { - QueryException createException(QueryOrigin queryOrigin, Optional queryStats, SQLException cause); + QueryException createException(QueryStage queryStage, Optional queryStats, SQLException cause); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/VerificationPrestoActionModule.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/VerificationPrestoActionModule.java new file mode 100644 index 0000000000000..41e7f67cc6350 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/VerificationPrestoActionModule.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.presto.verifier.annotation.ForControl; +import com.facebook.presto.verifier.annotation.ForTest; +import com.facebook.presto.verifier.retry.ForClusterConnection; +import com.facebook.presto.verifier.retry.RetryConfig; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.Singleton; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; +import static com.google.inject.Scopes.SINGLETON; +import static java.util.Objects.requireNonNull; + +public class VerificationPrestoActionModule + implements Module +{ + private final SqlExceptionClassifier exceptionClassifier; + + public VerificationPrestoActionModule(SqlExceptionClassifier exceptionClassifier) + { + this.exceptionClassifier = requireNonNull(exceptionClassifier, "exceptionClassifier is null"); + } + + @Override + public void configure(Binder binder) + { + configBinder(binder).bindConfig(PrestoClusterConfig.class, ForControl.class, "control"); + configBinder(binder).bindConfig(PrestoClusterConfig.class, ForTest.class, "test"); + + httpClientBinder(binder).bindHttpClient("test", ForTest.class); + binder.bind(PrestoActionFactory.class).to(RoutingPrestoActionFactory.class).in(SINGLETON); + binder.bind(SqlExceptionClassifier.class).toInstance(exceptionClassifier); + } + + @Provides + @ForTest + @Singleton + public NodeResourceClient providesTestHttpPrestoResourceClient( + @ForTest HttpClient httpClient, + @ForTest PrestoClusterConfig prestoClusterConfig, + @ForClusterConnection RetryConfig networkRetryConfig) + { + return new HttpNodeResourceClient(httpClient, prestoClusterConfig, networkRetryConfig); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/AbstractPrestoQueryFailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/AbstractPrestoQueryFailureResolver.java index cd00ab4e1e8bf..3796861db477e 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/AbstractPrestoQueryFailureResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/AbstractPrestoQueryFailureResolver.java @@ -15,8 +15,9 @@ import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.verifier.framework.QueryBundle; import com.facebook.presto.verifier.framework.QueryException; -import com.facebook.presto.verifier.framework.QueryOrigin; +import com.facebook.presto.verifier.framework.QueryStage; import java.util.Optional; @@ -26,25 +27,25 @@ public abstract class AbstractPrestoQueryFailureResolver implements FailureResolver { - private QueryOrigin expectedQueryOrigin; + private QueryStage expectedQueryStage; - public AbstractPrestoQueryFailureResolver(QueryOrigin expectedQueryOrigin) + public AbstractPrestoQueryFailureResolver(QueryStage expectedQueryStage) { - this.expectedQueryOrigin = requireNonNull(expectedQueryOrigin, "expectedQueryOrigin is null"); + this.expectedQueryStage = requireNonNull(expectedQueryStage, "expectedQueryOrigin is null"); } - public abstract Optional resolveTestQueryFailure(ErrorCodeSupplier errorCode, QueryStats controlQueryStats, QueryStats testQueryStats); + public abstract Optional resolveTestQueryFailure(ErrorCodeSupplier errorCode, QueryStats controlQueryStats, QueryStats testQueryStats, Optional test); @Override - public Optional resolve(QueryStats controlQueryStats, QueryException queryException) + public Optional resolve(QueryStats controlQueryStats, QueryException queryException, Optional test) { - if (!queryException.getQueryOrigin().equals(expectedQueryOrigin) || + if (queryException.getQueryStage() != expectedQueryStage || queryException.getType() != PRESTO || !queryException.getPrestoErrorCode().isPresent() || !queryException.getQueryStats().isPresent()) { return Optional.empty(); } - return resolveTestQueryFailure(queryException.getPrestoErrorCode().get(), controlQueryStats, queryException.getQueryStats().get()); + return resolveTestQueryFailure(queryException.getPrestoErrorCode().get(), controlQueryStats, queryException.getQueryStats().get(), test); } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededGlobalMemoryLimitFailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededGlobalMemoryLimitFailureResolver.java index 1e52d4ec6e2c2..3d442d0878130 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededGlobalMemoryLimitFailureResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededGlobalMemoryLimitFailureResolver.java @@ -15,23 +15,23 @@ import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.verifier.framework.QueryBundle; import java.util.Optional; import static com.facebook.presto.spi.StandardErrorCode.EXCEEDED_GLOBAL_MEMORY_LIMIT; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.TEST; -import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; +import static com.facebook.presto.verifier.framework.QueryStage.TEST_MAIN; public class ExceededGlobalMemoryLimitFailureResolver extends AbstractPrestoQueryFailureResolver { public ExceededGlobalMemoryLimitFailureResolver() { - super(forMain(TEST)); + super(TEST_MAIN); } @Override - public Optional resolveTestQueryFailure(ErrorCodeSupplier errorCode, QueryStats controlQueryStats, QueryStats testQueryStats) + public Optional resolveTestQueryFailure(ErrorCodeSupplier errorCode, QueryStats controlQueryStats, QueryStats testQueryStats, Optional test) { if (errorCode == EXCEEDED_GLOBAL_MEMORY_LIMIT && controlQueryStats.getPeakMemoryBytes() > testQueryStats.getPeakMemoryBytes()) { @@ -39,4 +39,14 @@ public Optional resolveTestQueryFailure(ErrorCodeSupplier errorCode, Que } return Optional.empty(); } + + public static class Factory + implements FailureResolverFactory + { + @Override + public FailureResolver create(FailureResolverFactoryContext context) + { + return new ExceededGlobalMemoryLimitFailureResolver(); + } + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java index ee9c0a2cbf896..6d94a0cef66b8 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/ExceededTimeLimitFailureResolver.java @@ -15,27 +15,37 @@ import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.verifier.framework.QueryBundle; import java.util.Optional; import static com.facebook.presto.spi.StandardErrorCode.EXCEEDED_TIME_LIMIT; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.TEST; -import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; +import static com.facebook.presto.verifier.framework.QueryStage.TEST_MAIN; public class ExceededTimeLimitFailureResolver extends AbstractPrestoQueryFailureResolver { public ExceededTimeLimitFailureResolver() { - super(forMain(TEST)); + super(TEST_MAIN); } @Override - public Optional resolveTestQueryFailure(ErrorCodeSupplier errorCode, QueryStats controlQueryStats, QueryStats testQueryStats) + public Optional resolveTestQueryFailure(ErrorCodeSupplier errorCode, QueryStats controlQueryStats, QueryStats testQueryStats, Optional test) { if (errorCode == EXCEEDED_TIME_LIMIT) { return Optional.of("Auto Resolved: Test cluster has less computing resource"); } return Optional.empty(); } + + public static class Factory + implements FailureResolverFactory + { + @Override + public FailureResolver create(FailureResolverFactoryContext context) + { + return new ExceededTimeLimitFailureResolver(); + } + } } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolver.java index c0dcbabfc81be..4dd1f163d59e9 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolver.java @@ -14,11 +14,12 @@ package com.facebook.presto.verifier.resolver; import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.verifier.framework.QueryBundle; import com.facebook.presto.verifier.framework.QueryException; import java.util.Optional; public interface FailureResolver { - Optional resolve(QueryStats controlQueryStats, QueryException queryException); + Optional resolve(QueryStats controlQueryStats, QueryException queryException, Optional test); } diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverConfig.java new file mode 100644 index 0000000000000..34f1d8d833786 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverConfig.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.resolver; + +import com.facebook.airlift.configuration.Config; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; + +import javax.validation.constraints.Min; + +import static java.util.concurrent.TimeUnit.MINUTES; + +public class FailureResolverConfig +{ + private boolean enabled = true; + private int maxBucketsPerWriter = 100; + private Duration clusterSizeExpiration = new Duration(30, MINUTES); + + public boolean isEnabled() + { + return enabled; + } + + @Config("failure-resolver.enabled") + public FailureResolverConfig setEnabled(boolean enabled) + { + this.enabled = enabled; + return this; + } + + @Min(1) + public int getMaxBucketsPerWriter() + { + return maxBucketsPerWriter; + } + + @Config("failure-resolver.max-buckets-per-writer") + public FailureResolverConfig setMaxBucketsPerWriter(int maxBucketsPerWriter) + { + this.maxBucketsPerWriter = maxBucketsPerWriter; + return this; + } + + @MinDuration("1m") + public Duration getClusterSizeExpiration() + { + return clusterSizeExpiration; + } + + @Config("failure-resolver.cluster-size-expiration") + public FailureResolverConfig setClusterSizeExpiration(Duration clusterSizeExpiration) + { + this.clusterSizeExpiration = clusterSizeExpiration; + return this; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverFactory.java new file mode 100644 index 0000000000000..aa4bc101cd519 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverFactory.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.resolver; + +public interface FailureResolverFactory +{ + FailureResolver create(FailureResolverFactoryContext context); +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverFactoryContext.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverFactoryContext.java new file mode 100644 index 0000000000000..79874ce3218d0 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverFactoryContext.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.resolver; + +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.verifier.prestoaction.NodeResourceClient; +import com.facebook.presto.verifier.prestoaction.PrestoAction; +import io.airlift.units.Duration; + +import static java.util.Objects.requireNonNull; + +public class FailureResolverFactoryContext +{ + private final SqlParser sqlParser; + private final PrestoAction prestoAction; + private final NodeResourceClient testResourceClient; + private final int maxBucketPerWriter; + private final Duration clusterSizeExpiration; + + public FailureResolverFactoryContext( + SqlParser sqlParser, + PrestoAction prestoAction, + NodeResourceClient testResourceClient, + FailureResolverConfig config) + { + this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + this.prestoAction = requireNonNull(prestoAction, "prestoAction is null"); + this.testResourceClient = requireNonNull(testResourceClient, "testResourceClient is null"); + this.maxBucketPerWriter = config.getMaxBucketsPerWriter(); + this.clusterSizeExpiration = config.getClusterSizeExpiration(); + } + + public SqlParser getSqlParser() + { + return sqlParser; + } + + public PrestoAction getPrestoAction() + { + return prestoAction; + } + + public NodeResourceClient getTestResourceClient() + { + return testResourceClient; + } + + public int getMaxBucketPerWriter() + { + return maxBucketPerWriter; + } + + public Duration getClusterSizeExpiration() + { + return clusterSizeExpiration; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverManager.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverManager.java new file mode 100644 index 0000000000000..c90c8b69bc750 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverManager.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.resolver; + +import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.verifier.framework.QueryBundle; +import com.facebook.presto.verifier.framework.QueryException; + +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class FailureResolverManager +{ + private final List failureResolvers; + + public FailureResolverManager(List failureResolvers) + { + this.failureResolvers = requireNonNull(failureResolvers, "failureResolvers is null"); + } + + public Optional resolve(QueryStats controlQueryStats, QueryException queryException, Optional test) + { + for (FailureResolver failureResolver : failureResolvers) { + Optional resolveMessage = failureResolver.resolve(controlQueryStats, queryException, test); + if (resolveMessage.isPresent()) { + return resolveMessage; + } + } + return Optional.empty(); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverManagerFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverManagerFactory.java new file mode 100644 index 0000000000000..d229465b93785 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverManagerFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.resolver; + +import com.google.common.collect.ImmutableList; + +import javax.inject.Inject; + +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +public class FailureResolverManagerFactory +{ + private final List failureResolverFactories; + private final boolean enabled; + + @Inject + public FailureResolverManagerFactory(List failureResolverFactories, FailureResolverConfig failureResolverConfig) + { + this.failureResolverFactories = requireNonNull(failureResolverFactories, "failureResolverFactories is null"); + this.enabled = failureResolverConfig.isEnabled(); + } + + public FailureResolverManager create(FailureResolverFactoryContext context) + { + List failureResolvers = ImmutableList.of(); + if (enabled) { + failureResolvers = failureResolverFactories.stream() + .map(factory -> factory.create(context)) + .collect(toImmutableList()); + } + return new FailureResolverManager(failureResolvers); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverModule.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverModule.java new file mode 100644 index 0000000000000..a9a6c302c6df7 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/FailureResolverModule.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.resolver; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.TypeLiteral; + +import java.util.List; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.google.inject.Scopes.SINGLETON; +import static java.util.Objects.requireNonNull; + +public class FailureResolverModule + implements Module +{ + private final List failureResolverFactories; + + public FailureResolverModule(List failureResolverFactories) + { + this.failureResolverFactories = requireNonNull(failureResolverFactories, "failureResolverFactories is null"); + } + + @Override + public void configure(Binder binder) + { + configBinder(binder).bindConfig(FailureResolverConfig.class); + + binder.bind(new TypeLiteral>() {}).toInstance(failureResolverFactories); + binder.bind(FailureResolverManagerFactory.class).in(SINGLETON); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/TooManyOpenPartitionsFailureResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/TooManyOpenPartitionsFailureResolver.java new file mode 100644 index 0000000000000..d7c300fe1e735 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/TooManyOpenPartitionsFailureResolver.java @@ -0,0 +1,118 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.resolver; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.sql.parser.ParsingOptions; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.tree.CreateTable; +import com.facebook.presto.sql.tree.LongLiteral; +import com.facebook.presto.sql.tree.Property; +import com.facebook.presto.sql.tree.ShowCreate; +import com.facebook.presto.verifier.framework.QueryBundle; +import com.facebook.presto.verifier.prestoaction.PrestoAction; + +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; + +import static com.facebook.presto.hive.HiveErrorCode.HIVE_TOO_MANY_OPEN_PARTITIONS; +import static com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY; +import static com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DOUBLE; +import static com.facebook.presto.sql.tree.ShowCreate.Type.TABLE; +import static com.facebook.presto.verifier.framework.QueryStage.DESCRIBE; +import static com.facebook.presto.verifier.framework.QueryStage.TEST_MAIN; +import static com.google.common.base.Suppliers.memoizeWithExpiration; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.Iterables.getOnlyElement; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +public class TooManyOpenPartitionsFailureResolver + extends AbstractPrestoQueryFailureResolver +{ + private static final Logger log = Logger.get(TooManyOpenPartitionsFailureResolver.class); + private static final String NODE_RESOURCE_PATH = "/v1/node"; + + private final SqlParser sqlParser; + private final PrestoAction prestoAction; + private final Supplier testClusterSizeSupplier; + private final int maxBucketPerWriter; + + public TooManyOpenPartitionsFailureResolver( + SqlParser sqlParser, + PrestoAction prestoAction, + Supplier testClusterSizeSupplier, + int maxBucketPerWriter) + { + super(TEST_MAIN); + this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + this.prestoAction = requireNonNull(prestoAction, "prestoAction is null"); + this.testClusterSizeSupplier = requireNonNull(testClusterSizeSupplier, "testClusterSizeSupplier is null"); + this.maxBucketPerWriter = maxBucketPerWriter; + } + + @Override + public Optional resolveTestQueryFailure(ErrorCodeSupplier errorCode, QueryStats controlQueryStats, QueryStats testQueryStats, Optional test) + { + if (errorCode != HIVE_TOO_MANY_OPEN_PARTITIONS || !test.isPresent()) { + return Optional.empty(); + } + + try { + ShowCreate showCreate = new ShowCreate(TABLE, test.get().getTableName()); + String showCreateResult = getOnlyElement(prestoAction.execute(showCreate, DESCRIBE, resultSet -> resultSet.getString(1)).getResults()); + CreateTable createTable = (CreateTable) sqlParser.createStatement(showCreateResult, ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build()); + List bucketCountProperty = createTable.getProperties().stream() + .filter(property -> property.getName().getValue().equals(BUCKET_COUNT_PROPERTY)) + .collect(toImmutableList()); + if (bucketCountProperty.size() != 1) { + return Optional.empty(); + } + long bucketCount = ((LongLiteral) getOnlyElement(bucketCountProperty).getValue()).getValue(); + + int testClusterSize = testClusterSizeSupplier.get(); + + if (testClusterSize * maxBucketPerWriter < bucketCount) { + return Optional.of("Auto Resolved: No enough worker on test cluster"); + } + return Optional.empty(); + } + catch (Throwable t) { + log.warn(t, "Exception when resolving HIVE_TOO_MANY_OPEN_PARTITIONS"); + return Optional.empty(); + } + } + + public static class Factory + implements FailureResolverFactory + { + @Override + public FailureResolver create(FailureResolverFactoryContext context) + { + Supplier testClusterSizeSupplier = memoizeWithExpiration( + () -> context.getTestResourceClient().getClusterSize(NODE_RESOURCE_PATH), + context.getClusterSizeExpiration().toMillis(), + MILLISECONDS); + + return new TooManyOpenPartitionsFailureResolver( + context.getSqlParser(), + context.getPrestoAction(), + testClusterSizeSupplier, + context.getMaxBucketPerWriter()); + } + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/retry/RetryConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/retry/RetryConfig.java index 59d636c1391d3..c6b8ae3161c64 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/retry/RetryConfig.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/retry/RetryConfig.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.verifier.retry; -import io.airlift.configuration.Config; +import com.facebook.airlift.configuration.Config; import io.airlift.units.Duration; import io.airlift.units.MinDuration; diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/retry/RetryDriver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/retry/RetryDriver.java index e7561e054ed37..76b79ce7b2804 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/retry/RetryDriver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/retry/RetryDriver.java @@ -13,14 +13,15 @@ */ package com.facebook.presto.verifier.retry; +import com.facebook.airlift.log.Logger; import com.facebook.presto.verifier.framework.QueryException; -import com.facebook.presto.verifier.framework.VerificationContext; -import io.airlift.log.Logger; import io.airlift.units.Duration; import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Consumer; import java.util.function.Predicate; +import static com.google.common.base.Throwables.throwIfUnchecked; import static java.lang.Math.max; import static java.lang.Math.min; import static java.lang.Math.pow; @@ -28,7 +29,7 @@ import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; -public class RetryDriver +public class RetryDriver { private static final Logger log = Logger.get(RetryDriver.class); @@ -36,28 +37,40 @@ public class RetryDriver private final Duration minBackoffDelay; private final Duration maxBackoffDelay; private final double scaleFactor; - private final Predicate retryPredicate; + private final Predicate retryPredicate; + private final Class exceptionClass; + private final Consumer exceptionCallback; public RetryDriver( RetryConfig config, - Predicate retryPredicate) + Predicate retryPredicate, + Class exceptionClass, + Consumer exceptionCallback) { this.maxAttempts = config.getMaxAttempts(); this.minBackoffDelay = requireNonNull(config.getMinBackoffDelay(), "minBackoffDelay is null"); this.maxBackoffDelay = requireNonNull(config.getMaxBackoffDelay(), "maxBackoffDelay is null"); this.scaleFactor = config.getScaleFactor(); this.retryPredicate = requireNonNull(retryPredicate, "retryPredicate is null"); + this.exceptionClass = requireNonNull(exceptionClass, "exceptionClass is null"); + this.exceptionCallback = requireNonNull(exceptionCallback, "exceptionCallback is null"); } - public V run(String callableName, VerificationContext context, RetryOperation operation) + @SuppressWarnings("unchecked") + public V run(String callableName, RetryOperation operation) { int attempt = 1; while (true) { try { return operation.run(); } - catch (QueryException qe) { - context.recordFailure(qe); + catch (Exception e) { + if (!exceptionClass.isInstance(e)) { + throwIfUnchecked(e); + throw new RuntimeException(e); + } + E qe = (E) e; + exceptionCallback.accept(qe); if (attempt >= maxAttempts || !retryPredicate.test(qe)) { throw qe; } @@ -66,13 +79,11 @@ public V run(String callableName, VerificationContext context, RetryOperatio int delayMillis = (int) min(minBackoffDelay.toMillis() * pow(scaleFactor, attempt - 1), maxBackoffDelay.toMillis()); int jitterMillis = ThreadLocalRandom.current().nextInt(max(1, (int) (delayMillis * 0.1))); log.debug( - "Failed on executing %s(%s, %s) with attempt %d. Retry after %sms. Cause: %s", + "Failed on executing %s with attempt %d. Retry after %sms. Cause: %s", callableName, - qe.getQueryOrigin().getCluster(), - qe.getQueryOrigin().getStage(), attempt - 1, delayMillis, - qe.getErrorCode()); + qe.getMessage()); try { MILLISECONDS.sleep(delayMillis + jitterMillis); diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriteConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriteConfig.java new file mode 100644 index 0000000000000..4fce470b8ee85 --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriteConfig.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.rewrite; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.presto.sql.tree.QualifiedName; +import com.google.common.base.Splitter; + +import javax.validation.constraints.NotNull; + +public class QueryRewriteConfig +{ + private QualifiedName tablePrefix = QualifiedName.of("tmp_verifier"); + + @NotNull + public QualifiedName getTablePrefix() + { + return tablePrefix; + } + + @ConfigDescription("The prefix to use for temporary shadow tables. May be fully qualified like 'tmp_catalog.tmp_schema.tmp_'") + @Config("table-prefix") + public QueryRewriteConfig setTablePrefix(String tablePrefix) + { + this.tablePrefix = tablePrefix == null ? + null : + QualifiedName.of(Splitter.on(".").splitToList(tablePrefix)); + return this; + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryRewriter.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java similarity index 83% rename from presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryRewriter.java rename to presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java index 45dc3ab817443..5f9a5b658e9db 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/QueryRewriter.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.verifier.framework; +package com.facebook.presto.verifier.rewrite; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.tree.CreateTable; @@ -26,14 +26,15 @@ import com.facebook.presto.sql.tree.Query; import com.facebook.presto.sql.tree.QuerySpecification; import com.facebook.presto.sql.tree.Statement; -import com.facebook.presto.verifier.framework.PrestoAction.ResultSetConverter; -import com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster; +import com.facebook.presto.verifier.framework.ClusterType; +import com.facebook.presto.verifier.framework.QueryBundle; +import com.facebook.presto.verifier.framework.QueryType; +import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.facebook.presto.verifier.prestoaction.PrestoAction.ResultSetConverter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.intellij.lang.annotations.Language; -import javax.inject.Inject; - import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -44,11 +45,11 @@ import java.util.stream.Stream; import static com.facebook.presto.sql.tree.LikeClause.PropertiesOption.INCLUDING; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.CONTROL; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.TEST; -import static com.facebook.presto.verifier.framework.QueryOrigin.forRewrite; +import static com.facebook.presto.verifier.framework.QueryStage.REWRITE; import static com.facebook.presto.verifier.framework.QueryType.Category.DATA_PRODUCING; import static com.facebook.presto.verifier.framework.VerifierUtil.PARSING_OPTIONS; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.lang.String.format; @@ -62,32 +63,29 @@ public class QueryRewriter private final SqlParser sqlParser; private final PrestoAction prestoAction; private final List tablePropertyOverrides; - private final Map prefixes; + private final Map prefixes; - @Inject - public QueryRewriter(SqlParser sqlParser, PrestoAction prestoAction, List tablePropertyOverrides, VerifierConfig config) + public QueryRewriter(SqlParser sqlParser, PrestoAction prestoAction, List tablePropertyOverrides, Map tablePrefixes) { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); this.prestoAction = requireNonNull(prestoAction, "prestoAction is null"); this.tablePropertyOverrides = requireNonNull(tablePropertyOverrides, "tablePropertyOverrides is null"); - this.prefixes = ImmutableMap.of( - CONTROL, config.getControlTablePrefix(), - TEST, config.getTestTablePrefix()); + this.prefixes = ImmutableMap.copyOf(tablePrefixes); } - public QueryBundle rewriteQuery(@Language("SQL") String query, TargetCluster cluster, QueryConfiguration controlConfiguration, VerificationContext context) + public QueryBundle rewriteQuery(@Language("SQL") String query, ClusterType clusterType) { + checkState(prefixes.containsKey(clusterType), "Unsupported cluster type: %s", clusterType); Statement statement = sqlParser.createStatement(query, PARSING_OPTIONS); - if (QueryType.of(statement).getCategory() != DATA_PRODUCING) { - return new QueryBundle(Optional.empty(), ImmutableList.of(), statement, ImmutableList.of()); - } + QueryType queryType = QueryType.of(statement); + checkArgument(queryType.getCategory() == DATA_PRODUCING, "Unsupported statement type: %s", queryType); - QualifiedName prefix = prefixes.get(cluster); + QualifiedName prefix = prefixes.get(clusterType); if (statement instanceof CreateTableAsSelect) { CreateTableAsSelect createTableAsSelect = (CreateTableAsSelect) statement; QualifiedName temporaryTableName = generateTemporaryTableName(Optional.of(createTableAsSelect.getName()), prefix); return new QueryBundle( - Optional.of(temporaryTableName), + temporaryTableName, ImmutableList.of(), new CreateTableAsSelect( temporaryTableName, @@ -97,14 +95,15 @@ public QueryBundle rewriteQuery(@Language("SQL") String query, TargetCluster clu createTableAsSelect.isWithData(), createTableAsSelect.getColumnAliases(), createTableAsSelect.getComment()), - ImmutableList.of(new DropTable(temporaryTableName, true))); + ImmutableList.of(new DropTable(temporaryTableName, true)), + clusterType); } if (statement instanceof Insert) { Insert insert = (Insert) statement; QualifiedName originalTableName = insert.getTarget(); QualifiedName temporaryTableName = generateTemporaryTableName(Optional.of(originalTableName), prefix); return new QueryBundle( - Optional.of(temporaryTableName), + temporaryTableName, ImmutableList.of( new CreateTable( temporaryTableName, @@ -116,12 +115,13 @@ public QueryBundle rewriteQuery(@Language("SQL") String query, TargetCluster clu temporaryTableName, insert.getColumns(), insert.getQuery()), - ImmutableList.of(new DropTable(temporaryTableName, true))); + ImmutableList.of(new DropTable(temporaryTableName, true)), + clusterType); } if (statement instanceof Query) { QualifiedName temporaryTableName = generateTemporaryTableName(Optional.empty(), prefix); return new QueryBundle( - Optional.of(temporaryTableName), + temporaryTableName, ImmutableList.of(), new CreateTableAsSelect( temporaryTableName, @@ -129,9 +129,10 @@ public QueryBundle rewriteQuery(@Language("SQL") String query, TargetCluster clu false, tablePropertyOverrides, true, - Optional.of(generateStorageColumnAliases((Query) statement, controlConfiguration, context)), + Optional.of(generateStorageColumnAliases((Query) statement)), Optional.empty()), - ImmutableList.of(new DropTable(temporaryTableName, true))); + ImmutableList.of(new DropTable(temporaryTableName, true)), + clusterType); } throw new IllegalStateException(format("Unsupported query type: %s", statement.getClass())); @@ -150,12 +151,12 @@ private QualifiedName generateTemporaryTableName(Optional origina return QualifiedName.of(parts); } - private List generateStorageColumnAliases(Query query, QueryConfiguration configuration, VerificationContext context) + private List generateStorageColumnAliases(Query query) { ImmutableList.Builder aliases = ImmutableList.builder(); Set usedAliases = new HashSet<>(); - for (String columnName : getColumnNames(query, configuration, context)) { + for (String columnName : getColumnNames(query)) { columnName = sanitizeColumnName(columnName); String alias = columnName; int postfix = 1; @@ -169,7 +170,7 @@ private List generateStorageColumnAliases(Query query, QueryConfigur return aliases.build(); } - private List getColumnNames(Query query, QueryConfiguration configuration, VerificationContext context) + private List getColumnNames(Query query) { Query zeroRowQuery; if (query.getQueryBody() instanceof QuerySpecification) { @@ -190,7 +191,7 @@ private List getColumnNames(Query query, QueryConfiguration configuratio else { zeroRowQuery = new Query(query.getWith(), query.getQueryBody(), Optional.empty(), Optional.of("0")); } - return prestoAction.execute(zeroRowQuery, configuration, forRewrite(), context, ResultSetConverter.DEFAULT).getColumnNames(); + return prestoAction.execute(zeroRowQuery, REWRITE, ResultSetConverter.DEFAULT).getColumnNames(); } private static String sanitizeColumnName(String columnName) diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriterFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriterFactory.java new file mode 100644 index 0000000000000..33f751cc7dbab --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriterFactory.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.rewrite; + +import com.facebook.presto.verifier.prestoaction.PrestoAction; + +public interface QueryRewriterFactory +{ + QueryRewriter create(PrestoAction prestoAction); +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java new file mode 100644 index 0000000000000..4cb52cc2ce7ee --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterFactory.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.rewrite; + +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.tree.Property; +import com.facebook.presto.verifier.annotation.ForControl; +import com.facebook.presto.verifier.annotation.ForTest; +import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.google.common.collect.ImmutableMap; + +import javax.inject.Inject; + +import java.util.List; + +import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; +import static com.facebook.presto.verifier.framework.ClusterType.TEST; +import static java.util.Objects.requireNonNull; + +public class VerificationQueryRewriterFactory + implements QueryRewriterFactory +{ + private final SqlParser sqlParser; + private final List tablePropertyOverrides; + private final QueryRewriteConfig controlConfig; + private final QueryRewriteConfig testConfig; + + @Inject + public VerificationQueryRewriterFactory( + SqlParser sqlParser, + List tablePropertyOverrides, + @ForControl QueryRewriteConfig controlConfig, + @ForTest QueryRewriteConfig testConfig) + { + this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + this.tablePropertyOverrides = requireNonNull(tablePropertyOverrides, "tablePropertyOverrides is null"); + this.controlConfig = requireNonNull(controlConfig, "controlConfig is null"); + this.testConfig = requireNonNull(testConfig, "testConfig is null"); + } + + @Override + public QueryRewriter create(PrestoAction prestoAction) + { + return new QueryRewriter( + sqlParser, + prestoAction, + tablePropertyOverrides, + ImmutableMap.of( + CONTROL, controlConfig.getTablePrefix(), + TEST, testConfig.getTablePrefix())); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterModule.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterModule.java new file mode 100644 index 0000000000000..9022368c3579f --- /dev/null +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/VerificationQueryRewriterModule.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.rewrite; + +import com.facebook.presto.verifier.annotation.ForControl; +import com.facebook.presto.verifier.annotation.ForTest; +import com.google.inject.Binder; +import com.google.inject.Module; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; + +public class VerificationQueryRewriterModule + implements Module +{ + @Override + public void configure(Binder binder) + { + configBinder(binder).bindConfig(QueryRewriteConfig.class, ForControl.class, "control"); + configBinder(binder).bindConfig(QueryRewriteConfig.class, ForTest.class, "test"); + + binder.bind(QueryRewriterFactory.class).to(VerificationQueryRewriterFactory.class); + } +} diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/MySqlSourceQueryConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/MySqlSourceQueryConfig.java index cdbf43577cf05..bba53faeaaacd 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/MySqlSourceQueryConfig.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/MySqlSourceQueryConfig.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.verifier.source; +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; import javax.validation.constraints.NotNull; diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/SourceQueryModule.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/SourceQueryModule.java index 9a348e0f5cd40..ca80fd899996e 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/SourceQueryModule.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/SourceQueryModule.java @@ -13,17 +13,17 @@ */ package com.facebook.presto.verifier.source; +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.facebook.presto.verifier.framework.VerifierConfig; import com.google.common.collect.ImmutableSet; import com.google.inject.Binder; -import io.airlift.configuration.AbstractConfigurationAwareModule; import java.util.Set; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; import static com.facebook.presto.verifier.source.MySqlSourceQuerySupplier.MYSQL_SOURCE_QUERY_SUPPLIER; import static com.google.common.base.Preconditions.checkArgument; import static com.google.inject.Scopes.SINGLETON; -import static io.airlift.configuration.ConfigBinder.configBinder; public class SourceQueryModule extends AbstractConfigurationAwareModule diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/StringToStringMapColumnMapper.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/StringToStringMapColumnMapper.java index 7f71b1c7206ed..17c61615d08bd 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/StringToStringMapColumnMapper.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/StringToStringMapColumnMapper.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.verifier.source; -import io.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodec; import org.jdbi.v3.core.mapper.ColumnMapper; import org.jdbi.v3.core.statement.StatementContext; @@ -21,7 +21,7 @@ import java.sql.SQLException; import java.util.Map; -import static io.airlift.json.JsonCodec.mapJsonCodec; +import static com.facebook.airlift.json.JsonCodec.mapJsonCodec; public class StringToStringMapColumnMapper implements ColumnMapper> diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java index 431b2ecfead90..58e4e0bb7bcc5 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/source/VerifierDao.java @@ -19,12 +19,31 @@ import org.jdbi.v3.sqlobject.customizer.Bind; import org.jdbi.v3.sqlobject.customizer.Define; import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; import java.util.List; @RegisterColumnMapper(StringToStringMapColumnMapper.class) public interface VerifierDao { + @SqlUpdate("CREATE TABLE verifier_queries (\n" + + " id int(11) unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT,\n" + + " suite varchar(256) NOT NULL,\n" + + " name varchar(256) DEFAULT NULL,\n" + + " control_catalog varchar(256) NOT NULL,\n" + + " control_schema varchar(256) NOT NULL,\n" + + " control_query text NOT NULL,\n" + + " control_username varchar(256) DEFAULT NULL,\n" + + " control_password varchar(256) DEFAULT NULL,\n" + + " control_session_properties text DEFAULT NULL,\n" + + " test_catalog varchar(256) NOT NULL,\n" + + " test_schema varchar(256) NOT NULL,\n" + + " test_query text NOT NULL,\n" + + " test_username varchar(256) DEFAULT NULL,\n" + + " test_password varchar(256) DEFAULT NULL,\n" + + " test_session_properties text DEFAULT NULL)") + void createVerifierQueriesTable(@Define("table_name") String tableName); + @SqlQuery("SELECT\n" + " suite,\n" + " name,\n" + @@ -33,13 +52,13 @@ public interface VerifierDao " control_schema,\n" + " control_username,\n" + " control_password,\n" + - " session_properties_json test_session_properties,\n" + + " control_session_properties,\n" + " test_query,\n" + " test_catalog,\n" + " test_schema,\n" + " test_username,\n" + " test_password,\n" + - " session_properties_json control_session_properties\n" + + " test_session_properties\n" + "FROM\n" + " \n" + "WHERE\n" + diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/TestPrestoVerifierIntegrationSmokeTest.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/TestPrestoVerifierIntegrationSmokeTest.java index 3a3a0f1273baa..ea8f89c89e832 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/TestPrestoVerifierIntegrationSmokeTest.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/TestPrestoVerifierIntegrationSmokeTest.java @@ -13,11 +13,11 @@ */ package com.facebook.presto.verifier; +import com.facebook.presto.testing.mysql.TestingMySqlServer; import com.facebook.presto.tests.StandaloneQueryRunner; import com.google.common.io.CharSink; import io.airlift.airline.Cli; import io.airlift.airline.Help; -import io.airlift.testing.mysql.TestingMySqlServer; import org.jdbi.v3.core.Handle; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; @@ -29,6 +29,7 @@ import java.io.IOException; import java.nio.file.Paths; +import static com.facebook.airlift.testing.Closeables.closeQuietly; import static com.facebook.presto.verifier.VerifierTestUtil.getHandle; import static com.facebook.presto.verifier.VerifierTestUtil.insertSourceQuery; import static com.facebook.presto.verifier.VerifierTestUtil.setupMySql; @@ -37,7 +38,6 @@ import static com.google.common.io.Files.createTempDir; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.airlift.testing.Closeables.closeQuietly; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.file.Files.createFile; @@ -76,8 +76,8 @@ public void setup() insertSourceQuery(handle, SUITE, "query_2", "SELECT * FROM table1 CROSS JOIN table2"); } - @AfterClass - public void teardown() + @AfterClass(alwaysRun = true) + public void destroy() { closeQuietly(queryRunner, mySqlServer, handle); } @@ -90,13 +90,16 @@ public void generateConfigFile() configFile = createFile(Paths.get(configDirectory.getAbsolutePath(), "config.properties")).toFile(); CharSink sink = asCharSink(configFile, UTF_8); - String gateway = queryRunner.getServer().getBaseUrl().toString().replace("http", "jdbc:presto"); + String host = queryRunner.getServer().getAddress().getHost(); + int port = queryRunner.getServer().getAddress().getPort(); jsonLogFile = Paths.get(configDirectory.getAbsolutePath(), "json.log").toFile(); humanReadableLogFile = Paths.get(configDirectory.getAbsolutePath(), "human-readable.log").toFile(); sink.write(format( "test-id=%s\n" + - "control.jdbc-url=%s\n" + - "test.jdbc-url=%s\n" + + "control.host=%s\n" + + "control.jdbc-port=%s\n" + + "test.host=%s\n" + + "test.jdbc-port=%s\n" + "control.table-prefix=local.tmp_verifier_c\n" + "test.table-prefix=local.tmp_verifier_t\n" + "source-query.database=%s\n" + @@ -107,8 +110,10 @@ public void generateConfigFile() "human-readable.log-file=%s\n" + "max-concurrency=50\n", TEST_ID, - gateway, - gateway, + host, + port, + host, + port, mySqlServer.getJdbcUrl(XDB), SUITE, jsonLogFile.getAbsolutePath(), diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java index b22dc472f75a5..3381bebda8661 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/VerifierTestUtil.java @@ -15,10 +15,12 @@ import com.facebook.presto.Session; import com.facebook.presto.plugin.memory.MemoryPlugin; +import com.facebook.presto.testing.mysql.MySqlOptions; +import com.facebook.presto.testing.mysql.TestingMySqlServer; import com.facebook.presto.tests.StandaloneQueryRunner; +import com.facebook.presto.verifier.source.MySqlSourceQueryConfig; +import com.facebook.presto.verifier.source.VerifierDao; import com.google.common.collect.ImmutableList; -import io.airlift.testing.mysql.MySqlOptions; -import io.airlift.testing.mysql.TestingMySqlServer; import io.airlift.units.Duration; import org.jdbi.v3.core.Handle; import org.jdbi.v3.core.Jdbi; @@ -55,31 +57,12 @@ public static StandaloneQueryRunner setupPresto() return queryRunner; } - public static String getJdbcUrl(StandaloneQueryRunner queryRunner) - { - return queryRunner.getServer().getBaseUrl().toString().replace("http", "jdbc:presto"); - } - public static TestingMySqlServer setupMySql() throws Exception { TestingMySqlServer mySqlServer = new TestingMySqlServer("testuser", "testpass", ImmutableList.of(XDB), MY_SQL_OPTIONS); try (Handle handle = getHandle(mySqlServer)) { - handle.execute("CREATE TABLE verifier_queries (\n" + - " id int(11) unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT,\n" + - " suite varchar(256) NOT NULL,\n" + - " name varchar(256) DEFAULT NULL,\n" + - " control_catalog varchar(256) NOT NULL,\n" + - " control_schema varchar(256) NOT NULL,\n" + - " control_query text NOT NULL,\n" + - " test_catalog varchar(256) NOT NULL,\n" + - " test_schema varchar(256) NOT NULL,\n" + - " test_query text NOT NULL,\n" + - " control_username varchar(256) NOT NULL DEFAULT 'verifier-test',\n" + - " control_password varchar(256) DEFAULT NULL,\n" + - " test_username varchar(256) NOT NULL DEFAULT 'verifier-test',\n" + - " test_password varchar(256) DEFAULT NULL,\n" + - " session_properties_json varchar(2048) DEFAULT NULL)"); + handle.attach(VerifierDao.class).createVerifierQueriesTable(new MySqlSourceQueryConfig().getTableName()); } return mySqlServer; } @@ -93,7 +76,7 @@ public static void insertSourceQuery(Handle handle, String suite, String name, S { handle.execute( "INSERT INTO verifier_queries(\n" + - " suite, name, control_catalog, control_schema, control_query, test_catalog, test_schema, test_query, control_username, test_username)\n" + + " suite, name, control_catalog, control_schema, control_query, test_catalog, test_schema, test_query)\n" + "SELECT\n" + " ?,\n" + " ?,\n" + @@ -102,9 +85,7 @@ public static void insertSourceQuery(Handle handle, String suite, String name, S " ?,\n" + " 'verifier',\n" + " 'default',\n" + - " ?,\n" + - " 'verifier_test',\n" + - " 'verifier_test'", + " ?", suite, name, query, diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java index cd32bd2391c48..b3a362530b243 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestDataVerification.java @@ -15,42 +15,62 @@ import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; +import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.tests.StandaloneQueryRunner; +import com.facebook.presto.type.TypeRegistry; import com.facebook.presto.verifier.checksum.ChecksumValidator; import com.facebook.presto.verifier.checksum.FloatingPointColumnValidator; import com.facebook.presto.verifier.checksum.OrderableArrayColumnValidator; import com.facebook.presto.verifier.checksum.SimpleColumnValidator; +import com.facebook.presto.verifier.event.DeterminismAnalysisRun; import com.facebook.presto.verifier.event.VerifierQueryEvent; import com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus; +import com.facebook.presto.verifier.prestoaction.JdbcPrestoAction; +import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.facebook.presto.verifier.prestoaction.PrestoClusterConfig; +import com.facebook.presto.verifier.prestoaction.PrestoExceptionClassifier; +import com.facebook.presto.verifier.resolver.FailureResolverManager; import com.facebook.presto.verifier.retry.RetryConfig; +import com.facebook.presto.verifier.rewrite.QueryRewriter; +import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.testng.annotations.BeforeClass; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import java.util.List; import java.util.Optional; import java.util.regex.Pattern; +import java.util.stream.IntStream; import static com.facebook.presto.sql.parser.IdentifierSymbol.AT_SIGN; import static com.facebook.presto.sql.parser.IdentifierSymbol.COLON; import static com.facebook.presto.verifier.VerifierTestUtil.CATALOG; import static com.facebook.presto.verifier.VerifierTestUtil.SCHEMA; -import static com.facebook.presto.verifier.VerifierTestUtil.getJdbcUrl; import static com.facebook.presto.verifier.VerifierTestUtil.setupPresto; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.FAILED; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SKIPPED; import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SUCCEEDED; +import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; +import static com.facebook.presto.verifier.framework.ClusterType.TEST; +import static com.facebook.presto.verifier.framework.DeterminismAnalysis.DETERMINISTIC; +import static com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_COLUMNS; import static com.facebook.presto.verifier.framework.SkippedReason.CONTROL_SETUP_QUERY_FAILED; import static com.facebook.presto.verifier.framework.SkippedReason.FAILED_BEFORE_CONTROL_QUERY; import static com.facebook.presto.verifier.framework.SkippedReason.NON_DETERMINISTIC; +import static com.facebook.presto.verifier.framework.SkippedReason.VERIFIER_LIMITATION; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.String.format; import static java.util.regex.Pattern.DOTALL; import static java.util.regex.Pattern.MULTILINE; +import static java.util.stream.Collectors.joining; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; +@Test(singleThreaded = true) public class TestDataVerification { private static final String SUITE = "test-suite"; @@ -59,11 +79,6 @@ public class TestDataVerification private static StandaloneQueryRunner queryRunner; - private VerifierConfig verifierConfig; - private PrestoAction prestoAction; - private QueryRewriter queryRewriter; - private ChecksumValidator checksumValidator; - @BeforeClass public void setupClass() throws Exception @@ -71,37 +86,46 @@ public void setupClass() queryRunner = setupPresto(); } - @BeforeMethod - public void setup() + private DataVerification createVerification(String controlQuery, String testQuery) { - String jdbcUrl = getJdbcUrl(queryRunner); + return createVerification(controlQuery, testQuery, new VerifierConfig().setTestId(TEST_ID)); + } + + private DataVerification createVerification(String controlQuery, String testQuery, VerifierConfig verifierConfig) + { + QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty()); + VerificationContext verificationContext = new VerificationContext(); RetryConfig retryConfig = new RetryConfig(); - verifierConfig = new VerifierConfig() - .setControlJdbcUrl(jdbcUrl) - .setTestJdbcUrl(jdbcUrl) - .setTestId(TEST_ID) - .setFailureResolverEnabled(false); - prestoAction = new JdbcPrestoAction( + PrestoAction prestoAction = new JdbcPrestoAction( new PrestoExceptionClassifier(ImmutableSet.of(), ImmutableSet.of()), - verifierConfig, + configuration, + verificationContext, + new PrestoClusterConfig() + .setHost(queryRunner.getServer().getAddress().getHost()) + .setJdbcPort(queryRunner.getServer().getAddress().getPort()), retryConfig, retryConfig); - queryRewriter = new QueryRewriter( + QueryRewriter queryRewriter = new QueryRewriter( new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)), prestoAction, ImmutableList.of(), - verifierConfig); - checksumValidator = new ChecksumValidator( + ImmutableMap.of(CONTROL, QualifiedName.of("tmp_verifier_c"), TEST, QualifiedName.of("tmp_verifier_t"))); + ChecksumValidator checksumValidator = new ChecksumValidator( new SimpleColumnValidator(), new FloatingPointColumnValidator(verifierConfig), new OrderableArrayColumnValidator()); - } - - private DataVerification createVerification(String controlQuery, String testQuery) - { - QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, "test-user", Optional.empty(), ImmutableMap.of()); SourceQuery sourceQuery = new SourceQuery(SUITE, NAME, controlQuery, testQuery, configuration, configuration); - return new DataVerification((verification, e) -> false, prestoAction, sourceQuery, queryRewriter, ImmutableList.of(), verifierConfig, checksumValidator); + return new DataVerification( + (verification, e) -> false, + prestoAction, + sourceQuery, + queryRewriter, + new FailureResolverManager(ImmutableList.of()), + verificationContext, + verifierConfig, + new TypeRegistry(), + checksumValidator, + new LimitQueryDeterminismAnalyzer(prestoAction, verifierConfig)); } @Test @@ -122,7 +146,7 @@ public void testSchemaMismatch() FAILED, Optional.empty(), Optional.of("SCHEMA_MISMATCH"), - Optional.of("Test state SUCCEEDED, Control state SUCCEEDED\n" + + Optional.of("Test state SUCCEEDED, Control state SUCCEEDED.\n\n" + "SCHEMA MISMATCH\n")); } @@ -136,9 +160,9 @@ public void testRowCountMismatch() assertEvent( event.get(), FAILED, - Optional.of(true), + Optional.of(DETERMINISTIC), Optional.of("ROW_COUNT_MISMATCH"), - Optional.of("Test state SUCCEEDED, Control state SUCCEEDED\n" + + Optional.of("Test state SUCCEEDED, Control state SUCCEEDED.\n\n" + "ROW COUNT MISMATCH\n" + "Control 1 rows, Test 2 rows\n")); } @@ -151,9 +175,9 @@ public void testColumnMismatch() assertEvent( event.get(), FAILED, - Optional.of(true), + Optional.of(DETERMINISTIC), Optional.of("COLUMN_MISMATCH"), - Optional.of("Test state SUCCEEDED, Control state SUCCEEDED\n" + + Optional.of("Test state SUCCEEDED, Control state SUCCEEDED.\n\n" + "COLUMN MISMATCH\n" + "Control 1 rows, Test 1 rows\n" + "Mismatched Columns:\n" + @@ -178,7 +202,8 @@ public void testRewriteFailed() SKIPPED, Optional.empty(), Optional.of("PRESTO(SYNTAX_ERROR)"), - Optional.of("Test state NOT_RUN, Control state NOT_RUN\n.*")); + Optional.of("Test state NOT_RUN, Control state NOT_RUN.\n\n" + + "REWRITE query failed on CONTROL cluster:\n.*")); } @Test @@ -192,7 +217,8 @@ public void testControlFailed() SKIPPED, Optional.empty(), Optional.of("PRESTO(SYNTAX_ERROR)"), - Optional.of("Test state NOT_RUN, Control state FAILED_TO_SETUP\n.*")); + Optional.of("Test state NOT_RUN, Control state FAILED_TO_SETUP.\n\n" + + "CONTROL SETUP query failed on CONTROL cluster:\n.*")); } @Test @@ -204,19 +230,25 @@ public void testNonDeterministic() assertEvent( event.get(), SKIPPED, - Optional.of(false), + Optional.of(NON_DETERMINISTIC_COLUMNS), Optional.of("COLUMN_MISMATCH"), - Optional.of("Test state SUCCEEDED, Control state SUCCEEDED\n" + + Optional.of("Test state SUCCEEDED, Control state SUCCEEDED.\n\n" + "COLUMN MISMATCH\n" + "Control 1 rows, Test 1 rows\n" + "Mismatched Columns:\n" + " _col0 \\(double\\): control\\(sum: .*\\) test\\(sum: 2.0\\) relative error: .*\n")); + + List runs = event.get().getDeterminismAnalysisDetails().getRuns(); + assertEquals(runs.size(), 1); + assertDeterminismAnalysisRun(runs.get(0)); } @Test public void testArrayOfRow() { - Optional event = createVerification("SELECT ARRAY[ROW(1, 'a'), ROW(2, null)]", "SELECT ARRAY[ROW(1, 'a'), ROW(2, null)]").run(); + Optional event = createVerification( + "SELECT ARRAY[ROW(1, 'a'), ROW(2, null)]", "SELECT ARRAY[ROW(1, 'a'), ROW(2, null)]", + new VerifierConfig().setTestId(TEST_ID).setMaxDeterminismAnalysisRuns(3)).run(); assertTrue(event.isPresent()); assertEvent(event.get(), SUCCEEDED, Optional.empty(), Optional.empty(), Optional.empty()); @@ -225,19 +257,42 @@ public void testArrayOfRow() assertEvent( event.get(), FAILED, - Optional.of(true), + Optional.of(DETERMINISTIC), Optional.of("COLUMN_MISMATCH"), - Optional.of("Test state SUCCEEDED, Control state SUCCEEDED\n" + + Optional.of("Test state SUCCEEDED, Control state SUCCEEDED.\n\n" + "COLUMN MISMATCH\n" + "Control 1 rows, Test 1 rows\n" + "Mismatched Columns:\n" + " _col0 \\(array\\(row\\(integer, varchar\\(1\\)\\)\\)\\): control\\(checksum: 71 b5 2f 7f 1e 9b a6 a4\\) test\\(checksum: b4 3c 7d 02 2b 14 77 12\\)\n")); + + List runs = event.get().getDeterminismAnalysisDetails().getRuns(); + assertEquals(runs.size(), 2); + assertDeterminismAnalysisRun(runs.get(0)); + assertDeterminismAnalysisRun(runs.get(1)); + } + + @Test + public void testChecksumQueryFailed() + { + List columns = IntStream.range(0, 1000).mapToObj(i -> "c" + i).collect(toImmutableList()); + queryRunner.execute(format("CREATE TABLE checksum_test (%s)", columns.stream().map(column -> column + " double").collect(joining(",")))); + + String query = format("SELECT %s FROM checksum_test", Joiner.on(",").join(columns)); + Optional event = createVerification(query, query).run(); + + assertTrue(event.isPresent()); + assertEquals(event.get().getStatus(), SKIPPED.name()); + assertEquals(event.get().getSkippedReason(), VERIFIER_LIMITATION.name()); + assertEquals(event.get().getErrorCode(), "PRESTO(COMPILER_ERROR)"); + assertNotNull(event.get().getControlQueryInfo().getChecksumQuery()); + assertNotNull(event.get().getControlQueryInfo().getChecksumQueryId()); + assertNotNull(event.get().getTestQueryInfo().getChecksumQuery()); } private void assertEvent( VerifierQueryEvent event, EventStatus expectedStatus, - Optional expectedDeterministic, + Optional expectedDeterminismAnalysis, Optional expectedErrorCode, Optional expectedErrorMessageRegex) { @@ -245,7 +300,7 @@ private void assertEvent( assertEquals(event.getTestId(), TEST_ID); assertEquals(event.getName(), NAME); assertEquals(event.getStatus(), expectedStatus.name()); - assertEquals(event.getDeterministic(), expectedDeterministic.orElse(null)); + assertEquals(event.getDeterminismAnalysis(), expectedDeterminismAnalysis.map(DeterminismAnalysis::name).orElse(null)); assertEquals(event.getErrorCode(), expectedErrorCode.orElse(null)); if (event.getErrorMessage() == null) { assertFalse(expectedErrorMessageRegex.isPresent()); @@ -255,4 +310,11 @@ private void assertEvent( assertTrue(Pattern.compile(expectedErrorMessageRegex.get(), MULTILINE + DOTALL).matcher(event.getErrorMessage()).matches()); } } + + private void assertDeterminismAnalysisRun(DeterminismAnalysisRun run) + { + assertNotNull(run.getTableName()); + assertNotNull(run.getQueryId()); + assertNotNull(run.getChecksumQueryId()); + } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestLimitQueryDeterminismAnalyzer.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestLimitQueryDeterminismAnalyzer.java new file mode 100644 index 0000000000000..ba49006a56f85 --- /dev/null +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestLimitQueryDeterminismAnalyzer.java @@ -0,0 +1,169 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.sql.parser.ParsingOptions; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.parser.SqlParserOptions; +import com.facebook.presto.sql.tree.QualifiedName; +import com.facebook.presto.sql.tree.Statement; +import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; + +import static com.facebook.presto.sql.SqlFormatter.formatSql; +import static com.facebook.presto.sql.parser.IdentifierSymbol.AT_SIGN; +import static com.facebook.presto.sql.parser.IdentifierSymbol.COLON; +import static com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DOUBLE; +import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.DETERMINISTIC; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.FAILED_DATA_CHANGED; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.NON_DETERMINISTIC; +import static com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.NOT_RUN; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestLimitQueryDeterminismAnalyzer +{ + private static class MockPrestoAction + implements PrestoAction + { + private final AtomicLong rowCount; + private Statement lastStatement; + + public MockPrestoAction(AtomicLong rowCount) + { + this.rowCount = requireNonNull(rowCount, "rowCount is null"); + } + + @Override + public QueryStats execute(Statement statement, QueryStage queryStage) + { + throw new UnsupportedOperationException(); + } + + @Override + @SuppressWarnings("unchecked") + public QueryResult execute(Statement statement, QueryStage queryStage, ResultSetConverter converter) + { + lastStatement = statement; + return new QueryResult(ImmutableList.of(rowCount.get()), ImmutableList.of("_col0"), QUERY_STATS); + } + + public Statement getLastStatement() + { + return lastStatement; + } + } + + private static final QualifiedName TABLE_NAME = QualifiedName.of("test"); + private static final long ROW_COUNT_WITH_LIMIT = 1000; + private static final QueryStats QUERY_STATS = new QueryStats("id", "", false, false, 1, 2, 3, 4, 5, 0, 7, 8, 9, 10, 11, 0, Optional.empty()); + private static final ParsingOptions PARSING_OPTIONS = ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build(); + private static final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); + + private AtomicLong rowCount = new AtomicLong(); + private MockPrestoAction prestoAction; + private LimitQueryDeterminismAnalyzer analyzer; + + @BeforeMethod + public void setup() + { + this.prestoAction = new MockPrestoAction(rowCount); + this.analyzer = new LimitQueryDeterminismAnalyzer(prestoAction, new VerifierConfig()); + } + + @Test + public void testNotRun() + { + // Unsupported statement types + assertAnalysis("CREATE TABLE test (x varchar, ds varhcar) WITH (partitioned_by = ARRAY[\"ds\"])", NOT_RUN); + assertAnalysis("SELECT * FROM source LIMIT 10", NOT_RUN); + + // Order by clause + assertAnalysis("INSERT INTO test SELECT * FROM source UNION ALL SELECT * FROM source ORDER BY 1 LIMIT 1000", NOT_RUN); + assertAnalysis("INSERT INTO test SELECT * FROM source ORDER BY 1 LIMIT 1000", NOT_RUN); + + // not outer limit clause + assertAnalysis("INSERT INTO test SELECT * FROM source UNION ALL SELECT * FROM source", NOT_RUN); + assertAnalysis("INSERT INTO test SELECT * FROM source", NOT_RUN); + assertAnalysis("INSERT INTO test SELECT * FROM (SELECT * FROM source LIMIT 1000)", NOT_RUN); + } + + @Test + public void testNonDeterministic() + { + rowCount.set(1001); + assertAnalysis("INSERT INTO test SELECT * FROM source LIMIT 1000", NON_DETERMINISTIC); + assertRowCountQuery("SELECT count(1) FROM (SELECT * FROM source)"); + + assertAnalysis("CREATE TABLE test AS (WITH f AS (select * from g) ((SELECT * FROM source UNION ALL SELECT * FROM source LIMIT 1000)))", NON_DETERMINISTIC); + assertRowCountQuery("SELECT count(1) FROM (WITH f AS (select * from g) SELECT * FROM source UNION ALL SELECT * FROM source)"); + + assertAnalysis("CREATE TABLE test AS (WITH f AS (select * from g) (SELECT * FROM source LIMIT 1000))", NON_DETERMINISTIC); + assertRowCountQuery("SELECT count(1) FROM (WITH f AS (select * from g) SELECT * FROM source)"); + } + + @Test + public void testDeterministic() + { + rowCount.set(1000); + assertAnalysis("INSERT INTO test SELECT * FROM source LIMIT 1000", DETERMINISTIC); + } + + @Test + public void testFailedDataChanged() + { + rowCount.set(999); + assertAnalysis("INSERT INTO test SELECT * FROM source LIMIT 1000", FAILED_DATA_CHANGED); + } + + private void assertAnalysis(String query, LimitQueryDeterminismAnalysis expectedAnalysis) + { + VerificationContext verificationContext = new VerificationContext(); + LimitQueryDeterminismAnalysis analysis = analyzer.analyze( + new QueryBundle( + TABLE_NAME, + ImmutableList.of(), + sqlParser.createStatement(query, PARSING_OPTIONS), + ImmutableList.of(), + CONTROL), + ROW_COUNT_WITH_LIMIT, + verificationContext); + + assertEquals(analysis, expectedAnalysis); + if (expectedAnalysis == NOT_RUN) { + assertFalse(verificationContext.getLimitQueryAnalysisQueryId().isPresent()); + } + else { + assertTrue(verificationContext.getLimitQueryAnalysisQueryId().isPresent()); + } + } + + private void assertRowCountQuery(String expectedQuery) + { + Statement expectedStatement = sqlParser.createStatement(expectedQuery, PARSING_OPTIONS); + Statement actualStatement = prestoAction.getLastStatement(); + String actualQuery = formatSql(actualStatement, Optional.empty()); + assertEquals(actualStatement, expectedStatement, format("expected:\n[%s]\nbut found:\n[%s]", formatSql(expectedStatement, Optional.empty()), actualQuery)); + } +} diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java new file mode 100644 index 0000000000000..b1fa7b5450cfb --- /dev/null +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfiguration.java @@ -0,0 +1,115 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.Optional; + +import static com.facebook.airlift.json.JsonCodec.mapJsonCodec; +import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static com.facebook.presto.verifier.framework.QueryConfigurationOverrides.SessionPropertiesOverrideStrategy.OVERRIDE; +import static com.facebook.presto.verifier.framework.QueryConfigurationOverrides.SessionPropertiesOverrideStrategy.SUBSTITUTE; + +public class TestQueryConfiguration +{ + private static final String CATALOG = "catalog"; + private static final String SCHEMA = "schema"; + private static final String USERNAME = "username"; + private static final String PASSWORD = "password"; + + private static final String CATALOG_OVERRIDE = "override_catalog"; + private static final String SCHEMA_OVERRIDE = "override_schema"; + private static final String USERNAME_OVERRIDE = "override_username"; + private static final String PASSWORD_OVERRIDE = "override_password"; + + private static final Map SESSION_PROPERTIES = ImmutableMap.of("property_1", "value_1", "property_2", "value_2"); + private static final Map SESSION_PROPERTIES_OVERRIDE = ImmutableMap.of("property_1", "value_x", "property_3", "value_3"); + private static final String SESSION_PROPERTIES_OVERRIDE_CONFIG = mapJsonCodec(String.class, String.class).toJson(SESSION_PROPERTIES_OVERRIDE); + + private static final QueryConfiguration CONFIGURATION_1 = new QueryConfiguration(CATALOG, SCHEMA, Optional.of(USERNAME), Optional.of(PASSWORD), Optional.of(SESSION_PROPERTIES)); + private static final QueryConfiguration CONFIGURATION_2 = new QueryConfiguration(CATALOG, SCHEMA, Optional.empty(), Optional.empty(), Optional.empty()); + private static final QueryConfiguration CONFIGURATION_FULL_OVERRIDE = new QueryConfiguration( + CATALOG_OVERRIDE, + SCHEMA_OVERRIDE, + Optional.of(USERNAME_OVERRIDE), + Optional.of(PASSWORD_OVERRIDE), + Optional.of(SESSION_PROPERTIES_OVERRIDE)); + + private QueryConfigurationOverridesConfig overrides; + + @BeforeMethod + public void setup() + { + overrides = new QueryConfigurationOverridesConfig() + .setCatalogOverride(CATALOG_OVERRIDE) + .setSchemaOverride(SCHEMA_OVERRIDE) + .setUsernameOverride(USERNAME_OVERRIDE) + .setPasswordOverride(PASSWORD_OVERRIDE) + .setSessionPropertiesOverride(SESSION_PROPERTIES_OVERRIDE_CONFIG); + } + + @Test + public void testEmptyOverrides() + { + assertEquals(CONFIGURATION_1.applyOverrides(new QueryConfigurationOverridesConfig()), CONFIGURATION_1); + assertEquals(CONFIGURATION_2.applyOverrides(new QueryConfigurationOverridesConfig()), CONFIGURATION_2); + } + + @Test + public void testOverrides() + { + assertEquals( + CONFIGURATION_1.applyOverrides(overrides), + new QueryConfiguration( + CATALOG_OVERRIDE, + SCHEMA_OVERRIDE, + Optional.of(USERNAME_OVERRIDE), + Optional.of(PASSWORD_OVERRIDE), + Optional.of(SESSION_PROPERTIES))); + assertEquals(CONFIGURATION_2.applyOverrides(overrides), + new QueryConfiguration( + CATALOG_OVERRIDE, + SCHEMA_OVERRIDE, + Optional.of(USERNAME_OVERRIDE), + Optional.of(PASSWORD_OVERRIDE), + Optional.empty())); + } + + @Test + public void testSessionPropertyOverride() + { + overrides.setSessionPropertiesOverrideStrategy(OVERRIDE); + assertEquals(CONFIGURATION_1.applyOverrides(overrides), CONFIGURATION_FULL_OVERRIDE); + assertEquals(CONFIGURATION_2.applyOverrides(overrides), CONFIGURATION_FULL_OVERRIDE); + } + + @Test + public void testSessionPropertySubstitute() + { + overrides.setSessionPropertiesOverrideStrategy(SUBSTITUTE); + QueryConfiguration substituted = new QueryConfiguration( + CATALOG_OVERRIDE, + SCHEMA_OVERRIDE, + Optional.of(USERNAME_OVERRIDE), + Optional.of(PASSWORD_OVERRIDE), + Optional.of(ImmutableMap.of("property_1", "value_x", "property_2", "value_2", "property_3", "value_3"))); + + assertEquals(CONFIGURATION_1.applyOverrides(overrides), substituted); + assertEquals(CONFIGURATION_2.applyOverrides(overrides), CONFIGURATION_FULL_OVERRIDE); + } +} diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfigurationOverridesConfig.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfigurationOverridesConfig.java new file mode 100644 index 0000000000000..0f0590c2c48a5 --- /dev/null +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryConfigurationOverridesConfig.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.framework; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.presto.verifier.framework.QueryConfigurationOverrides.SessionPropertiesOverrideStrategy.NO_ACTION; +import static com.facebook.presto.verifier.framework.QueryConfigurationOverrides.SessionPropertiesOverrideStrategy.SUBSTITUTE; + +public class TestQueryConfigurationOverridesConfig +{ + @Test + public void testDefault() + { + assertRecordedDefaults(recordDefaults(QueryConfigurationOverridesConfig.class) + .setCatalogOverride(null) + .setSchemaOverride(null) + .setUsernameOverride(null) + .setPasswordOverride(null) + .setSessionPropertiesOverrideStrategy(NO_ACTION) + .setSessionPropertiesOverride(null)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("catalog-override", "_catalog") + .put("schema-override", "_schema") + .put("username-override", "_username") + .put("password-override", "_password") + .put("session-properties-override-strategy", "SUBSTITUTE") + .put("session-properties-override", "{\"key\": \"value\"}") + .build(); + QueryConfigurationOverridesConfig expected = new QueryConfigurationOverridesConfig() + .setCatalogOverride("_catalog") + .setSchemaOverride("_schema") + .setUsernameOverride("_username") + .setPasswordOverride("_password") + .setSessionPropertiesOverrideStrategy(SUBSTITUTE) + .setSessionPropertiesOverride("{\"key\": \"value\"}"); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryException.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryException.java index e9f8db99ebf85..de66f353a9d44 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryException.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryException.java @@ -20,25 +20,24 @@ import java.util.Optional; import static com.facebook.presto.spi.StandardErrorCode.REMOTE_TASK_ERROR; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.CONTROL; -import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; +import static com.facebook.presto.verifier.framework.QueryStage.CONTROL_MAIN; import static org.testng.Assert.assertEquals; public class TestQueryException { - private static final QueryOrigin QUERY_ORIGIN = forMain(CONTROL); + private static final QueryStage QUERY_STAGE = CONTROL_MAIN; @Test public void testErrorCode() { assertEquals( - QueryException.forClusterConnection(new SocketTimeoutException(), QUERY_ORIGIN).getErrorCode(), + QueryException.forClusterConnection(new SocketTimeoutException(), QUERY_STAGE).getErrorCode(), "CLUSTER_CONNECTION(SocketTimeoutException)"); assertEquals( - QueryException.forPresto(new SQLException(), Optional.of(REMOTE_TASK_ERROR), false, Optional.empty(), QUERY_ORIGIN).getErrorCode(), + QueryException.forPresto(new SQLException(), Optional.of(REMOTE_TASK_ERROR), false, Optional.empty(), QUERY_STAGE).getErrorCode(), "PRESTO(REMOTE_TASK_ERROR)"); assertEquals( - QueryException.forPresto(new SQLException(), Optional.empty(), false, Optional.empty(), QUERY_ORIGIN).getErrorCode(), + QueryException.forPresto(new SQLException(), Optional.empty(), false, Optional.empty(), QUERY_STAGE).getErrorCode(), "PRESTO(UNKNOWN)"); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationContext.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationContext.java index 92acbafa055dd..c6f80e9968dda 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationContext.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationContext.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.verifier.framework; -import com.facebook.presto.verifier.event.FailureInfo; +import com.facebook.presto.verifier.event.QueryFailure; import org.testng.annotations.Test; import java.net.SocketTimeoutException; @@ -21,38 +21,42 @@ import java.util.Optional; import static com.facebook.presto.spi.StandardErrorCode.REMOTE_HOST_GONE; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.CONTROL; -import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; +import static com.facebook.presto.verifier.framework.QueryStage.CONTROL_MAIN; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; public class TestVerificationContext { - private static final QueryOrigin QUERY_ORIGIN = forMain(CONTROL); + private static final QueryStage QUERY_STAGE = CONTROL_MAIN; @Test public void testDuplicateExceptions() { VerificationContext context = new VerificationContext(); - QueryException queryException = QueryException.forPresto(new RuntimeException(), Optional.of(REMOTE_HOST_GONE), false, Optional.empty(), QUERY_ORIGIN); + QueryException queryException = QueryException.forPresto(new RuntimeException(), Optional.of(REMOTE_HOST_GONE), false, Optional.empty(), QUERY_STAGE); - context.recordFailure(queryException); - context.recordFailure(queryException); + context.addException(queryException); + context.addException(queryException); - List allFailures = context.getAllFailures(CONTROL); - assertEquals(allFailures.size(), 1); - assertEquals(allFailures.get(0).getErrorCode(), "PRESTO(REMOTE_HOST_GONE)"); + List queryFailures = context.getQueryFailures(); + assertEquals(queryFailures.size(), 1); + assertEquals(queryFailures.get(0).getErrorCode(), "PRESTO(REMOTE_HOST_GONE)"); + assertFalse(queryFailures.get(0).isRetryable()); } @Test public void testMultipleExceptions() { VerificationContext context = new VerificationContext(); - context.recordFailure(QueryException.forClusterConnection(new SocketTimeoutException(), QUERY_ORIGIN)); - context.recordFailure(QueryException.forClusterConnection(new SocketTimeoutException(), QUERY_ORIGIN)); - - List allFailures = context.getAllFailures(CONTROL); - assertEquals(allFailures.size(), 2); - assertEquals(allFailures.get(0).getErrorCode(), "CLUSTER_CONNECTION(SocketTimeoutException)"); - assertEquals(allFailures.get(1).getErrorCode(), "CLUSTER_CONNECTION(SocketTimeoutException)"); + context.addException(QueryException.forClusterConnection(new SocketTimeoutException(), QUERY_STAGE)); + context.addException(QueryException.forClusterConnection(new SocketTimeoutException(), QUERY_STAGE)); + + List queryFailures = context.getQueryFailures(); + assertEquals(queryFailures.size(), 2); + assertEquals(queryFailures.get(0).getErrorCode(), "CLUSTER_CONNECTION(SocketTimeoutException)"); + assertEquals(queryFailures.get(1).getErrorCode(), "CLUSTER_CONNECTION(SocketTimeoutException)"); + assertTrue(queryFailures.get(0).isRetryable()); + assertTrue(queryFailures.get(1).isRetryable()); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java index b430dae7f49ce..ccc1262be0879 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerificationManager.java @@ -13,29 +13,45 @@ */ package com.facebook.presto.verifier.framework; +import com.facebook.airlift.event.client.AbstractEventClient; import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; +import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Statement; +import com.facebook.presto.type.TypeRegistry; import com.facebook.presto.verifier.checksum.ChecksumValidator; import com.facebook.presto.verifier.checksum.FloatingPointColumnValidator; import com.facebook.presto.verifier.checksum.OrderableArrayColumnValidator; import com.facebook.presto.verifier.checksum.SimpleColumnValidator; -import com.facebook.presto.verifier.resolver.ExceededGlobalMemoryLimitFailureResolver; -import com.facebook.presto.verifier.resolver.ExceededTimeLimitFailureResolver; +import com.facebook.presto.verifier.event.VerifierQueryEvent; +import com.facebook.presto.verifier.prestoaction.NodeResourceClient; +import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.facebook.presto.verifier.resolver.FailureResolverConfig; +import com.facebook.presto.verifier.resolver.FailureResolverManagerFactory; +import com.facebook.presto.verifier.rewrite.QueryRewriter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import java.util.ArrayList; import java.util.List; import java.util.Optional; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.sql.parser.IdentifierSymbol.AT_SIGN; import static com.facebook.presto.sql.parser.IdentifierSymbol.COLON; +import static com.facebook.presto.verifier.event.VerifierQueryEvent.EventStatus.SKIPPED; +import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; +import static com.facebook.presto.verifier.framework.ClusterType.TEST; +import static com.facebook.presto.verifier.framework.SkippedReason.MISMATCHED_QUERY_TYPE; +import static com.facebook.presto.verifier.framework.SkippedReason.SYNTAX_ERROR; +import static com.facebook.presto.verifier.framework.SkippedReason.UNSUPPORTED_QUERY_TYPE; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import static org.testng.Assert.assertEquals; @@ -52,31 +68,54 @@ public MockPrestoAction(ErrorCodeSupplier errorCode) } @Override - public QueryStats execute( - Statement statement, - QueryConfiguration configuration, - QueryOrigin queryOrigin, - VerificationContext context) + public QueryStats execute(Statement statement, QueryStage queryStage) { - throw QueryException.forPresto(new RuntimeException(), Optional.of(errorCode), false, Optional.empty(), queryOrigin); + throw QueryException.forPresto(new RuntimeException(), Optional.of(errorCode), false, Optional.empty(), queryStage); } @Override public QueryResult execute( Statement statement, - QueryConfiguration configuration, - QueryOrigin queryOrigin, - VerificationContext context, + QueryStage queryStage, ResultSetConverter converter) { - throw QueryException.forPresto(new RuntimeException(), Optional.of(errorCode), false, Optional.empty(), queryOrigin); + throw QueryException.forPresto(new RuntimeException(), Optional.of(errorCode), false, Optional.empty(), queryStage); + } + } + + private static class MockNodeResourceClient + implements NodeResourceClient + { + @Override + public int getClusterSize(String path) + { + throw new UnsupportedOperationException(); + } + } + + private static class MockEventClient + extends AbstractEventClient + { + private final List events = new ArrayList<>(); + + @Override + protected void postEvent(T event) + { + checkArgument(event instanceof VerifierQueryEvent); + this.events.add((VerifierQueryEvent) event); + } + + public List getEvents() + { + return events; } } private static final String SUITE = "test-suite"; private static final String NAME = "test-query"; + private static final QualifiedName TABLE_PREFIX = QualifiedName.of("tmp_verifier"); private static final SqlParser SQL_PARSER = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(AT_SIGN, COLON)); - private static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration("test", "di", "user", Optional.empty(), ImmutableMap.of()); + private static final QueryConfiguration QUERY_CONFIGURATION = new QueryConfiguration("test", "di", Optional.of("user"), Optional.empty(), Optional.empty()); private static final SourceQuery SOURCE_QUERY = new SourceQuery( SUITE, NAME, @@ -86,6 +125,14 @@ public QueryResult execute( QUERY_CONFIGURATION); private static final VerifierConfig VERIFIER_CONFIG = new VerifierConfig().setTestId("test"); + private MockEventClient eventClient; + + @BeforeMethod + public void setup() + { + this.eventClient = new MockEventClient(); + } + @Test public void testFailureRequeued() { @@ -113,20 +160,65 @@ public void testFailureRequeueDisabled() assertEquals(manager.getQueriesSubmitted().get(), 1); } - private static VerificationManager getVerificationManager(List sourceQueries, PrestoAction prestoAction, VerifierConfig config) + @Test + public void testFilters() + { + List queries = ImmutableList.of( + createSourceQuery("q1", "CREATE TABLE t1 (x int)", "CREATE TABLE t1 (x int)"), + createSourceQuery("q2", "CREATE TABLE t1 (x int)", "CREATE TABLE t1 (x int)"), + createSourceQuery("q3", "CREATE TABLE t1 (x int)", "CREATE TABLE t1 (x int)"), + createSourceQuery("q4", "SHOW FUNCTIONS", "SHOW FUNCTIONS"), + createSourceQuery("q5", "SELECT * FROM t1", "INSERT INTO t2 SELECT * FROM t1"), + createSourceQuery("q6", "SELECT * FROM t1", "SELECT FROM t1")); + VerificationManager manager = getVerificationManager( + queries, + new MockPrestoAction(GENERIC_INTERNAL_ERROR), + new VerifierConfig() + .setTestId("test") + .setWhitelist("q2,q3,q4,q5,q6") + .setBlacklist("q2")); + manager.start(); + assertEquals(manager.getQueriesSubmitted().get(), 0); + + List events = eventClient.getEvents(); + assertEquals(events.size(), 4); + assertSkippedEvent(events.get(0), "q3", UNSUPPORTED_QUERY_TYPE); + assertSkippedEvent(events.get(1), "q4", UNSUPPORTED_QUERY_TYPE); + assertSkippedEvent(events.get(2), "q5", MISMATCHED_QUERY_TYPE); + assertSkippedEvent(events.get(3), "q6", SYNTAX_ERROR); + } + + private static SourceQuery createSourceQuery(String name, String controlQuery, String testQuery) + { + return new SourceQuery(SUITE, name, controlQuery, testQuery, QUERY_CONFIGURATION, QUERY_CONFIGURATION); + } + + private static void assertSkippedEvent(VerifierQueryEvent event, String name, SkippedReason skippedReason) + { + assertEquals(event.getName(), name); + assertEquals(event.getStatus(), SKIPPED.name()); + assertEquals(event.getSkippedReason(), skippedReason.name()); + } + + private VerificationManager getVerificationManager(List sourceQueries, PrestoAction prestoAction, VerifierConfig verifierConfig) { return new VerificationManager( () -> sourceQueries, new VerificationFactory( SQL_PARSER, - prestoAction, - new QueryRewriter(SQL_PARSER, prestoAction, ImmutableList.of(), config), - new ChecksumValidator(new SimpleColumnValidator(), new FloatingPointColumnValidator(config), new OrderableArrayColumnValidator()), - ImmutableList.of(new ExceededGlobalMemoryLimitFailureResolver(), new ExceededTimeLimitFailureResolver()), - config), + (sourceQuery, verificationContext) -> prestoAction, + presto -> new QueryRewriter(SQL_PARSER, presto, ImmutableList.of(), ImmutableMap.of(CONTROL, TABLE_PREFIX, TEST, TABLE_PREFIX)), + new FailureResolverManagerFactory(ImmutableList.of(), new FailureResolverConfig().setEnabled(false)), + new MockNodeResourceClient(), + new ChecksumValidator(new SimpleColumnValidator(), new FloatingPointColumnValidator(verifierConfig), new OrderableArrayColumnValidator()), + verifierConfig, + new TypeRegistry(), + new FailureResolverConfig().setEnabled(false)), SQL_PARSER, - ImmutableSet.of(), + ImmutableSet.of(eventClient), ImmutableList.of(), - config); + new QueryConfigurationOverridesConfig(), + new QueryConfigurationOverridesConfig(), + verifierConfig); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerifierConfig.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerifierConfig.java index c55d5adf089e3..bae405c84c73e 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerifierConfig.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestVerifierConfig.java @@ -14,16 +14,13 @@ package com.facebook.presto.verifier.framework; import com.google.common.collect.ImmutableMap; -import io.airlift.units.Duration; import org.testng.annotations.Test; import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MINUTES; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestVerifierConfig { @@ -31,25 +28,6 @@ public class TestVerifierConfig public void testDefault() { assertRecordedDefaults(recordDefaults(VerifierConfig.class) - .setAdditionalJdbcDriverPath(null) - .setControlJdbcDriverClass(null) - .setTestJdbcDriverClass(null) - .setControlJdbcUrl(null) - .setTestJdbcUrl(null) - .setControlTimeout(new Duration(10, MINUTES)) - .setTestTimeout(new Duration(30, MINUTES)) - .setMetadataTimeout(new Duration(3, MINUTES)) - .setChecksumTimeout(new Duration(20, MINUTES)) - .setControlTablePrefix("tmp_verifier_control") - .setTestTablePrefix("tmp_verifier_test") - .setControlCatalogOverride(null) - .setControlSchemaOverride(null) - .setControlUsernameOverride(null) - .setControlPasswordOverride(null) - .setTestCatalogOverride(null) - .setTestSchemaOverride(null) - .setTestUsernameOverride(null) - .setTestPasswordOverride(null) .setWhitelist(null) .setBlacklist(null) .setSourceQuerySupplier("mysql") @@ -62,8 +40,10 @@ public void testDefault() .setQueryRepetitions(1) .setRelativeErrorMargin(1e-4) .setAbsoluteErrorMargin(1e-12) - .setRunTearDownOnResultMismatch(false) - .setFailureResolverEnabled(true) + .setRunTeardownOnResultMismatch(false) + .setRunTeardownForDeterminismAnalysis(false) + .setMaxDeterminismAnalysisRuns(2) + .setEnableLimitQueryDeterminismAnalyzer(true) .setVerificationResubmissionLimit(2)); } @@ -71,25 +51,6 @@ public void testDefault() public void testExplicitPropertyMappings() { Map properties = new ImmutableMap.Builder() - .put("additional-jdbc-driver-path", "/path/to/file") - .put("control.jdbc-driver-class", "ControlDriver") - .put("test.jdbc-driver-class", "TestDriver") - .put("control.jdbc-url", "jdbc:presto://proxy.presto.fbinfra.net") - .put("test.jdbc-url", "jdbc:presto://proxy.presto.fbinfra.net") - .put("control.timeout", "1h") - .put("test.timeout", "2h") - .put("metadata.timeout", "3h") - .put("checksum.timeout", "4h") - .put("control.table-prefix", "local.control") - .put("test.table-prefix", "local.test") - .put("control.catalog-override", "control_catalog") - .put("control.schema-override", "control_schema") - .put("control.username-override", "control_username") - .put("control.password-override", "control_password") - .put("test.catalog-override", "test_catalog") - .put("test.schema-override", "test_schema") - .put("test.username-override", "test_username") - .put("test.password-override", "test_password") .put("whitelist", "a,b,c") .put("blacklist", "b,d,f") .put("source-query.supplier", "custom-supplier") @@ -103,29 +64,12 @@ public void testExplicitPropertyMappings() .put("relative-error-margin", "2e-5") .put("absolute-error-margin", "1e-14") .put("run-teardown-on-result-mismatch", "true") - .put("failure-resolver.enabled", "false") + .put("run-teardown-for-determinism-analysis", "true") + .put("max-determinism-analysis-runs", "3") + .put("enable-limit-query-determinism-analyzer", "false") .put("verification-resubmission.limit", "1") .build(); VerifierConfig expected = new VerifierConfig() - .setAdditionalJdbcDriverPath("/path/to/file") - .setControlJdbcDriverClass("ControlDriver") - .setTestJdbcDriverClass("TestDriver") - .setControlJdbcUrl("jdbc:presto://proxy.presto.fbinfra.net") - .setTestJdbcUrl("jdbc:presto://proxy.presto.fbinfra.net") - .setControlTimeout(new Duration(1, HOURS)) - .setTestTimeout(new Duration(2, HOURS)) - .setMetadataTimeout(new Duration(3, HOURS)) - .setChecksumTimeout(new Duration(4, HOURS)) - .setControlTablePrefix("local.control") - .setTestTablePrefix("local.test") - .setControlCatalogOverride("control_catalog") - .setControlSchemaOverride("control_schema") - .setControlUsernameOverride("control_username") - .setControlPasswordOverride("control_password") - .setTestCatalogOverride("test_catalog") - .setTestSchemaOverride("test_schema") - .setTestUsernameOverride("test_username") - .setTestPasswordOverride("test_password") .setWhitelist("a,b,c") .setBlacklist("b,d,f") .setSourceQuerySupplier("custom-supplier") @@ -138,8 +82,10 @@ public void testExplicitPropertyMappings() .setQueryRepetitions(3) .setRelativeErrorMargin(2e-5) .setAbsoluteErrorMargin(1e-14) - .setRunTearDownOnResultMismatch(true) - .setFailureResolverEnabled(false) + .setRunTeardownOnResultMismatch(true) + .setRunTeardownForDeterminismAnalysis(true) + .setMaxDeterminismAnalysisRuns(3) + .setEnableLimitQueryDeterminismAnalyzer(false) .setVerificationResubmissionLimit(1); assertFullMapping(properties, expected); diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestHttpNodeResourceClient.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestHttpNodeResourceClient.java new file mode 100644 index 0000000000000..31e2dd0b1aea4 --- /dev/null +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestHttpNodeResourceClient.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.facebook.airlift.http.client.jetty.JettyHttpClient; +import com.facebook.presto.server.testing.TestingPrestoServer; +import com.facebook.presto.verifier.retry.RetryConfig; +import org.testng.annotations.Test; + +public class TestHttpNodeResourceClient +{ + private final TestingPrestoServer server; + private final HttpNodeResourceClient client; + + public TestHttpNodeResourceClient() + throws Exception + { + server = new TestingPrestoServer(); + client = new HttpNodeResourceClient( + new JettyHttpClient(), + new PrestoClusterConfig() + .setHost(server.getAddress().getHost()) + .setHttpPort(server.getAddress().getPort()), + new RetryConfig()); + } + + @Test + public void testNodeResource() + { + client.getClusterSize("/v1/node"); + } +} diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestJdbcPrestoAction.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java similarity index 65% rename from presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestJdbcPrestoAction.java rename to presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java index a069fab8fc652..75ebaa3812948 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestJdbcPrestoAction.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestJdbcPrestoAction.java @@ -11,16 +11,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.verifier.framework; +package com.facebook.presto.verifier.prestoaction; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.tests.StandaloneQueryRunner; -import com.facebook.presto.verifier.event.FailureInfo; +import com.facebook.presto.verifier.event.QueryFailure; +import com.facebook.presto.verifier.framework.QueryConfiguration; +import com.facebook.presto.verifier.framework.QueryException; +import com.facebook.presto.verifier.framework.QueryResult; +import com.facebook.presto.verifier.framework.QueryStage; +import com.facebook.presto.verifier.framework.VerificationContext; import com.facebook.presto.verifier.retry.RetryConfig; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -35,12 +39,10 @@ import static com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DECIMAL; import static com.facebook.presto.verifier.VerifierTestUtil.CATALOG; import static com.facebook.presto.verifier.VerifierTestUtil.SCHEMA; -import static com.facebook.presto.verifier.VerifierTestUtil.getJdbcUrl; import static com.facebook.presto.verifier.VerifierTestUtil.setupPresto; +import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; import static com.facebook.presto.verifier.framework.QueryException.Type.PRESTO; -import static com.facebook.presto.verifier.framework.QueryOrigin.QueryStage.MAIN; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.CONTROL; -import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; +import static com.facebook.presto.verifier.framework.QueryStage.CONTROL_MAIN; import static com.google.common.collect.Iterables.getOnlyElement; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -48,12 +50,17 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +@Test(singleThreaded = true) public class TestJdbcPrestoAction { - private static final QueryOrigin QUERY_ORIGIN = forMain(CONTROL); + private static final QueryStage QUERY_STAGE = CONTROL_MAIN; + private static final QueryConfiguration CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty()); private static final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); + private static final ParsingOptions PARSING_OPTIONS = ParsingOptions.builder().setDecimalLiteralTreatment(AS_DECIMAL).build(); + private static StandaloneQueryRunner queryRunner; + private VerificationContext verificationContext; private PrestoAction prestoAction; @BeforeClass @@ -66,12 +73,14 @@ public void setupQueryRunner() @BeforeMethod public void setup() { - String jdbcUrl = getJdbcUrl(queryRunner); + verificationContext = new VerificationContext(); prestoAction = new JdbcPrestoAction( new PrestoExceptionClassifier(ImmutableSet.of(), ImmutableSet.of()), - new VerifierConfig() - .setControlJdbcUrl(jdbcUrl) - .setTestJdbcUrl(jdbcUrl), + CONFIGURATION, + verificationContext, + new PrestoClusterConfig() + .setHost(queryRunner.getServer().getAddress().getHost()) + .setJdbcPort(queryRunner.getServer().getAddress().getPort()), new RetryConfig(), new RetryConfig()); } @@ -81,18 +90,14 @@ public void testQuerySucceeded() { assertEquals( prestoAction.execute( - sqlParser.createStatement("SELECT 1", new ParsingOptions(AS_DECIMAL)), - new QueryConfiguration(CATALOG, SCHEMA, "user", Optional.empty(), ImmutableMap.of()), - QUERY_ORIGIN, - new VerificationContext()).getState(), + sqlParser.createStatement("SELECT 1", PARSING_OPTIONS), + QUERY_STAGE).getState(), FINISHED.name()); assertEquals( prestoAction.execute( - sqlParser.createStatement("CREATE TABLE test_table (x int)", new ParsingOptions(AS_DECIMAL)), - new QueryConfiguration(CATALOG, SCHEMA, "user", Optional.empty(), ImmutableMap.of()), - QUERY_ORIGIN, - new VerificationContext()).getState(), + sqlParser.createStatement("CREATE TABLE test_table (x int)", PARSING_OPTIONS), + QUERY_STAGE).getState(), FINISHED.name()); } @@ -100,10 +105,8 @@ public void testQuerySucceeded() public void testQuerySucceededWithConverter() { QueryResult result = prestoAction.execute( - sqlParser.createStatement("SELECT x FROM (VALUES (1), (2), (3)) t(x)", new ParsingOptions(AS_DECIMAL)), - new QueryConfiguration(CATALOG, SCHEMA, "user", Optional.empty(), ImmutableMap.of()), - QUERY_ORIGIN, - new VerificationContext(), + sqlParser.createStatement("SELECT x FROM (VALUES (1), (2), (3)) t(x)", PARSING_OPTIONS), + QUERY_STAGE, resultSet -> resultSet.getInt("x") * resultSet.getInt("x")); assertEquals(result.getQueryStats().getState(), FINISHED.name()); assertEquals(result.getResults(), ImmutableList.of(1, 4, 9)); @@ -112,13 +115,10 @@ public void testQuerySucceededWithConverter() @Test public void testQueryFailed() { - VerificationContext context = new VerificationContext(); try { prestoAction.execute( - sqlParser.createStatement("SELECT * FROM test_table", new ParsingOptions(AS_DECIMAL)), - new QueryConfiguration(CATALOG, SCHEMA, "user", Optional.empty(), ImmutableMap.of()), - QUERY_ORIGIN, - context); + sqlParser.createStatement("SELECT * FROM test_table", PARSING_OPTIONS), + QUERY_STAGE); fail("Expect QueryException"); } catch (QueryException qe) { @@ -128,10 +128,12 @@ public void testQueryFailed() assertTrue(qe.getQueryStats().isPresent()); assertEquals(qe.getQueryStats().get().getState(), FAILED.name()); - FailureInfo failureInfo = getOnlyElement(context.getAllFailures(CONTROL)); - assertEquals(failureInfo.getQueryStage(), MAIN.name()); - assertEquals(failureInfo.getErrorCode(), "PRESTO(SYNTAX_ERROR)"); - assertNotNull(failureInfo.getPrestoQueryId()); + QueryFailure queryFailure = getOnlyElement(verificationContext.getQueryFailures()); + assertEquals(queryFailure.getClusterType(), CONTROL.name()); + assertEquals(queryFailure.getQueryStage(), QUERY_STAGE.name()); + assertEquals(queryFailure.getErrorCode(), "PRESTO(SYNTAX_ERROR)"); + assertFalse(queryFailure.isRetryable()); + assertNotNull(queryFailure.getPrestoQueryId()); } } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoClusterConfig.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoClusterConfig.java new file mode 100644 index 0000000000000..7ace6fc84542b --- /dev/null +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoClusterConfig.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.prestoaction; + +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.net.URI; +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.MINUTES; + +public class TestPrestoClusterConfig +{ + @Test + public void testDefault() + { + assertRecordedDefaults(recordDefaults(PrestoClusterConfig.class) + .setHost(null) + .setJdbcPort(0) + .setHttpPort(0) + .setJdbcUrlParameters(null) + .setQueryTimeout(new Duration(60, MINUTES)) + .setMetadataTimeout(new Duration(3, MINUTES)) + .setChecksumTimeout(new Duration(30, MINUTES))); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("host", "proxy.presto.fbinfra.net") + .put("jdbc-port", "7778") + .put("http-port", "7777") + .put("jdbc-url-parameters", "{\"SSL\": false}") + .put("query-timeout", "2h") + .put("metadata-timeout", "1h") + .put("checksum-timeout", "3h") + .build(); + PrestoClusterConfig expected = new PrestoClusterConfig() + .setHost("proxy.presto.fbinfra.net") + .setJdbcPort(7778) + .setHttpPort(7777) + .setJdbcUrlParameters("{\"SSL\": false}") + .setQueryTimeout(new Duration(2, HOURS)) + .setMetadataTimeout(new Duration(1, HOURS)) + .setChecksumTimeout(new Duration(3, HOURS)); + + assertFullMapping(properties, expected); + } + + @Test + public void testJdbcUrl() + { + PrestoClusterConfig config = new PrestoClusterConfig() + .setHost("proxy.presto.fbinfra.net") + .setJdbcPort(7778) + .setJdbcUrlParameters("{\"SSL\": true, \"SSLTrustStorePath\": \"trust-store\", \"SSLKeyStorePath\": \"key-store\"}") + .setQueryTimeout(new Duration(60, MINUTES)); + assertEquals(config.getJdbcUrl(), "jdbc:presto://proxy.presto.fbinfra.net:7778?SSL=true&SSLTrustStorePath=trust-store&SSLKeyStorePath=key-store"); + } + + @Test + public void testHttpUri() + { + PrestoClusterConfig config = new PrestoClusterConfig() + .setHost("proxy.presto.fbinfra.net") + .setJdbcPort(7778) + .setHttpPort(7777) + .setQueryTimeout(new Duration(60, MINUTES)); + assertEquals(config.getHttpUri("/v1/node"), URI.create("http://proxy.presto.fbinfra.net:7777/v1/node")); + } +} diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestPrestoExceptionClassifier.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoExceptionClassifier.java similarity index 84% rename from presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestPrestoExceptionClassifier.java rename to presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoExceptionClassifier.java index a617ab74657a0..a21f032ed1c64 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestPrestoExceptionClassifier.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/prestoaction/TestPrestoExceptionClassifier.java @@ -11,11 +11,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.verifier.framework; +package com.facebook.presto.verifier.prestoaction; import com.facebook.presto.jdbc.QueryStats; import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.verifier.framework.QueryException; +import com.facebook.presto.verifier.framework.QueryStage; import com.google.common.collect.ImmutableSet; import org.testng.annotations.Test; @@ -29,7 +31,7 @@ import java.util.Optional; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_CORRUPTED_COLUMN_STATISTICS; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_CORRUPTED_COLUMN_STATISTICS; import static com.facebook.presto.spi.StandardErrorCode.EXCEEDED_TIME_LIMIT; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; import static com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE; @@ -38,12 +40,11 @@ import static com.facebook.presto.testing.assertions.Assert.assertEquals; import static com.facebook.presto.verifier.framework.QueryException.Type.CLUSTER_CONNECTION; import static com.facebook.presto.verifier.framework.QueryException.Type.PRESTO; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.CONTROL; -import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; +import static com.facebook.presto.verifier.framework.QueryStage.CONTROL_MAIN; public class TestPrestoExceptionClassifier { - private static final QueryOrigin QUERY_ORIGIN = forMain(CONTROL); + private static final QueryStage QUERY_STAGE = CONTROL_MAIN; private static final QueryStats QUERY_STATS = new QueryStats("id", "", false, false, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, Optional.empty()); private final SqlExceptionClassifier classifier = new PrestoExceptionClassifier(ImmutableSet.of(), ImmutableSet.of()); @@ -62,12 +63,12 @@ public void testNetworkException() private void testNetworkException(SQLException sqlException) { assertQueryException( - classifier.createException(QUERY_ORIGIN, Optional.empty(), sqlException), + classifier.createException(QUERY_STAGE, Optional.empty(), sqlException), CLUSTER_CONNECTION, Optional.empty(), true, Optional.empty(), - QUERY_ORIGIN); + QUERY_STAGE); } @Test @@ -87,12 +88,12 @@ private void testPrestoException(ErrorCodeSupplier errorCode, boolean expectedRe { SQLException sqlException = new SQLException("", "", errorCode.toErrorCode().getCode(), new PrestoException(errorCode, errorCode.toErrorCode().getName())); assertQueryException( - classifier.createException(QUERY_ORIGIN, Optional.of(QUERY_STATS), sqlException), + classifier.createException(QUERY_STAGE, Optional.of(QUERY_STATS), sqlException), PRESTO, Optional.of(errorCode), expectedRetryable, Optional.of(QUERY_STATS), - QUERY_ORIGIN); + QUERY_STAGE); } @Test @@ -100,12 +101,12 @@ public void testUnknownPrestoException() { SQLException sqlException = new SQLException("", "", 0xabcd_1234, new RuntimeException()); assertQueryException( - classifier.createException(QUERY_ORIGIN, Optional.of(QUERY_STATS), sqlException), + classifier.createException(QUERY_STAGE, Optional.of(QUERY_STATS), sqlException), PRESTO, Optional.empty(), false, Optional.of(QUERY_STATS), - QUERY_ORIGIN); + QUERY_STAGE); } private void assertQueryException( @@ -114,12 +115,12 @@ private void assertQueryException( Optional prestoErrorCode, boolean retryable, Optional queryStats, - QueryOrigin queryOrigin) + QueryStage queryStage) { assertEquals(queryException.getType(), type); assertEquals(queryException.getPrestoErrorCode(), prestoErrorCode); assertEquals(queryException.isRetryable(), retryable); assertEquals(queryException.getQueryStats(), queryStats); - assertEquals(queryException.getQueryOrigin(), queryOrigin); + assertEquals(queryException.getQueryStage(), queryStage); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/AbstractTestPrestoQueryFailureResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/AbstractTestPrestoQueryFailureResolver.java index 668f36f07d8ae..d159687499ffa 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/AbstractTestPrestoQueryFailureResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/AbstractTestPrestoQueryFailureResolver.java @@ -22,9 +22,8 @@ import static com.facebook.presto.spi.StandardErrorCode.EXCEEDED_GLOBAL_MEMORY_LIMIT; import static com.facebook.presto.spi.StandardErrorCode.EXCEEDED_LOCAL_MEMORY_LIMIT; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.TEST; -import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; -import static com.facebook.presto.verifier.framework.QueryOrigin.forSetup; +import static com.facebook.presto.verifier.framework.QueryStage.TEST_MAIN; +import static com.facebook.presto.verifier.framework.QueryStage.TEST_SETUP; import static java.util.Objects.requireNonNull; import static org.testng.Assert.assertFalse; @@ -51,7 +50,8 @@ public void testSetupFailure() Optional.of(EXCEEDED_GLOBAL_MEMORY_LIMIT), false, Optional.of(createQueryStats(CONTROL_CPU_TIME_MILLIS, CONTROL_PEAK_MEMORY_BYTES / 2)), - forSetup(TEST))) + TEST_SETUP), + Optional.empty()) .isPresent()); } @@ -60,7 +60,8 @@ public void testClusterConnectionFailure() { assertFalse(failureResolver.resolve( CONTROL_QUERY_STATS, - QueryException.forClusterConnection(new SocketTimeoutException(), forMain(TEST))) + QueryException.forClusterConnection(new SocketTimeoutException(), TEST_MAIN), + Optional.empty()) .isPresent()); } @@ -74,7 +75,8 @@ public void testNoQueryStats() Optional.of(EXCEEDED_GLOBAL_MEMORY_LIMIT), false, Optional.empty(), - forMain(TEST))) + TEST_MAIN), + Optional.empty()) .isPresent()); } @@ -88,7 +90,8 @@ public void testUnrelatedErrorCode() Optional.of(EXCEEDED_LOCAL_MEMORY_LIMIT), false, Optional.empty(), - forMain(TEST))) + TEST_MAIN), + Optional.empty()) .isPresent()); } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededGlobalMemoryLimitFailureResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededGlobalMemoryLimitFailureResolver.java index c009b81f1e1f2..10fe9147edb78 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededGlobalMemoryLimitFailureResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededGlobalMemoryLimitFailureResolver.java @@ -19,8 +19,7 @@ import java.util.Optional; import static com.facebook.presto.spi.StandardErrorCode.EXCEEDED_GLOBAL_MEMORY_LIMIT; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.TEST; -import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; +import static com.facebook.presto.verifier.framework.QueryStage.TEST_MAIN; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -42,7 +41,8 @@ public void testLowerControlMemory() Optional.of(EXCEEDED_GLOBAL_MEMORY_LIMIT), false, Optional.of(createQueryStats(CONTROL_CPU_TIME_MILLIS, 700L * 1024 * 1024 * 1024)), - forMain(TEST))) + TEST_MAIN), + Optional.empty()) .isPresent()); } @@ -57,7 +57,8 @@ public void testResolved() Optional.of(EXCEEDED_GLOBAL_MEMORY_LIMIT), false, Optional.of(createQueryStats(CONTROL_CPU_TIME_MILLIS, 500L * 1024 * 1024 * 1024)), - forMain(TEST))), + TEST_MAIN), + Optional.empty()), Optional.of("Auto Resolved: Control query uses more memory than test cluster limit")); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededTimeLimitFailureResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededTimeLimitFailureResolver.java index e16c710af0edd..e1f01e2345332 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededTimeLimitFailureResolver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestExceededTimeLimitFailureResolver.java @@ -19,8 +19,7 @@ import java.util.Optional; import static com.facebook.presto.spi.StandardErrorCode.EXCEEDED_TIME_LIMIT; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.TEST; -import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; +import static com.facebook.presto.verifier.framework.QueryStage.TEST_MAIN; import static org.testng.Assert.assertEquals; public class TestExceededTimeLimitFailureResolver @@ -42,7 +41,8 @@ public void testResolved() Optional.of(EXCEEDED_TIME_LIMIT), false, Optional.of(createQueryStats(CONTROL_CPU_TIME_MILLIS / 2, CONTROL_PEAK_MEMORY_BYTES)), - forMain(TEST))), + TEST_MAIN), + Optional.empty()), Optional.of("Auto Resolved: Test cluster has less computing resource")); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestFailureResolverConfig.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestFailureResolverConfig.java new file mode 100644 index 0000000000000..93020a4e27f46 --- /dev/null +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestFailureResolverConfig.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.resolver; + +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static java.util.concurrent.TimeUnit.MINUTES; + +public class TestFailureResolverConfig +{ + @Test + public void testDefault() + { + assertRecordedDefaults(recordDefaults(FailureResolverConfig.class) + .setEnabled(true) + .setMaxBucketsPerWriter(100) + .setClusterSizeExpiration(new Duration(30, MINUTES))); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("failure-resolver.enabled", "false") + .put("failure-resolver.max-buckets-per-writer", "50") + .put("failure-resolver.cluster-size-expiration", "10m") + .build(); + FailureResolverConfig expected = new FailureResolverConfig() + .setEnabled(false) + .setMaxBucketsPerWriter(50) + .setClusterSizeExpiration(new Duration(10, MINUTES)); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java new file mode 100644 index 0000000000000..810a2d62a4810 --- /dev/null +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/resolver/TestTooManyOpenPartitionsFailureResolver.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.resolver; + +import com.facebook.presto.jdbc.QueryStats; +import com.facebook.presto.sql.parser.IdentifierSymbol; +import com.facebook.presto.sql.parser.ParsingOptions; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.parser.SqlParserOptions; +import com.facebook.presto.sql.tree.QualifiedName; +import com.facebook.presto.sql.tree.Statement; +import com.facebook.presto.verifier.framework.QueryBundle; +import com.facebook.presto.verifier.framework.QueryException; +import com.facebook.presto.verifier.framework.QueryResult; +import com.facebook.presto.verifier.framework.QueryStage; +import com.facebook.presto.verifier.prestoaction.PrestoAction; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; + +import static com.facebook.presto.hive.HiveErrorCode.HIVE_TOO_MANY_OPEN_PARTITIONS; +import static com.facebook.presto.sql.parser.IdentifierSymbol.AT_SIGN; +import static com.facebook.presto.sql.parser.IdentifierSymbol.COLON; +import static com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DOUBLE; +import static com.facebook.presto.verifier.framework.ClusterType.TEST; +import static com.facebook.presto.verifier.framework.QueryStage.TEST_MAIN; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; + +@Test(singleThreaded = true) +public class TestTooManyOpenPartitionsFailureResolver + extends AbstractTestPrestoQueryFailureResolver +{ + private static class MockPrestoAction + implements PrestoAction + { + private final AtomicReference createTable; + + public MockPrestoAction(AtomicReference createTable) + { + this.createTable = requireNonNull(createTable, "createTable is null"); + } + + @Override + public QueryStats execute(Statement statement, QueryStage queryStage) + { + throw new UnsupportedOperationException(); + } + + @Override + @SuppressWarnings("unchecked") + public QueryResult execute(Statement statement, QueryStage queryStage, ResultSetConverter converter) + { + return new QueryResult( + ImmutableList.of(createTable.get()), + ImmutableList.of("Create Table"), + createQueryStats(0, 0)); + } + } + + private static final String TABLE_NAME = "test"; + private static final int MAX_BUCKETS_PER_WRITER = 100; + private static final QueryBundle TEST_BUNDLE = new QueryBundle( + QualifiedName.of(TABLE_NAME), + ImmutableList.of(), + new SqlParser(new SqlParserOptions().allowIdentifierSymbol(AT_SIGN, COLON)).createStatement( + "INSERT INTO test SELECT * FROM source", + ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build()), + ImmutableList.of(), + TEST); + private static final QueryException HIVE_TOO_MANY_OPEN_PARTITIONS_EXCEPTION = QueryException.forPresto( + new RuntimeException(), + Optional.of(HIVE_TOO_MANY_OPEN_PARTITIONS), + false, + Optional.of(createQueryStats(0, 0)), + TEST_MAIN); + + private static final AtomicReference createTable = new AtomicReference<>(); + + public TestTooManyOpenPartitionsFailureResolver() + { + super(new TooManyOpenPartitionsFailureResolver( + new SqlParser(new SqlParserOptions().allowIdentifierSymbol(IdentifierSymbol.AT_SIGN, COLON)), + new MockPrestoAction(createTable), + Suppliers.ofInstance(1), + MAX_BUCKETS_PER_WRITER)); + } + + @Test + public void testUnresolvedSufficientWorker() + { + createTable.set(format("CREATE TABLE %s (x varchar, ds varchar) WITH (partitioned_by = ARRAY[\"ds\"], bucket_count = 100)", TABLE_NAME)); + getFailureResolver().resolve(CONTROL_QUERY_STATS, HIVE_TOO_MANY_OPEN_PARTITIONS_EXCEPTION, Optional.of(TEST_BUNDLE)); + assertFalse(getFailureResolver().resolve(CONTROL_QUERY_STATS, HIVE_TOO_MANY_OPEN_PARTITIONS_EXCEPTION, Optional.of(TEST_BUNDLE)).isPresent()); + } + + @Test + public void testUnresolvedNonBucketed() + { + createTable.set(format("CREATE TABLE %s (x varchar, ds varchar) WITH (partitioned_by = ARRAY[\"ds\"])", TABLE_NAME)); + getFailureResolver().resolve(CONTROL_QUERY_STATS, HIVE_TOO_MANY_OPEN_PARTITIONS_EXCEPTION, Optional.of(TEST_BUNDLE)); + assertFalse(getFailureResolver().resolve(CONTROL_QUERY_STATS, HIVE_TOO_MANY_OPEN_PARTITIONS_EXCEPTION, Optional.of(TEST_BUNDLE)).isPresent()); + } + + @Test + public void testResolved() + { + createTable.set(format("CREATE TABLE %s (x varchar, ds varchar) WITH (partitioned_by = ARRAY[\"ds\"], bucket_count = 101)", TABLE_NAME)); + getFailureResolver().resolve(CONTROL_QUERY_STATS, HIVE_TOO_MANY_OPEN_PARTITIONS_EXCEPTION, Optional.of(TEST_BUNDLE)); + assertEquals( + getFailureResolver().resolve(CONTROL_QUERY_STATS, HIVE_TOO_MANY_OPEN_PARTITIONS_EXCEPTION, Optional.of(TEST_BUNDLE)), + Optional.of("Auto Resolved: No enough worker on test cluster")); + } +} diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/retry/TestRetryConfig.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/retry/TestRetryConfig.java index 466fe6823d36a..ae32577fc5b6c 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/retry/TestRetryConfig.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/retry/TestRetryConfig.java @@ -19,9 +19,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.NANOSECONDS; diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/retry/TestRetryDriver.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/retry/TestRetryDriver.java index d6e96cacdd429..1f37f7985a0b9 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/retry/TestRetryDriver.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/retry/TestRetryDriver.java @@ -13,25 +13,26 @@ */ package com.facebook.presto.verifier.retry; +import com.facebook.airlift.log.Logging; import com.facebook.presto.verifier.framework.QueryException; -import com.facebook.presto.verifier.framework.QueryOrigin; +import com.facebook.presto.verifier.framework.QueryStage; import com.facebook.presto.verifier.framework.VerificationContext; import com.facebook.presto.verifier.retry.RetryDriver.RetryOperation; -import io.airlift.log.Logging; import io.airlift.units.Duration; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.net.SocketTimeoutException; import java.util.Optional; +import static com.facebook.airlift.log.Level.DEBUG; import static com.facebook.presto.spi.StandardErrorCode.REMOTE_HOST_GONE; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.CONTROL; -import static com.facebook.presto.verifier.framework.QueryOrigin.forMain; -import static io.airlift.log.Level.DEBUG; +import static com.facebook.presto.verifier.framework.QueryStage.CONTROL_MAIN; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.testng.Assert.assertEquals; +@Test(singleThreaded = true) public class TestRetryDriver { private static class MockOperation @@ -63,47 +64,60 @@ public Integer run() Logging.initialize().setLevel(RetryDriver.class.getName(), DEBUG); } - private static final QueryOrigin QUERY_ORIGIN = forMain(CONTROL); - private static final QueryException RETRYABLE_EXCEPTION = QueryException.forClusterConnection(new SocketTimeoutException(), QUERY_ORIGIN); - private static final QueryException NON_RETRYABLE_EXCEPTION = QueryException.forPresto(new RuntimeException(), Optional.of(REMOTE_HOST_GONE), false, Optional.empty(), QUERY_ORIGIN); - private static final RetryDriver RETRY_DRIVER = new RetryDriver( - new RetryConfig() - .setMaxAttempts(5) - .setMinBackoffDelay(new Duration(10, MILLISECONDS)) - .setMaxBackoffDelay(new Duration(100, MILLISECONDS)) - .setScaleFactor(2), - QueryException::isRetryable); + private static final QueryStage QUERY_STAGE = CONTROL_MAIN; + private static final QueryException RETRYABLE_EXCEPTION = QueryException.forClusterConnection(new SocketTimeoutException(), QUERY_STAGE); + private static final QueryException NON_RETRYABLE_EXCEPTION = QueryException.forPresto(new RuntimeException(), Optional.of(REMOTE_HOST_GONE), false, Optional.empty(), QUERY_STAGE); + + private VerificationContext verificationContext; + private RetryDriver retryDriver; + + @BeforeMethod + public void setup() + { + verificationContext = new VerificationContext(); + retryDriver = new RetryDriver<>( + new RetryConfig() + .setMaxAttempts(5) + .setMinBackoffDelay(new Duration(10, MILLISECONDS)) + .setMaxBackoffDelay(new Duration(100, MILLISECONDS)) + .setScaleFactor(2), + QueryException::isRetryable, + QueryException.class, + verificationContext::addException); + } @Test public void testSuccess() { assertEquals( - RETRY_DRIVER.run("test", new VerificationContext(), new MockOperation(5, RETRYABLE_EXCEPTION)), + retryDriver.run("test", new MockOperation(5, RETRYABLE_EXCEPTION)), Integer.valueOf(5)); } @Test(expectedExceptions = QueryException.class) public void testMaxAttemptsExceeded() { - RETRY_DRIVER.run("test", new VerificationContext(), new MockOperation(6, RETRYABLE_EXCEPTION)); + retryDriver.run("test", new MockOperation(6, RETRYABLE_EXCEPTION)); } @Test(expectedExceptions = QueryException.class) public void testNonRetryableFailure() { - RETRY_DRIVER.run("test", new VerificationContext(), new MockOperation(3, NON_RETRYABLE_EXCEPTION)); + retryDriver.run("test", new MockOperation(3, NON_RETRYABLE_EXCEPTION)); } @Test(timeOut = 5000) public void testBackoffTimeCapped() { - RetryDriver retryDriver = new RetryDriver( + RetryDriver retryDriver = new RetryDriver<>( new RetryConfig() .setMaxAttempts(5) .setMinBackoffDelay(new Duration(10, MILLISECONDS)) .setMaxBackoffDelay(new Duration(100, MILLISECONDS)) .setScaleFactor(1000), - QueryException::isRetryable); - retryDriver.run("test", new VerificationContext(), new MockOperation(5, RETRYABLE_EXCEPTION)); + QueryException::isRetryable, + QueryException.class, + verificationContext::addException); + retryDriver.run("test", new MockOperation(5, RETRYABLE_EXCEPTION)); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriteConfig.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriteConfig.java new file mode 100644 index 0000000000000..cc53f954e5e6b --- /dev/null +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriteConfig.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.verifier.rewrite; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestQueryRewriteConfig +{ + @Test + public void testDefault() + { + assertRecordedDefaults(recordDefaults(QueryRewriteConfig.class) + .setTablePrefix("tmp_verifier")); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("table-prefix", "local.tmp") + .build(); + QueryRewriteConfig expected = new QueryRewriteConfig() + .setTablePrefix("local.tmp"); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryRewriter.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java similarity index 82% rename from presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryRewriter.java rename to presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java index af055299469b6..a84d2a5b27506 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/framework/TestQueryRewriter.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/rewrite/TestQueryRewriter.java @@ -11,16 +11,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.verifier.framework; +package com.facebook.presto.verifier.rewrite; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.tree.Identifier; import com.facebook.presto.sql.tree.LongLiteral; import com.facebook.presto.sql.tree.Property; +import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.tests.StandaloneQueryRunner; -import com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster; +import com.facebook.presto.verifier.framework.ClusterType; +import com.facebook.presto.verifier.framework.QueryBundle; +import com.facebook.presto.verifier.framework.QueryConfiguration; +import com.facebook.presto.verifier.framework.VerificationContext; +import com.facebook.presto.verifier.prestoaction.JdbcPrestoAction; +import com.facebook.presto.verifier.prestoaction.PrestoClusterConfig; +import com.facebook.presto.verifier.prestoaction.PrestoExceptionClassifier; import com.facebook.presto.verifier.retry.RetryConfig; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -38,7 +45,8 @@ import static com.facebook.presto.verifier.VerifierTestUtil.CATALOG; import static com.facebook.presto.verifier.VerifierTestUtil.SCHEMA; import static com.facebook.presto.verifier.VerifierTestUtil.setupPresto; -import static com.facebook.presto.verifier.framework.QueryOrigin.TargetCluster.CONTROL; +import static com.facebook.presto.verifier.framework.ClusterType.CONTROL; +import static com.facebook.presto.verifier.framework.ClusterType.TEST; import static com.facebook.presto.verifier.framework.VerifierUtil.PARSING_OPTIONS; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.String.format; @@ -48,8 +56,8 @@ @Test public class TestQueryRewriter { - private static final String DEFAULT_PREFIX = "local.tmp"; - private static final QueryConfiguration CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, "test_user", Optional.empty(), ImmutableMap.of()); + private static final QualifiedName DEFAULT_PREFIX = QualifiedName.of("local", "tmp"); + private static final QueryConfiguration CONFIGURATION = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty()); private static final List TABLE_PROPERTIES_OVERRIDE = ImmutableList.of(new Property(new Identifier("test_property"), new LongLiteral("21"))); private static final SqlParser sqlParser = new SqlParser(new SqlParserOptions().allowIdentifierSymbol(COLON, AT_SIGN)); @@ -136,9 +144,9 @@ public void testRewriteCreateTableAsSelect() @Test public void testTemporaryTableName() { - QueryRewriter tableNameRewriter = getQueryRewriter("tmp"); - QueryRewriter schemaRewriter = getQueryRewriter("local.tmp"); - QueryRewriter catalogRewriter = getQueryRewriter("verifier_batch.local.tmp"); + QueryRewriter tableNameRewriter = getQueryRewriter(QualifiedName.of("tmp")); + QueryRewriter schemaRewriter = getQueryRewriter(QualifiedName.of("local", "tmp")); + QueryRewriter catalogRewriter = getQueryRewriter(QualifiedName.of("verifier_batch", "local", "tmp")); @Language("SQL") String query = "INSERT INTO dest_table SELECT * FROM test_table"; assertTableName(tableNameRewriter, query, "tmp_"); @@ -164,8 +172,8 @@ private void assertShadowed( @Language("SQL") String expectedTemplates, List expectedTeardownTemplates) { - for (TargetCluster cluster : TargetCluster.values()) { - QueryBundle bundle = queryRewriter.rewriteQuery(query, cluster, CONFIGURATION, new VerificationContext()); + for (ClusterType cluster : ClusterType.values()) { + QueryBundle bundle = queryRewriter.rewriteQuery(query, cluster); String tableName = bundle.getTableName().toString(); assertTrue(tableName.startsWith(prefix + "_")); @@ -178,7 +186,7 @@ private void assertShadowed( private void assertTableName(QueryRewriter queryRewriter, @Language("SQL") String query, String expectedPrefix) { - QueryBundle bundle = queryRewriter.rewriteQuery(query, CONTROL, CONFIGURATION, new VerificationContext()); + QueryBundle bundle = queryRewriter.rewriteQuery(query, CONTROL); assertTrue(bundle.getTableName().toString().startsWith(expectedPrefix)); } @@ -190,22 +198,20 @@ private List templateToStatements(List templates, String tabl .collect(toImmutableList()); } - private QueryRewriter getQueryRewriter(String prefix) + private QueryRewriter getQueryRewriter(QualifiedName prefix) { - String gateway = queryRunner.getServer().getBaseUrl().toString().replace("http", "jdbc:presto"); - VerifierConfig config = new VerifierConfig() - .setControlJdbcUrl(gateway) - .setTestJdbcUrl(gateway) - .setControlTablePrefix(prefix) - .setTestTablePrefix(prefix); return new QueryRewriter( sqlParser, new JdbcPrestoAction( new PrestoExceptionClassifier(ImmutableSet.of(), ImmutableSet.of()), - config, + CONFIGURATION, + new VerificationContext(), + new PrestoClusterConfig() + .setHost(queryRunner.getServer().getAddress().getHost()) + .setJdbcPort(queryRunner.getServer().getAddress().getPort()), new RetryConfig(), new RetryConfig()), TABLE_PROPERTIES_OVERRIDE, - config); + ImmutableMap.of(CONTROL, prefix, TEST, prefix)); } } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestMySqlSourceQueryConfig.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestMySqlSourceQueryConfig.java index f2db57b6271b6..658d2dffba6a5 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestMySqlSourceQueryConfig.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestMySqlSourceQueryConfig.java @@ -18,9 +18,9 @@ import java.util.Map; -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; public class TestMySqlSourceQueryConfig { diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestMySqlSourceQuerySupplier.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestMySqlSourceQuerySupplier.java index 68a9a06fd8a27..2f4834beed7a8 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestMySqlSourceQuerySupplier.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestMySqlSourceQuerySupplier.java @@ -13,20 +13,20 @@ */ package com.facebook.presto.verifier.source; -import io.airlift.testing.mysql.TestingMySqlServer; +import com.facebook.presto.testing.mysql.TestingMySqlServer; import org.jdbi.v3.core.Handle; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static com.facebook.airlift.testing.Closeables.closeQuietly; import static com.facebook.presto.verifier.VerifierTestUtil.VERIFIER_QUERIES_TABLE; import static com.facebook.presto.verifier.VerifierTestUtil.XDB; import static com.facebook.presto.verifier.VerifierTestUtil.getHandle; import static com.facebook.presto.verifier.VerifierTestUtil.insertSourceQuery; import static com.facebook.presto.verifier.VerifierTestUtil.setupMySql; import static com.facebook.presto.verifier.VerifierTestUtil.truncateVerifierQueries; -import static io.airlift.testing.Closeables.closeQuietly; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -49,8 +49,8 @@ public void setup() .setTableName(VERIFIER_QUERIES_TABLE); } - @AfterClass - public void teardown() + @AfterClass(alwaysRun = true) + public void destroy() { closeQuietly(mySqlServer, handle); } diff --git a/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestSourceQueryModule.java b/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestSourceQueryModule.java index 02b15fbc24160..c9dd067224e75 100644 --- a/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestSourceQueryModule.java +++ b/presto-verifier/src/test/java/com/facebook/presto/verifier/source/TestSourceQueryModule.java @@ -13,13 +13,13 @@ */ package com.facebook.presto.verifier.source; +import com.facebook.airlift.bootstrap.Bootstrap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.inject.CreationException; import com.google.inject.Injector; import com.google.inject.Module; -import io.airlift.bootstrap.Bootstrap; import org.testng.annotations.Test; import java.util.Map; @@ -29,8 +29,6 @@ public class TestSourceQueryModule { private static final Map DEFAULT_CONFIGURATION_PROPERTIES = ImmutableMap.builder() - .put("control.jdbc-url", "test") - .put("test.jdbc-url", "test") .put("test-id", "test") .build(); diff --git a/pull_request_template.md b/pull_request_template.md index d7a279d32fd01..5a975275f0c85 100644 --- a/pull_request_template.md +++ b/pull_request_template.md @@ -1,4 +1,7 @@ -Please fill in the release notes towards the bottom of the PR description. See [README](https://github.com/prestodb/presto/blob/master/README.md#release-notes) for details. +Please make sure your submission complies with our [Development](https://github.com/prestodb/presto/wiki/Presto-Development-Guidelines#development), [Formatting](https://github.com/prestodb/presto/wiki/Presto-Development-Guidelines#formatting), and [Commit Message](https://github.com/prestodb/presto/wiki/Review-and-Commit-guidelines#commit-formatting-and-pull-requests) guidelines. + +Fill in the release notes towards the bottom of the PR description. +See [Release Notes Guidelines](https://github.com/prestodb/presto/wiki/Release-Notes-Guidelines) for details. ``` == RELEASE NOTES == From e06cb87acb7edc6e958806ba2b1034cf1ef47e90 Mon Sep 17 00:00:00 2001 From: Sreeni Viswanadha Date: Fri, 13 Dec 2019 15:33:29 -0800 Subject: [PATCH 2/2] Simplify CARDINALITY(FILTER(array,...)) ompared to zero to use none_match and any_match. --- .../presto/SystemSessionProperties.java | 9 +++ .../rule/SimplifyArrayOperationsRewriter.java | 78 +++---------------- .../presto/type/TestArrayOperators.java | 9 +++ 3 files changed, 28 insertions(+), 68 deletions(-) diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index 01bc88d7b0a50..2618a4480c35b 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -135,7 +135,10 @@ public final class SystemSessionProperties public static final String OPTIMIZE_FULL_OUTER_JOIN_WITH_COALESCE = "optimize_full_outer_join_with_coalesce"; public static final String INDEX_LOADER_TIMEOUT = "index_loader_timeout"; public static final String OPTIMIZED_REPARTITIONING_ENABLED = "optimized_repartitioning"; +<<<<<<< HEAD public static final String SIMPLIFY_ARRAY_OPERATIONS = "simplify_array_operations"; +======= +>>>>>>> bdd91af8cfac6a2d4a560f4dc92218881738f7a3 private final List> sessionProperties; @@ -665,11 +668,14 @@ public SystemSessionProperties( OPTIMIZED_REPARTITIONING_ENABLED, "Experimental: Use optimized repartitioning", featuresConfig.isOptimizedRepartitioningEnabled(), +<<<<<<< HEAD false), booleanProperty( SIMPLIFY_ARRAY_OPERATIONS, "Simplify and optimize array operations.", true, +======= +>>>>>>> bdd91af8cfac6a2d4a560f4dc92218881738f7a3 false)); } @@ -1140,8 +1146,11 @@ public static boolean isOptimizedRepartitioningEnabled(Session session) { return session.getSystemProperty(OPTIMIZED_REPARTITIONING_ENABLED, Boolean.class); } +<<<<<<< HEAD public static boolean isSimplifyArrayOperations(Session session) { return session.getSystemProperty(SIMPLIFY_ARRAY_OPERATIONS, Boolean.class); } +======= +>>>>>>> bdd91af8cfac6a2d4a560f4dc92218881738f7a3 } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyArrayOperationsRewriter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyArrayOperationsRewriter.java index 80278ca9785bd..a537560b4ffe9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyArrayOperationsRewriter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyArrayOperationsRewriter.java @@ -15,7 +15,6 @@ import com.facebook.presto.Session; import com.facebook.presto.SystemSessionProperties; -import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; import com.facebook.presto.sql.tree.Cast; import com.facebook.presto.sql.tree.ComparisonExpression; import com.facebook.presto.sql.tree.ComparisonExpression.Operator; @@ -23,15 +22,11 @@ import com.facebook.presto.sql.tree.ExpressionRewriter; import com.facebook.presto.sql.tree.ExpressionTreeRewriter; import com.facebook.presto.sql.tree.FunctionCall; -import com.facebook.presto.sql.tree.Identifier; -import com.facebook.presto.sql.tree.LambdaArgumentDeclaration; import com.facebook.presto.sql.tree.LambdaExpression; import com.facebook.presto.sql.tree.Literal; import com.facebook.presto.sql.tree.LongLiteral; import com.facebook.presto.sql.tree.NodeLocation; import com.facebook.presto.sql.tree.QualifiedName; -import com.facebook.presto.sql.tree.SearchedCaseExpression; -import com.facebook.presto.sql.tree.WhenClause; import com.google.common.collect.ImmutableList; import java.util.Optional; @@ -129,65 +124,7 @@ private Expression simplifyCardinalityOfFilter( return simplifyCardinalityOfFilterComparedToZero(cardinalityOfFilter.getLocation().orElse(new NodeLocation(0, 0)), array, operator.get(), origLambda); } - // Rewrite CARDINALITY(FILTER(arr, x -> f(x))) as REDUCE(arr, cast(0 as bigint), (s, x) -> case when f(x) then s + 1 else s, s -> s) - LambdaArgumentDeclaration origLambdaArgument = origLambda.getArguments().get(0); - Expression origLambdaBody = origLambda.getBody(); - NodeLocation location = filter.getLocation().orElse(new NodeLocation(0, 0)); - - // New lambda arguments - // TODO(viswanadha): Fix it to get a unique name that doesn't clash with existing ones. - String combineFunctionArgumentName = origLambdaArgument.getName().getValue() + "__1__"; - LambdaArgumentDeclaration combineFunctionArgument = new LambdaArgumentDeclaration(new Identifier(location, combineFunctionArgumentName, false)); - - // Final lambda arguments - String outputFunctionArgumentName = origLambdaArgument.getName().getValue() + "__2__"; - LambdaArgumentDeclaration outputFunctionArgument = new LambdaArgumentDeclaration(new Identifier(location, outputFunctionArgumentName, false)); - - ImmutableList.Builder builder = new ImmutableList.Builder(); - Expression elsePart = new Identifier(location, combineFunctionArgumentName, false); - - // New lambda body - ArithmeticBinaryExpression plus1 = - new ArithmeticBinaryExpression(location, - ArithmeticBinaryExpression.Operator.ADD, - new Identifier(location, combineFunctionArgumentName, false), - new LongLiteral(location, "1")); - - if (isComparison) { - // If it's a comparison, stop when the condition is true. So cardinality(filter(a, x -> f(x))) > 0 becomes reduce(a, 0, (s, x) ->tudligithfnithffeekfhuhrevjltijucase when s > 0 then s when f(x) then s + 1 else s, s->s > 0) - builder.add( - new WhenClause( - new ComparisonExpression( - location, - operator.get(), - new Identifier(location, combineFunctionArgumentName, false), - rhs.get()), - new Identifier(location, combineFunctionArgumentName, false))); - } - - builder.add(new WhenClause(origLambdaBody, plus1)); - LambdaExpression combineFunction = new LambdaExpression( - location, - ImmutableList.of(combineFunctionArgument, origLambdaArgument), - new SearchedCaseExpression(location, builder.build(), Optional.of(elsePart))); - - // Final argument to reduce. s -> s - Expression outputFunctionBody = new Identifier(location, outputFunctionArgumentName, false); - if (isComparison) { - // If it's a comparison, simply return the comparison - outputFunctionBody = new ComparisonExpression(cardinalityOfFilter.getLocation().orElse(new NodeLocation(0, 0)), operator.get(), outputFunctionBody, rhs.get()); - } - - LambdaExpression outputFunction = new LambdaExpression(location, ImmutableList.of(outputFunctionArgument), outputFunctionBody); - // Now make the reduce - return new FunctionCall( - location, - QualifiedName.of("reduce"), - ImmutableList.of( - array, - new Cast(location, new LongLiteral(location, "0"), "BIGINT"), - combineFunction, - outputFunction)); + return null; } @Override @@ -207,13 +144,18 @@ public Expression rewriteComparisonExpression(ComparisonExpression comparisonExp Expression right = comparisonExpression.getRight(); Operator operator = comparisonExpression.getOperator(); + Expression rewritten = null; if (isSimpleComaparison(comparisonExpression) && isCardinalityOfFilter(left) && isConstant(right)) { - return simplifyCardinalityOfFilter((FunctionCall) left, Optional.of(operator), Optional.of(right), context, treeRewriter); + rewritten = simplifyCardinalityOfFilter((FunctionCall) left, Optional.of(operator), Optional.of(right), context, treeRewriter); + } + else if (isSimpleComaparison(comparisonExpression) && isCardinalityOfFilter(right) && isConstant(left)) { + // If the left is a literal and right is cardinality, we simply normalize it to reverse the operation. + rewritten = simplifyCardinalityOfFilter((FunctionCall) left, Optional.of(operator.negate()), Optional.of(right), context, treeRewriter); } - // If the left is a literal and right is cardinality, we simply normalize it to reverse the operation. - if (isSimpleComaparison(comparisonExpression) && isCardinalityOfFilter(right) && isConstant(left)) { - return simplifyCardinalityOfFilter((FunctionCall) left, Optional.of(operator.negate()), Optional.of(right), context, treeRewriter); + if (rewritten != null) + { + return rewritten; } return treeRewriter.defaultRewrite(comparisonExpression, context); diff --git a/presto-main/src/test/java/com/facebook/presto/type/TestArrayOperators.java b/presto-main/src/test/java/com/facebook/presto/type/TestArrayOperators.java index ff704ce2008b0..71699e2a4e13c 100644 --- a/presto-main/src/test/java/com/facebook/presto/type/TestArrayOperators.java +++ b/presto-main/src/test/java/com/facebook/presto/type/TestArrayOperators.java @@ -1892,6 +1892,15 @@ private void assertArrayHashOperator(String inputArray, Type elementType, List x % 2 = 0))", BIGINT, 3L); + assertFunction("cardinality(filter(ARRAY[3,4,6,9,10], x -> x % 3 = 0)) = 3", BOOLEAN, true); + assertFunction("cardinality(filter(ARRAY[3,4,6,9,10], x -> x % 2 = 0)) > 0", BOOLEAN, true); + assertFunction("cardinality(filter(ARRAY[3,4,6,9,10], x -> x > 2)) = 0", BOOLEAN, false); + } + private static SqlDate sqlDate(String dateString) throws ParseException {